Extract JSON ingestion into a dedicated gem

CODEBASE_OVERVIEW.md

@@ -111,6 +111,7 @@ graph LR;
     rackup["rackup"];
     rake["rake"];
     webrick["webrick"];
+    elasticgraph-json_ingestion["eg-json_ingestion"];
     elasticgraph-schema_artifacts["eg-schema_artifacts"];
     graphql["graphql"];
     elasticgraph --> elasticgraph-support;
@@ -125,6 +126,7 @@ graph LR;
     elasticgraph-local --> webrick;
     elasticgraph-schema_definition --> elasticgraph-graphql;
     elasticgraph-schema_definition --> elasticgraph-indexer;
+    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
     elasticgraph-schema_definition --> elasticgraph-schema_artifacts;
     elasticgraph-schema_definition --> elasticgraph-support;
     elasticgraph-schema_definition --> graphql;
@@ -141,6 +143,7 @@ graph LR;
     class rackup externalGemCatStyle;
     class rake externalGemCatStyle;
     class webrick externalGemCatStyle;
+    class elasticgraph-json_ingestion otherEgGemStyle;
     class elasticgraph-schema_artifacts otherEgGemStyle;
     class graphql externalGemCatStyle;
     click thor href "https://rubygems.org/gems/thor" "Open on RubyGems.org" _blank;
@@ -192,12 +195,13 @@ graph LR;
     click opensearch-ruby href "https://rubygems.org/gems/opensearch-ruby" "Open on RubyGems.org" _blank;
 ```
 
-### Extensions (5 gems)
+### Extensions (6 gems)
 
 These libraries extend ElasticGraph to provide optional but commonly needed functionality.
 
 * [elasticgraph-apollo](elasticgraph-apollo/README.md): Transforms an ElasticGraph project into an Apollo subgraph.
 * [elasticgraph-health_check](elasticgraph-health_check/README.md): Provides a health check for high availability ElasticGraph deployments.
+* [elasticgraph-json_ingestion](elasticgraph-json_ingestion/README.md): JSON Schema ingestion support for ElasticGraph.
 * [elasticgraph-query_interceptor](elasticgraph-query_interceptor/README.md): Intercepts ElasticGraph datastore queries.
 * [elasticgraph-query_registry](elasticgraph-query_registry/README.md): Provides a source-controlled query registry for ElasticGraph applications.
 * [elasticgraph-warehouse](elasticgraph-warehouse/README.md): Extends ElasticGraph to support ingestion into a data warehouse.
@@ -216,6 +220,7 @@ graph LR;
     apollo-federation["apollo-federation"];
     elasticgraph-health_check["eg-health_check"];
     elasticgraph-datastore_core["eg-datastore_core"];
+    elasticgraph-json_ingestion["eg-json_ingestion"];
     elasticgraph-query_interceptor["eg-query_interceptor"];
     elasticgraph-schema_artifacts["eg-schema_artifacts"];
     elasticgraph-query_registry["eg-query_registry"];
@@ -228,6 +233,7 @@ graph LR;
     elasticgraph-health_check --> elasticgraph-datastore_core;
     elasticgraph-health_check --> elasticgraph-graphql;
     elasticgraph-health_check --> elasticgraph-support;
+    elasticgraph-json_ingestion --> elasticgraph-support;
     elasticgraph-query_interceptor --> elasticgraph-graphql;
     elasticgraph-query_interceptor --> elasticgraph-schema_artifacts;
     elasticgraph-query_registry --> elasticgraph-graphql;
@@ -242,6 +248,7 @@ graph LR;
     class apollo-federation externalGemCatStyle;
     class elasticgraph-health_check targetGemStyle;
     class elasticgraph-datastore_core otherEgGemStyle;
+    class elasticgraph-json_ingestion targetGemStyle;
     class elasticgraph-query_interceptor targetGemStyle;
     class elasticgraph-schema_artifacts otherEgGemStyle;
     class elasticgraph-query_registry targetGemStyle;

Gemfile.lock

@@ -128,6 +128,12 @@ PATH
       elasticgraph-support (= 1.1.1.pre)
       hashdiff (~> 1.2, >= 1.2.1)
 
+PATH
+  remote: elasticgraph-json_ingestion
+  specs:
+    elasticgraph-json_ingestion (1.1.1.pre)
+      elasticgraph-support (= 1.1.1.pre)
+
 PATH
   remote: elasticgraph-lambda_support
   specs:
@@ -192,6 +198,7 @@ PATH
     elasticgraph-schema_definition (1.1.1.pre)
       elasticgraph-graphql (= 1.1.1.pre)
       elasticgraph-indexer (= 1.1.1.pre)
+      elasticgraph-json_ingestion (= 1.1.1.pre)
       elasticgraph-schema_artifacts (= 1.1.1.pre)
       elasticgraph-support (= 1.1.1.pre)
       graphql (~> 2.5.22)
@@ -696,6 +703,7 @@ DEPENDENCIES
   elasticgraph-indexer (= 1.1.1.pre)!
   elasticgraph-indexer_autoscaler_lambda (= 1.1.1.pre)!
   elasticgraph-indexer_lambda (= 1.1.1.pre)!
+  elasticgraph-json_ingestion (= 1.1.1.pre)!
   elasticgraph-lambda_support (= 1.1.1.pre)!
   elasticgraph-local (= 1.1.1.pre)!
   elasticgraph-opensearch (= 1.1.1.pre)!
@@ -787,6 +795,7 @@ CHECKSUMS
   elasticgraph-indexer (1.1.1.pre)
   elasticgraph-indexer_autoscaler_lambda (1.1.1.pre)
   elasticgraph-indexer_lambda (1.1.1.pre)
+  elasticgraph-json_ingestion (1.1.1.pre)
   elasticgraph-lambda_support (1.1.1.pre)
   elasticgraph-local (1.1.1.pre)
   elasticgraph-opensearch (1.1.1.pre)

config/docker_demo/Dockerfile

@@ -16,6 +16,7 @@ COPY elasticgraph-datastore_core elasticgraph-datastore_core/
 COPY elasticgraph-graphiql elasticgraph-graphiql/
 COPY elasticgraph-graphql elasticgraph-graphql/
 COPY elasticgraph-indexer elasticgraph-indexer/
+COPY elasticgraph-json_ingestion elasticgraph-json_ingestion/
 COPY elasticgraph-local elasticgraph-local/
 COPY elasticgraph-opensearch elasticgraph-opensearch/
 COPY elasticgraph-query_registry elasticgraph-query_registry/

elasticgraph-apollo/apollo_tests_implementation/Dockerfile

@@ -14,6 +14,7 @@ COPY elasticgraph-elasticsearch /web/elasticgraph-elasticsearch
 COPY elasticgraph-graphiql /web/elasticgraph-graphiql
 COPY elasticgraph-graphql /web/elasticgraph-graphql
 COPY elasticgraph-indexer /web/elasticgraph-indexer
+COPY elasticgraph-json_ingestion /web/elasticgraph-json_ingestion
 COPY elasticgraph-rack /web/elasticgraph-rack
 COPY elasticgraph-schema_artifacts /web/elasticgraph-schema_artifacts
 COPY elasticgraph-schema_definition /web/elasticgraph-schema_definition

elasticgraph-apollo/apollo_tests_implementation/Gemfile

@@ -16,6 +16,7 @@ source "https://rubygems.org"
   graphiql
   graphql
   indexer
+  json_ingestion
   rack
   schema_artifacts
   schema_definition

elasticgraph-json_ingestion/.rspec

@@ -0,0 +1 @@
+../spec_support/subdir_dot_rspec

elasticgraph-json_ingestion/.yardopts

@@ -0,0 +1 @@
+../config/site/yardopts

elasticgraph-json_ingestion/Gemfile

@@ -0,0 +1 @@
+../Gemfile

elasticgraph-json_ingestion/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 - 2026 Block, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

elasticgraph-json_ingestion/README.md

@@ -0,0 +1,22 @@
+# ElasticGraph::JSONIngestion
+
+JSON Schema ingestion support for ElasticGraph.
+
+This gem contains ElasticGraph's JSON Schema generation and validation support for ingestion.
+
+## Dependency Diagram
+
+```mermaid
+graph LR;
+    classDef targetGemStyle fill:#FADBD8,stroke:#EC7063,color:#000,stroke-width:2px;
+    classDef otherEgGemStyle fill:#A9DFBF,stroke:#2ECC71,color:#000;
+    classDef externalGemStyle fill:#E0EFFF,stroke:#70A1D7,color:#2980B9;
+    elasticgraph-json_ingestion["elasticgraph-json_ingestion"];
+    class elasticgraph-json_ingestion targetGemStyle;
+    elasticgraph-support["elasticgraph-support"];
+    elasticgraph-json_ingestion --> elasticgraph-support;
+    class elasticgraph-support otherEgGemStyle;
+    elasticgraph-schema_definition["elasticgraph-schema_definition"];
+    elasticgraph-schema_definition --> elasticgraph-json_ingestion;
+    class elasticgraph-schema_definition otherEgGemStyle;
+```

elasticgraph-json_ingestion/elasticgraph-json_ingestion.gemspec

@@ -0,0 +1,41 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require_relative "../elasticgraph-support/lib/elastic_graph/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "elasticgraph-json_ingestion"
+  spec.version = ElasticGraph::VERSION
+  spec.authors = ["Josh Wilson", "Myron Marston", "Block Engineering"]
+  spec.email = ["joshuaw@squareup.com"]
+  spec.homepage = "https://block.github.io/elasticgraph/"
+  spec.license = "MIT"
+  spec.summary = "JSON Schema ingestion support for ElasticGraph."
+
+  spec.metadata = {
+    "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
+    "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
+    "documentation_uri" => "https://block.github.io/elasticgraph/api-docs/v#{ElasticGraph::VERSION}/",
+    "homepage_uri" => "https://block.github.io/elasticgraph/",
+    "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
+    "gem_category" => "extension"
+  }
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:test|spec|features|sig)/|\.(?:git|travis|circleci)|appveyor)})
+    end - [".rspec", "Gemfile", ".yardopts"]
+  end
+
+  spec.required_ruby_version = [">= 3.4", "< 4.1"]
+
+  # This extension is loaded by `elasticgraph-schema_definition` at schema-definition time, so we intentionally
+  # avoid a runtime dependency here to keep the dependency direction acyclic across gems.
+  spec.add_development_dependency "elasticgraph-schema_definition", ElasticGraph::VERSION
+  spec.add_dependency "elasticgraph-support", ElasticGraph::VERSION
+end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion.rb

@@ -0,0 +1,13 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module ElasticGraph
+  # JSON Schema ingestion support for ElasticGraph.
+  module JSONIngestion
+  end
+end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/api_extension.rb

@@ -0,0 +1,91 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+require "elastic_graph/json_ingestion/schema_definition/factory_extension"
+
+module ElasticGraph
+  module JSONIngestion
+    # Namespace for all JSON Schema schema definition support.
+    #
+    # {SchemaDefinition::APIExtension} is the primary entry point and should be used as a schema definition extension module.
+    module SchemaDefinition
+      # Module designed to be extended onto an {ElasticGraph::SchemaDefinition::API} instance
+      # to add JSON Schema ingestion support.
+      module APIExtension
+        # Wires up the factory extension when this module is extended onto an API instance.
+        #
+        # @param api [ElasticGraph::SchemaDefinition::API] the API instance to extend
+        # @return [void]
+        # @api private
+        def self.extended(api)
+          api.state.ingestion_serializer_state.tap do |state|
+            state[:allow_omitted_json_schema_fields] = false unless state.key?(:allow_omitted_json_schema_fields)
+            state[:allow_extra_json_schema_fields] = true unless state.key?(:allow_extra_json_schema_fields)
+            state[:reserved_type_names] = (state[:reserved_type_names] || ::Set.new).merge([EVENT_ENVELOPE_JSON_SCHEMA_NAME])
+          end
+
+          api.factory.extend FactoryExtension
+        end
+
+        # Defines the version number of the current JSON schema. Importantly, every time a change is made that impacts the JSON schema
+        # artifact, the version number must be incremented to ensure that each different version of the JSON schema is identified by a unique
+        # version number. The publisher will then include this version number in published events to identify the version of the schema it
+        # was using. This avoids the need to deploy the publisher and ElasticGraph indexer at the same time to keep them in sync.
+        #
+        # @note While this is an important part of how ElasticGraph is designed to support schema evolution, it can be annoying constantly
+        #   have to increment this while rapidly changing the schema during prototyping. You can disable the requirement to increment this
+        #   on every JSON schema change by setting `enforce_json_schema_version` to `false` in your `Rakefile`.
+        #
+        # @param version [Integer] current version number of the JSON schema artifact
+        # @return [void]
+        # @see Local::RakeTasks#enforce_json_schema_version
+        def json_schema_version(version)
+          if !version.is_a?(Integer) || version < 1
+            raise Errors::SchemaError, "`json_schema_version` must be a positive integer. Specified version: #{version}"
+          end
+
+          if @state.ingestion_serializer_state[:json_schema_version]
+            raise Errors::SchemaError, "`json_schema_version` can only be set once on a schema. Previously-set version: #{@state.ingestion_serializer_state[:json_schema_version]}"
+          end
+
+          @state.ingestion_serializer_state[:json_schema_version] = version
+          @state.ingestion_serializer_state[:json_schema_version_setter_location] = caller_locations(1, 1).to_a.first
+          nil
+        end
+
+        # Defines strictness of the JSON schema validation. By default, the JSON schema will require all fields to be provided by the
+        # publisher (but they can be nullable) and will ignore extra fields that are not defined in the schema. Use this method to
+        # configure this behavior.
+        #
+        # @param allow_omitted_fields [bool] Whether nullable fields can be omitted from indexing events.
+        # @param allow_extra_fields [bool] Whether extra fields (e.g. beyond fields defined in the schema) can be included in indexing events.
+        # @return [void]
+        #
+        # @note If you allow both omitted fields and extra fields, ElasticGraph's JSON schema validation will allow (and ignore) misspelled
+        #   field names in indexing events. For example, if the ElasticGraph schema has a nullable field named `parentId` but the publisher
+        #   accidentally provides it as `parent_id`, ElasticGraph would happily ignore the `parent_id` field entirely, because `parentId`
+        #   is allowed to be omitted and `parent_id` would be treated as an extra field. Therefore, we recommend that you only set one of
+        #   these to `true` (or none).
+        def json_schema_strictness(allow_omitted_fields: false, allow_extra_fields: true)
+          unless [true, false].include?(allow_omitted_fields)
+            raise Errors::SchemaError, "`allow_omitted_fields` must be true or false"
+          end
+
+          unless [true, false].include?(allow_extra_fields)
+            raise Errors::SchemaError, "`allow_extra_fields` must be true or false"
+          end
+
+          @state.ingestion_serializer_state[:allow_omitted_json_schema_fields] = allow_omitted_fields
+          @state.ingestion_serializer_state[:allow_extra_json_schema_fields] = allow_extra_fields
+          nil
+        end
+      end
+    end
+  end
+end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/schema_definition/built_in_types_extension.rb

@@ -0,0 +1,60 @@
+# Copyright 2024 - 2026 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+require "elastic_graph/constants"
+require "elastic_graph/graphql/scalar_coercion_adapters/valid_time_zones"
+
+module ElasticGraph
+  module JSONIngestion
+    module SchemaDefinition
+      # Extends ElasticGraph's built-in types with JSON ingestion configuration.
+      module BuiltInTypesExtension
+        # JSON Schema defaults applied to ElasticGraph's built-in scalar types.
+        BUILT_IN_SCALAR_JSON_SCHEMA_OPTIONS_BY_NAME = {
+          "Boolean" => {type: "boolean"},
+          "Float" => {type: "number"},
+          "ID" => {type: "string"},
+          "Int" => {type: "integer", minimum: INT_MIN, maximum: INT_MAX},
+          "String" => {type: "string"},
+          "Cursor" => {type: "string"},
+          "Date" => {type: "string", format: "date"},
+          "DateTime" => {type: "string", format: "date-time"},
+          "LocalTime" => {type: "string", pattern: VALID_LOCAL_TIME_JSON_SCHEMA_PATTERN},
+          "TimeZone" => {type: "string", enum: GraphQL::ScalarCoercionAdapters::VALID_TIME_ZONES.to_a.freeze},
+          "Untyped" => {type: ["array", "boolean", "integer", "number", "object", "string"].freeze},
+          "JsonSafeLong" => {type: "integer", minimum: JSON_SAFE_LONG_MIN, maximum: JSON_SAFE_LONG_MAX},
+          "LongString" => {type: "integer", minimum: LONG_STRING_MIN, maximum: LONG_STRING_MAX}
+        }.freeze
+
+        # Returns JSON schema defaults for a built-in scalar type.
+        #
+        # These are applied eagerly while the scalar is being instantiated so built-in scalar types still satisfy
+        # `validate_json_schema_configuration!` during schema-definition time.
+        #
+        # @param type_name [String]
+        # @return [Hash<Symbol, Object>, nil]
+        def self.json_schema_options_for_scalar(type_name)
+          BUILT_IN_SCALAR_JSON_SCHEMA_OPTIONS_BY_NAME[type_name]
+        end
+
+        private
+
+        def register_standard_elastic_graph_types
+          super
+
+          geo_location = schema_def_state.object_types_by_name.fetch(schema_def_state.type_ref("GeoLocation").to_final_form.name)
+
+          # We use `nullable: false` because `GeoLocation` is indexed as a single `geo_point` field,
+          # and therefore can't support a `latitude` without a `longitude` or vice-versa.
+          geo_location.graphql_fields_by_name.fetch(names.latitude).json_schema minimum: -90, maximum: 90, nullable: false
+          geo_location.graphql_fields_by_name.fetch(names.longitude).json_schema minimum: -180, maximum: 180, nullable: false
+        end
+      end
+    end
+  end
+end