From afef7fdd746f765f073eeb6ea0405e51d61a106c Mon Sep 17 00:00:00 2001 From: Garik Kalashyan Date: Fri, 22 May 2026 08:34:01 +0400 Subject: [PATCH 1/6] Add GEOGRAPHY logical type model Add DataTypes.GEOGRAPHY Java API Runtime representation and serializer Cover geography serializer with SerializerTestBase TASK-3: Add parser string and JSON serialization support TASK 4: Add planner validation for unsupported conversions TASK 8: Add GeographyTypeSerializer and snapshot TASK 9:Add runtime row and codegen integration --- docs/static/generated/rest_v1_sql_gateway.yml | 1 + docs/static/generated/rest_v2_sql_gateway.yml | 1 + docs/static/generated/rest_v3_sql_gateway.yml | 1 + docs/static/generated/rest_v4_sql_gateway.yml | 1 + ...stractAsyncRunnableStreamOperatorTest.java | 7 +- .../src/main/codegen/data/Parser.tdd | 3 + .../src/main/codegen/includes/parserImpls.ftl | 11 + .../parser/type/SqlGeographyTypeNameSpec.java | 59 ++++ .../table/calcite/ExtendedRelTypeFactory.java | 3 + .../sql/parser/FlinkSqlParserImplTest.java | 19 + .../sql/parser/TestRelDataTypeFactory.java | 17 + .../org/apache/flink/table/api/DataTypes.java | 10 + .../apache/flink/table/data/ArrayData.java | 9 + .../flink/table/data/GenericArrayData.java | 5 + .../flink/table/data/GenericRowData.java | 5 + .../flink/table/data/GeographyData.java | 82 +++++ .../org/apache/flink/table/data/RowData.java | 11 + .../table/data/binary/BinaryArrayData.java | 10 + .../data/binary/BinaryGeographyData.java | 269 ++++++++++++++ .../table/data/binary/BinaryRowData.java | 9 + .../table/data/binary/BinarySegmentUtils.java | 28 ++ .../table/data/binary/NestedRowData.java | 9 + .../data/columnar/ColumnarArrayData.java | 7 + .../table/data/columnar/ColumnarRowData.java | 7 + .../flink/table/data/utils/JoinedRowData.java | 10 + .../table/data/utils/ProjectedRowData.java | 6 + .../table/types/logical/GeographyType.java | 83 +++++ .../table/types/logical/LogicalTypeRoot.java | 4 +- .../types/logical/LogicalTypeVisitor.java | 4 + .../utils/LogicalTypeDefaultVisitor.java | 6 + .../logical/utils/LogicalTypeParser.java | 6 +- .../types/logical/utils/LogicalTypeUtils.java | 3 + .../utils/LogicalTypeDataTypeConverter.java | 6 + .../apache/flink/table/api/SchemaTest.java | 16 + .../data/binary/BinaryGeographyDataTest.java | 331 ++++++++++++++++++ .../flink/table/types/DataTypesTest.java | 6 + .../table/types/LogicalTypeCastsTest.java | 9 +- .../table/types/LogicalTypeParserTest.java | 3 + .../flink/table/types/LogicalTypesTest.java | 17 + flink-table/flink-table-planner/pom.xml | 7 + .../calcite/sql/fun/SqlCastFunction.java | 8 +- .../planner/calcite/FlinkTypeFactory.java | 49 ++- .../exec/serde/LogicalTypeJsonSerializer.java | 1 + .../plan/schema/GeographyRelDataType.java | 62 ++++ .../LogicalRelDataTypeConverter.java | 9 + .../table/planner/codegen/CodeGenUtils.scala | 7 + .../flink/table/api/CompiledPlanITCase.java | 27 ++ .../calcite/FlinkCalciteSqlValidatorTest.java | 56 +++ .../planner/calcite/FlinkTypeFactoryTest.java | 33 ++ .../ProjectionCodeGeneratorGeographyTest.java | 65 ++++ .../functions/CastFunctionMiscITCase.java | 57 +++ .../exec/serde/DataTypeJsonSerdeTest.java | 1 + .../exec/serde/LogicalTypeJsonSerdeTest.java | 2 + .../flink/table/data/BoxedWrapperRowData.java | 5 + .../apache/flink/table/data/RowDataTest.java | 29 +- flink-table/flink-table-test-utils/pom.xml | 6 + .../conversion/DataStructureConverters.java | 4 + .../data/writer/AbstractBinaryWriter.java | 6 + .../table/data/writer/BinaryArrayWriter.java | 1 + .../flink/table/data/writer/BinaryWriter.java | 8 + .../typeutils/GeographyTypeSerializer.java | 127 +++++++ .../GeographyTypeSerializerSnapshot.java | 63 ++++ .../typeutils/InternalSerializers.java | 2 + .../GeographyTypeSerializerTest.java | 215 ++++++++++++ .../typeutils/RowDataSerializerTest.java | 29 ++ flink-tests/pom.xml | 14 + .../TypeSerializerTestCoverageTest.java | 3 + 67 files changed, 1980 insertions(+), 10 deletions(-) create mode 100644 flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java create mode 100644 flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java create mode 100644 flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java create mode 100644 flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java create mode 100644 flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java create mode 100644 flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java diff --git a/docs/static/generated/rest_v1_sql_gateway.yml b/docs/static/generated/rest_v1_sql_gateway.yml index fecf0576a011c..baee191c2a042 100644 --- a/docs/static/generated/rest_v1_sql_gateway.yml +++ b/docs/static/generated/rest_v1_sql_gateway.yml @@ -403,6 +403,7 @@ components: - DESCRIPTOR - VARIANT - BITMAP + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v2_sql_gateway.yml b/docs/static/generated/rest_v2_sql_gateway.yml index e39feaaea8ed5..0b6b70219283b 100644 --- a/docs/static/generated/rest_v2_sql_gateway.yml +++ b/docs/static/generated/rest_v2_sql_gateway.yml @@ -477,6 +477,7 @@ components: - DESCRIPTOR - VARIANT - BITMAP + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v3_sql_gateway.yml b/docs/static/generated/rest_v3_sql_gateway.yml index 6d171ac2152f3..bc46614b4248b 100644 --- a/docs/static/generated/rest_v3_sql_gateway.yml +++ b/docs/static/generated/rest_v3_sql_gateway.yml @@ -506,6 +506,7 @@ components: - DESCRIPTOR - VARIANT - BITMAP + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/docs/static/generated/rest_v4_sql_gateway.yml b/docs/static/generated/rest_v4_sql_gateway.yml index d39da5e007c2b..b1d917452af64 100644 --- a/docs/static/generated/rest_v4_sql_gateway.yml +++ b/docs/static/generated/rest_v4_sql_gateway.yml @@ -516,6 +516,7 @@ components: - DESCRIPTOR - VARIANT - BITMAP + - GEOGRAPHY OpenSessionRequestBody: type: object properties: diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java index b073f6cbed121..1601b06bda998 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java @@ -253,8 +253,11 @@ void testCheckpointDrain() throws Exception { }); ((AbstractAsyncRunnableStreamOperator) testHarness.getOperator()) .postProcessElement(); - assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(1); - unblockAsyncRequest.complete(null); + try { + assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(1); + } finally { + unblockAsyncRequest.complete(null); + } testHarness.drainAsyncRequests(); assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(0); } diff --git a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd index b7e6d7ab9ea3f..db08bbfbf2c32 100644 --- a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd +++ b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd @@ -173,6 +173,7 @@ "org.apache.flink.sql.parser.type.ExtendedSqlCollectionTypeNameSpec" "org.apache.flink.sql.parser.type.ExtendedSqlRowTypeNameSpec" "org.apache.flink.sql.parser.type.SqlBitmapTypeNameSpec" + "org.apache.flink.sql.parser.type.SqlGeographyTypeNameSpec" "org.apache.flink.sql.parser.type.SqlRawTypeNameSpec" "org.apache.flink.sql.parser.type.SqlStructuredTypeNameSpec" "org.apache.flink.sql.parser.type.SqlTimestampLtzTypeNameSpec" @@ -225,6 +226,7 @@ "FROM_TIMESTAMP" "FUNCTIONS" "FRESHNESS" + "GEOGRAPHY" "HASH" "IF" "JSON_EXECUTION_PLAN" @@ -719,6 +721,7 @@ "ExtendedSqlRowTypeName()" "SqlStructuredTypeName()" "SqlBitmapTypeName()" + "SqlGeographyTypeName()" ] # List of methods for parsing builtin function calls. diff --git a/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl b/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl index 2b8250ea98ed9..ede54c27ca86e 100644 --- a/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl +++ b/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl @@ -2666,6 +2666,17 @@ SqlTypeNameSpec SqlBitmapTypeName() : } } +/** Parses GEOGRAPHY type. */ +SqlTypeNameSpec SqlGeographyTypeName() : +{ +} +{ + + { + return new SqlGeographyTypeNameSpec(getPos()); + } +} + /** * Parse a "name1 type1 [ NULL | NOT NULL] [ comment ] * [, name2 type2 [ NULL | NOT NULL] [ comment ] ]* ..." list. diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java new file mode 100644 index 0000000000000..55a3d7ff4fd7d --- /dev/null +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.sql.parser.type; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.calcite.ExtendedRelTypeFactory; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.util.Litmus; + +/** Represents the GEOGRAPHY data type. */ +@Internal +public final class SqlGeographyTypeNameSpec extends SqlTypeNameSpec { + + private static final String GEOGRAPHY_TYPE_NAME = "GEOGRAPHY"; + + public SqlGeographyTypeNameSpec(SqlParserPos pos) { + super(new SqlIdentifier(GEOGRAPHY_TYPE_NAME, pos), pos); + } + + @Override + public RelDataType deriveType(SqlValidator validator) { + return ((ExtendedRelTypeFactory) validator.getTypeFactory()).createGeographyType(); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + writer.keyword(GEOGRAPHY_TYPE_NAME); + } + + @Override + public boolean equalsDeep(SqlTypeNameSpec spec, Litmus litmus) { + if (!(spec instanceof SqlGeographyTypeNameSpec)) { + return litmus.fail("{} != {}", this, spec); + } + return litmus.succeed(); + } +} diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java index 57b9bc615ec93..57be54e4cb6bd 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java @@ -44,4 +44,7 @@ RelDataType createStructuredType( /** Creates a BITMAP type. */ RelDataType createBitmapType(); + + /** Creates a GEOGRAPHY type. */ + RelDataType createGeographyType(); } diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java index 92f8e34c770a0..ee33ea2e37daf 100644 --- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java +++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java @@ -4074,4 +4074,23 @@ void testBitmapType() { sql("CREATE TABLE t (\n" + "^bitmap^ INT" + "\n)") .fails("(?s).*Encountered \"bitmap\" at line 2, column 1.\n.*"); } + + @Test + void testGeographyType() { + sql("CREATE TABLE t (\n" + "g geography" + "\n)") + .ok("CREATE TABLE `T` (\n" + " `G` GEOGRAPHY\n" + ")"); + + sql("CREATE TABLE t (\n" + "g geography NOT NULL" + "\n)") + .ok("CREATE TABLE `T` (\n" + " `G` GEOGRAPHY NOT NULL\n" + ")"); + + // GEOGRAPHY takes no parameters + sql("CREATE TABLE t (\n" + "g geography^(^1)" + "\n)") + .fails("(?s).*Encountered \"\\(\" at line 2, column 12.\n.*"); + sql("CREATE TABLE t (\n" + "g geography^(^4326)" + "\n)") + .fails("(?s).*Encountered \"\\(\" at line 2, column 12.\n.*"); + + // GEOGRAPHY is a reserved keyword and cannot be used as an identifier without escaping + sql("CREATE TABLE t (\n" + "^geography^ INT" + "\n)") + .fails("(?s).*Encountered \"geography\" at line 2, column 1.\n.*"); + } } diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java index 34fe9926079b3..070c77905a335 100644 --- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java +++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java @@ -53,6 +53,11 @@ public RelDataType createBitmapType() { return canonize(new DummyBitmapType()); } + @Override + public RelDataType createGeographyType() { + return canonize(new DummyGeographyType()); + } + private static class DummyRawType extends RelDataTypeImpl { private final String className; @@ -117,4 +122,16 @@ protected void generateTypeString(StringBuilder sb, boolean withDetail) { sb.append("BITMAP"); } } + + private static class DummyGeographyType extends RelDataTypeImpl { + + DummyGeographyType() { + computeDigest(); + } + + @Override + protected void generateTypeString(StringBuilder sb, boolean withDetail) { + sb.append("GEOGRAPHY"); + } + } } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java index af1e718d90e04..3912c17832852 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java @@ -44,6 +44,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -1075,6 +1076,15 @@ public static DataType BITMAP() { return new AtomicDataType(new BitmapType()); } + /** + * Data type of geography data. + * + * @see GeographyType + */ + public static DataType GEOGRAPHY() { + return new AtomicDataType(new GeographyType()); + } + // -------------------------------------------------------------------------------------------- // Helper functions // -------------------------------------------------------------------------------------------- diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java index 811c9be70e6b8..728d3c228f05d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java @@ -123,6 +123,12 @@ default Bitmap getBitmap(int pos) { "This ArrayData implementation does not support Bitmap type."); } + /** Returns the geography value at the given position. */ + default GeographyData getGeography(int pos) { + throw new UnsupportedOperationException( + "This ArrayData implementation does not support Geography type."); + } + // ------------------------------------------------------------------------------------------ // Conversion Utilities // ------------------------------------------------------------------------------------------ @@ -225,6 +231,9 @@ static ElementGetter createElementGetter(LogicalType elementType) { case BITMAP: elementGetter = ArrayData::getBitmap; break; + case GEOGRAPHY: + elementGetter = ArrayData::getGeography; + break; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java index f609c2f22f131..08856cdb0ceb1 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java @@ -265,6 +265,11 @@ public Bitmap getBitmap(int pos) { return (Bitmap) getObject(pos); } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) getObject(pos); + } + private Object getObject(int pos) { return ((Object[]) array)[pos]; } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java index b234edef62d05..98c77bca28f06 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java @@ -192,6 +192,11 @@ public byte[] getBinary(int pos) { return (byte[]) this.fields[pos]; } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) this.fields[pos]; + } + @Override public ArrayData getArray(int pos) { return (ArrayData) this.fields[pos]; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java new file mode 100644 index 0000000000000..f65075be637c8 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.data.binary.BinaryGeographyData; +import org.apache.flink.table.types.logical.GeographyType; + +/** An internal data structure representing data of {@link GeographyType}. */ +@PublicEvolving +public interface GeographyData { + + /** ISO WKB subtype ID for Point geometries. */ + int POINT = 1; + + /** ISO WKB subtype ID for LineString geometries. */ + int LINE_STRING = 2; + + /** ISO WKB subtype ID for Polygon geometries. */ + int POLYGON = 3; + + /** ISO WKB subtype ID for MultiPoint geometries. */ + int MULTI_POINT = 4; + + /** ISO WKB subtype ID for MultiLineString geometries. */ + int MULTI_LINE_STRING = 5; + + /** ISO WKB subtype ID for MultiPolygon geometries. */ + int MULTI_POLYGON = 6; + + /** ISO WKB subtype ID for GeometryCollection geometries. */ + int GEOMETRY_COLLECTION = 7; + + /** + * Converts this {@link GeographyData} object to an ISO WKB byte array. + * + *

Note: The returned byte array may be reused. + */ + byte[] toBytes(); + + /** Returns the ISO WKB subtype ID. */ + int subtypeId(); + + /** Returns the size in bytes of the ISO WKB payload. */ + int sizeInBytes(); + + // ------------------------------------------------------------------------------------------ + // Construction Utilities + // ------------------------------------------------------------------------------------------ + + /** + * Creates an instance of {@link GeographyData} from the given ISO WKB byte array. Returns + * {@code null} if the input is {@code null}. + */ + static GeographyData fromBytes(byte[] bytes) { + return BinaryGeographyData.fromBytes(bytes); + } + + /** + * Creates an instance of {@link GeographyData} from the given ISO WKB byte range. Returns + * {@code null} if the input is {@code null}. + */ + static GeographyData fromBytes(byte[] bytes, int offset, int numBytes) { + return BinaryGeographyData.fromBytes(bytes, offset, numBytes); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java index ca43f1608f99e..a783ecb572a69 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java @@ -111,6 +111,8 @@ * +--------------------------------+-----------------------------------------+ * | BITMAP | {@link Bitmap} | * +--------------------------------+-----------------------------------------+ + * | GEOGRAPHY | {@link GeographyData} | + * +--------------------------------+-----------------------------------------+ * * *

Nullability is always handled by the container data structure. @@ -214,6 +216,12 @@ default Bitmap getBitmap(int pos) { "This RowData implementation does not support Bitmap type."); } + /** Returns the geography value at the given position. */ + default GeographyData getGeography(int pos) { + throw new UnsupportedOperationException( + "This RowData implementation does not support Geography type."); + } + // ------------------------------------------------------------------------------------------ // Access Utilities // ------------------------------------------------------------------------------------------ @@ -299,6 +307,9 @@ static FieldGetter createFieldGetter(LogicalType fieldType, int fieldPos) { case BITMAP: fieldGetter = row -> row.getBitmap(fieldPos); break; + case GEOGRAPHY: + fieldGetter = row -> row.getGeography(fieldPos); + break; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java index 10a8b3e6ef71c..f09f07d25996c 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java @@ -23,6 +23,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -95,6 +96,7 @@ public static int calculateFixLengthPartSize(LogicalType type) { case RAW: case VARIANT: case BITMAP: + case GEOGRAPHY: // long and double are 8 bytes; // otherwise it stores the length and offset of the variable-length part for types // such as is string, map, etc. @@ -269,6 +271,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndSize); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getElementOffset(pos, 8); + final long offsetAndSize = BinarySegmentUtils.getLong(segments, fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndSize); + } + @Override public ArrayData getArray(int pos) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java new file mode 100644 index 0000000000000..4256f786a8681 --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java @@ -0,0 +1,269 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data.binary; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GeographyData; + +import java.util.Arrays; + +/** + * A binary implementation of {@link GeographyData} backed by raw ISO WKB bytes. + * + *

GEOGRAPHY uses OGC:CRS84 by contract, but ISO WKB does not encode CRS or SRID metadata. This + * container stores the raw ISO WKB payload only; CRS validation, CRS transformation, and EWKB/SRID + * handling belong to constructors, functions, and connector schema mapping. + */ +@Internal +public final class BinaryGeographyData extends BinarySection implements GeographyData { + + private static final int MIN_WKB_HEADER_SIZE = 5; + private static final int WKB_COUNT_SIZE = 4; + private static final int WKB_POINT_COORDINATE_SIZE = 16; + private static final int BIG_ENDIAN = 0; + private static final int LITTLE_ENDIAN = 1; + + private final int subtypeId; + + private BinaryGeographyData(MemorySegment[] segments, int offset, int sizeInBytes) { + super(segments, offset, sizeInBytes); + this.subtypeId = readSubtypeId(segments, offset, sizeInBytes); + } + + /** Creates a {@link BinaryGeographyData} instance from the given address and length. */ + public static BinaryGeographyData fromAddress( + MemorySegment[] segments, int offset, int numBytes) { + return new BinaryGeographyData(segments, offset, numBytes); + } + + /** Creates a {@link BinaryGeographyData} instance from the given ISO WKB bytes. */ + public static BinaryGeographyData fromBytes(byte[] bytes) { + return bytes == null ? null : fromBytes(bytes, 0, bytes.length); + } + + /** Creates a {@link BinaryGeographyData} instance from the given ISO WKB byte range. */ + public static BinaryGeographyData fromBytes(byte[] bytes, int offset, int numBytes) { + if (bytes == null) { + return null; + } + checkRange(bytes, offset, numBytes); + byte[] copy = Arrays.copyOfRange(bytes, offset, offset + numBytes); + return fromAddress(new MemorySegment[] {MemorySegmentFactory.wrap(copy)}, 0, copy.length); + } + + @Override + public byte[] toBytes() { + return BinarySegmentUtils.copyToBytes(segments, offset, sizeInBytes); + } + + @Override + public int subtypeId() { + return subtypeId; + } + + @Override + public int sizeInBytes() { + return sizeInBytes; + } + + private static void checkRange(byte[] bytes, int offset, int numBytes) { + if (offset < 0 || numBytes < 0 || offset > bytes.length - numBytes) { + throw new TableRuntimeException( + String.format( + "Invalid ISO WKB byte range: offset %d, length %d, array length %d.", + offset, numBytes, bytes.length)); + } + } + + private static int readSubtypeId(MemorySegment[] segments, int offset, int sizeInBytes) { + final long endOffset = (long) offset + sizeInBytes; + final GeometryHeader header = readHeader(segments, offset, endOffset); + final long consumedOffset = validateGeometry(segments, offset, endOffset); + if (consumedOffset != endOffset) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Found %d trailing byte(s).", + endOffset - consumedOffset)); + } + return header.subtypeId; + } + + private static long validateGeometry( + MemorySegment[] segments, long geometryOffset, long endOffset) { + final GeometryHeader header = readHeader(segments, geometryOffset, endOffset); + long cursor = geometryOffset + MIN_WKB_HEADER_SIZE; + + switch (header.subtypeId) { + case GeographyData.POINT: + return requireBytes( + cursor, WKB_POINT_COORDINATE_SIZE, endOffset, "POINT coordinates") + + WKB_POINT_COORDINATE_SIZE; + case GeographyData.LINE_STRING: + return skipCoordinateSequence(segments, cursor, endOffset, header.byteOrder); + case GeographyData.POLYGON: + return skipPolygon(segments, cursor, endOffset, header.byteOrder); + case GeographyData.MULTI_POINT: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.POINT); + case GeographyData.MULTI_LINE_STRING: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.LINE_STRING); + case GeographyData.MULTI_POLYGON: + return skipTypedGeometryCollection( + segments, cursor, endOffset, header.byteOrder, GeographyData.POLYGON); + case GeographyData.GEOMETRY_COLLECTION: + return skipGeometryCollection(segments, cursor, endOffset, header.byteOrder); + default: + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported geography subtype ID %d.", + header.subtypeId)); + } + } + + private static long skipCoordinateSequence( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numPoints = + readUnsignedInt(segments, offset, endOffset, byteOrder, "point count"); + long cursor = offset + WKB_COUNT_SIZE; + final long coordinateSequenceSize = + multiplyExact(numPoints, WKB_POINT_COORDINATE_SIZE, "coordinate sequence"); + return requireBytes(cursor, coordinateSequenceSize, endOffset, "coordinate sequence") + + coordinateSequenceSize; + } + + private static long skipPolygon( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numRings = readUnsignedInt(segments, offset, endOffset, byteOrder, "ring count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numRings; i++) { + cursor = skipCoordinateSequence(segments, cursor, endOffset, byteOrder); + } + return cursor; + } + + private static long skipTypedGeometryCollection( + MemorySegment[] segments, + long offset, + long endOffset, + int byteOrder, + int expectedSubtypeId) { + final long numGeometries = + readUnsignedInt(segments, offset, endOffset, byteOrder, "geometry count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numGeometries; i++) { + final GeometryHeader nestedHeader = readHeader(segments, cursor, endOffset); + if (nestedHeader.subtypeId != expectedSubtypeId) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Expected nested subtype ID %d but found %d.", + expectedSubtypeId, nestedHeader.subtypeId)); + } + cursor = validateGeometry(segments, cursor, endOffset); + } + return cursor; + } + + private static long skipGeometryCollection( + MemorySegment[] segments, long offset, long endOffset, int byteOrder) { + final long numGeometries = + readUnsignedInt(segments, offset, endOffset, byteOrder, "geometry count"); + long cursor = offset + WKB_COUNT_SIZE; + for (long i = 0; i < numGeometries; i++) { + cursor = validateGeometry(segments, cursor, endOffset); + } + return cursor; + } + + private static GeometryHeader readHeader( + MemorySegment[] segments, long offset, long endOffset) { + requireBytes(offset, MIN_WKB_HEADER_SIZE, endOffset, "WKB header"); + + final int byteOrder = BinarySegmentUtils.getByte(segments, (int) offset) & 0xFF; + if (byteOrder != BIG_ENDIAN && byteOrder != LITTLE_ENDIAN) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported byte order %d.", byteOrder)); + } + + final long subtypeId = + readUnsignedInt(segments, offset + 1, endOffset, byteOrder, "subtype ID"); + + if (subtypeId < GeographyData.POINT || subtypeId > GeographyData.GEOMETRY_COLLECTION) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Unsupported geography subtype ID %d.", + subtypeId)); + } + return new GeometryHeader(byteOrder, (int) subtypeId); + } + + private static long readUnsignedInt( + MemorySegment[] segments, + long offset, + long endOffset, + int byteOrder, + String fieldName) { + requireBytes(offset, WKB_COUNT_SIZE, endOffset, fieldName); + + if (byteOrder == LITTLE_ENDIAN) { + return (BinarySegmentUtils.getByte(segments, (int) offset) & 0xFFL) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 1) & 0xFFL) << 8) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 2) & 0xFFL) << 16) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 3) & 0xFFL) << 24); + } + return ((BinarySegmentUtils.getByte(segments, (int) offset) & 0xFFL) << 24) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 1) & 0xFFL) << 16) + | ((BinarySegmentUtils.getByte(segments, (int) offset + 2) & 0xFFL) << 8) + | (BinarySegmentUtils.getByte(segments, (int) offset + 3) & 0xFFL); + } + + private static long requireBytes( + long offset, long numBytes, long endOffset, String componentName) { + if (offset > endOffset || endOffset - offset < numBytes) { + throw new TableRuntimeException( + String.format( + "Malformed ISO WKB payload. Incomplete %s: expected %d byte(s) but found %d.", + componentName, numBytes, Math.max(0, endOffset - offset))); + } + return offset; + } + + private static long multiplyExact(long value, int factor, String componentName) { + final long result = value * factor; + if (value != 0 && result / value != factor) { + throw new TableRuntimeException( + String.format("Malformed ISO WKB payload. %s size overflows.", componentName)); + } + return result; + } + + private static final class GeometryHeader { + private final int byteOrder; + private final int subtypeId; + + private GeometryHeader(int byteOrder, int subtypeId) { + this.byteOrder = byteOrder; + this.subtypeId = subtypeId; + } + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java index 45bcf7cd3f112..04c9a0e50ed43 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java @@ -22,6 +22,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -372,6 +373,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getFieldOffset(pos); + final long offsetAndLen = segments[0].getLong(fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndLen); + } + @Override public ArrayData getArray(int pos) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java index 653fc559df9bc..2dd12978d6bf5 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java @@ -1056,6 +1056,34 @@ public static byte[] readBinary( } } + /** + * Get geography data, if len less than 8, it will be included in variablePartOffsetAndLen. + * + * @param baseOffset base offset of composite binary format. + * @param fieldOffset absolute start offset of 'variablePartOffsetAndLen'. + * @param variablePartOffsetAndLen a long value, real data or offset and len. + */ + public static BinaryGeographyData readGeographyData( + MemorySegment[] segments, + int baseOffset, + int fieldOffset, + long variablePartOffsetAndLen) { + long mark = variablePartOffsetAndLen & HIGHEST_FIRST_BIT; + if (mark == 0) { + final int subOffset = (int) (variablePartOffsetAndLen >> 32); + final int len = (int) variablePartOffsetAndLen; + return BinaryGeographyData.fromAddress(segments, baseOffset + subOffset, len); + } else { + int len = (int) ((variablePartOffsetAndLen & HIGHEST_SECOND_TO_EIGHTH_BIT) >>> 56); + if (BinarySegmentUtils.LITTLE_ENDIAN) { + return BinaryGeographyData.fromAddress(segments, fieldOffset, len); + } else { + // fieldOffset + 1 to skip header. + return BinaryGeographyData.fromAddress(segments, fieldOffset + 1, len); + } + } + } + /** * Get binary string, if len less than 8, will be include in variablePartOffsetAndLen. * diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java index 27085b487ae50..225d3e103fe10 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java @@ -21,6 +21,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -294,6 +295,14 @@ public byte[] getBinary(int pos) { return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen); } + @Override + public GeographyData getGeography(int pos) { + assertIndexIsValid(pos); + int fieldOffset = getFieldOffset(pos); + final long offsetAndLen = BinarySegmentUtils.getLong(segments, fieldOffset); + return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndLen); + } + @Override public RowData getRow(int pos, int numFields) { assertIndexIsValid(pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java index a3b8e7dd56daf..9c6e6ad8c317d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -147,6 +148,12 @@ public byte[] getBinary(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + BytesColumnVector.Bytes byteArray = getByteArray(pos); + return GeographyData.fromBytes(byteArray.data, byteArray.offset, byteArray.len); + } + @Override public ArrayData getArray(int pos) { return ((ArrayColumnVector) data).getArray(offset + pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java index bd3dce1edf6ba..0413ac980f737 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -152,6 +153,12 @@ public byte[] getBinary(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + Bytes byteArray = vectorizedColumnBatch.getByteArray(rowId, pos); + return GeographyData.fromBytes(byteArray.data, byteArray.offset, byteArray.len); + } + @Override public RowData getRow(int pos, int numFields) { return vectorizedColumnBatch.getRow(rowId, pos); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java index b9eff05a6c388..d1420debfdb07 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -269,6 +270,15 @@ public Bitmap getBitmap(int pos) { } } + @Override + public GeographyData getGeography(int pos) { + if (pos < row1.getArity()) { + return row1.getGeography(pos); + } else { + return row2.getGeography(pos - row1.getArity()); + } + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java index 7f29ca5e61323..e6a173bb4cd99 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java @@ -22,6 +22,7 @@ import org.apache.flink.table.connector.Projection; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -178,6 +179,11 @@ public Bitmap getBitmap(int pos) { return row.getBitmap(indexMapping[pos]); } + @Override + public GeographyData getGeography(int pos) { + return row.getGeography(indexMapping[pos]); + } + @Override public boolean equals(Object o) { throw new UnsupportedOperationException("Projected row data cannot be compared"); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java new file mode 100644 index 0000000000000..403ee88e0cfcc --- /dev/null +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.types.logical; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.table.data.GeographyData; + +import java.util.Collections; +import java.util.List; + +/** + * Data type of geography data. + * + *

The serializable string representation of this type is {@code GEOGRAPHY}. + */ +@PublicEvolving +public final class GeographyType extends LogicalType { + + private static final long serialVersionUID = 1L; + + private static final String FORMAT = "GEOGRAPHY"; + + private static final Class INPUT_OUTPUT_CONVERSION = GeographyData.class; + + public GeographyType(boolean isNullable) { + super(isNullable, LogicalTypeRoot.GEOGRAPHY); + } + + public GeographyType() { + this(true); + } + + @Override + public LogicalType copy(boolean isNullable) { + return new GeographyType(isNullable); + } + + @Override + public String asSerializableString() { + return withNullability(FORMAT); + } + + @Override + public boolean supportsInputConversion(Class clazz) { + return INPUT_OUTPUT_CONVERSION.isAssignableFrom(clazz); + } + + @Override + public boolean supportsOutputConversion(Class clazz) { + return INPUT_OUTPUT_CONVERSION.isAssignableFrom(clazz); + } + + @Override + public Class getDefaultConversion() { + return INPUT_OUTPUT_CONVERSION; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public R accept(LogicalTypeVisitor visitor) { + return visitor.visit(this); + } +} diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java index 6c823add433bf..c21abce4af746 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java @@ -145,7 +145,9 @@ public enum LogicalTypeRoot { VARIANT(LogicalTypeFamily.EXTENSION), - BITMAP(LogicalTypeFamily.EXTENSION); + BITMAP(LogicalTypeFamily.EXTENSION), + + GEOGRAPHY(LogicalTypeFamily.EXTENSION); private final Set families; diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java index 6a0e5614466d1..b35ffe05541f4 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java @@ -101,5 +101,9 @@ default R visit(BitmapType bitmapType) { return visit((LogicalType) bitmapType); } + default R visit(GeographyType geographyType) { + return visit((LogicalType) geographyType); + } + R visit(LogicalType other); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java index 056ec5953ccdc..43237c443f56e 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java @@ -31,6 +31,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -203,6 +204,11 @@ public R visit(BitmapType bitmapType) { return defaultMethod(bitmapType); } + @Override + public R visit(GeographyType geographyType) { + return defaultMethod(geographyType); + } + @Override public R visit(LogicalType other) { return defaultMethod(other); diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java index 3711ac76f4fd2..dfa2e7cf35814 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java @@ -36,6 +36,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -336,7 +337,8 @@ private enum Keyword { DESCRIPTOR, STRUCTURED, VARIANT, - BITMAP + BITMAP, + GEOGRAPHY } private static final Set KEYWORDS = @@ -586,6 +588,8 @@ private LogicalType parseTypeByKeyword() { return new VariantType(); case BITMAP: return new BitmapType(); + case GEOGRAPHY: + return new GeographyType(); default: throw parsingError("Unsupported type: " + token().value); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java index 98629b15fc177..fa994ed160f3c 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java @@ -21,6 +21,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -115,6 +116,8 @@ public static Class toInternalConversionClass(LogicalType type) { return Variant.class; case BITMAP: return Bitmap.class; + case GEOGRAPHY: + return GeographyData.class; case SYMBOL: case UNRESOLVED: default: diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java index 9aa83f3cbee1d..929aaec786f49 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java @@ -38,6 +38,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -265,6 +266,11 @@ public DataType visit(BitmapType bitmapType) { return new AtomicDataType(bitmapType); } + @Override + public DataType visit(GeographyType geographyType) { + return new AtomicDataType(geographyType); + } + @Override public DataType visit(LogicalType other) { if (other.is(LogicalTypeRoot.UNRESOLVED)) { diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java index 7b54e40f38793..7b4aef14cc3a8 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java @@ -51,5 +51,21 @@ void testFromResolvedSchema() { assertThat(newSchema.resolve(new TestSchemaResolver())).isEqualTo(originalSchema); } + + @Test + void testGeographyColumn() { + final Schema schema = + Schema.newBuilder() + .column("location", DataTypes.GEOGRAPHY()) + .column("required_location", DataTypes.GEOGRAPHY().notNull()) + .build(); + + assertThat(schema.resolve(new TestSchemaResolver())) + .isEqualTo( + ResolvedSchema.of( + Column.physical("location", DataTypes.GEOGRAPHY()), + Column.physical( + "required_location", DataTypes.GEOGRAPHY().notNull()))); + } } } diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java new file mode 100644 index 0000000000000..4e45d2d28549a --- /dev/null +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.data.binary; + +import org.apache.flink.core.memory.MemorySegment; +import org.apache.flink.core.memory.MemorySegmentFactory; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.GeographyType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Arrays; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link BinaryGeographyData}. */ +class BinaryGeographyDataTest { + + private static final int BIG_ENDIAN = 0; + private static final int LITTLE_ENDIAN = 1; + private static final byte[] POINT_WKB = pointWkb(LITTLE_ENDIAN); + + @Test + void testValidWkbRoundTrip() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + + assertThat(geography).isInstanceOf(BinaryGeographyData.class); + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(geography.sizeInBytes()).isEqualTo(POINT_WKB.length); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("supportedSubtypePayloads") + void testSupported2dSubtypePayloads(String name, int subtypeId, byte[] wkb) { + final BinaryGeographyData geography = BinaryGeographyData.fromBytes(wkb); + + assertThat(geography.subtypeId()).isEqualTo(subtypeId); + assertThat(geography.toBytes()).isEqualTo(wkb); + assertThat(geography.sizeInBytes()).isEqualTo(wkb.length); + } + + static Stream supportedSubtypePayloads() { + return Stream.of( + Arguments.of("Point", GeographyData.POINT, pointWkb(LITTLE_ENDIAN)), + Arguments.of("LineString", GeographyData.LINE_STRING, lineStringWkb(LITTLE_ENDIAN)), + Arguments.of("Polygon", GeographyData.POLYGON, polygonWkb(LITTLE_ENDIAN)), + Arguments.of("MultiPoint", GeographyData.MULTI_POINT, multiPointWkb(LITTLE_ENDIAN)), + Arguments.of( + "MultiLineString", + GeographyData.MULTI_LINE_STRING, + multiLineStringWkb(LITTLE_ENDIAN)), + Arguments.of( + "MultiPolygon", + GeographyData.MULTI_POLYGON, + multiPolygonWkb(LITTLE_ENDIAN)), + Arguments.of( + "GeometryCollection", + GeographyData.GEOMETRY_COLLECTION, + geometryCollectionWkb(LITTLE_ENDIAN))); + } + + @Test + void testBigEndianSubtypeExtraction() { + final BinaryGeographyData point = BinaryGeographyData.fromBytes(pointWkb(BIG_ENDIAN)); + final BinaryGeographyData lineString = + BinaryGeographyData.fromBytes(lineStringWkb(BIG_ENDIAN)); + + assertThat(point.subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(lineString.subtypeId()).isEqualTo(GeographyData.LINE_STRING); + } + + @Test + void testNullHandling() { + assertThat(GeographyData.fromBytes((byte[]) null)).isNull(); + assertThat(GeographyData.fromBytes(null, 0, 0)).isNull(); + assertThat(BinaryGeographyData.fromBytes((byte[]) null)).isNull(); + assertThat(BinaryGeographyData.fromBytes(null, 0, 0)).isNull(); + } + + @Test + void testGenericRowDataNullPath() { + final RowData.FieldGetter getter = RowData.createFieldGetter(new GeographyType(), 0); + final GenericRowData nullRow = GenericRowData.of((Object) null); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GenericRowData valueRow = GenericRowData.of(geography); + + assertThat(getter.getFieldOrNull(nullRow)).isNull(); + assertThat(getter.getFieldOrNull(valueRow)).isSameAs(geography); + } + + @Test + void testBinaryRowDataNullPath() { + final RowData.FieldGetter getter = RowData.createFieldGetter(new GeographyType(), 0); + final int fixedLength = BinaryRowData.calculateFixPartSizeInBytes(1); + final BinaryRowData nullRow = new BinaryRowData(1); + nullRow.pointTo(MemorySegmentFactory.wrap(new byte[fixedLength]), 0, fixedLength); + nullRow.setNullAt(0); + + assertThat(getter.getFieldOrNull(nullRow)).isNull(); + } + + @Test + void testBinaryRowDataValuePath() { + final BinaryRowData row = binaryRowWithGeography(POINT_WKB); + + assertThat(row.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(row.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testByteRangeCopiesOnlySelectedPayload() { + final byte[] bytes = concat(bytes(42), POINT_WKB, bytes(99)); + final BinaryGeographyData geography = + BinaryGeographyData.fromBytes(bytes, 1, POINT_WKB.length); + + bytes[1] = 0; + + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testFromAddressSupportsSegmentBoundary() { + final byte[] first = concat(new byte[14], Arrays.copyOfRange(POINT_WKB, 0, 2)); + final byte[] second = Arrays.copyOfRange(POINT_WKB, 2, 18); + final byte[] third = + concat(Arrays.copyOfRange(POINT_WKB, 18, POINT_WKB.length), new byte[13]); + final MemorySegment[] segments = + new MemorySegment[] { + MemorySegmentFactory.wrap(first), + MemorySegmentFactory.wrap(second), + MemorySegmentFactory.wrap(third) + }; + + final BinaryGeographyData geography = + BinaryGeographyData.fromAddress(segments, 14, POINT_WKB.length); + + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testMalformedPayloadBoundaries() { + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes())) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete WKB header"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes(1, 1, 0, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete WKB header"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes(2, 1, 0, 0, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported byte order 2"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(header(LITTLE_ENDIAN, 0))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported geography subtype ID 0"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(header(LITTLE_ENDIAN, 8))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Unsupported geography subtype ID 8"); + } + + @Test + void testStructurallyIncompletePayloads() { + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + header(LITTLE_ENDIAN, GeographyData.POINT))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete POINT coordinates"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat( + header(LITTLE_ENDIAN, GeographyData.LINE_STRING), + unsignedInt(LITTLE_ENDIAN, 1), + new byte[15]))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Incomplete coordinate sequence"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat( + header(LITTLE_ENDIAN, GeographyData.MULTI_POINT), + unsignedInt(LITTLE_ENDIAN, 1), + lineStringWkb(LITTLE_ENDIAN)))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Expected nested subtype ID 1 but found 2"); + + assertThatThrownBy( + () -> + BinaryGeographyData.fromBytes( + concat(pointWkb(LITTLE_ENDIAN), bytes(42)))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("trailing byte"); + } + + @Test + void testInvalidByteRanges() { + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(POINT_WKB, -1, POINT_WKB.length)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid ISO WKB byte range"); + + assertThatThrownBy(() -> BinaryGeographyData.fromBytes(POINT_WKB, 0, POINT_WKB.length + 1)) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid ISO WKB byte range"); + } + + private static BinaryRowData binaryRowWithGeography(byte[] wkb) { + final int fixedLength = BinaryRowData.calculateFixPartSizeInBytes(1); + final byte[] rowBytes = new byte[fixedLength + wkb.length]; + final MemorySegment segment = MemorySegmentFactory.wrap(rowBytes); + segment.putLong(8, ((long) fixedLength << 32) | wkb.length); + segment.put(fixedLength, wkb, 0, wkb.length); + + final BinaryRowData row = new BinaryRowData(1); + row.pointTo(segment, 0, rowBytes.length); + return row; + } + + private static byte[] pointWkb(int byteOrder) { + return concat(header(byteOrder, GeographyData.POINT), new byte[16]); + } + + private static byte[] lineStringWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.LINE_STRING), + unsignedInt(byteOrder, 2), + new byte[32]); + } + + private static byte[] polygonWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.POLYGON), + unsignedInt(byteOrder, 1), + unsignedInt(byteOrder, 4), + new byte[64]); + } + + private static byte[] multiPointWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_POINT), + unsignedInt(byteOrder, 1), + pointWkb(byteOrder)); + } + + private static byte[] multiLineStringWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_LINE_STRING), + unsignedInt(byteOrder, 1), + lineStringWkb(byteOrder)); + } + + private static byte[] multiPolygonWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.MULTI_POLYGON), + unsignedInt(byteOrder, 1), + polygonWkb(byteOrder)); + } + + private static byte[] geometryCollectionWkb(int byteOrder) { + return concat( + header(byteOrder, GeographyData.GEOMETRY_COLLECTION), + unsignedInt(byteOrder, 2), + pointWkb(byteOrder), + lineStringWkb(byteOrder)); + } + + private static byte[] header(int byteOrder, int subtypeId) { + return concat(bytes(byteOrder), unsignedInt(byteOrder, subtypeId)); + } + + private static byte[] unsignedInt(int byteOrder, int value) { + if (byteOrder == LITTLE_ENDIAN) { + return bytes(value, value >>> 8, value >>> 16, value >>> 24); + } + return bytes(value >>> 24, value >>> 16, value >>> 8, value); + } + + private static byte[] concat(byte[]... values) { + int length = 0; + for (byte[] value : values) { + length += value.length; + } + + final byte[] result = new byte[length]; + int offset = 0; + for (byte[] value : values) { + System.arraycopy(value, 0, result, offset, value.length); + offset += value.length; + } + return result; + } + + private static byte[] bytes(int... values) { + final byte[] result = new byte[values.length]; + for (int i = 0; i < values.length; i++) { + result[i] = (byte) values[i]; + } + return result; + } +} diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java index 8c274ed719137..3c0bf06363145 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.common.typeutils.base.VoidSerializer; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.BinaryType; @@ -32,6 +33,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -82,6 +84,7 @@ import static org.apache.flink.table.api.DataTypes.DOUBLE; import static org.apache.flink.table.api.DataTypes.FIELD; import static org.apache.flink.table.api.DataTypes.FLOAT; +import static org.apache.flink.table.api.DataTypes.GEOGRAPHY; import static org.apache.flink.table.api.DataTypes.INT; import static org.apache.flink.table.api.DataTypes.INTERVAL; import static org.apache.flink.table.api.DataTypes.MAP; @@ -231,6 +234,9 @@ private static Stream testData() { TestSpec.forDataType(BITMAP()) .expectLogicalType(new BitmapType()) .expectConversionClass(Bitmap.class), + TestSpec.forDataType(GEOGRAPHY()) + .expectLogicalType(new GeographyType()) + .expectConversionClass(GeographyData.class), TestSpec.forUnresolvedDataType(RAW(Types.VOID)) .expectUnresolvedString("[RAW('java.lang.Void', '?')]") .lookupReturns(dummyRaw(Void.class)) diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java index 3d0dec239974f..81995b2f2a7e5 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java @@ -31,6 +31,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -262,7 +263,13 @@ private static Stream testData() { new RawType<>(Integer.class, IntSerializer.INSTANCE), VarCharType.STRING_TYPE, false, - true)); + true), + + // GEOGRAPHY construction and serialization require explicit functions. + Arguments.of(new GeographyType(), VarCharType.STRING_TYPE, false, false), + Arguments.of(VarCharType.STRING_TYPE, new GeographyType(), false, false), + Arguments.of(new GeographyType(), new VarBinaryType(), false, false), + Arguments.of(new VarBinaryType(), new GeographyType(), false, false)); } @ParameterizedTest(name = "{index}: [From: {0}, To: {1}, Implicit: {2}, Explicit: {3}]") diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java index 258b397c87187..e1c96b1fbc9e7 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java @@ -39,6 +39,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -309,6 +310,8 @@ private static Stream testData() { TestSpec.forString("VARIANT NOT NULL").expectType(new VariantType(false)), TestSpec.forString("BITMAP").expectType(new BitmapType()), TestSpec.forString("BITMAP NOT NULL").expectType(new BitmapType(false)), + TestSpec.forString("GEOGRAPHY").expectType(new GeographyType()), + TestSpec.forString("GEOGRAPHY NOT NULL").expectType(new GeographyType(false)), // error message testing diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java index ee40463cf0b92..b608d923cc3b6 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java @@ -26,6 +26,8 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.catalog.ObjectIdentifier; import org.apache.flink.table.catalog.UnresolvedIdentifier; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.expressions.TimeIntervalUnit; import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.types.logical.ArrayType; @@ -40,6 +42,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -629,6 +632,20 @@ void testBitmapType() { new BitmapType(false))); } + @Test + void testGeographyType() { + assertThat(new GeographyType()) + .isJavaSerializable() + .satisfies( + baseAssertions( + "GEOGRAPHY", + "GEOGRAPHY", + new Class[] {GeographyData.class, BinaryGeographyData.class}, + new Class[] {GeographyData.class, BinaryGeographyData.class}, + new LogicalType[] {}, + new GeographyType(false))); + } + @Test void testTypeInformationRawType() { final TypeInformationRawType rawType = diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml index b8f76f7f96992..b57ab89c943fa 100644 --- a/flink-table/flink-table-planner/pom.xml +++ b/flink-table/flink-table-planner/pom.xml @@ -183,6 +183,13 @@ under the License. test + + org.apache.flink + flink-table-code-splitter + ${project.version} + test + + org.apache.flink diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java index 7b6112f9672dc..d819a145e94cd 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java @@ -269,16 +269,22 @@ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFail private boolean canCastFrom(RelDataType toType, RelDataType fromType) { SqlTypeName fromTypeName = fromType.getSqlTypeName(); + SqlTypeName toTypeName = toType.getSqlTypeName(); // Cast to Variant is not support at the moment. // TODO: Support cast to variant (FLINK-37925，FLINK-37926) - if (toType.getSqlTypeName() == SqlTypeName.VARIANT) { + if (toTypeName == SqlTypeName.VARIANT) { return false; } // Cast to BITMAP is not supported at the moment. if (toType instanceof BitmapRelDataType) { return false; } + if (toTypeName == SqlTypeName.OTHER) { + return LogicalTypeCasts.supportsExplicitCast( + FlinkTypeFactory.toLogicalType(fromType), + FlinkTypeFactory.toLogicalType(toType)); + } switch (fromTypeName) { case ARRAY: case MAP: diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java index bb99230775c85..c857cf07c23ea 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java @@ -28,6 +28,7 @@ import org.apache.flink.table.legacy.api.TableSchema; import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.planner.plan.schema.BitmapRelDataType; +import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.table.planner.plan.schema.GenericRelDataType; import org.apache.flink.table.planner.plan.schema.RawRelDataType; import org.apache.flink.table.planner.plan.schema.StructuredRelDataType; @@ -46,6 +47,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LegacyTypeInformationType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; @@ -65,6 +67,7 @@ import org.apache.flink.table.types.logical.VarBinaryType; import org.apache.flink.table.types.logical.VarCharType; import org.apache.flink.table.types.logical.VariantType; +import org.apache.flink.table.types.logical.utils.LogicalTypeMerging; import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo; import org.apache.flink.table.utils.TableSchemaUtils; import org.apache.flink.util.Preconditions; @@ -160,6 +163,11 @@ public RelDataType createBitmapType() { return canonize(new BitmapRelDataType(new BitmapType())); } + @Override + public RelDataType createGeographyType() { + return canonize(new GeographyRelDataType(new GeographyType())); + } + @Override public RelDataType createArrayType(RelDataType elementType, long maxCardinality) { // Just validate type, make sure there is a failure in validate phase. @@ -230,6 +238,8 @@ public RelDataType createTypeWithNullability(RelDataType relDataType, boolean is newType = ((StructuredRelDataType) relDataType).createWithNullability(isNullable); } else if (relDataType instanceof BitmapRelDataType) { newType = ((BitmapRelDataType) relDataType).createWithNullability(isNullable); + } else if (relDataType instanceof GeographyRelDataType) { + newType = ((GeographyRelDataType) relDataType).createWithNullability(isNullable); } else if (relDataType instanceof GenericRelDataType) { final GenericRelDataType generic = (GenericRelDataType) relDataType; newType = new GenericRelDataType(generic.genericType(), isNullable, getTypeSystem()); @@ -254,8 +264,20 @@ public RelDataType createTypeWithNullability(RelDataType relDataType, boolean is @Override public RelDataType leastRestrictive(List types) { final Optional resolved = resolveAllIdenticalTypes(types); - final RelDataType leastRestrictive = - resolved.orElseGet(() -> super.leastRestrictive(types)); + if (resolved.isPresent()) { + return normalizeLeastRestrictive(resolved.get()); + } + + if (containsFlinkExtensionType(types)) { + return normalizeLeastRestrictive( + resolveCommonTypeForFlinkExtensions(types).orElse(null)); + } + + final RelDataType leastRestrictive = super.leastRestrictive(types); + return normalizeLeastRestrictive(leastRestrictive); + } + + private RelDataType normalizeLeastRestrictive(RelDataType leastRestrictive) { // NULL is reserved for untyped literals only if (leastRestrictive == null || leastRestrictive.getSqlTypeName() == SqlTypeName.NULL) { return null; @@ -263,6 +285,24 @@ public RelDataType leastRestrictive(List types) { return leastRestrictive; } + private Optional resolveCommonTypeForFlinkExtensions(List types) { + return LogicalTypeMerging.findCommonType( + types.stream() + .map(FlinkTypeFactory::toLogicalType) + .collect(Collectors.toList())) + .map(this::createFieldTypeFromLogicalType); + } + + private boolean containsFlinkExtensionType(List types) { + return types.stream().anyMatch(FlinkTypeFactory::isFlinkExtensionType); + } + + private static boolean isFlinkExtensionType(RelDataType type) { + return type instanceof RawRelDataType + || type instanceof BitmapRelDataType + || type instanceof GeographyRelDataType; + } + private Optional resolveAllIdenticalTypes(List types) { final RelDataType head = types.get(0); // check if all types are the same @@ -492,6 +532,9 @@ private RelDataType newRelDataType(LogicalType logicalType) { case BITMAP: return new BitmapRelDataType((BitmapType) logicalType); + case GEOGRAPHY: + return new GeographyRelDataType((GeographyType) logicalType); + default: throw new TableException("Type is not supported: " + logicalType); } @@ -865,6 +908,8 @@ private static LogicalType toLogicalTypeWithoutNullability(RelDataType relDataTy return ((RawRelDataType) relDataType).getRawType(); } else if (relDataType instanceof BitmapRelDataType) { return ((BitmapRelDataType) relDataType).getBitmapType(); + } else if (relDataType instanceof GeographyRelDataType) { + return ((GeographyRelDataType) relDataType).getGeographyType(); } else { throw new TableException("Type is not supported: " + relDataType); } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java index 2857c4dbaa3bc..6429a2c5586a8 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerializer.java @@ -544,6 +544,7 @@ protected Boolean defaultMethod(LogicalType logicalType) { case NULL: case DESCRIPTOR: case BITMAP: + case GEOGRAPHY: return true; default: // fall back to generic serialization diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java new file mode 100644 index 0000000000000..41b586f586324 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/plan/schema/GeographyRelDataType.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.schema; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.types.logical.GeographyType; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.AbstractSqlType; +import org.apache.calcite.sql.type.SqlTypeName; + +/** The {@link RelDataType} representation of a {@link GeographyType}. */ +@Internal +public final class GeographyRelDataType extends AbstractSqlType { + + private final GeographyType geographyType; + + public GeographyRelDataType(GeographyType geographyType) { + super(SqlTypeName.OTHER, geographyType.isNullable(), null); + this.geographyType = geographyType; + computeDigest(); + } + + public GeographyType getGeographyType() { + return geographyType; + } + + public GeographyRelDataType createWithNullability(boolean nullable) { + if (nullable == isNullable()) { + return this; + } + return new GeographyRelDataType((GeographyType) geographyType.copy(nullable)); + } + + @Override + protected void generateTypeString(StringBuilder sb, boolean withDetail) { + sb.append(geographyType.asSummaryString()); + } + + @Override + protected void computeDigest() { + final StringBuilder sb = new StringBuilder(); + generateTypeString(sb, true); + digest = sb.toString(); + } +} diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java index 12093cfe1e571..145376f781a07 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/typeutils/LogicalRelDataTypeConverter.java @@ -23,6 +23,7 @@ import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; import org.apache.flink.table.planner.plan.schema.BitmapRelDataType; +import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.table.planner.plan.schema.RawRelDataType; import org.apache.flink.table.planner.plan.schema.StructuredRelDataType; import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; @@ -40,6 +41,7 @@ import org.apache.flink.table.types.logical.DistinctType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -466,6 +468,11 @@ public RelDataType visit(BitmapType bitmapType) { return new BitmapRelDataType(bitmapType); } + @Override + public RelDataType visit(GeographyType geographyType) { + return new GeographyRelDataType(geographyType); + } + @Override public RelDataType visit(LogicalType other) { throw new TableException( @@ -596,6 +603,8 @@ private static LogicalType toLogicalTypeNotNull( return ((RawRelDataType) relDataType).getRawType(); } else if (relDataType instanceof BitmapRelDataType) { return ((BitmapRelDataType) relDataType).getBitmapType(); + } else if (relDataType instanceof GeographyRelDataType) { + return ((GeographyRelDataType) relDataType).getGeographyType(); } // fall through case REAL: diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala index ff924fe0f30f8..148d85ad969d2 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/planner/codegen/CodeGenUtils.scala @@ -279,6 +279,7 @@ object CodeGenUtils { case DESCRIPTOR => className[ColumnList] case VARIANT => className[Variant] case BITMAP => className[Bitmap] + case GEOGRAPHY => className[GeographyData] case SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -386,6 +387,8 @@ object CodeGenUtils { s"$term.toObject($serTerm).hashCode()" case BITMAP => s"$term.hashCode()" + case GEOGRAPHY => + s"$term.hashCode()" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -538,6 +541,8 @@ object CodeGenUtils { s"$rowTerm.getVariant($indexTerm)" case BITMAP => s"$rowTerm.getBitmap($indexTerm)" + case GEOGRAPHY => + s"$rowTerm.getGeography($indexTerm)" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t) } @@ -835,6 +840,8 @@ object CodeGenUtils { s"$writerTerm.writeVariant($indexTerm, $fieldValTerm)" case BITMAP => s"$writerTerm.writeBitmap($indexTerm, $fieldValTerm)" + case GEOGRAPHY => + s"$writerTerm.writeGeography($indexTerm, $fieldValTerm)" case NULL | SYMBOL | UNRESOLVED => throw new IllegalArgumentException("Illegal type: " + t); } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java index 0d5c13d48a538..aa0039b7bd16b 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/api/CompiledPlanITCase.java @@ -86,6 +86,33 @@ void testCompilePlanSql() throws IOException { .isEqualTo(getPreparedToCompareCompiledPlan(expected)); } + @Test + void testCompilePlanSqlWithGeographyColumns() { + tableEnv.executeSql( + "CREATE TABLE GeoSource (" + + "id INT, " + + "location GEOGRAPHY, " + + "required_location GEOGRAPHY NOT NULL" + + ") WITH (" + + "'connector' = 'values', " + + "'bounded' = 'false')"); + tableEnv.executeSql( + "CREATE TABLE GeoSink (" + + "id INT, " + + "location GEOGRAPHY, " + + "required_location GEOGRAPHY NOT NULL" + + ") WITH (" + + "'connector' = 'values', " + + "'table-sink-class' = 'DEFAULT')"); + + final CompiledPlan compiledPlan = + tableEnv.compilePlanSql("INSERT INTO GeoSink SELECT * FROM GeoSource"); + final String planJson = compiledPlan.asJsonString(); + + assertThat(planJson).contains("\"GEOGRAPHY\"", "\"GEOGRAPHY NOT NULL\""); + assertThat(tableEnv.loadPlan(PlanReference.fromJsonString(planJson))).isNotNull(); + } + @Test void testSourceTableWithHints() { CompiledPlan compiledPlan = diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java index 831b3b6ecc480..de08fb2c8e631 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkCalciteSqlValidatorTest.java @@ -26,12 +26,21 @@ import org.apache.flink.table.api.ValidationException; import org.apache.flink.table.functions.ScalarFunction; import org.apache.flink.table.planner.utils.PlannerMocks; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.GeographyType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.sql.SqlNode; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.catchThrowable; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; /** Test for {@link FlinkCalciteSqlValidator}. */ @@ -158,6 +167,53 @@ void testMixedPositionalAndNamedArguments() { .hasMessageContaining("Cannot mix positional and named arguments"); } + @Test + void testArrayConstructorInfersGeographyElementType() { + LogicalType logicalType = + projectedLogicalType( + "ARRAY[ST_GEOGFROMTEXT('POINT (0 0)'), ST_GEOGFROMTEXT('POINT (1 1)')]"); + + assertThat(logicalType).isEqualTo(new ArrayType(new GeographyType())); + } + + @Test + void testArrayConstructorInfersNullableGeographyElementType() { + LogicalType logicalType = + projectedLogicalType("ARRAY[ST_GEOGFROMTEXT('POINT (0 0)'), NULL]"); + + assertThat(logicalType).isEqualTo(new ArrayType(new GeographyType())); + } + + @Test + void testMapConstructorInfersGeographyValueType() { + LogicalType logicalType = + projectedLogicalType( + "MAP['a', ST_GEOGFROMTEXT('POINT (0 0)'), 'b', ST_GEOGFROMTEXT('POINT (1 1)')]"); + + assertThat(logicalType) + .isEqualTo(new MapType(VarCharType.STRING_TYPE, new GeographyType())); + } + + @Test + void testIncompatibleExtensionTypesFailValidation() { + Throwable thrown = + catchThrowable( + () -> + projectedLogicalType( + "ARRAY[ST_GEOGFROMTEXT('POINT (0 0)'), BITMAP_BUILD(1)]")); + + assertThat(thrown).isInstanceOf(ValidationException.class); + assertThat(thrown.getCause()).isNotInstanceOf(AssertionError.class); + } + + private LogicalType projectedLogicalType(String expression) { + SqlNode parsed = plannerMocks.getPlanner().parser().parse("SELECT " + expression + " AS c"); + SqlNode validated = plannerMocks.getPlanner().validate(parsed); + RelRoot relRoot = plannerMocks.getPlanner().rel(validated); + return FlinkTypeFactory.toLogicalType( + relRoot.rel.getRowType().getFieldList().get(0).getType()); + } + /** Scalar function with named arguments for the mixed-argument validation test. */ public static class NamedArgsScalarFunction extends ScalarFunction { @FunctionHint( diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java index 7715f5bacddcb..ae775d7446c7b 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/calcite/FlinkTypeFactoryTest.java @@ -25,12 +25,14 @@ import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.BitmapType; import org.apache.flink.table.types.logical.BooleanType; import org.apache.flink.table.types.logical.CharType; import org.apache.flink.table.types.logical.DateType; import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -221,6 +223,37 @@ void testLeastRestrictive(List input, LogicalType expected) { .isEqualTo(typeFactory.createFieldTypeFromLogicalType(expected)); } + @Test + void testLeastRestrictiveGeographyNullability() { + FlinkTypeFactory typeFactory = + new FlinkTypeFactory( + Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + + assertThat( + typeFactory.leastRestrictive( + Stream.of( + new GeographyType(false), + new GeographyType(true), + new NullType()) + .map(typeFactory::createFieldTypeFromLogicalType) + .collect(Collectors.toList()))) + .isEqualTo(typeFactory.createFieldTypeFromLogicalType(new GeographyType(true))); + } + + @Test + void testLeastRestrictiveIncompatibleExtensionTypes() { + FlinkTypeFactory typeFactory = + new FlinkTypeFactory( + Thread.currentThread().getContextClassLoader(), FlinkTypeSystem.INSTANCE); + + assertThat( + typeFactory.leastRestrictive( + Stream.of(new GeographyType(), new BitmapType()) + .map(typeFactory::createFieldTypeFromLogicalType) + .collect(Collectors.toList()))) + .isNull(); + } + public static class TestClass { public int f0; public String f1; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java new file mode 100644 index 0000000000000..652d0af25d642 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/codegen/ProjectionCodeGeneratorGeographyTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.codegen; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.runtime.generated.Projection; +import org.apache.flink.table.types.logical.GeographyType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for generated projections involving {@link GeographyData}. */ +class ProjectionCodeGeneratorGeographyTest { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Test + void testGeneratedProjectionForGeography() { + RowType inputType = RowType.of(new IntType(), new GeographyType(), new GeographyType()); + RowType outputType = RowType.of(new GeographyType(), new GeographyType(), new IntType()); + GenericRowData input = GenericRowData.of(7, GeographyData.fromBytes(POINT_WKB), null); + + Projection projection = + ProjectionCodeGenerator.generateProjection( + new CodeGeneratorContext( + new Configuration(), + Thread.currentThread().getContextClassLoader()), + "GeographyProjection", + inputType, + outputType, + new int[] {1, 2, 0}) + .newInstance(Thread.currentThread().getContextClassLoader()); + + BinaryRowData output = (BinaryRowData) projection.apply(input); + + assertThat(output.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(output.isNullAt(1)).isTrue(); + assertThat(output.getInt(2)).isEqualTo(7); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java index 218d911e881db..f37a8acfa4ac4 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/CastFunctionMiscITCase.java @@ -39,6 +39,7 @@ import static org.apache.flink.table.api.DataTypes.BOOLEAN; import static org.apache.flink.table.api.DataTypes.BYTES; import static org.apache.flink.table.api.DataTypes.FIELD; +import static org.apache.flink.table.api.DataTypes.GEOGRAPHY; import static org.apache.flink.table.api.DataTypes.INT; import static org.apache.flink.table.api.DataTypes.MAP; import static org.apache.flink.table.api.DataTypes.ROW; @@ -260,6 +261,62 @@ Stream getTestSetSpecs() { .testSqlValidationError( "CAST(CreateMultiset(f0) AS BITMAP)", "Cast function cannot convert value of type VARCHAR(2147483647) MULTISET to type BITMAP"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast STRING to GEOGRAPHY") + .onFieldsWithData("POINT (0 0)") + .andDataTypes(STRING()) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'STRING' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARCHAR(2147483647) to type GEOGRAPHY"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast BYTES to GEOGRAPHY") + .onFieldsWithData(new byte[] {1, 2, 3}) + .andDataTypes(BYTES()) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'BYTES' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARBINARY(2147483647) to type GEOGRAPHY"), + TestSetSpec.forFunction( + BuiltInFunctionDefinitions.CAST, "cast VARBINARY to GEOGRAPHY") + .onFieldsWithData(new byte[] {1, 2, 3}) + .andDataTypes(VARBINARY(3)) + .testTableApiValidationError( + $("f0").cast(GEOGRAPHY()), + "Unsupported cast from 'VARBINARY(3)' to 'GEOGRAPHY'") + .testSqlValidationError( + "CAST(f0 AS GEOGRAPHY)", + "Cast function cannot convert value of type VARBINARY(3) to type GEOGRAPHY"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to STRING") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(STRING()), + "Unsupported cast from 'GEOGRAPHY' to 'STRING'") + .testSqlValidationError( + "CAST(f0 AS STRING)", + "Cast function cannot convert value of type GEOGRAPHY to type VARCHAR(2147483647)"), + TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to BYTES") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(BYTES()), + "Unsupported cast from 'GEOGRAPHY' to 'BYTES'") + .testSqlValidationError( + "CAST(f0 AS BYTES)", + "Cast function cannot convert value of type GEOGRAPHY to type VARBINARY(2147483647)"), + TestSetSpec.forFunction( + BuiltInFunctionDefinitions.CAST, "cast GEOGRAPHY to VARBINARY") + .onFieldsWithData((Object) null) + .andDataTypes(GEOGRAPHY()) + .testTableApiValidationError( + $("f0").cast(VARBINARY(3)), + "Unsupported cast from 'GEOGRAPHY' to 'VARBINARY(3)'") + .testSqlValidationError( + "CAST(f0 AS VARBINARY(3))", + "Cast function cannot convert value of type GEOGRAPHY to type VARBINARY(3)"), TestSetSpec.forFunction(BuiltInFunctionDefinitions.CAST, "cast RAW to STRING") .onFieldsWithData("2020-11-11T18:08:01.123") .andDataTypes(STRING()) diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java index 3c24b472bac93..766edc9786837 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/DataTypeJsonSerdeTest.java @@ -59,6 +59,7 @@ private static Stream testDataTypeSerde() { DataTypes.TIMESTAMP_LTZ(3).toInternal(), DataTypes.TIMESTAMP_LTZ(9).bridgedTo(long.class), DataTypes.BITMAP(), + DataTypes.GEOGRAPHY(), DataTypes.ROW( DataTypes.TIMESTAMP_LTZ(3).toInternal(), DataTypes.TIMESTAMP_LTZ(9).bridgedTo(long.class), diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java index 0742134ebd64b..4e891871b9c0b 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/plan/nodes/exec/serde/LogicalTypeJsonSerdeTest.java @@ -43,6 +43,7 @@ import org.apache.flink.table.types.logical.DescriptorType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -267,6 +268,7 @@ private static List testLogicalTypeSerde() { new MultisetType(BinaryType.ofEmptyLiteral()), new MultisetType(VarBinaryType.ofEmptyLiteral()), new BitmapType(), + new GeographyType(), RowType.of(new BigIntType(), new IntType(false), new VarCharType(200)), RowType.of( new LogicalType[] { diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java index d5e7f361d7fa9..33030f2e35775 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/data/BoxedWrapperRowData.java @@ -154,6 +154,11 @@ public Bitmap getBitmap(int pos) { return (Bitmap) this.fields[pos]; } + @Override + public GeographyData getGeography(int pos) { + return (GeographyData) this.fields[pos]; + } + @Override public void setNullAt(int pos) { this.fields[pos] = null; diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java index 787e54957c749..37e5d492a5577 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/data/RowDataTest.java @@ -33,11 +33,13 @@ import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.BinaryType; +import org.apache.flink.table.types.logical.BitmapType; import org.apache.flink.table.types.logical.BooleanType; import org.apache.flink.table.types.logical.CharType; import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.MultisetType; @@ -63,7 +65,11 @@ /** Test for {@link RowData}s. */ class RowDataTest { - private static final int NUM_FIELDS = 19; + private static final int NUM_FIELDS = 20; + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; private StringData str; private RawValueData generic; @@ -77,6 +83,7 @@ class RowDataTest { private TimestampData timestamp1; private TimestampData timestamp2; private Bitmap bitmap; + private GeographyData geography; @BeforeEach void before() { @@ -105,6 +112,7 @@ void before() { timestamp2 = TimestampData.fromLocalDateTime(LocalDateTime.of(1969, 1, 1, 0, 0, 0, 123456789)); bitmap = Bitmap.fromArray(new int[] {1, 2, 3}); + geography = GeographyData.fromBytes(POINT_WKB); } @Test @@ -152,6 +160,7 @@ private BinaryRowData getBinaryRow() { writer.writeTimestamp(16, timestamp1, 3); writer.writeTimestamp(17, timestamp2, 9); writer.writeBitmap(18, bitmap); + writer.writeGeography(19, geography); return row; } @@ -177,6 +186,7 @@ void testGenericRow() { row.setField(16, timestamp1); row.setField(17, timestamp2); row.setField(18, bitmap); + row.setField(19, geography); testGetters(row); } @@ -201,6 +211,7 @@ public void testBoxedWrapperRow() { row.setNonPrimitiveValue(16, timestamp1); row.setNonPrimitiveValue(17, timestamp2); row.setNonPrimitiveValue(18, bitmap); + row.setNonPrimitiveValue(19, geography); testGetters(row); testSetters(row); } @@ -230,6 +241,7 @@ public void testJoinedRow() { row2.setField(11, timestamp1); row2.setField(12, timestamp2); row2.setField(13, bitmap); + row2.setField(14, geography); testGetters(new JoinedRowData(row1, row2)); } @@ -277,6 +289,14 @@ void testFieldGetters() { .isEqualTo(timestamp1); assertThat(RowData.createFieldGetter(new TimestampType(9), 17).getFieldOrNull(row)) .isEqualTo(timestamp2); + assertThat(RowData.createFieldGetter(new BitmapType(), 18).getFieldOrNull(row)) + .isEqualTo(bitmap); + assertThat( + ((GeographyData) + RowData.createFieldGetter(new GeographyType(), 19) + .getFieldOrNull(row)) + .toBytes()) + .isEqualTo(geography.toBytes()); } @Test @@ -355,6 +375,10 @@ private void testFieldGettersWithNull(boolean nullable) { RowData.createFieldGetter(new TimestampType(nullable, 9), 17) .getFieldOrNull(row)) .isNull(); + assertThat(RowData.createFieldGetter(new BitmapType(nullable), 18).getFieldOrNull(row)) + .isNull(); + assertThat(RowData.createFieldGetter(new GeographyType(nullable), 19).getFieldOrNull(row)) + .isNull(); } private void testGetters(RowData row) { @@ -385,6 +409,7 @@ private void testGetters(RowData row) { assertThat(row.getTimestamp(16, 3)).isEqualTo(timestamp1); assertThat(row.getTimestamp(17, 9)).isEqualTo(timestamp2); assertThat(row.getBitmap(18)).isEqualTo(bitmap); + assertThat(row.getGeography(19).toBytes()).isEqualTo(geography.toBytes()); } private void testSetters(RowData row) { @@ -435,7 +460,7 @@ private void testSetters(RowData row) { } private static BinaryRowData getNullBinaryRow() { - BinaryRowData row = new BinaryRowData(18); + BinaryRowData row = new BinaryRowData(NUM_FIELDS); BinaryRowWriter binaryRowWriter = new BinaryRowWriter(row); for (int i = 0; i < row.getArity(); i++) { binaryRowWriter.setNullAt(i); diff --git a/flink-table/flink-table-test-utils/pom.xml b/flink-table/flink-table-test-utils/pom.xml index 590f4d52359bf..9460a52e034ae 100644 --- a/flink-table/flink-table-test-utils/pom.xml +++ b/flink-table/flink-table-test-utils/pom.xml @@ -59,6 +59,12 @@ under the License. flink-table-type-utils ${project.version} + + org.apache.flink + flink-table-type-utils + ${project.version} + test + org.apache.flink flink-table-planner-loader diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java index f01a1aca779da..3123edcb3b844 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/conversion/DataStructureConverters.java @@ -22,11 +22,13 @@ import org.apache.flink.table.api.TableException; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.binary.BinaryGeographyData; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalType; import org.apache.flink.table.types.logical.LogicalTypeRoot; @@ -200,6 +202,8 @@ public final class DataStructureConverters { putConverter(LogicalTypeRoot.VARIANT, Variant.class, identity()); putConverter(LogicalTypeRoot.BITMAP, Bitmap.class, constructor(BitmapBitmapConverter::new)); putConverter(LogicalTypeRoot.BITMAP, RoaringBitmapData.class, identity()); + putConverter(LogicalTypeRoot.GEOGRAPHY, GeographyData.class, identity()); + putConverter(LogicalTypeRoot.GEOGRAPHY, BinaryGeographyData.class, identity()); } /** Returns a converter for the given {@link DataType}. */ diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java index 73f817e64815e..0564d0b0cc634 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/AbstractBinaryWriter.java @@ -24,6 +24,7 @@ import org.apache.flink.core.memory.MemorySegmentFactory; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -141,6 +142,11 @@ public void writeBitmap(int pos, Bitmap bitmap) { writeBytesToVarLenPart(pos, bytes, bytes.length); } + @Override + public void writeGeography(int pos, GeographyData geography) { + writeBinary(pos, geography.toBytes()); + } + private DataOutputViewStreamWrapper getOutputView() { if (outputView == null) { outputView = new DataOutputViewStreamWrapper(new BinaryRowWriterOutputView()); diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java index 857c2ecd42466..68540b9fc93e2 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryArrayWriter.java @@ -247,6 +247,7 @@ public static NullSetter createNullSetter(LogicalType elementType) { case RAW: case VARIANT: case BITMAP: + case GEOGRAPHY: return BinaryArrayWriter::setNullLong; case BOOLEAN: return BinaryArrayWriter::setNullBoolean; diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java index 4db8977ccd2f0..482540b1d5331 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/data/writer/BinaryWriter.java @@ -21,6 +21,7 @@ import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.table.data.ArrayData; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.MapData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; @@ -91,6 +92,8 @@ public interface BinaryWriter { void writeBitmap(int pos, Bitmap bitmap); + void writeGeography(int pos, GeographyData geography); + /** Finally, complete write to set real size to binary. */ void complete(); @@ -174,6 +177,9 @@ static void write( case BITMAP: writer.writeBitmap(pos, (Bitmap) o); break; + case GEOGRAPHY: + writer.writeGeography(pos, (GeographyData) o); + break; default: throw new UnsupportedOperationException("Not support type: " + type); } @@ -253,6 +259,8 @@ static ValueSetter createValueSetter(LogicalType elementType) { return (writer, pos, value) -> writer.writeVariant(pos, (Variant) value); case BITMAP: return (writer, pos, value) -> writer.writeBitmap(pos, (Bitmap) value); + case GEOGRAPHY: + return (writer, pos, value) -> writer.writeGeography(pos, (GeographyData) value); case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java new file mode 100644 index 0000000000000..3e54f24009eb8 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializer.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.base.TypeSerializerSingleton; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.table.data.GeographyData; + +import java.io.IOException; + +/** + * Serializer for {@link GeographyData} values of {@link + * org.apache.flink.table.types.logical.GeographyType}. + */ +@Internal +public final class GeographyTypeSerializer extends TypeSerializerSingleton { + + private static final long serialVersionUID = 1L; + + private static final int FORMAT_VERSION = 1; + + private static final byte[] EMPTY_GEOMETRY_COLLECTION = + new byte[] {1, GeographyData.GEOMETRY_COLLECTION, 0, 0, 0, 0, 0, 0, 0}; + + public static final GeographyTypeSerializer INSTANCE = new GeographyTypeSerializer(); + + private GeographyTypeSerializer() {} + + @Override + public boolean isImmutableType() { + return true; + } + + @Override + public GeographyData createInstance() { + return GeographyData.fromBytes(EMPTY_GEOMETRY_COLLECTION); + } + + @Override + public GeographyData copy(GeographyData from) { + return GeographyData.fromBytes(from.toBytes()); + } + + @Override + public GeographyData copy(GeographyData from, GeographyData reuse) { + return copy(from); + } + + @Override + public int getLength() { + return -1; + } + + @Override + public void serialize(GeographyData record, DataOutputView target) throws IOException { + final byte[] bytes = record.toBytes(); + target.writeByte(FORMAT_VERSION); + target.writeInt(bytes.length); + target.write(bytes); + } + + @Override + public GeographyData deserialize(DataInputView source) throws IOException { + readFormatVersion(source); + final int length = readPayloadLength(source); + final byte[] bytes = new byte[length]; + source.readFully(bytes); + return GeographyData.fromBytes(bytes); + } + + @Override + public GeographyData deserialize(GeographyData reuse, DataInputView source) throws IOException { + return deserialize(source); + } + + @Override + public void copy(DataInputView source, DataOutputView target) throws IOException { + final int version = readFormatVersion(source); + final int length = readPayloadLength(source); + target.writeByte(version); + target.writeInt(length); + target.write(source, length); + } + + @Override + public TypeSerializerSnapshot snapshotConfiguration() { + return new GeographyTypeSerializerSnapshot(); + } + + private static int readFormatVersion(DataInputView source) throws IOException { + final int version = source.readUnsignedByte(); + if (version != FORMAT_VERSION) { + throw new IOException( + String.format( + "Unsupported GEOGRAPHY serializer format version %d. Expected %d.", + version, FORMAT_VERSION)); + } + return version; + } + + private static int readPayloadLength(DataInputView source) throws IOException { + final int length = source.readInt(); + if (length < 0) { + throw new IOException(String.format("Invalid GEOGRAPHY payload length %d.", length)); + } + return length; + } +} diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java new file mode 100644 index 0000000000000..a315901647989 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerSnapshot.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSchemaCompatibility; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.core.memory.DataInputView; +import org.apache.flink.core.memory.DataOutputView; +import org.apache.flink.table.data.GeographyData; + +import java.io.IOException; + +/** Serializer snapshot for {@link GeographyTypeSerializer}. */ +@Internal +public final class GeographyTypeSerializerSnapshot + implements TypeSerializerSnapshot { + + private static final int CURRENT_VERSION = 1; + + @Override + public int getCurrentVersion() { + return CURRENT_VERSION; + } + + @Override + public void writeSnapshot(DataOutputView out) throws IOException {} + + @Override + public void readSnapshot(int readVersion, DataInputView in, ClassLoader userCodeClassLoader) + throws IOException {} + + @Override + public TypeSerializer restoreSerializer() { + return GeographyTypeSerializer.INSTANCE; + } + + @Override + public TypeSerializerSchemaCompatibility resolveSchemaCompatibility( + TypeSerializerSnapshot oldSerializerSnapshot) { + if (oldSerializerSnapshot instanceof GeographyTypeSerializerSnapshot) { + return TypeSerializerSchemaCompatibility.compatibleAsIs(); + } + return TypeSerializerSchemaCompatibility.incompatible(); + } +} diff --git a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java index 371dcd136e1d5..dc99d6477d7d5 100644 --- a/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java +++ b/flink-table/flink-table-type-utils/src/main/java/org/apache/flink/table/runtime/typeutils/InternalSerializers.java @@ -129,6 +129,8 @@ private static TypeSerializer createInternal(LogicalType type) { return VariantSerializer.INSTANCE; case BITMAP: return BitmapSerializer.INSTANCE; + case GEOGRAPHY: + return GeographyTypeSerializer.INSTANCE; case NULL: case SYMBOL: case UNRESOLVED: diff --git a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java new file mode 100644 index 0000000000000..8a695c5c020f5 --- /dev/null +++ b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/GeographyTypeSerializerTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.typeutils; + +import org.apache.flink.api.common.typeutils.SerializerTestBase; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot; +import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil; +import org.apache.flink.core.memory.DataInputViewStreamWrapper; +import org.apache.flink.core.memory.DataOutputViewStreamWrapper; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.binary.BinaryArrayData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.types.logical.GeographyType; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link GeographyTypeSerializer}. */ +class GeographyTypeSerializerTest extends SerializerTestBase { + + private static final int FORMAT_VERSION = 1; + + private static final byte[] POINT_WKB = + new byte[] { + 1, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + private static final byte[] BIG_ENDIAN_POINT_WKB = + new byte[] { + 0, 0, 0, 0, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + @Override + protected TypeSerializer createSerializer() { + return GeographyTypeSerializer.INSTANCE; + } + + @Override + protected int getLength() { + return -1; + } + + @Override + protected Class getTypeClass() { + return GeographyData.class; + } + + @Override + protected GeographyData[] getTestData() { + return new GeographyData[] { + GeographyData.fromBytes(POINT_WKB), + GeographyData.fromBytes(BIG_ENDIAN_POINT_WKB), + GeographyTypeSerializer.INSTANCE.createInstance() + }; + } + + @Override + protected void deepEquals(String message, GeographyData should, GeographyData is) { + assertThat(is.toBytes()).as(message).isEqualTo(should.toBytes()); + } + + @Test + void testInternalSerializerRoundTripsRawWkb() throws Exception { + final TypeSerializer serializer = + InternalSerializers.create(new GeographyType()); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + serializer.serialize(geography, new DataOutputViewStreamWrapper(bytes)); + final GeographyData deserialized = + serializer.deserialize( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray()))); + + assertThat(deserialized.toBytes()).isEqualTo(POINT_WKB); + assertThat(deserialized.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testSerializedFormUsesVersionedLengthEnvelope() throws Exception { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + GeographyTypeSerializer.INSTANCE.serialize( + GeographyData.fromBytes(POINT_WKB), new DataOutputViewStreamWrapper(bytes)); + final DataInputViewStreamWrapper input = + new DataInputViewStreamWrapper(new ByteArrayInputStream(bytes.toByteArray())); + final byte[] payload = new byte[POINT_WKB.length]; + + assertThat(input.readUnsignedByte()).isEqualTo(FORMAT_VERSION); + assertThat(input.readInt()).isEqualTo(POINT_WKB.length); + input.readFully(payload); + assertThat(payload).isEqualTo(POINT_WKB); + } + + @Test + void testRejectsUnsupportedPayloadVersion() throws Exception { + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + final DataOutputViewStreamWrapper output = new DataOutputViewStreamWrapper(bytes); + output.writeByte(2); + output.writeInt(POINT_WKB.length); + output.write(POINT_WKB); + + assertThatThrownBy( + () -> + GeographyTypeSerializer.INSTANCE.deserialize( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray())))) + .isInstanceOf(IOException.class) + .hasMessageContaining("Unsupported GEOGRAPHY serializer format version 2"); + } + + @Test + void testSnapshotSelfCompatibility() throws Exception { + final TypeSerializerSnapshot snapshot = + GeographyTypeSerializer.INSTANCE.snapshotConfiguration(); + final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot( + new DataOutputViewStreamWrapper(bytes), snapshot); + final TypeSerializerSnapshot restoredSnapshot = + TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot( + new DataInputViewStreamWrapper( + new ByteArrayInputStream(bytes.toByteArray())), + getClass().getClassLoader()); + + assertThat( + GeographyTypeSerializer.INSTANCE + .snapshotConfiguration() + .resolveSchemaCompatibility(restoredSnapshot) + .isCompatibleAsIs()) + .isTrue(); + assertThat(restoredSnapshot.restoreSerializer()).isSameAs(GeographyTypeSerializer.INSTANCE); + } + + @Test + void testCopyPreservesRawWkbBytes() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GeographyData copied = GeographyTypeSerializer.INSTANCE.copy(geography); + + assertThat(copied).isNotSameAs(geography); + assertThat(copied.toBytes()).isEqualTo(POINT_WKB); + assertThat(copied.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testCreateInstanceReturnsValidGeographyData() { + final GeographyData instance = GeographyTypeSerializer.INSTANCE.createInstance(); + + assertThat(instance.subtypeId()).isEqualTo(GeographyData.GEOMETRY_COLLECTION); + assertThat(instance.sizeInBytes()).isEqualTo(9); + } + + @Test + void testRowDataSerializerConvertsGenericRowToBinaryRow() { + final RowDataSerializer serializer = + InternalTypeInfo.ofFields(new GeographyType()).toRowSerializer(); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final GenericRowData row = GenericRowData.of(geography); + + final BinaryRowData binaryRow = serializer.toBinaryRow(row, true); + + assertThat(binaryRow.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(binaryRow.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testRowDataSerializerPreservesNullGeography() { + final RowDataSerializer serializer = + InternalTypeInfo.ofFields(new GeographyType()).toRowSerializer(); + final GenericRowData row = GenericRowData.of((Object) null); + + final BinaryRowData binaryRow = serializer.toBinaryRow(row, true); + + assertThat(binaryRow.isNullAt(0)).isTrue(); + } + + @Test + void testArrayDataSerializerConvertsGenericArrayToBinaryArray() { + final ArrayDataSerializer serializer = new ArrayDataSerializer(new GeographyType()); + final GenericArrayData array = + new GenericArrayData(new Object[] {GeographyData.fromBytes(POINT_WKB), null}); + + final BinaryArrayData binaryArray = serializer.toBinaryArray(array); + + assertThat(binaryArray.getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(binaryArray.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(binaryArray.isNullAt(1)).isTrue(); + } +} diff --git a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java index 6d6379ed04696..e5a09b407e45b 100644 --- a/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java +++ b/flink-table/flink-table-type-utils/src/test/java/org/apache/flink/table/runtime/typeutils/RowDataSerializerTest.java @@ -24,6 +24,7 @@ import org.apache.flink.api.java.typeutils.runtime.kryo.KryoSerializer; import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.RawValueData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.StringData; @@ -41,6 +42,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.LocalZonedTimestampType; import org.apache.flink.table.types.logical.LogicalType; @@ -62,6 +64,11 @@ /** Test for {@link RowDataSerializer}. */ abstract class RowDataSerializerTest extends SerializerTestInstance { + private static final byte[] POINT_WKB = + new byte[] { + 1, GeographyData.POINT, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + private final RowDataSerializer serializer; private final RowData[] testData; @@ -233,6 +240,28 @@ private static RowDataSerializer getRowSerializer() { } } + static final class RowDataSerializerWithGeographyTest extends RowDataSerializerTest { + public RowDataSerializerWithGeographyTest() { + super(getRowSerializer(), getData()); + } + + private static RowData[] getData() { + GenericRowData row1 = new GenericRowData(1); + row1.setField(0, GeographyData.fromBytes(POINT_WKB)); + + GenericRowData row2 = new GenericRowData(1); + row2.setField(0, null); + + return new RowData[] {row1, row2}; + } + + private static RowDataSerializer getRowSerializer() { + InternalTypeInfo typeInfo = InternalTypeInfo.ofFields(new GeographyType()); + + return typeInfo.toRowSerializer(); + } + } + static final class LargeRowDataSerializerTest extends RowDataSerializerTest { public LargeRowDataSerializerTest() { super(getRowSerializer(), getData()); diff --git a/flink-tests/pom.xml b/flink-tests/pom.xml index 4ef3c5f273d53..ee49aca935c6d 100644 --- a/flink-tests/pom.xml +++ b/flink-tests/pom.xml @@ -69,6 +69,13 @@ under the License. test + + org.apache.flink + flink-connector-base + ${project.version} + test + + org.apache.flink flink-connector-files @@ -205,6 +212,13 @@ under the License. test + + org.apache.flink + flink-table-type-utils + ${project.version} + test + + org.apache.flink flink-runtime diff --git a/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java b/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java index 0ac026145139e..84ada2f5b4271 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java +++ b/flink-tests/src/test/java/org/apache/flink/test/completeness/TypeSerializerTestCoverageTest.java @@ -83,6 +83,7 @@ import org.apache.flink.table.runtime.typeutils.BinaryRowDataSerializer; import org.apache.flink.table.runtime.typeutils.DecimalDataSerializer; import org.apache.flink.table.runtime.typeutils.ExternalSerializer; +import org.apache.flink.table.runtime.typeutils.GeographyTypeSerializer; import org.apache.flink.table.runtime.typeutils.LinkedListSerializer; import org.apache.flink.table.runtime.typeutils.MapDataSerializer; import org.apache.flink.table.runtime.typeutils.RawValueDataSerializer; @@ -196,6 +197,7 @@ void testTypeSerializerTestCoverage() { SharedBufferEdge.SharedBufferEdgeSerializer.class.getName(), RowDataSerializer.class.getName(), DecimalDataSerializer.class.getName(), + GeographyTypeSerializer.class.getName(), AvroSerializer.class.getName()); // type serializer whitelist for TypeSerializerUpgradeTestBase test coverage @@ -258,6 +260,7 @@ void testTypeSerializerTestCoverage() { SharedBufferEdge.SharedBufferEdgeSerializer.class.getName(), RowDataSerializer.class.getName(), DecimalDataSerializer.class.getName(), + GeographyTypeSerializer.class.getName(), AvroSerializer.class.getName(), // KeyAndValueSerializer shouldn't be used to serialize data to state and // doesn't need to ensure upgrade compatibility. From 7f4990ae7d94d45427c1471f2dbb34e2249b44fe Mon Sep 17 00:00:00 2001 From: Garik Kalashyan Date: Mon, 15 Jun 2026 20:09:03 +0400 Subject: [PATCH 2/6] [FLINK-xxxxx] Add SQL geography construction/functions --- .../functions/BuiltInFunctionDefinitions.java | 57 ++++++ .../converter/ExpressionConverter.java | 29 +++ .../functions/sql/FlinkSqlOperatorTable.java | 47 +++++ ...LiteralExpressionsSerializationITCase.java | 82 ++++++++ .../converter/ExpressionConverterTest.java | 44 +++++ .../GeographyAccessorFunctionsITCase.java | 59 ++++++ .../GeographyConstructorFunctionsITCase.java | 83 ++++++++ flink-table/flink-table-runtime/pom.xml | 8 + .../scalar/GeographyConversionUtils.java | 187 ++++++++++++++++++ .../functions/scalar/StAsTextFunction.java | 40 ++++ .../functions/scalar/StAsWkbFunction.java | 39 ++++ .../scalar/StGeogFromTextFunction.java | 40 ++++ .../scalar/StGeogFromWkbFunction.java | 39 ++++ .../src/main/resources/META-INF/NOTICE | 9 + .../GeographyAccessorFunctionsTest.java | 67 +++++++ .../GeographyConstructorFunctionsTest.java | 108 ++++++++++ 16 files changed, 938 insertions(+) create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyAccessorFunctionsITCase.java create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyConstructorFunctionsITCase.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/GeographyConversionUtils.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsTextFunction.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsWkbFunction.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromTextFunction.java create mode 100644 flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromWkbFunction.java create mode 100644 flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyAccessorFunctionsTest.java create mode 100644 flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyConstructorFunctionsTest.java diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java index 0828f2bb8b4e9..337bfeeba973d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java @@ -3115,6 +3115,63 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL) "org.apache.flink.table.runtime.functions.scalar.TryParseJsonFunction") .build(); + // -------------------------------------------------------------------------------------------- + // Geography functions + // -------------------------------------------------------------------------------------------- + + public static final BuiltInFunctionDefinition ST_GEOGFROMTEXT = + BuiltInFunctionDefinition.newBuilder() + .name("ST_GEOGFROMTEXT") + .kind(SCALAR) + .inputTypeStrategy( + sequence( + Collections.singletonList("text"), + Collections.singletonList( + logical(LogicalTypeFamily.CHARACTER_STRING)))) + .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.GEOGRAPHY()))) + .runtimeClass( + "org.apache.flink.table.runtime.functions.scalar.StGeogFromTextFunction") + .build(); + + public static final BuiltInFunctionDefinition ST_GEOGFROMWKB = + BuiltInFunctionDefinition.newBuilder() + .name("ST_GEOGFROMWKB") + .kind(SCALAR) + .inputTypeStrategy( + sequence( + Collections.singletonList("bytes"), + Collections.singletonList( + logical(LogicalTypeFamily.BINARY_STRING)))) + .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.GEOGRAPHY()))) + .runtimeClass( + "org.apache.flink.table.runtime.functions.scalar.StGeogFromWkbFunction") + .build(); + + public static final BuiltInFunctionDefinition ST_ASTEXT = + BuiltInFunctionDefinition.newBuilder() + .name("ST_ASTEXT") + .kind(SCALAR) + .inputTypeStrategy( + sequence( + Collections.singletonList("geography"), + Collections.singletonList(logical(LogicalTypeRoot.GEOGRAPHY)))) + .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.STRING()))) + .runtimeClass( + "org.apache.flink.table.runtime.functions.scalar.StAsTextFunction") + .build(); + + public static final BuiltInFunctionDefinition ST_ASWKB = + BuiltInFunctionDefinition.newBuilder() + .name("ST_ASWKB") + .kind(SCALAR) + .inputTypeStrategy( + sequence( + Collections.singletonList("geography"), + Collections.singletonList(logical(LogicalTypeRoot.GEOGRAPHY)))) + .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.BYTES()))) + .runtimeClass("org.apache.flink.table.runtime.functions.scalar.StAsWkbFunction") + .build(); + // -------------------------------------------------------------------------------------------- // Bitmap functions // -------------------------------------------------------------------------------------------- diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java index 3a532d6f15f5d..9e9b0da7a20e1 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java @@ -18,11 +18,13 @@ package org.apache.flink.table.planner.expressions.converter; +import org.apache.flink.table.api.DataTypes; import org.apache.flink.table.api.TableException; import org.apache.flink.table.catalog.Catalog; import org.apache.flink.table.catalog.ContextResolvedModel; import org.apache.flink.table.catalog.DataTypeFactory; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.expressions.CallExpression; import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.expressions.ExpressionVisitor; @@ -36,6 +38,7 @@ import org.apache.flink.table.expressions.ValueLiteralExpression; import org.apache.flink.table.factories.FactoryUtil; import org.apache.flink.table.factories.ModelProviderFactory; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; import org.apache.flink.table.ml.ModelProvider; import org.apache.flink.table.module.Module; import org.apache.flink.table.planner.calcite.FlinkContext; @@ -141,6 +144,10 @@ public RexNode visit(ValueLiteralExpression valueLiteral) { .collect(Collectors.toList())); } + if (type.getTypeRoot() == LogicalTypeRoot.GEOGRAPHY) { + return convertGeographyLiteral(valueLiteral); + } + Object value; switch (type.getTypeRoot()) { case DECIMAL: @@ -220,6 +227,28 @@ public RexNode visit(ValueLiteralExpression valueLiteral) { true); } + private RexNode convertGeographyLiteral(ValueLiteralExpression valueLiteral) { + final GeographyData geography = + valueLiteral + .getValueAs(GeographyData.class) + .orElseThrow( + () -> + new TableException( + String.format( + "GEOGRAPHY literals require values of class '%s' but found '%s'.", + GeographyData.class.getName(), + extractValue(valueLiteral, Object.class) + .getClass() + .getName()))); + return visit( + CallExpression.permanent( + BuiltInFunctionDefinitions.ST_GEOGFROMWKB, + List.of( + new ValueLiteralExpression( + geography.toBytes(), DataTypes.BYTES().notNull())), + valueLiteral.getOutputDataType())); + } + @Override public RexNode visit(FieldReferenceExpression fieldReference) { // We can not use inputCount+inputIndex+FieldIndex to construct field of calcite. diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java index 32a2a20d67966..33da605f632aa 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java @@ -25,6 +25,7 @@ import org.apache.flink.table.planner.functions.sql.ml.SqlVectorSearchTableFunction; import org.apache.flink.table.planner.plan.type.FlinkReturnTypes; import org.apache.flink.table.planner.plan.type.NumericExceptFirstOperandChecker; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlFunction; @@ -72,6 +73,16 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable { /** The table of contains Flink-specific operators. */ private static final Map cachedInstances = new HashMap<>(); + private static final SqlReturnTypeInference GEOGRAPHY_NULLABLE_IF_ARGS = + opBinding -> { + boolean nullable = false; + for (int i = 0; i < opBinding.getOperandCount(); i++) { + nullable |= opBinding.getOperandType(i).isNullable(); + } + return ((FlinkTypeFactory) opBinding.getTypeFactory()) + .createFieldTypeFromLogicalType(new GeographyType(nullable)); + }; + /** Returns the Flink operator table, creating it if necessary. */ public static synchronized FlinkSqlOperatorTable instance(boolean isBatchMode) { FlinkSqlOperatorTable instance = cachedInstances.get(isBatchMode); @@ -867,6 +878,42 @@ public SqlSyntax getSyntax() { OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.CHARACTER), SqlFunctionCategory.STRING); + // GEOGRAPHY FUNCTIONS + public static final SqlFunction ST_GEOGFROMTEXT = + BuiltInSqlFunction.newBuilder() + .name(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT.getName()) + .returnType(GEOGRAPHY_NULLABLE_IF_ARGS) + .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY)) + .category(SqlFunctionCategory.STRING) + .build(); + + public static final SqlFunction ST_GEOGFROMWKB = + BuiltInSqlFunction.newBuilder() + .name(BuiltInFunctionDefinitions.ST_GEOGFROMWKB.getName()) + .returnType(GEOGRAPHY_NULLABLE_IF_ARGS) + .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY)) + .category(SqlFunctionCategory.STRING) + .build(); + + public static final SqlFunction ST_ASTEXT = + BuiltInSqlFunction.newBuilder() + .name(BuiltInFunctionDefinitions.ST_ASTEXT.getName()) + .returnType(VARCHAR_FORCE_NULLABLE) + .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY)) + .category(SqlFunctionCategory.STRING) + .build(); + + public static final SqlFunction ST_ASWKB = + BuiltInSqlFunction.newBuilder() + .name(BuiltInFunctionDefinitions.ST_ASWKB.getName()) + .returnType( + ReturnTypes.cascade( + ReturnTypes.explicit(SqlTypeName.VARBINARY), + SqlTypeTransforms.TO_NULLABLE)) + .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY)) + .category(SqlFunctionCategory.STRING) + .build(); + public static final SqlFunction INSTR = new SqlFunction( "INSTR", diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/LiteralExpressionsSerializationITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/LiteralExpressionsSerializationITCase.java index c24b84b1b0016..93d1e62cbea86 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/LiteralExpressionsSerializationITCase.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/LiteralExpressionsSerializationITCase.java @@ -23,6 +23,7 @@ import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.expressions.DefaultSqlFactory; import org.apache.flink.table.expressions.ResolvedExpression; import org.apache.flink.table.expressions.SqlFactory; @@ -42,17 +43,93 @@ import java.time.LocalTime; import java.time.Period; import java.time.temporal.ChronoUnit; +import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import static org.apache.flink.table.api.Expressions.array; import static org.apache.flink.table.api.Expressions.lit; +import static org.apache.flink.table.api.Expressions.map; import static org.apache.flink.table.api.Expressions.nullOf; +import static org.apache.flink.table.api.Expressions.row; import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link ResolvedExpression#asSerializableString(SqlFactory)}. */ @ExtendWith(MiniClusterExtension.class) public class LiteralExpressionsSerializationITCase { + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final String POINT_WKB_HEX = "0101000000000000000000F03F0000000000000040"; + + @Test + void testGeographySqlSerialization() { + final TableEnvironment env = TableEnvironment.create(EnvironmentSettings.inStreamingMode()); + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + final Table t = + env.fromValues(1) + .select( + lit(geography, DataTypes.GEOGRAPHY().notNull()), + lit(null, DataTypes.GEOGRAPHY()), + array( + lit(geography, DataTypes.GEOGRAPHY().notNull()), + lit(null, DataTypes.GEOGRAPHY())), + map( + lit("a", DataTypes.STRING().notNull()), + lit(geography, DataTypes.GEOGRAPHY().notNull())), + row( + lit(geography, DataTypes.GEOGRAPHY().notNull()), + lit(null, DataTypes.GEOGRAPHY()))); + + final ProjectQueryOperation operation = (ProjectQueryOperation) t.getQueryOperation(); + final List expressions = + operation.getProjectList().stream() + .map( + resolvedExpression -> + resolvedExpression.asSerializableString( + DefaultSqlFactory.INSTANCE)) + .collect(Collectors.toList()); + + assertThat(expressions.get(0)).isEqualTo("ST_GEOGFROMWKB(X'" + POINT_WKB_HEX + "')"); + assertThat(expressions.get(1)).isEqualTo("CAST(NULL AS GEOGRAPHY)"); + assertThat(expressions.get(2)) + .contains("ST_GEOGFROMWKB(X'" + POINT_WKB_HEX + "')") + .contains("CAST(NULL AS GEOGRAPHY)"); + assertThat(expressions.get(3)) + .isEqualTo("MAP['a', ST_GEOGFROMWKB(X'" + POINT_WKB_HEX + "')]"); + assertThat(expressions.get(4)) + .contains("ST_GEOGFROMWKB(X'" + POINT_WKB_HEX + "')") + .contains("CAST(NULL AS GEOGRAPHY)"); + + final TableResult tableResult = + env.sqlQuery(String.format("SELECT %s", String.join(", ", expressions))).execute(); + final Row result = CollectionUtil.iteratorToList(tableResult.collect()).get(0); + + assertGeography(result.getField(0)); + assertThat(result.getField(1)).isNull(); + + final Object arrayField = result.getField(2); + final List arrayValue = + arrayField instanceof List + ? (List) arrayField + : Arrays.asList((Object[]) arrayField); + assertThat(arrayValue).hasSize(2); + assertGeography(arrayValue.get(0)); + assertThat(arrayValue.get(1)).isNull(); + + final Map mapValue = (Map) result.getField(3); + assertThat(mapValue).hasSize(1); + assertGeography(mapValue.get("a")); + + final Row rowValue = (Row) result.getField(4); + assertGeography(rowValue.getField(0)); + assertThat(rowValue.getField(1)).isNull(); + } + @Test void testSqlSerialization() { final TableEnvironment env = TableEnvironment.create(EnvironmentSettings.inStreamingMode()); @@ -161,4 +238,9 @@ void testSqlSerialization() { duration, period)); } + + private static void assertGeography(Object value) { + assertThat(value).isInstanceOf(GeographyData.class); + assertThat(((GeographyData) value).toBytes()).isEqualTo(POINT_WKB); + } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverterTest.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverterTest.java index ffea6f95d5e2d..588db6b5bc3d5 100644 --- a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverterTest.java +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverterTest.java @@ -19,11 +19,14 @@ package org.apache.flink.table.planner.expressions.converter; import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.expressions.TimePointUnit; import org.apache.flink.table.planner.delegation.PlannerContext; import org.apache.flink.table.planner.utils.PlannerMocks; import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.type.SqlTypeName; @@ -42,10 +45,16 @@ import static org.apache.flink.table.expressions.ApiExpressionUtils.valueLiteral; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** Test for {@link ExpressionConverter}. */ class ExpressionConverterTest { + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + private final PlannerContext plannerContext = PlannerMocks.create().getPlannerContext(); private final ExpressionConverter converter = new ExpressionConverter(plannerContext.createRelBuilder()); @@ -194,4 +203,39 @@ void testSymbolLiteral() { assertThat(((RexLiteral) rex).getValueAs(TimeUnit.class)).isEqualTo(TimeUnit.MICROSECOND); assertThat(rex.getType().getSqlTypeName()).isEqualTo(SqlTypeName.SYMBOL); } + + @Test + void testGeographyLiteralUsesConstructorCall() { + RexNode rex = + converter.visit( + valueLiteral( + GeographyData.fromBytes(POINT_WKB), + DataTypes.GEOGRAPHY().notNull())); + + assertThat(rex).isInstanceOf(RexCall.class); + assertThat(rex.getType().getSqlTypeName()).isEqualTo(SqlTypeName.OTHER); + assertThat(rex.getType().isNullable()).isFalse(); + + RexCall call = (RexCall) rex; + assertThat(call.getOperator().getName()).isEqualTo("ST_GEOGFROMWKB"); + assertThat(((RexLiteral) call.getOperands().get(0)).getValueAs(byte[].class)) + .isEqualTo(POINT_WKB); + } + + @Test + void testNullableGeographyLiteral() { + RexNode rex = converter.visit(valueLiteral(null, DataTypes.GEOGRAPHY())); + + assertThat(rex).isInstanceOf(RexLiteral.class); + assertThat(rex.getType().getSqlTypeName()).isEqualTo(SqlTypeName.OTHER); + assertThat(rex.getType().isNullable()).isTrue(); + assertThat(((RexLiteral) rex).getValue()).isNull(); + } + + @Test + void testInvalidGeographyLiteralFailsValidation() { + assertThatThrownBy(() -> valueLiteral("POINT (1 2)", DataTypes.GEOGRAPHY().notNull())) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("does not support a value literal"); + } } diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyAccessorFunctionsITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyAccessorFunctionsITCase.java new file mode 100644 index 0000000000000..a4f40e0d54f8c --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyAccessorFunctionsITCase.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.functions; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; + +import java.util.stream.Stream; + +/** Tests for GEOGRAPHY accessor functions. */ +class GeographyAccessorFunctionsITCase extends BuiltInFunctionTestBase { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Override + Stream getTestSetSpecs() { + return Stream.of(stAsTextCases(), stAsWkbCases()); + } + + private TestSetSpec stAsTextCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_ASTEXT) + .onFieldsWithData(null, "POINT (1 2)", "POINT EMPTY") + .andDataTypes(DataTypes.STRING(), DataTypes.STRING().notNull(), DataTypes.STRING()) + .testSqlResult("ST_ASTEXT(ST_GEOGFROMTEXT(f0))", null, DataTypes.STRING()) + .testSqlResult( + "ST_ASTEXT(ST_GEOGFROMTEXT(f1))", + "POINT (1 2)", + DataTypes.STRING().notNull()) + .testSqlResult("ST_ASTEXT(ST_GEOGFROMTEXT(f2))", "POINT EMPTY", DataTypes.STRING()); + } + + private TestSetSpec stAsWkbCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_ASWKB) + .onFieldsWithData(null, POINT_WKB) + .andDataTypes(DataTypes.BYTES(), DataTypes.BYTES().notNull()) + .testSqlResult("ST_ASWKB(ST_GEOGFROMWKB(f0))", null, DataTypes.BYTES()) + .testSqlResult( + "ST_ASWKB(ST_GEOGFROMWKB(f1))", POINT_WKB, DataTypes.BYTES().notNull()); + } +} diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyConstructorFunctionsITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyConstructorFunctionsITCase.java new file mode 100644 index 0000000000000..0788a87ffa005 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/GeographyConstructorFunctionsITCase.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.functions; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; + +import java.util.stream.Stream; + +/** Tests for GEOGRAPHY constructor functions. */ +class GeographyConstructorFunctionsITCase extends BuiltInFunctionTestBase { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Override + Stream getTestSetSpecs() { + return Stream.of(stGeogFromTextCases(), stGeogFromWkbCases(), runtimeErrorCases()); + } + + private TestSetSpec stGeogFromTextCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT) + .onFieldsWithData(null, "POINT (1 2)", "POINT EMPTY") + .andDataTypes(DataTypes.STRING(), DataTypes.STRING().notNull(), DataTypes.STRING()) + .testSqlResult("ST_GEOGFROMTEXT(f0)", null, DataTypes.GEOGRAPHY()) + .testSqlResult("ST_GEOGFROMTEXT(f1)", DataTypes.GEOGRAPHY().notNull()) + .testSqlResult("ST_GEOGFROMTEXT(f2)", DataTypes.GEOGRAPHY()); + } + + private TestSetSpec stGeogFromWkbCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_GEOGFROMWKB) + .onFieldsWithData(null, POINT_WKB) + .andDataTypes(DataTypes.BYTES(), DataTypes.BYTES().notNull()) + .testSqlResult("ST_GEOGFROMWKB(f0)", null, DataTypes.GEOGRAPHY()) + .testSqlResult("ST_GEOGFROMWKB(f1)", DataTypes.GEOGRAPHY().notNull()); + } + + private TestSetSpec runtimeErrorCases() { + return TestSetSpec.forFunction(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT, "Runtime errors") + .onFieldsWithData( + "not wkt", "POINT Z (1 2 3)", "POINT (181 2)", new byte[] {1, 1, 0, 0}) + .andDataTypes( + DataTypes.STRING(), + DataTypes.STRING(), + DataTypes.STRING(), + DataTypes.BYTES()) + .testSqlRuntimeError( + "ST_GEOGFROMTEXT(f0)", + TableRuntimeException.class, + "Invalid GEOGRAPHY WKT.") + .testSqlRuntimeError( + "ST_GEOGFROMTEXT(f1)", + TableRuntimeException.class, + "Only 2D coordinates are supported") + .testSqlRuntimeError( + "ST_GEOGFROMTEXT(f2)", + TableRuntimeException.class, + "Expected range is [-180, 180]") + .testSqlRuntimeError( + "ST_GEOGFROMWKB(f3)", + TableRuntimeException.class, + "Invalid GEOGRAPHY WKB."); + } +} diff --git a/flink-table/flink-table-runtime/pom.xml b/flink-table/flink-table-runtime/pom.xml index 9958fc66f1b93..65b5609969b00 100644 --- a/flink-table/flink-table-runtime/pom.xml +++ b/flink-table/flink-table-runtime/pom.xml @@ -78,6 +78,13 @@ under the License. ${flink.markBundledAsOptional} + + org.locationtech.jts + jts-core + 1.19.0 + ${flink.markBundledAsOptional} + + org.apache.flink @@ -186,6 +193,7 @@ under the License. org.codehaus.janino:* org.apache.flink:flink-table-code-splitter org.apache.flink:flink-table-type-utils + org.locationtech.jts:jts-core diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/GeographyConversionUtils.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/GeographyConversionUtils.java new file mode 100644 index 0000000000000..06c2f9736190f --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/GeographyConversionUtils.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; + +import org.locationtech.jts.geom.CoordinateSequence; +import org.locationtech.jts.geom.CoordinateSequenceFilter; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.ByteOrderValues; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKBWriter; +import org.locationtech.jts.io.WKTReader; +import org.locationtech.jts.io.WKTWriter; + +import java.util.regex.Pattern; + +/** Utilities for converting GEOGRAPHY values to and from portable OGC encodings. */ +@Internal +final class GeographyConversionUtils { + + private static final Pattern WKT_DIMENSION_TOKEN = + Pattern.compile( + "\\b(POINT|LINESTRING|POLYGON|MULTIPOINT|MULTILINESTRING|MULTIPOLYGON|GEOMETRYCOLLECTION)\\s+(ZM|Z|M)\\b", + Pattern.CASE_INSENSITIVE); + private static final WKTReader WKT_READER = createWktReader(); + private static final WKBReader WKB_READER = new WKBReader(); + private static final WKBWriter WKB_WRITER = + new WKBWriter(2, ByteOrderValues.LITTLE_ENDIAN, false); + private static final WKTWriter WKT_WRITER = new WKTWriter(2); + + private GeographyConversionUtils() {} + + static GeographyData fromWkt(StringData text) { + if (text == null) { + return null; + } + + final String wkt = text.toString(); + if (WKT_DIMENSION_TOKEN.matcher(wkt).find()) { + throw new TableRuntimeException( + "Invalid GEOGRAPHY WKT. Only 2D coordinates are supported."); + } + + final Geometry geometry; + try { + geometry = WKT_READER.read(wkt); + } catch (ParseException | IllegalArgumentException e) { + throw new TableRuntimeException("Invalid GEOGRAPHY WKT.", e); + } + validateGeometry(geometry); + return fromValidatedGeometry(geometry); + } + + static GeographyData fromWkb(byte[] bytes) { + if (bytes == null) { + return null; + } + + final GeographyData geography; + try { + geography = GeographyData.fromBytes(bytes); + } catch (TableRuntimeException e) { + throw new TableRuntimeException("Invalid GEOGRAPHY WKB.", e); + } + + validateGeometry(readGeometry(geography)); + return geography; + } + + static StringData asText(GeographyData geography) { + if (geography == null) { + return null; + } + + return StringData.fromString(WKT_WRITER.write(readGeometry(geography))); + } + + static byte[] asWkb(GeographyData geography) { + if (geography == null) { + return null; + } + + return geography.toBytes(); + } + + private static GeographyData fromValidatedGeometry(Geometry geometry) { + try { + return GeographyData.fromBytes(WKB_WRITER.write(geometry)); + } catch (TableRuntimeException e) { + throw new TableRuntimeException("Invalid GEOGRAPHY WKT.", e); + } + } + + private static Geometry readGeometry(GeographyData geography) { + try { + return WKB_READER.read(geography.toBytes()); + } catch (ParseException e) { + throw new TableRuntimeException("Invalid GEOGRAPHY WKB.", e); + } + } + + private static WKTReader createWktReader() { + final WKTReader reader = new WKTReader(); + reader.setIsOldJtsCoordinateSyntaxAllowed(false); + return reader; + } + + private static void validateGeometry(Geometry geometry) { + validateSubtype(geometry); + if (!geometry.isEmpty()) { + geometry.apply(new Crs84CoordinateValidator()); + } + } + + private static void validateSubtype(Geometry geometry) { + switch (geometry.getGeometryType()) { + case Geometry.TYPENAME_POINT: + case Geometry.TYPENAME_LINESTRING: + case Geometry.TYPENAME_POLYGON: + case Geometry.TYPENAME_MULTIPOINT: + case Geometry.TYPENAME_MULTILINESTRING: + case Geometry.TYPENAME_MULTIPOLYGON: + case Geometry.TYPENAME_GEOMETRYCOLLECTION: + return; + default: + throw new TableRuntimeException( + "Unsupported GEOGRAPHY subtype: " + geometry.getGeometryType() + "."); + } + } + + private static final class Crs84CoordinateValidator implements CoordinateSequenceFilter { + + @Override + public void filter(CoordinateSequence sequence, int i) { + if (sequence.hasZ() || sequence.hasM()) { + throw new TableRuntimeException( + "Invalid GEOGRAPHY coordinates. Only 2D coordinates are supported."); + } + + final double longitude = sequence.getX(i); + final double latitude = sequence.getY(i); + if (longitude < -180D || longitude > 180D) { + throw new TableRuntimeException( + String.format( + "Invalid GEOGRAPHY longitude %.12f. Expected range is [-180, 180].", + longitude)); + } + if (latitude < -90D || latitude > 90D) { + throw new TableRuntimeException( + String.format( + "Invalid GEOGRAPHY latitude %.12f. Expected range is [-90, 90].", + latitude)); + } + } + + @Override + public boolean isDone() { + return false; + } + + @Override + public boolean isGeometryChanged() { + return false; + } + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsTextFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsTextFunction.java new file mode 100644 index 0000000000000..17940f9ebc18a --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsTextFunction.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; + +import javax.annotation.Nullable; + +/** Implementation of {@link BuiltInFunctionDefinitions#ST_ASTEXT}. */ +@Internal +public class StAsTextFunction extends BuiltInScalarFunction { + + public StAsTextFunction(SpecializedContext context) { + super(BuiltInFunctionDefinitions.ST_ASTEXT, context); + } + + public @Nullable StringData eval(@Nullable GeographyData geography) { + return GeographyConversionUtils.asText(geography); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsWkbFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsWkbFunction.java new file mode 100644 index 0000000000000..27baaac1637eb --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StAsWkbFunction.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; + +import javax.annotation.Nullable; + +/** Implementation of {@link BuiltInFunctionDefinitions#ST_ASWKB}. */ +@Internal +public class StAsWkbFunction extends BuiltInScalarFunction { + + public StAsWkbFunction(SpecializedContext context) { + super(BuiltInFunctionDefinitions.ST_ASWKB, context); + } + + public @Nullable byte[] eval(@Nullable GeographyData geography) { + return GeographyConversionUtils.asWkb(geography); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromTextFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromTextFunction.java new file mode 100644 index 0000000000000..4781ad2d5a7ea --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromTextFunction.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; + +import javax.annotation.Nullable; + +/** Implementation of {@link BuiltInFunctionDefinitions#ST_GEOGFROMTEXT}. */ +@Internal +public class StGeogFromTextFunction extends BuiltInScalarFunction { + + public StGeogFromTextFunction(SpecializedContext context) { + super(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT, context); + } + + public @Nullable GeographyData eval(@Nullable StringData text) { + return GeographyConversionUtils.fromWkt(text); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromWkbFunction.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromWkbFunction.java new file mode 100644 index 0000000000000..c3a17ac96f206 --- /dev/null +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/StGeogFromWkbFunction.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.functions.BuiltInFunctionDefinitions; +import org.apache.flink.table.functions.SpecializedFunction.SpecializedContext; + +import javax.annotation.Nullable; + +/** Implementation of {@link BuiltInFunctionDefinitions#ST_GEOGFROMWKB}. */ +@Internal +public class StGeogFromWkbFunction extends BuiltInScalarFunction { + + public StGeogFromWkbFunction(SpecializedContext context) { + super(BuiltInFunctionDefinitions.ST_GEOGFROMWKB, context); + } + + public @Nullable GeographyData eval(@Nullable byte[] bytes) { + return GeographyConversionUtils.fromWkb(bytes); + } +} diff --git a/flink-table/flink-table-runtime/src/main/resources/META-INF/NOTICE b/flink-table/flink-table-runtime/src/main/resources/META-INF/NOTICE index 0a2b4b723f1e9..4de8b43653c20 100644 --- a/flink-table/flink-table-runtime/src/main/resources/META-INF/NOTICE +++ b/flink-table/flink-table-runtime/src/main/resources/META-INF/NOTICE @@ -7,5 +7,14 @@ The Apache Software Foundation (http://www.apache.org/). This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - com.jayway.jsonpath:json-path:2.7.0 +<<<<<<< HEAD - org.codehaus.janino:janino:3.1.12 - org.codehaus.janino:commons-compiler:3.1.12 +======= +- org.codehaus.janino:janino:3.1.11 +- org.codehaus.janino:commons-compiler:3.1.11 + +This project bundles the following dependencies under the Eclipse Public License - v 2.0 (https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt) + +- org.locationtech.jts:jts-core:1.19.0 +>>>>>>> 83088b18ec1 (TASK 10: Add GEOGRAPHY SQL constructor functions) diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyAccessorFunctionsTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyAccessorFunctionsTest.java new file mode 100644 index 0000000000000..d25dfd265fc12 --- /dev/null +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyAccessorFunctionsTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for GEOGRAPHY accessor conversion utilities. */ +class GeographyAccessorFunctionsTest { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Test + void testAsTextReturnsWkt() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + + assertThat(GeographyConversionUtils.asText(geography)) + .isEqualTo(StringData.fromString("POINT (1 2)")); + } + + @Test + void testAsTextSupportsEmptyGeometry() { + final GeographyData geography = + GeographyConversionUtils.fromWkt(StringData.fromString("POINT EMPTY")); + + assertThat(GeographyConversionUtils.asText(geography)) + .isEqualTo(StringData.fromString("POINT EMPTY")); + } + + @Test + void testAsWkbReturnsRawIsoWkb() { + final GeographyData geography = GeographyData.fromBytes(POINT_WKB); + + assertThat(GeographyConversionUtils.asWkb(geography)) + .hasSize(POINT_WKB.length) + .isEqualTo(POINT_WKB); + } + + @Test + void testAccessorsPropagateNull() { + assertThat(GeographyConversionUtils.asText(null)).isNull(); + assertThat(GeographyConversionUtils.asWkb(null)).isNull(); + } +} diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyConstructorFunctionsTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyConstructorFunctionsTest.java new file mode 100644 index 0000000000000..af64f32329f81 --- /dev/null +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyConstructorFunctionsTest.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.table.api.TableRuntimeException; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.StringData; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for GEOGRAPHY constructor conversion utilities. */ +class GeographyConstructorFunctionsTest { + + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + @Test + void testFromWktCreatesIsoWkb() { + final GeographyData geography = + GeographyConversionUtils.fromWkt(StringData.fromString("POINT (1 2)")); + + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + } + + @Test + void testFromWktAllowsEmptyGeometry() { + final GeographyData geography = + GeographyConversionUtils.fromWkt(StringData.fromString("POINT EMPTY")); + + assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT); + } + + @Test + void testFromWkbKeepsRawIsoWkb() { + final GeographyData geography = GeographyConversionUtils.fromWkb(POINT_WKB); + + assertThat(geography.toBytes()).isEqualTo(POINT_WKB); + } + + @Test + void testConstructorsPropagateNull() { + assertThat(GeographyConversionUtils.fromWkt(null)).isNull(); + assertThat(GeographyConversionUtils.fromWkb(null)).isNull(); + } + + @Test + void testFromWktRejectsMalformedInput() { + assertThatThrownBy(() -> GeographyConversionUtils.fromWkt(StringData.fromString("not wkt"))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid GEOGRAPHY WKT."); + } + + @Test + void testFromWktRejectsThreeDimensionalInput() { + assertThatThrownBy( + () -> + GeographyConversionUtils.fromWkt( + StringData.fromString("POINT Z (1 2 3)"))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Only 2D coordinates are supported"); + } + + @Test + void testFromWktRejectsCoordinatesOutsideCrs84Range() { + assertThatThrownBy( + () -> + GeographyConversionUtils.fromWkt( + StringData.fromString("POINT (181 2)"))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Expected range is [-180, 180]"); + + assertThatThrownBy( + () -> + GeographyConversionUtils.fromWkt( + StringData.fromString("POINT (1 91)"))) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Expected range is [-90, 90]"); + } + + @Test + void testFromWkbRejectsMalformedInput() { + assertThatThrownBy(() -> GeographyConversionUtils.fromWkb(new byte[] {1, 1, 0, 0})) + .isInstanceOf(TableRuntimeException.class) + .hasMessageContaining("Invalid GEOGRAPHY WKB."); + } +} From e97171c543ab9219539b74a68de93765d425cfb5 Mon Sep 17 00:00:00 2001 From: Garik Kalashyan Date: Wed, 24 Jun 2026 11:43:41 +0400 Subject: [PATCH 3/6] TASK 13: PyFlink users need access to the new user-facing type. --- .../pyflink/table/tests/test_schema.py | 45 +++++++++++-------- .../pyflink/table/tests/test_types.py | 7 ++- flink-python/pyflink/table/types.py | 36 +++++++++++++++ 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/flink-python/pyflink/table/tests/test_schema.py b/flink-python/pyflink/table/tests/test_schema.py index 46968c9fad324..345eef8a7748e 100644 --- a/flink-python/pyflink/table/tests/test_schema.py +++ b/flink-python/pyflink/table/tests/test_schema.py @@ -23,33 +23,40 @@ class SchemaTest(PyFlinkTestCase): def test_schema_basic(self): - old_schema = Schema.new_builder() \ - .from_row_data_type(DataTypes.ROW( - [DataTypes.FIELD("a", DataTypes.TINYINT()), - DataTypes.FIELD("b", DataTypes.SMALLINT()), - DataTypes.FIELD("c", DataTypes.INT())])) \ - .from_fields(["d", "e"], [DataTypes.STRING(), DataTypes.BOOLEAN()]) \ - .build() - self.schema = Schema.new_builder() \ - .from_schema(old_schema) \ - .primary_key_named("primary_constraint", "id") \ - .column("id", DataTypes.INT().not_null()) \ - .column("counter", DataTypes.INT().not_null()) \ - .column("payload", "ROW") \ - .column_by_metadata("topic", DataTypes.STRING(), None, True) \ - .column_by_expression("ts", call_sql("orig_ts - INTERVAL '60' MINUTE")) \ - .column_by_metadata("orig_ts", DataTypes.TIMESTAMP(3), "timestamp") \ - .watermark("ts", "ts - INTERVAL '5' SECOND") \ - .column_by_expression("proctime", "PROCTIME()") \ - .build() + old_schema = ( + Schema.new_builder() + .from_row_data_type( + DataTypes.ROW( + [DataTypes.FIELD("a", DataTypes.TINYINT()), + DataTypes.FIELD("b", DataTypes.SMALLINT()), + DataTypes.FIELD("c", DataTypes.INT()), + DataTypes.FIELD("f", DataTypes.GEOGRAPHY())])) + .from_fields(["d", "e"], [DataTypes.STRING(), DataTypes.BOOLEAN()]) + .build()) + self.schema = ( + Schema.new_builder() + .from_schema(old_schema) + .primary_key_named("primary_constraint", "id") + .column("id", DataTypes.INT().not_null()) + .column("counter", DataTypes.INT().not_null()) + .column("location", DataTypes.GEOGRAPHY()) + .column("payload", "ROW") + .column_by_metadata("topic", DataTypes.STRING(), None, True) + .column_by_expression("ts", call_sql("orig_ts - INTERVAL '60' MINUTE")) + .column_by_metadata("orig_ts", DataTypes.TIMESTAMP(3), "timestamp") + .watermark("ts", "ts - INTERVAL '5' SECOND") + .column_by_expression("proctime", "PROCTIME()") + .build()) self.assertEqual("""( `a` TINYINT, `b` SMALLINT, `c` INT, + `f` GEOGRAPHY, `d` STRING, `e` BOOLEAN, `id` INT NOT NULL, `counter` INT NOT NULL, + `location` GEOGRAPHY, `payload` [ROW], `topic` STRING METADATA VIRTUAL, `ts` AS [orig_ts - INTERVAL '60' MINUTE], diff --git a/flink-python/pyflink/table/tests/test_types.py b/flink-python/pyflink/table/tests/test_types.py index d3bd37c6ffd8f..699d8b5f9d631 100644 --- a/flink-python/pyflink/table/tests/test_types.py +++ b/flink-python/pyflink/table/tests/test_types.py @@ -806,6 +806,7 @@ def test_basic_type(self): test_types = [DataTypes.STRING(), DataTypes.BOOLEAN(), DataTypes.BYTES(), + DataTypes.GEOGRAPHY(), DataTypes.TINYINT(), DataTypes.SMALLINT(), DataTypes.INT(), @@ -831,7 +832,8 @@ def test_atomic_type_with_data_type_with_parameters(self): JDataTypes.BINARY(2).notNull(), JDataTypes.VARCHAR(30).notNull(), JDataTypes.CHAR(50).notNull(), - JDataTypes.DECIMAL(20, 10).notNull()] + JDataTypes.DECIMAL(20, 10).notNull(), + JDataTypes.GEOGRAPHY().notNull()] converted_python_types = [_from_java_data_type(item) for item in java_types] @@ -841,7 +843,8 @@ def test_atomic_type_with_data_type_with_parameters(self): DataTypes.BINARY(2, False), DataTypes.VARCHAR(30, False), DataTypes.CHAR(50, False), - DataTypes.DECIMAL(20, 10, False)] + DataTypes.DECIMAL(20, 10, False), + DataTypes.GEOGRAPHY(False)] self.assertEqual(converted_python_types, expected) def test_array_type(self): diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index b62b55af9e1f3..1a428cab20bae 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -248,6 +248,28 @@ def __repr__(self): return "VarBinaryType(%d, %s)" % (self.length, str(self._nullable).lower()) +class GeographyType(AtomicType): + """ + Geography data type. SQL GEOGRAPHY. + + The Python representation uses WKB bytes. + + :param nullable: boolean, whether the field can be null (None) or not. + """ + + def __init__(self, nullable=True): + super(GeographyType, self).__init__(nullable) + + def need_conversion(self): + return True + + def to_sql_type(self, obj): + return bytearray(obj) if obj is not None else obj + + def from_sql_type(self, obj): + return bytes(obj) if obj is not None else obj + + class BooleanType(AtomicType): """ Boolean data types. SQL BOOLEAN @@ -1700,6 +1722,8 @@ def _from_java_data_type(j_data_type): data_type = DataTypes.BINARY(logical_type.getLength(), logical_type.isNullable()) elif is_instance_of(logical_type, gateway.jvm.VarBinaryType): data_type = DataTypes.VARBINARY(logical_type.getLength(), logical_type.isNullable()) + elif is_instance_of(logical_type, gateway.jvm.GeographyType): + data_type = DataTypes.GEOGRAPHY(logical_type.isNullable()) elif is_instance_of(logical_type, gateway.jvm.DecimalType): data_type = DataTypes.DECIMAL(logical_type.getPrecision(), logical_type.getScale(), @@ -1850,6 +1874,8 @@ def _to_java_data_type(data_type: DataType): j_data_type = JDataTypes.VARBINARY(data_type.length) elif isinstance(data_type, BinaryType): j_data_type = JDataTypes.BINARY(data_type.length) + elif isinstance(data_type, GeographyType): + j_data_type = JDataTypes.GEOGRAPHY() elif isinstance(data_type, DecimalType): j_data_type = JDataTypes.DECIMAL(data_type.precision, data_type.scale) elif isinstance(data_type, DateType): @@ -1955,6 +1981,7 @@ def _to_java_data_type(data_type: DataType): VarCharType: (str,), BinaryType: (bytearray,), VarBinaryType: (bytearray,), + GeographyType: (bytes, bytearray), DateType: (datetime.date, datetime.datetime), TimeType: (datetime.time,), TimestampType: (datetime.datetime,), @@ -2444,6 +2471,15 @@ def BYTES(nullable: bool = True) -> VarBinaryType: """ return DataTypes.VARBINARY(0x7fffffff, nullable) + @staticmethod + def GEOGRAPHY(nullable: bool = True) -> GeographyType: + """ + Data type of geography data represented as WKB bytes in Python. + + :param nullable: boolean, whether the type can be null (None) or not. + """ + return GeographyType(nullable) + @staticmethod def DECIMAL(precision: int, scale: int, nullable: bool = True) -> DecimalType: """ From 4975c0e1a649988bbac599ef69d6527b47dba593 Mon Sep 17 00:00:00 2001 From: Garik Kalashyan Date: Wed, 24 Jun 2026 12:01:17 +0400 Subject: [PATCH 4/6] TASK 14: Support SQL usage and WKB bytes round-trip from PyFlink. --- flink-python/pyflink/table/tests/test_sql.py | 22 ++++++++++++++++++ .../table/tests/test_table_environment_api.py | 2 +- .../pyflink/table/tests/test_types.py | 7 ++++++ flink-python/pyflink/table/types.py | 23 +++++++++++++++++-- 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/flink-python/pyflink/table/tests/test_sql.py b/flink-python/pyflink/table/tests/test_sql.py index 549c48133d60a..3639119e52c01 100644 --- a/flink-python/pyflink/table/tests/test_sql.py +++ b/flink-python/pyflink/table/tests/test_sql.py @@ -101,6 +101,28 @@ def test_execute_sql(self): table_result.print() + def test_geography_sql_wkb_bytes_round_trip(self): + point_wkb = bytes([ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + ]) + source = self.t_env.from_elements( + [(point_wkb, 'POINT (1 2)')], + ['wkb', 'wkt']) + self.t_env.create_temporary_view('geography_source', source) + + result = self.t_env.sql_query( + 'SELECT ST_ASTEXT(ST_GEOGFROMWKB(wkb)), ' + 'ST_ASWKB(ST_GEOGFROMTEXT(wkt)), ' + 'ST_ASWKB(ST_GEOGFROMWKB(wkb)) ' + 'FROM geography_source') + collected = list(result.execute().collect()) + + self.assertEqual(1, len(collected)) + self.assertEqual('POINT (1 2)', collected[0][0]) + self.assertEqual(point_wkb, collected[0][1]) + self.assertEqual(point_wkb, collected[0][2]) + + class JavaSqlTests(PyFlinkTestCase): """ We need to start these Java tests from python process to make sure that Python environment is diff --git a/flink-python/pyflink/table/tests/test_table_environment_api.py b/flink-python/pyflink/table/tests/test_table_environment_api.py index 505d1dbd26fb4..b11ac88dbafe2 100644 --- a/flink-python/pyflink/table/tests/test_table_environment_api.py +++ b/flink-python/pyflink/table/tests/test_table_environment_api.py @@ -697,7 +697,7 @@ def test_collect_with_retract(self): def test_collect_for_all_data_types(self): expected_result = [Row(1, None, 1, True, 32767, -2147483648, 1.23, - 1.98932, bytearray(b'pyflink'), 'pyflink', + 1.98932, b'pyflink', 'pyflink', datetime.date(2014, 9, 13), datetime.time(12, 0, 0, 123000), datetime.datetime(2018, 3, 11, 3, 0, 0, 123000), [['a', 'b'], ['c', 'd'], ['e', 'f']], diff --git a/flink-python/pyflink/table/tests/test_types.py b/flink-python/pyflink/table/tests/test_types.py index 699d8b5f9d631..42004a178b7fc 100644 --- a/flink-python/pyflink/table/tests/test_types.py +++ b/flink-python/pyflink/table/tests/test_types.py @@ -677,7 +677,14 @@ def __init__(self, **kwargs): (decimal.Decimal("1.0"), DataTypes.DECIMAL(10, 0)), # Binary + (b"\x01", DataTypes.BINARY(1)), (bytearray([1]), DataTypes.BINARY(1)), + (b"\x01\x02", DataTypes.VARBINARY(2)), + + # Geography + (bytes([ + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xF0, 0x3F, + 0, 0, 0, 0, 0, 0, 0, 0x40]), DataTypes.GEOGRAPHY()), # Date/Time/Timestamp (datetime.date(2000, 1, 2), DataTypes.DATE()), diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index 1a428cab20bae..f5021e626f87d 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -228,6 +228,15 @@ def __init__(self, length=1, nullable=True): def __repr__(self): return "BinaryType(%d, %s)" % (self.length, str(self._nullable).lower()) + def need_conversion(self): + return True + + def to_sql_type(self, obj): + return bytearray(obj) if obj is not None else obj + + def from_sql_type(self, obj): + return bytes(obj) if obj is not None else obj + class VarBinaryType(AtomicType): """ @@ -247,6 +256,15 @@ def __init__(self, length=1, nullable=True): def __repr__(self): return "VarBinaryType(%d, %s)" % (self.length, str(self._nullable).lower()) + def need_conversion(self): + return True + + def to_sql_type(self, obj): + return bytearray(obj) if obj is not None else obj + + def from_sql_type(self, obj): + return bytes(obj) if obj is not None else obj + class GeographyType(AtomicType): """ @@ -1407,6 +1425,7 @@ def deserialize(self, datum): int: BigIntType(), float: DoubleType(), str: VarCharType(0x7fffffff), + bytes: VarBinaryType(0x7fffffff), bytearray: VarBinaryType(0x7fffffff), decimal.Decimal: DecimalType(38, 18), datetime.date: DateType(), @@ -1979,8 +1998,8 @@ def _to_java_data_type(data_type: DataType): DecimalType: (decimal.Decimal,), CharType: (str,), VarCharType: (str,), - BinaryType: (bytearray,), - VarBinaryType: (bytearray,), + BinaryType: (bytes, bytearray), + VarBinaryType: (bytes, bytearray), GeographyType: (bytes, bytearray), DateType: (datetime.date, datetime.datetime), TimeType: (datetime.time,), From ddf536c82ff9384e27396558300a6f10d5169ce7 Mon Sep 17 00:00:00 2001 From: "david.chaava" Date: Fri, 19 Jun 2026 16:10:06 +0200 Subject: [PATCH 5/6] Add Parquet mapping for GEOGRAPHY type --- .../parquet/row/ParquetRowDataWriter.java | 19 +++ .../parquet/utils/ParquetSchemaConverter.java | 4 + .../vector/ParquetColumnarRowSplitReader.java | 38 +++--- .../vector/ParquetSplitReaderUtil.java | 12 +- .../reader/NestedPrimitiveColumnReader.java | 3 + .../parquet/row/ParquetRowDataWriterTest.java | 111 ++++++++++++++++++ 6 files changed, 165 insertions(+), 22 deletions(-) diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriter.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriter.java index 28ae2d8b62a66..0b9616eef1dbd 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriter.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriter.java @@ -119,6 +119,8 @@ private FieldWriter createWriter(LogicalType t, Type type) { case BINARY: case VARBINARY: return new BinaryWriter(); + case GEOGRAPHY: + return new GeographyWriter(); case DECIMAL: DecimalType decimalType = (DecimalType) t; return createDecimalWriter(decimalType.getPrecision(), decimalType.getScale()); @@ -311,6 +313,23 @@ private void writeBinary(byte[] value) { } } + private class GeographyWriter implements FieldWriter { + + @Override + public void write(RowData row, int ordinal) { + writeGeography(row.getGeography(ordinal).toBytes()); + } + + @Override + public void write(ArrayData arrayData, int ordinal) { + writeGeography(arrayData.getGeography(ordinal).toBytes()); + } + + private void writeGeography(byte[] value) { + recordConsumer.addBinary(Binary.fromReusedByteArray(value)); + } + } + private class IntWriter implements FieldWriter { @Override diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetSchemaConverter.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetSchemaConverter.java index 0101fc13f428e..0a6524232ca5b 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetSchemaConverter.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetSchemaConverter.java @@ -81,6 +81,10 @@ private static Type convertToParquetType( case VARBINARY: return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) .named(name); + case GEOGRAPHY: + return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition) + .as(LogicalTypeAnnotation.geographyType()) + .named(name); case DECIMAL: int precision = ((DecimalType) type).getPrecision(); int scale = ((DecimalType) type).getScale(); diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetColumnarRowSplitReader.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetColumnarRowSplitReader.java index 7634978e91640..18ac1058178d9 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetColumnarRowSplitReader.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetColumnarRowSplitReader.java @@ -18,6 +18,8 @@ package org.apache.flink.formats.parquet.vector; +import org.apache.flink.core.fs.FileSystem; +import org.apache.flink.formats.parquet.ParquetInputFile; import org.apache.flink.formats.parquet.vector.reader.ColumnReader; import org.apache.flink.formats.parquet.vector.type.ParquetField; import org.apache.flink.table.data.columnar.ColumnarRowData; @@ -31,12 +33,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.page.PageReadStore; import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.metadata.BlockMetaData; -import org.apache.parquet.hadoop.metadata.ParquetMetadata; import org.apache.parquet.io.ColumnIOFactory; import org.apache.parquet.io.MessageColumnIO; import org.apache.parquet.schema.GroupType; @@ -55,9 +57,6 @@ import static org.apache.flink.formats.parquet.vector.ParquetSplitReaderUtil.buildFieldsList; import static org.apache.flink.formats.parquet.vector.ParquetSplitReaderUtil.createColumnReader; import static org.apache.flink.formats.parquet.vector.ParquetSplitReaderUtil.createWritableColumnVector; -import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups; -import static org.apache.parquet.format.converter.ParquetMetadataConverter.range; -import static org.apache.parquet.hadoop.ParquetFileReader.readFooter; import static org.apache.parquet.hadoop.ParquetInputFormat.getFilter; /** This reader is used to read a {@link VectorizedColumnBatch} from input split. */ @@ -122,24 +121,25 @@ public ParquetColumnarRowSplitReader( this.utcTimestamp = utcTimestamp; this.selectedTypes = selectedTypes; this.batchSize = batchSize; - // then we need to apply the predicate push down filter - ParquetMetadata footer = - readFooter(conf, path, range(splitStart, splitStart + splitLength)); - MessageType fileSchema = footer.getFileMetaData().getSchema(); FilterCompat.Filter filter = getFilter(conf); - List blocks = filterRowGroups(filter, footer.getBlocks(), fileSchema); - - this.fileSchema = footer.getFileMetaData().getSchema(); + ParquetReadOptions parquetReadOptions = + ParquetReadOptions.builder() + .withRange(splitStart, splitStart + splitLength) + .withRecordFilter(filter) + .build(); + org.apache.flink.core.fs.Path flinkPath = + new org.apache.flink.core.fs.Path(path.toUri().toString()); + FileSystem fs = flinkPath.getFileSystem(); + ParquetInputFile inputFile = + new ParquetInputFile(fs.open(flinkPath), fs.getFileStatus(flinkPath).getLen()); + + this.reader = ParquetFileReader.open(inputFile, parquetReadOptions); + MessageType fileSchema = reader.getFooter().getFileMetaData().getSchema(); + this.fileSchema = fileSchema; this.requestedSchema = clipParquetSchema(fileSchema, selectedFieldNames, caseSensitive); - this.reader = - new ParquetFileReader( - conf, footer.getFileMetaData(), path, blocks, requestedSchema.getColumns()); + this.reader.setRequestedSchema(requestedSchema); - long totalRowCount = 0; - for (BlockMetaData block : blocks) { - totalRowCount += block.getRowCount(); - } - this.totalRowCount = totalRowCount; + this.totalRowCount = reader.getRecordCount(); this.nextRow = 0; this.rowsInBatch = 0; this.rowsReturned = 0; diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetSplitReaderUtil.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetSplitReaderUtil.java index fc757738f01dd..ed22354bb2ad7 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetSplitReaderUtil.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/ParquetSplitReaderUtil.java @@ -36,6 +36,7 @@ import org.apache.flink.formats.parquet.vector.type.ParquetGroupField; import org.apache.flink.formats.parquet.vector.type.ParquetPrimitiveField; import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.TimestampData; import org.apache.flink.table.data.columnar.vector.ColumnVector; import org.apache.flink.table.data.columnar.vector.VectorizedColumnBatch; @@ -169,14 +170,17 @@ public static ColumnVector createVectorFromConstant( case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: HeapBytesVector bsv = new HeapBytesVector(batchSize); if (value == null) { bsv.fillWithNulls(); } else { bsv.fill( - value instanceof byte[] - ? (byte[]) value - : value.toString().getBytes(StandardCharsets.UTF_8)); + value instanceof GeographyData + ? ((GeographyData) value).toBytes() + : value instanceof byte[] + ? (byte[]) value + : value.toString().getBytes(StandardCharsets.UTF_8)); } return bsv; case BOOLEAN: @@ -344,6 +348,7 @@ public static ColumnReader createColumnReader( case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: return new BytesColumnReader( descriptors.get(0), pages.getPageReader(descriptors.get(0))); case TIMESTAMP_WITHOUT_TIME_ZONE: @@ -438,6 +443,7 @@ public static WritableColumnVector createWritableColumnVector( case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: checkArgument( typeName == PrimitiveType.PrimitiveTypeName.BINARY, "Unexpected type: %s", diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/reader/NestedPrimitiveColumnReader.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/reader/NestedPrimitiveColumnReader.java index ceff4d118a5b5..6727a484a9b73 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/reader/NestedPrimitiveColumnReader.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/vector/reader/NestedPrimitiveColumnReader.java @@ -238,6 +238,7 @@ private Object readPrimitiveTypedRow(LogicalType category) { case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: return dataColumn.readBytes(); case BOOLEAN: return dataColumn.readBoolean(); @@ -283,6 +284,7 @@ private Object dictionaryDecodeValue(LogicalType category, Integer dictionaryVal case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: return dictionary.readBytes(dictionaryValue); case DATE: case TIME_WITHOUT_TIME_ZONE: @@ -324,6 +326,7 @@ private WritableColumnVector fillColumnVector(int total, List valueList) { case VARCHAR: case BINARY: case VARBINARY: + case GEOGRAPHY: HeapBytesVector heapBytesVector = new HeapBytesVector(total); for (int i = 0; i < valueList.size(); i++) { byte[] src = ((List) valueList).get(i); diff --git a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java index cd0bf0016672d..fce77ba996e59 100644 --- a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java +++ b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java @@ -24,6 +24,9 @@ import org.apache.flink.formats.parquet.ParquetWriterFactory; import org.apache.flink.formats.parquet.vector.ParquetColumnarRowSplitReader; import org.apache.flink.formats.parquet.vector.ParquetSplitReaderUtil; +import org.apache.flink.table.data.GenericArrayData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; import org.apache.flink.table.data.RowData; import org.apache.flink.table.data.util.DataFormatConverters; import org.apache.flink.table.types.DataType; @@ -33,6 +36,7 @@ import org.apache.flink.table.types.logical.DecimalType; import org.apache.flink.table.types.logical.DoubleType; import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.GeographyType; import org.apache.flink.table.types.logical.IntType; import org.apache.flink.table.types.logical.MapType; import org.apache.flink.table.types.logical.RowType; @@ -49,10 +53,14 @@ import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.hadoop.ParquetOutputFormat; import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.util.HadoopInputFile; import org.apache.parquet.io.InputFile; +import org.apache.parquet.io.LocalInputFile; +import org.apache.parquet.schema.LogicalTypeAnnotation; +import org.apache.parquet.schema.PrimitiveType; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -77,6 +85,16 @@ /** Test for {@link ParquetRowDataBuilder} and {@link ParquetRowDataWriter}. */ class ParquetRowDataWriterTest { + private static final byte[] POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, (byte) 0xF0, 0x3F, 0, 0, 0, 0, 0, 0, 0, 0x40 + }; + + private static final byte[] SECOND_POINT_WKB = + new byte[] { + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x40, 0, 0, 0, 0, 0, 0, 8, 0x40 + }; + private static final RowType ROW_TYPE = RowType.of( new VarCharType(VarCharType.MAX_LENGTH), @@ -192,6 +210,99 @@ public void testInt64Timestamp(@TempDir java.nio.file.Path folder) throws Except invalidTypeTest(folder, conf, false); } + @Test + void testGeographyType(@TempDir java.nio.file.Path folder) throws Exception { + RowType rowType = RowType.of(new GeographyType()); + Path path = new Path(folder.toString(), UUID.randomUUID().toString()); + Configuration conf = new Configuration(); + + ParquetWriterFactory factory = + ParquetRowDataBuilder.createWriterFactory(rowType, conf, true); + BulkWriter writer = + factory.create(path.getFileSystem().create(path, FileSystem.WriteMode.OVERWRITE)); + writer.addElement(GenericRowData.of(GeographyData.fromBytes(POINT_WKB))); + writer.flush(); + writer.finish(); + + try (ParquetFileReader fileReader = + ParquetFileReader.open(new LocalInputFile(new File(path.getPath()).toPath()))) { + PrimitiveType field = + fileReader.getFileMetaData().getSchema().getType(0).asPrimitiveType(); + assertThat(field.getPrimitiveTypeName()) + .isEqualTo(PrimitiveType.PrimitiveTypeName.BINARY); + assertThat(field.getLogicalTypeAnnotation()) + .isInstanceOf(LogicalTypeAnnotation.GeographyLogicalTypeAnnotation.class); + } + + ParquetColumnarRowSplitReader reader = + ParquetSplitReaderUtil.genPartColumnarRowReader( + true, + true, + conf, + rowType.getFieldNames().toArray(new String[0]), + rowType.getChildren().stream() + .map(TypeConversions::fromLogicalToDataType) + .toArray(DataType[]::new), + new HashMap<>(), + IntStream.range(0, rowType.getFieldCount()).toArray(), + 50, + path, + 0, + Long.MAX_VALUE); + + assertThat(reader.reachedEnd()).isFalse(); + assertThat(reader.nextRecord().getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(reader.reachedEnd()).isTrue(); + } + + @Test + void testNestedGeographyType(@TempDir java.nio.file.Path folder) throws Exception { + RowType rowType = RowType.of(new BigIntType(), new ArrayType(true, new GeographyType())); + Path path = new Path(folder.toString(), UUID.randomUUID().toString()); + Configuration conf = new Configuration(); + + ParquetWriterFactory factory = + ParquetRowDataBuilder.createWriterFactory(rowType, conf, true); + BulkWriter writer = + factory.create(path.getFileSystem().create(path, FileSystem.WriteMode.OVERWRITE)); + writer.addElement( + GenericRowData.of( + 1L, + new GenericArrayData( + new Object[] { + GeographyData.fromBytes(POINT_WKB), + GeographyData.fromBytes(SECOND_POINT_WKB), + null + }))); + writer.flush(); + writer.finish(); + + try (ParquetColumnarRowSplitReader reader = + ParquetSplitReaderUtil.genPartColumnarRowReader( + true, + true, + conf, + rowType.getFieldNames().toArray(new String[0]), + rowType.getChildren().stream() + .map(TypeConversions::fromLogicalToDataType) + .toArray(DataType[]::new), + new HashMap<>(), + IntStream.range(0, rowType.getFieldCount()).toArray(), + 50, + path, + 0, + Long.MAX_VALUE)) { + assertThat(reader.reachedEnd()).isFalse(); + RowData row = reader.nextRecord(); + assertThat(row.getLong(0)).isEqualTo(1L); + assertThat(row.getArray(1).size()).isEqualTo(3); + assertThat(row.getArray(1).getGeography(0).toBytes()).isEqualTo(POINT_WKB); + assertThat(row.getArray(1).getGeography(1).toBytes()).isEqualTo(SECOND_POINT_WKB); + assertThat(row.getArray(1).isNullAt(2)).isTrue(); + assertThat(reader.reachedEnd()).isTrue(); + } + } + private void innerTest(java.nio.file.Path folder, Configuration conf, boolean utcTimestamp) throws IOException { Path path = new Path(folder.toString(), UUID.randomUUID().toString()); From 294dc87444cfb7ebba7ba6d03464f70e1f54619c Mon Sep 17 00:00:00 2001 From: Garik Kalashyan Date: Thu, 25 Jun 2026 10:44:58 +0400 Subject: [PATCH 6/6] TASK 16: Unsupported connectors should fail clearly for GEOGRAPHY. --- .../docs/sql/functions/built-in-functions.md | 43 +++ docs/content/docs/sql/reference/data-types.md | 95 ++++++ .../formats/avro/AvroFormatFactoryTest.java | 28 ++ .../ParquetFormatStatisticsReportUtil.java | 86 ++++-- .../parquet/row/ParquetRowDataWriterTest.java | 4 +- .../scheduler/DefaultSchedulerTest.java | 3 +- .../TestExecutionOperationsDecorator.java | 5 + flink-table/flink-table-planner/pom.xml | 22 ++ .../calcite/sql2rel/SqlToRelConverter.java | 56 +++- .../planner/calcite/FlinkTypeFactory.java | 2 +- .../runtime/batch/sql/GeographySqlITCase.java | 245 +++++++++++++++ .../types/TypeInfoDataTypeConverter.java | 2 + .../formats/raw/RawFormatFactoryTest.java | 12 + .../scalar/GeographyWkbBenchmark.java | 289 ++++++++++++++++++ .../scalar/GeographyWkbBenchmarkTest.java | 52 ++++ .../process/GeographyStateRestoreTest.java | 192 ++++++++++++ flink-table/flink-table-test-utils/pom.xml | 7 +- 17 files changed, 1100 insertions(+), 43 deletions(-) create mode 100644 flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/GeographySqlITCase.java create mode 100644 flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyWkbBenchmark.java create mode 100644 flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyWkbBenchmarkTest.java create mode 100644 flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/GeographyStateRestoreTest.java diff --git a/docs/content/docs/sql/functions/built-in-functions.md b/docs/content/docs/sql/functions/built-in-functions.md index 42808103674bb..a131706e06eaf 100644 --- a/docs/content/docs/sql/functions/built-in-functions.md +++ b/docs/content/docs/sql/functions/built-in-functions.md @@ -69,6 +69,44 @@ The scalar functions take zero, one or more values as the input and return a sin {{< sql_functions "collection" >}} +### Geography Functions + +The v1 geography surface is intentionally small and focuses on schema declaration plus portable WKT +and WKB boundaries. + +| Function | Description | +|:---------|:------------| +| `ST_GEOGFROMTEXT(wkt)` | Parses 2D WKT text into a `GEOGRAPHY` value. Returns `NULL` for `NULL` input. | +| `ST_GEOGFROMWKB(wkb)` | Parses ISO/OGC WKB bytes into a `GEOGRAPHY` value. Returns `NULL` for `NULL` input. | +| `ST_ASTEXT(geography)` | Serializes a `GEOGRAPHY` value to WKT text. Returns `NULL` for `NULL` input. | +| `ST_ASWKB(geography)` | Returns the raw WKB bytes stored in a `GEOGRAPHY` value. Returns `NULL` for `NULL` input. | + +Example: + +```sql +SELECT + ST_ASTEXT(ST_GEOGFROMTEXT('POINT (1 2)')), + ST_ASWKB(ST_GEOGFROMTEXT('POINT (1 2)')), + ST_ASTEXT( + ST_GEOGFROMWKB( + ST_ASWKB(ST_GEOGFROMTEXT('LINESTRING (0 0, 1 1)')))); +``` + +Notes: + +- `GEOGRAPHY` coordinates are validated as 2D CRS84 longitude/latitude values. Longitudes must be in `[-180, 180]`; latitudes must be in `[-90, 90]`. +- `ST_ASWKB` returns the raw WKB bytes stored in the value. `ST_GEOGFROMWKB` followed by `ST_ASWKB` therefore preserves user-provided bytes after validation. +- Values created from `ST_GEOGFROMTEXT` are stored as 2D WKB without SRID metadata. +- PyFlink uses `bytes` for WKB boundaries, so no extra Python geospatial dependency is required for round-trip handling. + +#### Current Limitations And Follow-up Work + +- Spatial predicates and measurements such as `ST_INTERSECTS`, `ST_WITHIN`, `ST_LENGTH`, and `ST_DISTANCE` are not part of v1 yet. +- `GEOMETRY`, typed geography literals, and a broader geography function set remain follow-up work. +- Unsupported connector mappings fail explicitly instead of silently degrading `GEOGRAPHY` to `VARBINARY`; broader mappings for Iceberg, Avro, JDBC, Debezium, and additional external systems remain follow-up work. +- Spatial pruning, spatial joins, and related optimizer/runtime work are intentionally deferred. +- Published-version savepoint restore tests remain a follow-up because `GEOGRAPHY` does not yet have a released serializer baseline. Current coverage is limited to local SQL/runtime validation plus serializer versioning tests. + ### JSON Functions JSON functions make use of JSON path expressions as described in ISO/IEC TR 19075-6 of the SQL @@ -294,3 +332,8 @@ table.select( {{< /tabs >}} {{< top >}} + + + + + diff --git a/docs/content/docs/sql/reference/data-types.md b/docs/content/docs/sql/reference/data-types.md index a849ee5dac8de..2f6e4d224044a 100644 --- a/docs/content/docs/sql/reference/data-types.md +++ b/docs/content/docs/sql/reference/data-types.md @@ -121,6 +121,7 @@ For vectorized Python UDFs, the input types and output type are `pandas.Series`. | `DOUBLE` | `float` | `numpy.float64` | | `VARCHAR` | `str` | `str` | | `VARBINARY` | `bytes` | `bytes` | +| `GEOGRAPHY` | `bytes` | `bytes` | | `DECIMAL` | `decimal.Decimal` | `decimal.Decimal` | | `DATE` | `datetime.date` | `datetime.date` | | `TIME` | `datetime.time` | `datetime.time` | @@ -215,6 +216,7 @@ The default planner supports the following set of SQL types: | `MULTISET` | | | `MAP` | | | `ROW` | | +| `GEOGRAPHY` | 2D CRS84 longitude/latitude geospatial type. | | `RAW` | | | Structured types | Only exposed in user-defined functions yet. | | `VARIANT` | | @@ -1591,6 +1593,94 @@ DataTypes.BITMAP() {{< /tab >}} {{< /tabs >}} +#### `GEOGRAPHY` + +Data type of 2D geospatial values in CRS84 longitude/latitude coordinates. + +`GEOGRAPHY` is a user-facing type for SQL and Table API schemas. In v1, values are created and +accessed through the built-in geography functions. `ST_GEOGFROMTEXT` parses WKT text, +`ST_GEOGFROMWKB` parses WKB bytes, `ST_ASTEXT` serializes to WKT, and `ST_ASWKB` exposes WKB +bytes. + +**Declaration** + +{{< tabs "geography-data-type" >}} +{{< tab "SQL" >}} +```text +GEOGRAPHY +``` +{{< /tab >}} +{{< tab "Java/Scala" >}} +```java +DataTypes.GEOGRAPHY() +``` +{{< /tab >}} +{{< tab "Python" >}} +```python +DataTypes.GEOGRAPHY() +``` +{{< /tab >}} +{{< /tabs >}} + +{{< tabs "geography-notes" >}} +{{< tab "SQL/Java/Scala/Python" >}} +Coordinates are validated as 2D CRS84 longitude/latitude values. Longitudes must be in `[-180, 180]` +and latitudes in `[-90, 90]`. Z and M coordinates are rejected. + +`ST_ASWKB` returns the raw ISO/OGC WKB bytes stored in the value. When a value originates from +`ST_GEOGFROMWKB`, the bytes round-trip unchanged after validation. When a value originates from +`ST_GEOGFROMTEXT`, Flink stores a 2D WKB representation without SRID metadata. + +`NULL` inputs to `ST_GEOGFROMTEXT`, `ST_GEOGFROMWKB`, `ST_ASTEXT`, and `ST_ASWKB` return `NULL`. +{{< /tab >}} +{{< tab "Java" >}} +```java +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.Schema; + +Schema schema = + Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .column("location", DataTypes.GEOGRAPHY()) + .build(); + +// Use SQL functions to construct or access GEOGRAPHY values. +tableEnv.executeSql( + "INSERT INTO sink " + + "SELECT id, ST_GEOGFROMTEXT(wkt) " + + "FROM source"); +``` +{{< /tab >}} +{{< tab "Python" >}} +```python +from pyflink.table import Schema +from pyflink.table.types import DataTypes + +schema = ( + Schema.new_builder() + .column("id", DataTypes.BIGINT()) + .column("location", DataTypes.GEOGRAPHY()) + .build() +) + +# PyFlink uses bytes at WKB boundaries. +result = table_env.execute_sql( + "SELECT ST_ASWKB(ST_GEOGFROMTEXT('POINT (1 2)'))") +wkb = next(result.collect())[0] +``` +{{< /tab >}} +{{< /tabs >}} + +Migration from `VARBINARY` plus custom WKB UDFs to `GEOGRAPHY` is usually a boundary refactor: + +```sql +INSERT INTO new_places +SELECT id, ST_GEOGFROMWKB(old_location_wkb) +FROM old_places; +``` + +Use `ST_ASWKB(location)` when a sink or downstream system still expects WKB bytes. Unsupported +connector mappings fail explicitly instead of silently degrading `GEOGRAPHY` to `VARBINARY`. #### `RAW` Data type of an arbitrary serialized type. This type is a black box within the table ecosystem @@ -1897,3 +1987,8 @@ Not supported. {{< /tabs >}} {{< top >}} + + + + + diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroFormatFactoryTest.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroFormatFactoryTest.java index 0b9d572a13b07..22bff21a87839 100644 --- a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroFormatFactoryTest.java +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroFormatFactoryTest.java @@ -129,6 +129,34 @@ void testNewSeDeNewSchema() { testSeDeSchema(NEW_ROW_TYPE, NEW_SCHEMA, false); } + @Test + void testGeographyTypeIsRejected() { + ResolvedSchema geographySchema = + ResolvedSchema.of(Column.physical("g", DataTypes.GEOGRAPHY())); + + DynamicTableSource source = + FactoryMocks.createTableSource(geographySchema, getAllOptions(true)); + assertThat(source).isInstanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class); + TestDynamicTableFactory.DynamicTableSourceMock sourceMock = + (TestDynamicTableFactory.DynamicTableSourceMock) source; + assertThatThrownBy( + () -> + sourceMock.valueFormat.createRuntimeDecoder( + ScanRuntimeProviderContext.INSTANCE, + geographySchema.toPhysicalRowDataType())) + .hasMessageContaining("Unsupported to derive Schema for type: GEOGRAPHY"); + + DynamicTableSink sink = FactoryMocks.createTableSink(geographySchema, getAllOptions(true)); + assertThat(sink).isInstanceOf(TestDynamicTableFactory.DynamicTableSinkMock.class); + TestDynamicTableFactory.DynamicTableSinkMock sinkMock = + (TestDynamicTableFactory.DynamicTableSinkMock) sink; + assertThatThrownBy( + () -> + sinkMock.valueFormat.createRuntimeEncoder( + null, geographySchema.toPhysicalRowDataType())) + .hasMessageContaining("Unsupported to derive Schema for type: GEOGRAPHY"); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) void testSeDeSchema(boolean legacyTimestampMapping) { diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetFormatStatisticsReportUtil.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetFormatStatisticsReportUtil.java index 9eed138bab748..9c33809499660 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetFormatStatisticsReportUtil.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/utils/ParquetFormatStatisticsReportUtil.java @@ -61,7 +61,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.stream.Collectors; import static org.apache.flink.util.Preconditions.checkArgument; @@ -103,20 +102,22 @@ public static TableStats getTableStatistics( for (Path file : files) { fileRowCountFutures.add( executorService.submit( - new ParquetFileRowCountCalculator( - hadoopConfig, file, columnStatisticsMap))); + new ParquetFileRowCountCalculator(hadoopConfig, file))); } for (Future fileCountFuture : fileRowCountFutures) { FileParquetStatistics fileStatistics = fileCountFuture.get(); - List columns = fileStatistics.getColumns(); - List blocks = fileStatistics.blocks; + List columns = fileStatistics.getLeafColumnNames(); + List blocks = fileStatistics.getBlocks(); for (BlockMetaData block : blocks) { rowCount += block.getRowCount(); - for (int i = 0; i < columns.size(); ++i) { - updateStatistics( - block.getColumns().get(i).getStatistics(), - columns.get(i), - columnStatisticsMap); + for (int i = 0; i < Math.min(columns.size(), block.getColumns().size()); ++i) { + String column = columns.get(i); + if (column != null) { + updateStatistics( + block.getColumns().get(i).getStatistics(), + column, + columnStatisticsMap); + } } } } @@ -151,20 +152,33 @@ private static Map convertToColumnStats( boolean isUtcTimestamp) { Map columnStatMap = new HashMap<>(); for (String column : producedRowType.getFieldNames()) { + LogicalType logicalType = + producedRowType.getTypeAt(producedRowType.getFieldIndex(column)); Statistics statistics = columnStatisticsMap.get(column); if (statistics == null) { + if (shouldReportNullColumnStats(logicalType)) { + columnStatMap.put(column, null); + } continue; } - ColumnStats columnStats = - convertToColumnStats( - producedRowType.getTypeAt(producedRowType.getFieldIndex(column)), - statistics, - isUtcTimestamp); + ColumnStats columnStats = convertToColumnStats(logicalType, statistics, isUtcTimestamp); columnStatMap.put(column, columnStats); } return columnStatMap; } + private static boolean shouldReportNullColumnStats(LogicalType logicalType) { + switch (logicalType.getTypeRoot()) { + case ARRAY: + case MAP: + case MULTISET: + case ROW: + return true; + default: + return false; + } + } + private static ColumnStats convertToColumnStats( LogicalType logicalType, Statistics statistics, boolean isUtcTimestamp) { ColumnStats.Builder builder = @@ -176,6 +190,7 @@ private static ColumnStats convertToColumnStats( case BOOLEAN: case BINARY: case VARBINARY: + case GEOGRAPHY: break; case TINYINT: case SMALLINT: @@ -328,19 +343,39 @@ private static Timestamp binaryToTimestamp(Binary timestamp, boolean utcTimestam return timestampData.toTimestamp(); } + private static List getLeafColumnNames(MessageType schema) { + List leafColumnNames = new ArrayList<>(); + for (Type field : schema.getFields()) { + addLeafColumnNames(field, field.getName(), leafColumnNames); + } + return leafColumnNames; + } + + private static void addLeafColumnNames( + Type field, String topLevelFieldName, List leafColumnNames) { + if (field.isPrimitive()) { + leafColumnNames.add(topLevelFieldName); + return; + } + + for (Type nestedField : field.asGroupType().getFields()) { + addLeafColumnNames(nestedField, null, leafColumnNames); + } + } + private static class FileParquetStatistics { - private final List columns; + private final List leafColumnNames; private final List blocks; - public FileParquetStatistics(List columns, List blocks) { - this.columns = columns; + public FileParquetStatistics(List leafColumnNames, List blocks) { + this.leafColumnNames = leafColumnNames; this.blocks = blocks; } - public List getColumns() { - return columns; + public List getLeafColumnNames() { + return leafColumnNames; } public List getBlocks() { @@ -352,10 +387,7 @@ private static class ParquetFileRowCountCalculator implements Callable> columnStatisticsMap) { + public ParquetFileRowCountCalculator(Configuration hadoopConfig, Path file) { this.hadoopConfig = hadoopConfig; this.file = file; } @@ -365,12 +397,8 @@ public FileParquetStatistics call() throws Exception { org.apache.hadoop.fs.Path hadoopPath = new org.apache.hadoop.fs.Path(file.toUri()); ParquetMetadata metadata = ParquetFileReader.readFooter(hadoopConfig, hadoopPath); MessageType schema = metadata.getFileMetaData().getSchema(); - List columns = - schema.asGroupType().getFields().stream() - .map(Type::getName) - .collect(Collectors.toList()); List blocks = metadata.getBlocks(); - return new FileParquetStatistics(columns, blocks); + return new FileParquetStatistics(getLeafColumnNames(schema), blocks); } } } diff --git a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java index fce77ba996e59..75ee34cb0481a 100644 --- a/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java +++ b/flink-formats/flink-parquet/src/test/java/org/apache/flink/formats/parquet/row/ParquetRowDataWriterTest.java @@ -91,9 +91,7 @@ class ParquetRowDataWriterTest { }; private static final byte[] SECOND_POINT_WKB = - new byte[] { - 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x40, 0, 0, 0, 0, 0, 0, 8, 0x40 - }; + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x40, 0, 0, 0, 0, 0, 0, 8, 0x40}; private static final RowType ROW_TYPE = RowType.of( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultSchedulerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultSchedulerTest.java index 543203ad48203..98fe257363e2e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultSchedulerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultSchedulerTest.java @@ -504,7 +504,7 @@ private void testRestartVerticesOnFailuresInScheduling( } @Test - void skipDeploymentIfVertexVersionOutdated() { + void skipDeploymentIfVertexVersionOutdated() throws InterruptedException { testExecutionSlotAllocator.disableAutoCompletePendingRequests(); final JobGraph jobGraph = nonParallelSourceSinkJobGraph(); @@ -532,6 +532,7 @@ void skipDeploymentIfVertexVersionOutdated() { testExecutionSlotAllocator.enableAutoCompletePendingRequests(); taskRestartExecutor.triggerScheduledTasks(); + testExecutionOperations.awaitDeployedExecutions(2); assertThat(testExecutionOperations.getDeployedVertices()) .contains(sourceExecutionVertexId, sinkExecutionVertexId); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionOperationsDecorator.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionOperationsDecorator.java index cd4205bba1500..3ae55a28ef133 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionOperationsDecorator.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionOperationsDecorator.java @@ -110,6 +110,11 @@ public void awaitCanceledExecutions(int count) throws InterruptedException { canceledExecutions.await(count); } + /** Waits until the given number of executions have been deployed. */ + public void awaitDeployedExecutions(int count) throws InterruptedException { + deployedExecutions.await(count); + } + /** Waits until the given number of executions have been failed. */ public void awaitFailedExecutions(int count) throws InterruptedException { failedExecutions.await(count); diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml index b57ab89c943fa..6f22e8cbcfc04 100644 --- a/flink-table/flink-table-planner/pom.xml +++ b/flink-table/flink-table-planner/pom.xml @@ -158,6 +158,11 @@ under the License. flink-table-runtime ${project.version} + + org.apache.flink + flink-table-type-utils + ${project.version} + @@ -190,6 +195,20 @@ under the License. test + + org.antlr + antlr4-runtime + 4.7 + test + + + + io.airlift + aircompressor + 0.27 + test + + org.apache.flink @@ -521,3 +540,6 @@ under the License. + + + diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java index a1cb5e56cb5ed..64d556a5d6fbc 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java @@ -4918,7 +4918,8 @@ private RexNode convertIdentifier(Blackboard bb, SqlIdentifier identifier) { * *

The field {@code r.order_id} is declared {@code NOT NULL} inside a {@code NOT NULL} ROW. * However, in a LEFT JOIN when there is no match on the right side, the entire {@code b} row is - * null-padded — so {@code b.r} is null, and {@code b.r.order_id} must produce {@code null}. + * null-padded — so {@code b.r} is null, and {@code b.r.order_id} must produce + * {@code null}. * *

Without this adjustment, the {@link RexFieldAccess} built by {@code convertIdentifier} * still carries the pre-join {@link RexInputRef} type ({@code NOT NULL}), so the field access @@ -6036,8 +6037,7 @@ public RexNode convertExpression(SqlNode expr) { } requireNonNull(subQuery, "subQuery"); rex = requireNonNull(subQuery.expr); - return StandardConvertletTable.castToValidatedType( - expr.getParserPosition(), expr, rex, validator(), rexBuilder, false); + return castToValidatedTypeCompat(expr, rex, validator(), rexBuilder); case SELECT: case EXISTS: @@ -7136,4 +7136,54 @@ private class MeasureBlackboard extends Blackboard { return super.lookupMeasure(identifier); } } + + private static RexNode castToValidatedTypeCompat( + SqlNode expr, RexNode rex, SqlValidator validator, RexBuilder rexBuilder) { + try { + try { + final java.lang.reflect.Method method = + StandardConvertletTable.class.getMethod( + "castToValidatedType", + org.apache.calcite.sql.parser.SqlParserPos.class, + SqlNode.class, + RexNode.class, + SqlValidator.class, + RexBuilder.class, + boolean.class); + return (RexNode) + method.invoke( + null, + expr.getParserPosition(), + expr, + rex, + validator, + rexBuilder, + false); + } catch (NoSuchMethodException ignored) { + try { + final java.lang.reflect.Method method = + StandardConvertletTable.class.getMethod( + "castToValidatedType", + SqlNode.class, + RexNode.class, + SqlValidator.class, + RexBuilder.class, + boolean.class); + return (RexNode) method.invoke(null, expr, rex, validator, rexBuilder, false); + } catch (NoSuchMethodException ignoredAgain) { + final java.lang.reflect.Method method = + StandardConvertletTable.class.getMethod( + "castToValidatedType", + SqlNode.class, + RexNode.class, + SqlValidator.class, + RexBuilder.class); + return (RexNode) method.invoke(null, expr, rex, validator, rexBuilder); + } + } + } catch (ReflectiveOperationException e) { + throw new RuntimeException( + "Unable to invoke StandardConvertletTable.castToValidatedType.", e); + } + } } diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java index c857cf07c23ea..703c570758397 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java @@ -28,8 +28,8 @@ import org.apache.flink.table.legacy.api.TableSchema; import org.apache.flink.table.legacy.types.logical.TypeInformationRawType; import org.apache.flink.table.planner.plan.schema.BitmapRelDataType; -import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.table.planner.plan.schema.GenericRelDataType; +import org.apache.flink.table.planner.plan.schema.GeographyRelDataType; import org.apache.flink.table.planner.plan.schema.RawRelDataType; import org.apache.flink.table.planner.plan.schema.StructuredRelDataType; import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType; diff --git a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/GeographySqlITCase.java b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/GeographySqlITCase.java new file mode 100644 index 0000000000000..0400cf0912cff --- /dev/null +++ b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/runtime/batch/sql/GeographySqlITCase.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.runtime.batch.sql; + +import org.apache.flink.table.api.Table; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.planner.factories.utils.TestCollectionTableFactory; +import org.apache.flink.table.planner.runtime.utils.BatchTestBase; +import org.apache.flink.types.Row; +import org.apache.flink.util.CollectionUtil; + +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import static org.apache.flink.table.api.Expressions.$; +import static org.apache.flink.table.api.Expressions.callSql; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** End-to-end SQL coverage for GEOGRAPHY values. */ +class GeographySqlITCase extends BatchTestBase { + + private static final byte[] LINESTRING_WKB = + new byte[] { + 1, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, -16, 63 + }; + + @Test + void testGeographySqlInsertAndRoundTripAccessors() throws Exception { + final List sourceData = + Arrays.asList( + Row.of(1L, "POINT (1 2)"), + Row.of(2L, "LINESTRING (0 0, 1 1)"), + Row.of(3L, null)); + + TestCollectionTableFactory.reset(); + TestCollectionTableFactory.initData(sourceData); + + tEnv().executeSql( + "CREATE TABLE GeographySource(id BIGINT, wkt STRING) WITH " + + "('connector' = 'COLLECTION')") + .await(); + + final List geographies = + CollectionUtil.iteratorToList( + tEnv().executeSql( + "SELECT id, ST_GEOGFROMTEXT(wkt) AS geog " + + "FROM GeographySource ORDER BY id") + .collect()); + + assertThat(geographies).hasSize(3); + assertThat(geographies.get(0).getField(0)).isEqualTo(1L); + assertThat(((GeographyData) geographies.get(0).getField(1)).subtypeId()) + .isEqualTo(GeographyData.POINT); + assertThat(geographies.get(1).getField(0)).isEqualTo(2L); + assertThat(((GeographyData) geographies.get(1).getField(1)).toBytes()) + .isEqualTo(LINESTRING_WKB); + assertThat(geographies.get(2).getField(0)).isEqualTo(3L); + assertThat(geographies.get(2).getField(1)).isNull(); + + final List selectedRows = + CollectionUtil.iteratorToList( + tEnv().executeSql( + "SELECT id, " + + "ST_ASTEXT(geog), " + + "ST_ASWKB(geog), " + + "ST_ASTEXT(ST_GEOGFROMWKB(ST_ASWKB(geog))) " + + "FROM (" + + "SELECT id, ST_GEOGFROMTEXT(wkt) AS geog " + + "FROM GeographySource) " + + "WHERE geog IS NOT NULL " + + "ORDER BY id") + .collect()); + + assertThat(selectedRows).hasSize(2); + assertThat(selectedRows.get(0).getField(0)).isEqualTo(1L); + assertThat(selectedRows.get(0).getField(1)).isEqualTo("POINT (1 2)"); + assertThat(((byte[]) selectedRows.get(0).getField(2)).length).isEqualTo(21); + assertThat(selectedRows.get(0).getField(3)).isEqualTo("POINT (1 2)"); + assertThat(selectedRows.get(1).getField(0)).isEqualTo(2L); + assertThat(selectedRows.get(1).getField(1)).isEqualTo("LINESTRING (0 0, 1 1)"); + assertThat((byte[]) selectedRows.get(1).getField(2)).isEqualTo(LINESTRING_WKB); + assertThat(selectedRows.get(1).getField(3)).isEqualTo("LINESTRING (0 0, 1 1)"); + } + + @Test + void testSpatialPredicatesAndMeasurementsRemainUnavailableInV1() { + final String distanceSql = + "SELECT ST_DISTANCE(ST_GEOGFROMTEXT('POINT (0 0)'), " + + "ST_GEOGFROMTEXT('POINT (1 1)'))"; + final String intersectsSql = + "SELECT ST_INTERSECTS(ST_GEOGFROMTEXT('POINT (0 0)'), " + + "ST_GEOGFROMTEXT('POINT (1 1)'))"; + + assertThatThrownBy(() -> tEnv().executeSql(distanceSql)) + .hasMessageContaining("ST_DISTANCE"); + assertThatThrownBy(() -> tEnv().executeSql(intersectsSql)) + .hasMessageContaining("ST_INTERSECTS"); + } + + @Test + void testMixedGeographySchemaAcrossSqlAndTableRuntime() throws Exception { + final List sourceData = + Arrays.asList( + Row.of( + 1L, + "alpha", + true, + LocalDateTime.parse("2024-01-02T03:04:05.123"), + null, + "POINT (1 2)", + null), + Row.of( + 2L, + "beta", + false, + LocalDateTime.parse("2024-02-03T04:05:06.789"), + "second row", + null, + LINESTRING_WKB), + Row.of(3L, null, null, null, "missing geography", null, null)); + + TestCollectionTableFactory.reset(); + TestCollectionTableFactory.initData(sourceData); + + tEnv().executeSql( + "CREATE TABLE MixedGeographySource(" + + "id BIGINT, " + + "name STRING, " + + "active BOOLEAN, " + + "event_ts TIMESTAMP(3), " + + "note STRING, " + + "wkt STRING, " + + "wkb BYTES" + + ") WITH ('connector' = 'COLLECTION')") + .await(); + tEnv().createTemporaryView( + "MixedGeographyInput", + tEnv().sqlQuery( + "SELECT id, name, active, event_ts, note, " + + "CASE " + + "WHEN wkt IS NOT NULL THEN ST_GEOGFROMTEXT(wkt) " + + "WHEN wkb IS NOT NULL THEN ST_GEOGFROMWKB(wkb) " + + "ELSE CAST(NULL AS GEOGRAPHY) " + + "END AS geog " + + "FROM MixedGeographySource")); + + final List sinkRows = + CollectionUtil.iteratorToList( + tEnv().executeSql( + "SELECT id, name, active, event_ts, note, geog " + + "FROM MixedGeographyInput ORDER BY id") + .collect()); + + assertThat(sinkRows).hasSize(3); + + assertThat(sinkRows.get(0).getField(0)).isEqualTo(1L); + assertThat(sinkRows.get(0).getField(1)).isEqualTo("alpha"); + assertThat(sinkRows.get(0).getField(2)).isEqualTo(true); + assertThat(String.valueOf(sinkRows.get(0).getField(3))) + .isEqualTo("2024-01-02T03:04:05.123"); + assertThat(sinkRows.get(0).getField(4)).isNull(); + assertThat(((GeographyData) sinkRows.get(0).getField(5)).subtypeId()) + .isEqualTo(GeographyData.POINT); + + assertThat(sinkRows.get(1).getField(0)).isEqualTo(2L); + assertThat(sinkRows.get(1).getField(1)).isEqualTo("beta"); + assertThat(sinkRows.get(1).getField(2)).isEqualTo(false); + assertThat(String.valueOf(sinkRows.get(1).getField(3))) + .isEqualTo("2024-02-03T04:05:06.789"); + assertThat(sinkRows.get(1).getField(4)).isEqualTo("second row"); + assertThat(((GeographyData) sinkRows.get(1).getField(5)).toBytes()) + .isEqualTo(LINESTRING_WKB); + + assertThat(sinkRows.get(2).getField(0)).isEqualTo(3L); + assertThat(sinkRows.get(2).getField(1)).isNull(); + assertThat(sinkRows.get(2).getField(2)).isNull(); + assertThat(sinkRows.get(2).getField(3)).isNull(); + assertThat(sinkRows.get(2).getField(4)).isEqualTo("missing geography"); + assertThat(sinkRows.get(2).getField(5)).isNull(); + + final Table projectedTable = + tEnv().from("MixedGeographyInput") + .select( + $("id"), + $("name"), + $("active"), + $("event_ts"), + $("note"), + callSql("ST_ASTEXT(geog)"), + callSql("ST_ASWKB(geog)")); + + final List projectedRows = + CollectionUtil.iteratorToList(projectedTable.execute().collect()); + projectedRows.sort(Comparator.comparing(row -> (Long) row.getField(0))); + + assertThat(projectedRows).hasSize(3); + assertThat(projectedRows.get(0).getField(0)).isEqualTo(1L); + assertThat(projectedRows.get(0).getField(1)).isEqualTo("alpha"); + assertThat(projectedRows.get(0).getField(2)).isEqualTo(true); + assertThat(String.valueOf(projectedRows.get(0).getField(3))) + .isEqualTo("2024-01-02T03:04:05.123"); + assertThat(projectedRows.get(0).getField(4)).isNull(); + assertThat(projectedRows.get(0).getField(5)).isEqualTo("POINT (1 2)"); + assertThat(((byte[]) projectedRows.get(0).getField(6)).length).isEqualTo(21); + + assertThat(projectedRows.get(1).getField(0)).isEqualTo(2L); + assertThat(projectedRows.get(1).getField(1)).isEqualTo("beta"); + assertThat(projectedRows.get(1).getField(2)).isEqualTo(false); + assertThat(String.valueOf(projectedRows.get(1).getField(3))) + .isEqualTo("2024-02-03T04:05:06.789"); + assertThat(projectedRows.get(1).getField(4)).isEqualTo("second row"); + assertThat(projectedRows.get(1).getField(5)).isEqualTo("LINESTRING (0 0, 1 1)"); + assertThat((byte[]) projectedRows.get(1).getField(6)).isEqualTo(LINESTRING_WKB); + + assertThat(projectedRows.get(2).getField(0)).isEqualTo(3L); + assertThat(projectedRows.get(2).getField(1)).isNull(); + assertThat(projectedRows.get(2).getField(2)).isNull(); + assertThat(projectedRows.get(2).getField(3)).isNull(); + assertThat(projectedRows.get(2).getField(4)).isEqualTo("missing geography"); + assertThat(projectedRows.get(2).getField(5)).isNull(); + assertThat(projectedRows.get(2).getField(6)).isNull(); + } +} diff --git a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/types/TypeInfoDataTypeConverter.java b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/types/TypeInfoDataTypeConverter.java index f1b65271204d6..7ba97ff18f616 100644 --- a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/types/TypeInfoDataTypeConverter.java +++ b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/types/TypeInfoDataTypeConverter.java @@ -201,6 +201,8 @@ public static TypeInformation fromDataTypeToTypeInfo(DataType dataType) { return ExternalTypeInfo.of(dataType); } return TypeConversions.fromDataTypeToLegacyInfo(dataType); + case GEOGRAPHY: + return ExternalTypeInfo.of(dataType); default: return TypeConversions.fromDataTypeToLegacyInfo(dataType); } diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/formats/raw/RawFormatFactoryTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/formats/raw/RawFormatFactoryTest.java index 17ad29f92254b..0c9862d0e8045 100644 --- a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/formats/raw/RawFormatFactoryTest.java +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/formats/raw/RawFormatFactoryTest.java @@ -175,6 +175,18 @@ void testInvalidFieldTypes() { .hasMessage("The 'raw' format doesn't supports 'MAP' as column type."); } + @Test + void testGeographyTypeIsRejected() { + ResolvedSchema geographySchema = + ResolvedSchema.of(Column.physical("field1", DataTypes.GEOGRAPHY())); + + assertThatThrownBy(() -> createDeserializationSchema(geographySchema, getBasicOptions())) + .hasMessage("The 'raw' format doesn't supports 'GEOGRAPHY' as column type."); + + assertThatThrownBy(() -> createSerializationSchema(geographySchema, getBasicOptions())) + .hasMessage("The 'raw' format doesn't supports 'GEOGRAPHY' as column type."); + } + @Test void testLineDelimiterOption() { final Map tableOptions = diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyWkbBenchmark.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyWkbBenchmark.java new file mode 100644 index 0000000000000..6122110d06630 --- /dev/null +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyWkbBenchmark.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.apache.flink.core.memory.DataInputDeserializer; +import org.apache.flink.core.memory.DataOutputSerializer; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.data.binary.BinaryRowData; +import org.apache.flink.table.runtime.typeutils.GeographyTypeSerializer; +import org.apache.flink.table.runtime.typeutils.RowDataSerializer; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.GeographyType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.table.types.logical.VarCharType; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Benchmark-style baseline coverage for GEOGRAPHY WKB runtime hot paths. + * + *

This class is intentionally placed under {@code src/test/java}. It is not a correctness test + * and does not enforce wall-clock assertions. Instead, it provides a runnable baseline that a + * developer can execute directly via {@link #main(String[])} or from an IDE while CI keeps a small + * smoke test over the wiring. + */ +public class GeographyWkbBenchmark { + + private static final int ID_POS = 0; + private static final int NAME_POS = 1; + private static final int ACTIVE_POS = 2; + private static final int EVENT_TIME_POS = 3; + private static final int GEOGRAPHY_POS = 4; + + private static final byte[] POINT_WKB = + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 0, 64}; + + private static final byte[] LINESTRING_WKB = + new byte[] { + 1, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, -16, 63 + }; + + private static final RowType MIXED_ROW_TYPE = + RowType.of( + new LogicalType[] { + new BigIntType(false), + new VarCharType(VarCharType.MAX_LENGTH), + new BooleanType(), + new TimestampType(3), + new GeographyType() + }); + + private static final int DEFAULT_ITERATIONS = 100_000; + + private final RowDataSerializer rowDataSerializer = new RowDataSerializer(MIXED_ROW_TYPE); + private final RowDataSerializer binaryRowAccessSerializer = + new RowDataSerializer(MIXED_ROW_TYPE); + private final DataOutputSerializer geographyOutput = new DataOutputSerializer(128); + private final DataInputDeserializer geographyInput = new DataInputDeserializer(); + private final DataOutputSerializer rowOutput = new DataOutputSerializer(256); + private final DataInputDeserializer rowInput = new DataInputDeserializer(); + + private PayloadKind payloadKind; + private byte[] geographyBytes; + private GeographyData geography; + private GenericRowData genericRow; + private BinaryRowData binaryRow; + + public void setup(PayloadKind payloadKind) { + this.payloadKind = payloadKind; + this.geographyBytes = payloadKind.createWkb(); + this.geography = GeographyData.fromBytes(geographyBytes); + this.genericRow = createRow(geography); + this.binaryRow = binaryRowAccessSerializer.toBinaryRow(genericRow, true); + } + + public PayloadKind getPayloadKind() { + return payloadKind; + } + + public Map runAllBenchmarks(int iterations) throws IOException { + final LinkedHashMap results = new LinkedHashMap<>(); + results.put( + "geographySerializerRoundTrip", + measure(iterations, this::benchmarkSerializerRoundTrip)); + results.put("geographySerializerCopy", measure(iterations, this::benchmarkSerializerCopy)); + results.put( + "rowDataSerializerRoundTrip", + measure(iterations, this::benchmarkRowDataSerializerRoundTrip)); + results.put( + "genericRowGeographyAccess", + measure(iterations, this::benchmarkGenericRowGeographyAccess)); + results.put( + "binaryRowGeographyAccess", + measure(iterations, this::benchmarkBinaryRowGeographyAccess)); + results.put("stGeogFromWkb", measure(iterations, this::benchmarkStGeogFromWkb)); + results.put("stAsWkb", measure(iterations, this::benchmarkStAsWkb)); + return results; + } + + public long benchmarkSerializerRoundTrip(int iterations) throws IOException { + long checksum = 0L; + for (int i = 0; i < iterations; i++) { + geographyOutput.clear(); + GeographyTypeSerializer.INSTANCE.serialize(geography, geographyOutput); + geographyInput.setBuffer(geographyOutput.wrapAsByteBuffer()); + checksum += + GeographyTypeSerializer.INSTANCE.deserialize(geographyInput).toBytes().length; + } + return checksum; + } + + public long benchmarkSerializerCopy(int iterations) { + long checksum = 0L; + for (int i = 0; i < iterations; i++) { + checksum += GeographyTypeSerializer.INSTANCE.copy(geography).toBytes().length; + } + return checksum; + } + + public long benchmarkRowDataSerializerRoundTrip(int iterations) throws IOException { + long checksum = 0L; + for (int i = 0; i < iterations; i++) { + rowOutput.clear(); + rowDataSerializer.serialize(genericRow, rowOutput); + rowInput.setBuffer(rowOutput.wrapAsByteBuffer()); + final RowData deserialized = rowDataSerializer.deserialize(rowInput); + checksum += deserialized.getLong(ID_POS); + checksum += deserialized.getString(NAME_POS).toBytes().length; + checksum += deserialized.getBoolean(ACTIVE_POS) ? 1 : 0; + checksum += deserialized.getTimestamp(EVENT_TIME_POS, 3).getMillisecond(); + checksum += deserialized.getGeography(GEOGRAPHY_POS).toBytes().length; + } + return checksum; + } + + public long benchmarkGenericRowGeographyAccess(int iterations) { + long checksum = 0L; + for (int i = 0; i < iterations; i++) { + checksum += genericRow.getGeography(GEOGRAPHY_POS).toBytes().length; + } + return checksum; + } + + public long benchmarkBinaryRowGeographyAccess(int iterations) { + long checksum = 0L; + for (int i = 0; i < iterations; i++) { + checksum += binaryRow.getGeography(GEOGRAPHY_POS).toBytes().length; + } + return checksum; + } + + public long benchmarkStGeogFromWkb(int iterations) { + long checksum = 0L; + for (int i = 0; i < iterations; i++) { + final GeographyData converted = GeographyConversionUtils.fromWkb(geographyBytes); + checksum += converted.subtypeId(); + checksum += converted.toBytes().length; + } + return checksum; + } + + public long benchmarkStAsWkb(int iterations) { + long checksum = 0L; + for (int i = 0; i < iterations; i++) { + checksum += GeographyConversionUtils.asWkb(geography).length; + } + return checksum; + } + + public static void main(String[] args) throws IOException { + final int iterations = args.length > 0 ? Integer.parseInt(args[0]) : DEFAULT_ITERATIONS; + + for (PayloadKind payloadKind : PayloadKind.values()) { + final GeographyWkbBenchmark benchmark = new GeographyWkbBenchmark(); + benchmark.setup(payloadKind); + + System.out.printf("Payload: %s (%d iterations)%n", payloadKind.displayName, iterations); + for (Map.Entry entry : + benchmark.runAllBenchmarks(iterations).entrySet()) { + final BenchmarkResult result = entry.getValue(); + System.out.printf( + " %-30s %8d ns/op checksum=%d%n", + entry.getKey(), result.getNanosPerOperation(), result.getChecksum()); + } + } + } + + private BenchmarkResult measure(int iterations, CheckedBenchmark benchmark) throws IOException { + benchmark.run(Math.max(1, Math.min(iterations, 256))); + + final long startNanos = System.nanoTime(); + final long checksum = benchmark.run(iterations); + final long elapsedNanos = System.nanoTime() - startNanos; + return new BenchmarkResult(elapsedNanos / Math.max(1, iterations), checksum); + } + + private static GenericRowData createRow(GeographyData geography) { + final GenericRowData row = new GenericRowData(5); + row.setField(ID_POS, 42L); + row.setField(NAME_POS, StringData.fromString("benchmark-geography")); + row.setField(ACTIVE_POS, true); + row.setField( + EVENT_TIME_POS, + TimestampData.fromLocalDateTime( + LocalDateTime.of(2025, 1, 2, 3, 4, 5, 123_000_000))); + row.setField(GEOGRAPHY_POS, geography); + return row; + } + + private interface CheckedBenchmark { + long run(int iterations) throws IOException; + } + + /** Representative payloads for serializer and conversion hot paths. */ + public enum PayloadKind { + SMALL_POINT("small-point") { + @Override + byte[] createWkb() { + return POINT_WKB; + } + }, + EMPTY_COLLECTION("empty-geometry-collection") { + @Override + byte[] createWkb() { + return GeographyTypeSerializer.INSTANCE.createInstance().toBytes(); + } + }, + LARGE_LINESTRING("large-linestring") { + @Override + byte[] createWkb() { + return LINESTRING_WKB; + } + }; + + private final String displayName; + + PayloadKind(String displayName) { + this.displayName = displayName; + } + + abstract byte[] createWkb(); + } + + /** Immutable benchmark result for one measured hot path. */ + public static final class BenchmarkResult { + private final long nanosPerOperation; + private final long checksum; + + private BenchmarkResult(long nanosPerOperation, long checksum) { + this.nanosPerOperation = nanosPerOperation; + this.checksum = checksum; + } + + public long getNanosPerOperation() { + return nanosPerOperation; + } + + public long getChecksum() { + return checksum; + } + } +} diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyWkbBenchmarkTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyWkbBenchmarkTest.java new file mode 100644 index 0000000000000..1d0a007c2af8b --- /dev/null +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/functions/scalar/GeographyWkbBenchmarkTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.functions.scalar; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Smoke tests for {@link GeographyWkbBenchmark}. */ +class GeographyWkbBenchmarkTest { + + @Test + void testBenchmarksRunForRepresentativePayloads() throws IOException { + for (GeographyWkbBenchmark.PayloadKind payloadKind : + GeographyWkbBenchmark.PayloadKind.values()) { + final GeographyWkbBenchmark benchmark = new GeographyWkbBenchmark(); + benchmark.setup(payloadKind); + + final Map results = + benchmark.runAllBenchmarks(128); + + assertThat(benchmark.getPayloadKind()).isEqualTo(payloadKind); + assertThat(results).hasSize(7); + assertThat(results.values()) + .allSatisfy( + result -> { + assertThat(result.getNanosPerOperation()) + .isGreaterThanOrEqualTo(0L); + assertThat(result.getChecksum()).isGreaterThan(0L); + }); + } + } +} diff --git a/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/GeographyStateRestoreTest.java b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/GeographyStateRestoreTest.java new file mode 100644 index 0000000000000..2c61d2606a50f --- /dev/null +++ b/flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/process/GeographyStateRestoreTest.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.process; + +import org.apache.flink.api.common.functions.OpenContext; +import org.apache.flink.api.common.state.ValueState; +import org.apache.flink.api.common.state.ValueStateDescriptor; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.functions.KeyedProcessFunction; +import org.apache.flink.streaming.api.operators.KeyedProcessOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.KeyedOneInputStreamOperatorTestHarness; +import org.apache.flink.streaming.util.TestHarnessUtil; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.GeographyData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.GeographyType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.table.types.logical.VarCharType; +import org.apache.flink.util.Collector; + +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests GEOGRAPHY values across operator checkpoint and restore paths. */ +class GeographyStateRestoreTest { + + private static final byte[] POINT_1_WKB = + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, -16, 63, 0, 0, 0, 0, 0, 0, 0, 64}; + + private static final byte[] POINT_2_WKB = + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 64, 0, 0, 0, 0, 0, 0, 24, 64}; + + private static final byte[] POINT_3_WKB = + new byte[] {1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 64, 0, 0, 0, 0, 0, 0, 36, 64}; + + private static final RowType MIXED_ROW_TYPE = + RowType.of( + new LogicalType[] { + new BigIntType(false), + new VarCharType(), + new BooleanType(), + new TimestampType(3), + new VarCharType(), + new GeographyType() + }, + new String[] {"id", "name", "active", "event_ts", "note", "geog"}); + + @Test + void testGeographyStateSurvivesCheckpointAndRestore() throws Exception { + final RowData firstRow = + rowOf( + 1L, + "alpha", + true, + LocalDateTime.of(2024, 1, 2, 3, 4, 5, 123_000_000), + "before restore", + POINT_1_WKB); + final RowData secondRow = + rowOf( + 1L, + "beta", + false, + LocalDateTime.of(2024, 2, 3, 4, 5, 6, 789_000_000), + null, + POINT_2_WKB); + final RowData thirdRow = + rowOf( + 1L, + "gamma", + true, + LocalDateTime.of(2024, 3, 4, 5, 6, 7, 456_000_000), + "after restore", + POINT_3_WKB); + + final OperatorSubtaskState snapshot; + try (KeyedOneInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + harness.processElement(new StreamRecord<>(firstRow)); + assertThat(harness.getOutput()).isEmpty(); + snapshot = harness.snapshot(1L, 1L); + } + + try (KeyedOneInputStreamOperatorTestHarness restoredHarness = + createHarness()) { + restoredHarness.initializeState(snapshot); + restoredHarness.open(); + + restoredHarness.processElement(new StreamRecord<>(secondRow)); + restoredHarness.processElement(new StreamRecord<>(thirdRow)); + + final List restoredOutput = + TestHarnessUtil.getRawElementsFromOutput(restoredHarness.getOutput()); + assertThat(restoredOutput).hasSize(2); + + assertRowMatches(firstRow, (RowData) restoredOutput.get(0)); + assertRowMatches(secondRow, (RowData) restoredOutput.get(1)); + } + } + + private static KeyedOneInputStreamOperatorTestHarness createHarness() + throws Exception { + return new KeyedOneInputStreamOperatorTestHarness<>( + new KeyedProcessOperator<>(new RememberingGeographyProcessFunction()), + row -> row.getLong(0), + Types.LONG); + } + + private static RowData rowOf( + long id, + String name, + Boolean active, + LocalDateTime eventTime, + String note, + byte[] geographyWkb) { + return GenericRowData.of( + id, + name == null ? null : StringData.fromString(name), + active, + eventTime == null ? null : TimestampData.fromLocalDateTime(eventTime), + note == null ? null : StringData.fromString(note), + geographyWkb == null ? null : GeographyData.fromBytes(geographyWkb)); + } + + private static void assertRowMatches(RowData expected, RowData actual) { + assertThat(actual.getLong(0)).isEqualTo(expected.getLong(0)); + assertThat(actual.isNullAt(1) ? null : actual.getString(1).toString()) + .isEqualTo(expected.isNullAt(1) ? null : expected.getString(1).toString()); + assertThat(actual.isNullAt(2) ? null : actual.getBoolean(2)) + .isEqualTo(expected.isNullAt(2) ? null : expected.getBoolean(2)); + assertThat(actual.isNullAt(3) ? null : actual.getTimestamp(3, 3)) + .isEqualTo(expected.isNullAt(3) ? null : expected.getTimestamp(3, 3)); + assertThat(actual.isNullAt(4) ? null : actual.getString(4).toString()) + .isEqualTo(expected.isNullAt(4) ? null : expected.getString(4).toString()); + assertThat(actual.isNullAt(5) ? null : actual.getGeography(5).toBytes()) + .isEqualTo(expected.isNullAt(5) ? null : expected.getGeography(5).toBytes()); + } + + private static final class RememberingGeographyProcessFunction + extends KeyedProcessFunction { + + private transient ValueState previousRowState; + + @Override + public void open(OpenContext openContext) { + previousRowState = + getRuntimeContext() + .getState( + new ValueStateDescriptor<>( + "previous-row", InternalTypeInfo.of(MIXED_ROW_TYPE))); + } + + @Override + public void processElement(RowData value, Context ctx, Collector out) + throws Exception { + final RowData previous = previousRowState.value(); + if (previous != null) { + out.collect(previous); + } + previousRowState.update(value); + } + } +} diff --git a/flink-table/flink-table-test-utils/pom.xml b/flink-table/flink-table-test-utils/pom.xml index 9460a52e034ae..9844348377a62 100644 --- a/flink-table/flink-table-test-utils/pom.xml +++ b/flink-table/flink-table-test-utils/pom.xml @@ -59,12 +59,6 @@ under the License. flink-table-type-utils ${project.version} - - org.apache.flink - flink-table-type-utils - ${project.version} - test - org.apache.flink flink-table-planner-loader @@ -136,3 +130,4 @@ under the License. +