diff --git a/core/src/main/java/org/polypheny/db/adapter/Adapter.java b/core/src/main/java/org/polypheny/db/adapter/Adapter.java index 858c4f3a71..a048b18810 100644 --- a/core/src/main/java/org/polypheny/db/adapter/Adapter.java +++ b/core/src/main/java/org/polypheny/db/adapter/Adapter.java @@ -312,4 +312,8 @@ protected void resetDockerConnection() { } + public List getActiveFeatureNames() { + return List.of(); + } + } diff --git a/core/src/main/java/org/polypheny/db/adapter/AdapterManager.java b/core/src/main/java/org/polypheny/db/adapter/AdapterManager.java index 132690e3a8..f0a6fcc37b 100644 --- a/core/src/main/java/org/polypheny/db/adapter/AdapterManager.java +++ b/core/src/main/java/org/polypheny/db/adapter/AdapterManager.java @@ -31,6 +31,7 @@ import org.apache.calcite.linq4j.tree.Expressions; import org.jetbrains.annotations.NotNull; import org.polypheny.db.adapter.annotations.AdapterProperties; +import org.polypheny.db.adapter.annotations.AdapterSettingsPreset; import org.polypheny.db.adapter.java.AdapterTemplate; import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.entity.LogicalAdapter; @@ -65,8 +66,10 @@ private AdapterManager() { public static long addAdapterTemplate( Class> clazz, String adapterName, DeployFn deployer ) { List settings = AdapterTemplate.getAllSettings( clazz ); AdapterProperties properties = clazz.getAnnotation( AdapterProperties.class ); + List modes = List.of( properties.usedModes() ); + List presets = AdapterTemplate.getAllPresets( clazz, settings, modes ); long id = AdapterManager.getInstance().idBuilder.getAndIncrement(); - AdapterManager.getInstance().adapterTemplates.put( id, new AdapterTemplate( id, clazz, adapterName, settings, List.of( properties.usedModes() ), properties.description(), deployer ) ); + AdapterManager.getInstance().adapterTemplates.put( id, new AdapterTemplate( id, clazz, adapterName, settings, modes, presets, properties.description(), deployer ) ); return id; } diff --git a/core/src/main/java/org/polypheny/db/adapter/DataStore.java b/core/src/main/java/org/polypheny/db/adapter/DataStore.java index 0084043a52..27c035debd 100644 --- a/core/src/main/java/org/polypheny/db/adapter/DataStore.java +++ b/core/src/main/java/org/polypheny/db/adapter/DataStore.java @@ -27,6 +27,7 @@ import org.polypheny.db.catalog.Catalog; import org.polypheny.db.catalog.catalogs.AdapterCatalog; import org.polypheny.db.catalog.entity.logical.LogicalTable; +import org.polypheny.db.catalog.logistic.IndexCategory; @Slf4j public abstract class DataStore extends Adapter implements Modifiable, ExtensionPoint { @@ -52,11 +53,27 @@ public DataStore( final long adapterId, final String uniqueName, final Map getFunctionalIndexes( LogicalTable catalogTable ); - public record IndexMethodModel( @JsonProperty String name, @JsonProperty String displayName ) { - + public record IndexMethodModel( + @JsonProperty String name, + @JsonProperty String displayName, + @JsonProperty IndexCategory category, + @JsonProperty List parameters + ) { + public IndexMethodModel( String name, String displayName ) { + this( name, displayName, IndexCategory.REGULAR, List.of() ); + } } + public record IndexParameterModel( + @JsonProperty String name, + @JsonProperty String displayName, + @JsonProperty String type, //INTEGER, BOOLEAN, ENUM + @JsonProperty List options, + @JsonProperty String defaultValue + ){} + + public record FunctionalIndexInfo( List columnIds, String methodDisplayName ) { public List getColumnNames() { diff --git a/core/src/main/java/org/polypheny/db/adapter/RelationalDataSource.java b/core/src/main/java/org/polypheny/db/adapter/RelationalDataSource.java index 5ceec1aede..9174b03a72 100644 --- a/core/src/main/java/org/polypheny/db/adapter/RelationalDataSource.java +++ b/core/src/main/java/org/polypheny/db/adapter/RelationalDataSource.java @@ -24,7 +24,7 @@ public interface RelationalDataSource { Map> getExportedColumns(); - record ExportedColumn( String name, PolyType type, PolyType collectionsType, Integer length, Integer scale, Integer dimension, Integer cardinality, boolean nullable, String physicalSchemaName, String physicalTableName, String physicalColumnName, int physicalPosition, boolean primary ) { + record ExportedColumn( String name, PolyType type, PolyType collectionsType, Integer length, Integer scale, Integer dimension, Integer cardinality, boolean nullable, boolean elementsNullable, String physicalSchemaName, String physicalTableName, String physicalColumnName, int physicalPosition, boolean primary ) { public String getDisplayType() { String typeStr = type.getName(); diff --git a/core/src/main/java/org/polypheny/db/adapter/RelationalModifyDelegate.java b/core/src/main/java/org/polypheny/db/adapter/RelationalModifyDelegate.java index 8e37fe5e51..b8fdb5f2b5 100644 --- a/core/src/main/java/org/polypheny/db/adapter/RelationalModifyDelegate.java +++ b/core/src/main/java/org/polypheny/db/adapter/RelationalModifyDelegate.java @@ -112,8 +112,8 @@ public void dropGraph( Context context, AllocationGraph allocation ) { @Override public List createCollection( Context context, LogicalCollection logical, AllocationCollection allocation ) { PhysicalTable physical = Scannable.createSubstitutionTable( modifiable, context, logical, allocation, "_doc_", List.of( - new ColumnContext( DocumentType.DOCUMENT_ID, null, PolyType.TEXT, false ), - new ColumnContext( DocumentType.DOCUMENT_DATA, null, PolyType.TEXT, true ) ), 1 ); + new ColumnContext( DocumentType.DOCUMENT_ID, null, PolyType.TEXT, false, true ), + new ColumnContext( DocumentType.DOCUMENT_DATA, null, PolyType.TEXT, true, true ) ), 1 ); catalog.addPhysical( allocation, physical ); return List.of( physical ); diff --git a/core/src/main/java/org/polypheny/db/adapter/Scannable.java b/core/src/main/java/org/polypheny/db/adapter/Scannable.java index 609a87cbd3..0f4cfb5ce9 100644 --- a/core/src/main/java/org/polypheny/db/adapter/Scannable.java +++ b/core/src/main/java/org/polypheny/db/adapter/Scannable.java @@ -60,7 +60,7 @@ static PhysicalEntity createSubstitutionEntity( Scannable scannable, Context con int i = 0; for ( ColumnContext col : columnsInformations ) { - LogicalColumn column = new LogicalColumn( builder.getNewFieldId(), col.name, table.id, table.namespaceId, i, col.type, null, col.precision, null, null, null, col.nullable, Collation.getDefaultCollation(), null ); + LogicalColumn column = new LogicalColumn( builder.getNewFieldId(), col.name, table.id, table.namespaceId, i, col.type, null, col.precision, null, null, null, col.nullable, col.elementsNullable, Collation.getDefaultCollation(), null ); columns.add( column ); i++; } @@ -170,24 +170,24 @@ static AlgNode getDocumentScanSubstitute( Scannable scannable, long allocId, Alg static List createGraphSubstitute( Scannable scannable, Context context, LogicalGraph logical, AllocationGraph allocation ) { PhysicalEntity node = createSubstitutionEntity( scannable, context, logical, allocation, "_node_", List.of( - new ColumnContext( "id", GraphType.ID_SIZE, PolyType.VARCHAR, false ), - new ColumnContext( "label", null, PolyType.TEXT, false ) ), 2 ); + new ColumnContext( "id", GraphType.ID_SIZE, PolyType.VARCHAR, false, true ), + new ColumnContext( "label", null, PolyType.TEXT, false, true ) ), 2 ); PhysicalEntity nProperties = createSubstitutionEntity( scannable, context, logical, allocation, "_nProperties_", List.of( - new ColumnContext( "id", GraphType.ID_SIZE, PolyType.VARCHAR, false ), - new ColumnContext( "key", null, PolyType.TEXT, false ), - new ColumnContext( "value", null, PolyType.TEXT, true ) ), 2 ); + new ColumnContext( "id", GraphType.ID_SIZE, PolyType.VARCHAR, false, true ), + new ColumnContext( "key", null, PolyType.TEXT, false, true ), + new ColumnContext( "value", null, PolyType.TEXT, true, true ) ), 2 ); PhysicalEntity edge = createSubstitutionEntity( scannable, context, logical, allocation, "_edge_", List.of( - new ColumnContext( "id", GraphType.ID_SIZE, PolyType.VARCHAR, false ), - new ColumnContext( "label", null, PolyType.TEXT, true ), - new ColumnContext( "_l_id_", GraphType.ID_SIZE, PolyType.VARCHAR, true ), - new ColumnContext( "_r_id_", GraphType.ID_SIZE, PolyType.VARCHAR, true ) ), 1 ); + new ColumnContext( "id", GraphType.ID_SIZE, PolyType.VARCHAR, false, true ), + new ColumnContext( "label", null, PolyType.TEXT, true, true ), + new ColumnContext( "_l_id_", GraphType.ID_SIZE, PolyType.VARCHAR, true, true ), + new ColumnContext( "_r_id_", GraphType.ID_SIZE, PolyType.VARCHAR, true, true ) ), 1 ); PhysicalEntity eProperties = createSubstitutionEntity( scannable, context, logical, allocation, "_eProperties_", List.of( - new ColumnContext( "id", GraphType.ID_SIZE, PolyType.VARCHAR, false ), - new ColumnContext( "key", null, PolyType.TEXT, false ), - new ColumnContext( "value", null, PolyType.TEXT, true ) ), 2 ); + new ColumnContext( "id", GraphType.ID_SIZE, PolyType.VARCHAR, false, true ), + new ColumnContext( "key", null, PolyType.TEXT, false, true ), + new ColumnContext( "value", null, PolyType.TEXT, true, true ) ), 2 ); scannable.getCatalog().addPhysical( allocation, node, nProperties, edge, eProperties ); return List.of( node, nProperties, edge, eProperties ); @@ -222,8 +222,8 @@ static void dropGraphSubstitute( Scannable scannable, Context context, Allocatio static List createCollectionSubstitute( Scannable scannable, Context context, LogicalCollection logical, AllocationCollection allocation ) { PhysicalEntity doc = createSubstitutionEntity( scannable, context, logical, allocation, "_doc_", List.of( - new ColumnContext( DocumentType.DOCUMENT_ID, null, PolyType.TEXT, false ), - new ColumnContext( DocumentType.DOCUMENT_DATA, null, PolyType.TEXT, false ) ), 1 ); + new ColumnContext( DocumentType.DOCUMENT_ID, null, PolyType.TEXT, false, true ), + new ColumnContext( DocumentType.DOCUMENT_DATA, null, PolyType.TEXT, false, true ) ), 1 ); scannable.getCatalog().addPhysical( allocation, doc ); return List.of( doc ); @@ -250,7 +250,7 @@ static void dropCollectionSubstitute( Scannable scannable, Context context, Allo void renameLogicalColumn( long id, String newColumnName ); - record ColumnContext( String name, Integer precision, PolyType type, boolean nullable ) { + record ColumnContext( String name, Integer precision, PolyType type, boolean nullable, boolean elementsNullable ) { } diff --git a/core/src/main/java/org/polypheny/db/adapter/annotations/AdapterSettingsPreset.java b/core/src/main/java/org/polypheny/db/adapter/annotations/AdapterSettingsPreset.java new file mode 100644 index 0000000000..35a17a7d34 --- /dev/null +++ b/core/src/main/java/org/polypheny/db/adapter/annotations/AdapterSettingsPreset.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.polypheny.db.adapter.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Repeatable; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import org.polypheny.db.adapter.DeployMode; +import org.polypheny.db.adapter.annotations.AdapterSettingsPreset.List; + +/** + * Declares a named deployment preset for an adapter: a deploy mode together with + * predefined values for some of its settings. Presets are offered in the UI as + * one-click deploy options, all settings not covered by the preset keep their + * default values. + */ +@Inherited +@Repeatable(List.class) +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface AdapterSettingsPreset { + + String name(); + + String description() default ""; + + DeployMode mode(); + + Setting[] settings(); + + @Inherited + @Target(ElementType.TYPE) + @Retention(RetentionPolicy.RUNTIME) + @interface List { + + AdapterSettingsPreset[] value(); + + } + + @Target({}) + @Retention(RetentionPolicy.RUNTIME) + @interface Setting { + + String name(); + + String value(); + + } + +} diff --git a/core/src/main/java/org/polypheny/db/adapter/java/AdapterTemplate.java b/core/src/main/java/org/polypheny/db/adapter/java/AdapterTemplate.java index 10b4668c95..a399456225 100644 --- a/core/src/main/java/org/polypheny/db/adapter/java/AdapterTemplate.java +++ b/core/src/main/java/org/polypheny/db/adapter/java/AdapterTemplate.java @@ -31,6 +31,7 @@ import org.polypheny.db.adapter.DeployMode; import org.polypheny.db.adapter.DeployMode.DeploySetting; import org.polypheny.db.adapter.annotations.AdapterProperties; +import org.polypheny.db.adapter.annotations.AdapterSettingsPreset; import org.polypheny.db.catalog.entity.LogicalAdapter.AdapterType; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; import org.polypheny.db.docker.DockerManager; @@ -44,17 +45,19 @@ public class AdapterTemplate { DeployFn deployer; public List settings; public List modes; + public List presets; public long id; public String description; - public AdapterTemplate( long id, Class clazz, String adapterName, List settings, List modes, String description, DeployFn deployer ) { + public AdapterTemplate( long id, Class clazz, String adapterName, List settings, List modes, List presets, String description, DeployFn deployer ) { this.id = id; this.adapterName = adapterName; this.description = description; this.clazz = clazz; this.settings = settings; this.modes = modes; + this.presets = presets; this.adapterType = getAdapterType( clazz ); this.deployer = deployer; } @@ -85,6 +88,30 @@ public static List getAllSettings( Class getAllPresets( Class> clazz, List settings, List modes ) { + List presets = Arrays.asList( clazz.getAnnotationsByType( AdapterSettingsPreset.class ) ); + for ( AdapterSettingsPreset preset : presets ) { + if ( !modes.contains( preset.mode() ) ) { + throw new GenericRuntimeException( "Preset '%s' of adapter %s uses deploy mode %s, which the adapter does not support.", preset.name(), clazz.getSimpleName(), preset.mode() ); + } + for ( AdapterSettingsPreset.Setting entry : preset.settings() ) { + AbstractAdapterSetting setting = settings.stream() + .filter( s -> s.name.equals( entry.name() ) ) + .findFirst() + .orElseThrow( () -> new GenericRuntimeException( "Preset '%s' of adapter %s references the unknown setting '%s'.", preset.name(), clazz.getSimpleName(), entry.name() ) ); + if ( setting instanceof AbstractAdapterSettingList list && !list.options.contains( entry.value() ) ) { + throw new GenericRuntimeException( "Preset '%s' of adapter %s uses the value '%s' for setting '%s', which is not one of its options.", preset.name(), clazz.getSimpleName(), entry.value(), entry.name() ); + } + } + } + return presets; + } + + public Map getDefaultSettings() { Map map = new HashMap<>(); for ( AbstractAdapterSetting s : settings ) { diff --git a/core/src/main/java/org/polypheny/db/algebra/constant/FunctionCategory.java b/core/src/main/java/org/polypheny/db/algebra/constant/FunctionCategory.java index e1988b17db..af4865360c 100644 --- a/core/src/main/java/org/polypheny/db/algebra/constant/FunctionCategory.java +++ b/core/src/main/java/org/polypheny/db/algebra/constant/FunctionCategory.java @@ -45,6 +45,12 @@ public enum FunctionCategory { USER_DEFINED_TABLE_SPECIFIC_FUNCTION( "TABLE_UDF_SPECIFIC", "User-defined table function with SPECIFIC name", USER_DEFINED, TABLE_FUNCTION, SPECIFIC ), MATCH_RECOGNIZE( "MATCH_RECOGNIZE", "MATCH_RECOGNIZE function", TABLE_FUNCTION ), DISTANCE( "DISTANCE", "distance function", DISTANCE_FUNCTION ), + L1_DISTANCE( "L1_DISTANCE", "l1 distance function", DISTANCE_FUNCTION ), + L2_DISTANCE( "L2_DISTANCE", "l2 distance function", DISTANCE_FUNCTION ), + COS_DISTANCE( "COS_DISTANCE", "cosine distance function", DISTANCE_FUNCTION ), + HAMMING_DISTANCE( "HAMMING_DISTANCE", "hamming distance function", DISTANCE_FUNCTION ), + JACCARD_DISTANCE( "JACCARD_DISTANCE", "jaccard distance function", DISTANCE_FUNCTION ), + INNER_PRODUCT_DISTANCE( "INNER_PRODUCT_DISTANCE", "inner product distance function", DISTANCE_FUNCTION ), MULTIMEDIA( "MULTIMEDIA", "Multimedia function", MULTIMEDIA_FUNCTION ), GEOMETRY( "GEOMETRY", "Geo function", GEO_FUNCTION ); diff --git a/core/src/main/java/org/polypheny/db/algebra/constant/Kind.java b/core/src/main/java/org/polypheny/db/algebra/constant/Kind.java index 9b7d6403c7..067ae832b5 100644 --- a/core/src/main/java/org/polypheny/db/algebra/constant/Kind.java +++ b/core/src/main/java/org/polypheny/db/algebra/constant/Kind.java @@ -117,6 +117,18 @@ public enum Kind { */ DISTANCE, + L1_DISTANCE, + + L2_DISTANCE, + + COS_DISTANCE, + + HAMMING_DISTANCE, + + JACCARD_DISTANCE, + + INNER_PRODUCT_DISTANCE, + /** * GEO functions. */ diff --git a/core/src/main/java/org/polypheny/db/algebra/enumerable/RexImpTable.java b/core/src/main/java/org/polypheny/db/algebra/enumerable/RexImpTable.java index 1c40720801..6d210ef7d8 100644 --- a/core/src/main/java/org/polypheny/db/algebra/enumerable/RexImpTable.java +++ b/core/src/main/java/org/polypheny/db/algebra/enumerable/RexImpTable.java @@ -225,6 +225,12 @@ public Expression implement( RexToLixTranslator translator, RexCall call, List PolyDouble.of( Math.PI ).asExpression() ); diff --git a/core/src/main/java/org/polypheny/db/algebra/operators/OperatorName.java b/core/src/main/java/org/polypheny/db/algebra/operators/OperatorName.java index 188325ad64..e39b0ce7b1 100644 --- a/core/src/main/java/org/polypheny/db/algebra/operators/OperatorName.java +++ b/core/src/main/java/org/polypheny/db/algebra/operators/OperatorName.java @@ -1577,6 +1577,40 @@ public enum OperatorName { */ ST_GEOMETRYN( Function.class ), + //------------------------------------------------------------ + // VECTOR DISTANCE FUNCTIONS + //------------------------------------------------------------ + + L2_DISTANCE( Function.class ), + + L1_DISTANCE( Function.class ), + + COS_DISTANCE( Function.class ), + + HAMMING_DISTANCE( Function.class ), + + JACCARD_DISTANCE( Function.class ), + + INNER_PRODUCT_DISTANCE( Function.class ), + + // Cypher specific operator + VECTOR_DISTANCE( Function.class), + + + // PostgreSQL pgvector Operators + + PGVECTOR_L2( BinaryOperator.class ), + + PGVECTOR_L1( BinaryOperator.class ), + + PGVECTOR_COS( BinaryOperator.class ), + + PGVECTOR_HAMMING( BinaryOperator.class ), + + PGVECTOR_JACCARD( BinaryOperator.class ), + + PGVECTOR_INNER_PRODUCT( BinaryOperator.class ), + //------------------------------------------------------------- // SET OPERATORS //------------------------------------------------------------- diff --git a/core/src/main/java/org/polypheny/db/algebra/type/AlgDataTypeFactory.java b/core/src/main/java/org/polypheny/db/algebra/type/AlgDataTypeFactory.java index 9fbaaede59..718843fbbd 100644 --- a/core/src/main/java/org/polypheny/db/algebra/type/AlgDataTypeFactory.java +++ b/core/src/main/java/org/polypheny/db/algebra/type/AlgDataTypeFactory.java @@ -135,6 +135,16 @@ public interface AlgDataTypeFactory { */ AlgDataType createArrayType( AlgDataType elementType, long maxCardinality, long dimension ); + /** + * Creates a vector type. A vector is a fixed-length, single-dimension array + of numeric values (i.e. dim=1, card=k). + * + * @param elementType the element type (typically {@code FLOAT/REAL} or {@code BOOLEAN (BIT)}) + * @param dimension the fixed number of elements in the vector, corresponds to cardinality of an array. + * @return canonical vector type descriptor + */ + AlgDataType createVectorType( AlgDataType elementType, long dimension ); + /** * Creates a map type. Maps are unordered collections of key/value pairs. * diff --git a/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalRelationalCatalog.java b/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalRelationalCatalog.java index b1350707df..b688c76ba8 100644 --- a/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalRelationalCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/catalogs/LogicalRelationalCatalog.java @@ -121,7 +121,7 @@ public interface LogicalRelationalCatalog extends LogicalCatalog { * @param collation The collation of the field (if applicable, else null) * @return The id of the inserted column */ - LogicalColumn addColumn( String name, long tableId, int position, PolyType type, PolyType collectionsType, Integer length, Integer scale, Integer dimension, Integer cardinality, boolean nullable, Collation collation ); + LogicalColumn addColumn( String name, long tableId, int position, PolyType type, PolyType collectionsType, Integer length, Integer scale, Integer dimension, Integer cardinality, boolean nullable, boolean elementsNullable, Collation collation ); /** @@ -146,7 +146,7 @@ public interface LogicalRelationalCatalog extends LogicalCatalog { * @param columnId The id of the column * @param type The new type of the column */ - void setColumnType( long columnId, PolyType type, PolyType collectionsType, Integer length, Integer precision, Integer dimension, Integer cardinality ); + void setColumnType( long columnId, PolyType type, PolyType collectionsType, Integer length, Integer precision, Integer dimension, Integer cardinality, Boolean elementsNullable ); /** * Change nullability of the column (weather the column allows null values). @@ -299,7 +299,7 @@ public interface LogicalRelationalCatalog extends LogicalCatalog { * @param indexName The name of the index * @return The id of the created index */ - LogicalIndex addIndex( long tableId, List columnIds, boolean unique, String method, String methodDisplayName, long adapterId, IndexType type, String indexName ); + LogicalIndex addIndex( long tableId, List columnIds, boolean unique, String method, String methodDisplayName, long adapterId, IndexType type, String indexName, Map options ); /** * Set physical index name. diff --git a/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalColumn.java b/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalColumn.java index 18b75cc056..e228102c04 100644 --- a/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalColumn.java +++ b/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalColumn.java @@ -91,6 +91,10 @@ public class LogicalColumn implements PolyObject, Comparable { @JsonProperty public boolean nullable; + @Serialize + @JsonProperty + public boolean elementsNullable; + @Serialize @JsonProperty public @SerializeNullable Collation collation; @@ -116,6 +120,7 @@ public LogicalColumn( @Deserialize("dimension") final Integer dimension, @Deserialize("cardinality") final Integer cardinality, @Deserialize("nullable") final boolean nullable, + @Deserialize( "elementsNullable" ) final boolean elementsNullable, @Deserialize("collation") final Collation collation, @Deserialize("defaultValue") final LogicalDefaultValue defaultValue ) { this.id = id; @@ -130,17 +135,18 @@ public LogicalColumn( this.dimension = dimension; this.cardinality = cardinality; this.nullable = nullable; + this.elementsNullable = elementsNullable; this.collation = collation; this.defaultValue = defaultValue; } public AlgDataType getAlgDataType( final AlgDataTypeFactory typeFactory ) { - return getAlgDataType( typeFactory, this.length, this.scale, this.type, collectionsType, cardinality, dimension, nullable ); + return getAlgDataType( typeFactory, this.length, this.scale, this.type, collectionsType, cardinality, dimension, nullable, elementsNullable ); } - public static AlgDataType getAlgDataType( AlgDataTypeFactory typeFactory, Integer length, Integer scale, PolyType type, PolyType collectionsType, Integer cardinality, Integer dimension, boolean nullable ) { + public static AlgDataType getAlgDataType( AlgDataTypeFactory typeFactory, Integer length, Integer scale, PolyType type, PolyType collectionsType, Integer cardinality, Integer dimension, boolean nullable, boolean elementsNullable ) { AlgDataType elementType; if ( length != null && scale != null && type.allowsPrecScale( true, true ) ) { elementType = typeFactory.createPolyType( type, length, scale ); @@ -150,9 +156,17 @@ public static AlgDataType getAlgDataType( AlgDataTypeFactory typeFactory, Intege assert type.allowsNoPrecNoScale(); elementType = typeFactory.createPolyType( type ); } + elementType = typeFactory.createTypeWithNullability( elementType, elementsNullable ); if ( collectionsType == PolyType.ARRAY ) { - elementType = typeFactory.createArrayType( elementType, cardinality != null ? cardinality : -1, dimension != null ? dimension : -1 ); + if ( !elementsNullable && (elementType.getPolyType() == PolyType.FLOAT + || elementType.getPolyType() == PolyType.REAL + || elementType.getPolyType() == PolyType.BOOLEAN) + && dimension == 1 && cardinality != null && cardinality > 0 ) { + elementType = typeFactory.createVectorType( elementType, cardinality ); + } else { + elementType = typeFactory.createArrayType( elementType, cardinality != null ? cardinality : -1, dimension != null ? dimension : -1 ); + } } else if ( collectionsType == PolyType.MAP ) { elementType = typeFactory.createMapType( typeFactory.createPolyType( PolyType.ANY ), elementType ); } diff --git a/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalIndex.java b/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalIndex.java index e7fb949705..bf741ebf89 100644 --- a/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalIndex.java +++ b/core/src/main/java/org/polypheny/db/catalog/entity/logical/LogicalIndex.java @@ -23,6 +23,8 @@ import io.activej.serializer.annotations.SerializeNullable; import java.io.Serial; import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; import lombok.NonNull; import lombok.Value; import lombok.experimental.SuperBuilder; @@ -67,6 +69,10 @@ public class LogicalIndex implements Serializable { @Serialize @JsonProperty public long keyId; + @Serialize + @JsonProperty + @SerializeNullable + public Map options; public LogicalIndex( @@ -79,7 +85,8 @@ public LogicalIndex( @Deserialize("location") final Long location, @Deserialize("keyId") final long keyId, @Deserialize("key") final LogicalKey key, - @Deserialize("physicalName") final String physicalName ) { + @Deserialize("physicalName") final String physicalName, + @Deserialize("options") final Map options ) { this.id = id; this.name = name; this.unique = unique; @@ -90,6 +97,7 @@ public LogicalIndex( this.keyId = keyId; this.key = key; this.physicalName = physicalName; + this.options = options == null ? new HashMap<>() : options; } } diff --git a/core/src/main/java/org/polypheny/db/catalog/entity/physical/PhysicalColumn.java b/core/src/main/java/org/polypheny/db/catalog/entity/physical/PhysicalColumn.java index f7c3a36204..2feb98770e 100644 --- a/core/src/main/java/org/polypheny/db/catalog/entity/physical/PhysicalColumn.java +++ b/core/src/main/java/org/polypheny/db/catalog/entity/physical/PhysicalColumn.java @@ -71,6 +71,9 @@ public class PhysicalColumn extends PhysicalField { @Serialize public boolean nullable; + @Serialize + public boolean elementsNullable; + @Serialize @Nullable @SerializeNullable @@ -97,6 +100,7 @@ public PhysicalColumn( @Deserialize("dimension") final @Nullable Integer dimension, @Deserialize("cardinality") final @Nullable Integer cardinality, @Deserialize("nullable") final boolean nullable, + @Deserialize( "elementsNullable" ) final boolean elementsNullable, @Deserialize("collation") final @Nullable Collation collation, @Deserialize("defaultValue") @Nullable LogicalDefaultValue defaultValue ) { super( id, name, logicalName, allocId, logicalEntityId, adapterId, DataModel.RELATIONAL, true ); @@ -108,6 +112,7 @@ public PhysicalColumn( this.dimension = dimension; this.cardinality = cardinality; this.nullable = nullable; + this.elementsNullable = elementsNullable; this.collation = collation; this.defaultValue = defaultValue; } @@ -135,13 +140,14 @@ public PhysicalColumn( column.dimension, column.cardinality, column.nullable, + column.elementsNullable, column.collation, column.defaultValue ); } public AlgDataType getAlgDataType( final AlgDataTypeFactory typeFactory ) { - return LogicalColumn.getAlgDataType( typeFactory, this.length, this.scale, this.type, collectionsType, cardinality, dimension, nullable ); + return LogicalColumn.getAlgDataType( typeFactory, this.length, this.scale, this.type, collectionsType, cardinality, dimension, nullable, elementsNullable ); } } diff --git a/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java b/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java index 8640125ee8..e00286e858 100644 --- a/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java +++ b/core/src/main/java/org/polypheny/db/catalog/impl/logical/RelationalCatalog.java @@ -228,7 +228,7 @@ public void setPrimaryKey( long tableId, @Nullable Long keyId ) { @Override - public LogicalIndex addIndex( long tableId, List columnIds, boolean unique, String method, String methodDisplayName, long adapterId, IndexType type, String indexName ) { + public LogicalIndex addIndex( long tableId, List columnIds, boolean unique, String method, String methodDisplayName, long adapterId, IndexType type, String indexName, Map options ) { long keyId = getOrAddKey( tableId, columnIds, EnforcementTime.ON_QUERY ); if ( unique ) { // TODO: Check if the current values are unique @@ -244,7 +244,8 @@ public LogicalIndex addIndex( long tableId, List columnIds, boolean unique adapterId, keyId, Objects.requireNonNull( keys.get( keyId ) ), - null ); + null, + options ); synchronized ( this ) { indexes.put( id, index ); } @@ -301,9 +302,9 @@ public void deleteKey( long id ) { @Override - public LogicalColumn addColumn( String name, long tableId, int position, PolyType type, PolyType collectionsType, Integer length, Integer scale, Integer dimension, Integer cardinality, boolean nullable, Collation collation ) { + public LogicalColumn addColumn( String name, long tableId, int position, PolyType type, PolyType collectionsType, Integer length, Integer scale, Integer dimension, Integer cardinality, boolean nullable, boolean elementsNullable, Collation collation ) { long id = idBuilder.getNewFieldId(); - LogicalColumn column = new LogicalColumn( id, name, tableId, logicalNamespace.id, position, type, collectionsType, length, scale, dimension, cardinality, nullable, collation, null ); + LogicalColumn column = new LogicalColumn( id, name, tableId, logicalNamespace.id, position, type, collectionsType, length, scale, dimension, cardinality, nullable, elementsNullable, collation, null ); columns.put( id, column ); change( CatalogEvent.LOGICAL_REL_FIELD_CREATED, null, id ); return column; @@ -325,12 +326,12 @@ public void setColumnPosition( long columnId, int position ) { @Override - public void setColumnType( long columnId, PolyType type, PolyType collectionsType, Integer length, Integer scale, Integer dimension, Integer cardinality ) { + public void setColumnType( long columnId, PolyType type, PolyType collectionsType, Integer length, Integer scale, Integer dimension, Integer cardinality, Boolean elementsNullable ) { if ( scale != null && scale > length ) { throw new RuntimeException( "Invalid scale! Scale can not be larger than length." ); } - columns.put( columnId, columns.get( columnId ).toBuilder().type( type ).collectionsType( collectionsType ).length( length ).scale( scale ).dimension( dimension ).cardinality( cardinality ).build() ); + columns.put( columnId, columns.get( columnId ).toBuilder().type( type ).collectionsType( collectionsType ).length( length ).scale( scale ).dimension( dimension ).cardinality( cardinality ).elementsNullable( elementsNullable ).build() ); change( CatalogEvent.LOGICAL_REL_FIELD_TYPE_CHANGED, columnId, type ); } diff --git a/core/src/main/java/org/polypheny/db/catalog/logistic/IndexCategory.java b/core/src/main/java/org/polypheny/db/catalog/logistic/IndexCategory.java new file mode 100644 index 0000000000..5910336fa9 --- /dev/null +++ b/core/src/main/java/org/polypheny/db/catalog/logistic/IndexCategory.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.catalog.logistic; + +public enum IndexCategory { + REGULAR, + VECTOR, + SPATIAL, +} diff --git a/core/src/main/java/org/polypheny/db/ddl/DdlManager.java b/core/src/main/java/org/polypheny/db/ddl/DdlManager.java index a3f3adef0b..00f90904e8 100644 --- a/core/src/main/java/org/polypheny/db/ddl/DdlManager.java +++ b/core/src/main/java/org/polypheny/db/ddl/DdlManager.java @@ -195,7 +195,7 @@ public static DdlManager getInstance() { * @param location instance of the data store on which to create the index; if null, default strategy is being used * @param statement the initial query statement */ - public abstract void createIndex( LogicalTable table, String indexMethodName, List columnNames, String indexName, boolean isUnique, DataStore location, Statement statement ) throws TransactionException; + public abstract void createIndex( LogicalTable table, String indexMethodName, List columnNames, String indexName, boolean isUnique, DataStore location, Statement statement, Map options ) throws TransactionException; /** * Adds an index located in Polypheny to a table @@ -618,7 +618,7 @@ public ConstraintInformation( String name, ConstraintType type, List col * decoupled from the used query language */ - public record ColumnTypeInformation( PolyType type, @Nullable PolyType collectionType, Integer precision, Integer scale, Integer dimension, Integer cardinality, Boolean nullable ) { + public record ColumnTypeInformation( PolyType type, @Nullable PolyType collectionType, Integer precision, Integer scale, Integer dimension, Integer cardinality, Boolean nullable, Boolean elementsNullable ) { public ColumnTypeInformation( PolyType type, @@ -627,7 +627,8 @@ public ColumnTypeInformation( Integer scale, Integer dimension, Integer cardinality, - Boolean nullable ) { + Boolean nullable, + Boolean elementsNullable ) { this.type = type; this.collectionType = collectionType == type ? null : collectionType; this.precision = precision == null || precision == -1 ? null : precision; @@ -635,10 +636,12 @@ public ColumnTypeInformation( this.dimension = dimension == null || dimension == -1 ? null : dimension; this.cardinality = cardinality == null || cardinality == -1 ? null : cardinality; this.nullable = nullable; + this.elementsNullable = elementsNullable; } public static ColumnTypeInformation fromDataTypeSpec( DataTypeSpec sqlDataType ) { + Boolean en = sqlDataType.getElementsNullable(); return new ColumnTypeInformation( sqlDataType.getType(), sqlDataType.getCollectionsType(), @@ -646,7 +649,8 @@ public static ColumnTypeInformation fromDataTypeSpec( DataTypeSpec sqlDataType ) sqlDataType.getScale(), sqlDataType.getDimension(), sqlDataType.getCardinality(), - sqlDataType.getNullable() ); + sqlDataType.getNullable(), + en == null || en ); } } diff --git a/core/src/main/java/org/polypheny/db/functions/DistanceFunctions.java b/core/src/main/java/org/polypheny/db/functions/DistanceFunctions.java index 95dde05591..1d702a8210 100644 --- a/core/src/main/java/org/polypheny/db/functions/DistanceFunctions.java +++ b/core/src/main/java/org/polypheny/db/functions/DistanceFunctions.java @@ -18,10 +18,9 @@ import java.util.List; -import java.util.stream.Collectors; +import org.polypheny.db.type.entity.PolyBoolean; import org.polypheny.db.type.entity.category.PolyNumber; import org.polypheny.db.type.entity.numerical.PolyDouble; -import org.polypheny.db.util.Pair; public class DistanceFunctions { @@ -105,11 +104,17 @@ protected static PolyDouble cosineMetric( List value, List value, List target, List weights ) { - List valueWeighted = Pair.zip( value, weights ).stream().map( p -> PolyDouble.of( p.left.doubleValue() * p.right.doubleValue() ) ).collect( Collectors.toList() ); - List targetWeighted = Pair.zip( target, weights ).stream().map( p -> PolyDouble.of( p.left.doubleValue() * p.right.doubleValue() ) ).collect( Collectors.toList() ); - return cosineMetric( valueWeighted, targetWeighted ); + double dot = 0, normV = 0, normT = 0; + for ( int i = 0; i < value.size(); i++ ) { + double v = value.get( i ).doubleValue() * weights.get( i ).doubleValue(); + double t = target.get( i ).doubleValue() * weights.get( i ).doubleValue(); + dot += v * t; + normV += v * v; + normT += t * t; + } + return PolyDouble.of( 1 - dot / (Math.sqrt( normV ) * Math.sqrt( normT )) ); } @@ -122,19 +127,65 @@ private static PolyDouble dot( List a, List b ) { } + protected static PolyDouble hammingMetric( List value, List target ) { + double result = 0; + for ( int i = 0; i < value.size(); i++ ) { + if ( value.get( i ).asBoolean().getValue() != target.get( i ).asBoolean().getValue() ) { + result++; + } + } + return PolyDouble.of( result ); + } + + + protected static PolyDouble jaccardMetric( List value, List target ) { + double intersection = 0; + double union = 0; + for ( int i = 0; i < value.size(); i++ ) { + boolean a = value.get( i ).asBoolean().getValue(); + boolean b = target.get( i ).asBoolean().getValue(); + if ( a && b ) { + intersection++; + } + if ( a || b ) { + union++; + } + } + + if ( union == 0.0 ) { + return PolyDouble.of( 0.0 ); + } + + return PolyDouble.of( 1.0 - (intersection / union) ); + } + + + protected static PolyDouble innerProductMetric( List value, List target ) { + double result = 0; + for ( int i = 0; i < value.size(); i++ ) { + result += value.get( i ).doubleValue() * target.get( i ).doubleValue(); + } + return PolyDouble.of( -result ); + } + + + protected static PolyDouble innerProductMetricWeighted( List value, List target, List weights ) { + double result = 0; + for ( int i = 0; i < value.size(); i++ ) { + result += value.get( i ).doubleValue() * target.get( i ).doubleValue() * weights.get( i ).doubleValue(); + } + return PolyDouble.of( -result ); + } + + private static double norm2( List list ) { return Math.sqrt( list.stream().mapToDouble( a -> Math.pow( a.doubleValue(), 2.0 ) ).sum() ); } protected static void verifyInputs( List a, List b, List w ) { - if ( a.isEmpty() && b.isEmpty() && (w == null || w.isEmpty()) ) { + if ( emptyArgument( a, b, w ) ) return; - } - - if ( (a.size() != b.size()) || (w != null && a.size() != w.size()) ) { - throw new RuntimeException( "Sizes of inputs do not match." ); - } if ( !a.get( 0 ).getClass().isArray() || !b.get( 0 ).getClass().isArray() || (w != null && !w.get( 0 ).getClass().isArray()) ) { if ( !(a.get( 0 ) instanceof PolyNumber) || !(b.get( 0 ) instanceof PolyNumber) || (w != null && !(w.get( 0 ) instanceof PolyNumber)) ) { @@ -145,4 +196,16 @@ protected static void verifyInputs( List a, List b, List w ) { } } + + private static boolean emptyArgument( List a, List b, List w ) { + if ( a.isEmpty() && b.isEmpty() && (w == null || w.isEmpty()) ) { + return true; + } + + if ( (a.size() != b.size()) || (w != null && a.size() != w.size()) ) { + throw new RuntimeException( "Sizes of inputs do not match." ); + } + return false; + } + } diff --git a/core/src/main/java/org/polypheny/db/functions/Functions.java b/core/src/main/java/org/polypheny/db/functions/Functions.java index eb4e57c0dc..89e0bac9c5 100644 --- a/core/src/main/java/org/polypheny/db/functions/Functions.java +++ b/core/src/main/java/org/polypheny/db/functions/Functions.java @@ -169,7 +169,7 @@ private Functions() { } - public static PolyDouble distance( List value, List target, PolyString metric, List weights ) { + private static PolyDouble distance( List value, List target, PolyString metric, List weights ) { DistanceFunctions.verifyInputs( value, target, weights ); return switch ( metric.value ) { case "L2" -> DistanceFunctions.l2MetricWeighted( value, target, weights ); @@ -177,12 +177,13 @@ public static PolyDouble distance( List value, List targ case "L2SQUARED" -> DistanceFunctions.l2SquaredMetricWeighted( value, target, weights ); case "CHISQUARED" -> DistanceFunctions.chiSquaredMetricWeighted( value, target, weights ); case "COSINE" -> DistanceFunctions.cosineMetricWeighted( value, target, weights ); + case "INNER_PRODUCT" -> DistanceFunctions.innerProductMetricWeighted( value, target, weights ); default -> PolyDouble.of( 0.0 ); }; } - public static PolyDouble distance( List value, List target, PolyString metric ) { + private static PolyDouble distance( List value, List target, PolyString metric ) { DistanceFunctions.verifyInputs( value, target, null ); return switch ( metric.value ) { case "L2" -> DistanceFunctions.l2Metric( value, target ); @@ -190,11 +191,80 @@ public static PolyDouble distance( List value, List targ case "L2SQUARED" -> DistanceFunctions.l2SquaredMetric( value, target ); case "CHISQUARED" -> DistanceFunctions.chiSquaredMetric( value, target ); case "COSINE" -> DistanceFunctions.cosineMetric( value, target ); + case "INNER_PRODUCT" -> DistanceFunctions.innerProductMetric( value, target ); default -> PolyDouble.of( 0.0 ); }; } + public static PolyDouble l1Distance( PolyValue value, PolyValue target ) { + return DistanceFunctions.l1Metric( toNumberList( value ), toNumberList( target ) + ); + } + + + public static PolyDouble l2Distance( PolyValue value, PolyValue target ) { + return DistanceFunctions.l2Metric( toNumberList( value ), toNumberList( target ) + ); + } + + + public static PolyDouble cosDistance( PolyValue value, PolyValue target ) { + return DistanceFunctions.cosineMetric( toNumberList( value ), toNumberList( target + ) ); + } + + + public static PolyDouble hammingDistance( PolyValue value, PolyValue target ) { + return DistanceFunctions.hammingMetric( + value.asList().stream().map( e -> (PolyBoolean) e ).toList(), + target.asList().stream().map( e -> (PolyBoolean) e ).toList() ); + } + + + public static PolyDouble jaccardDistance( PolyValue value, PolyValue target ) { + return DistanceFunctions.jaccardMetric( + value.asList().stream().map( e -> (PolyBoolean) e ).toList(), + target.asList().stream().map( e -> (PolyBoolean) e ).toList() ); + } + + + public static PolyDouble innerProductDistance( PolyValue value, PolyValue target ) { + return DistanceFunctions.innerProductMetric( toNumberList( value ), toNumberList( target ) ); + } + + + public static PolyDouble distance( PolyValue value, PolyValue target, PolyValue metric ) { + return distance( toNumberList( value ), toNumberList( target ), metric.asString() ); + } + + + public static PolyDouble distance( PolyValue value, PolyValue target, PolyValue metric, PolyValue weights ) { + return distance( toNumberList( value ), toNumberList( target ), metric.asString(), + toNumberList( weights ) ); + } + + + private static List toNumberList( PolyValue v ) { + if ( v.isList() ) { + return v.asList().value.stream().map( e -> { + if ( e instanceof PolyNumber n ) return n; + if ( e instanceof PolyString s ) return (PolyNumber) PolyDouble.of( Double.parseDouble( s.value ) ); + throw new GenericRuntimeException( "Cannot convert list element " + e + " to number" ); + } ).toList(); + } + + if ( v.isString() ) { + String raw = v.asString().value.trim().replaceAll( "^\\[|\\]$", "" ); + return Arrays.stream( raw.split( "," ) ) + .map( s -> (PolyNumber) PolyDouble.of( Double.parseDouble( s.trim() ) + ) ) + .toList(); + } + throw new GenericRuntimeException( "Cannot convert " + v + " to number list" ); + } + + private static class MetadataModel { String name; diff --git a/core/src/main/java/org/polypheny/db/nodes/DataTypeSpec.java b/core/src/main/java/org/polypheny/db/nodes/DataTypeSpec.java index a47e4b2a33..9973367696 100644 --- a/core/src/main/java/org/polypheny/db/nodes/DataTypeSpec.java +++ b/core/src/main/java/org/polypheny/db/nodes/DataTypeSpec.java @@ -35,6 +35,8 @@ public interface DataTypeSpec extends Visitable { Boolean getNullable(); + Boolean getElementsNullable(); + Identifier getCollectionsTypeName(); PolyType getCollectionsType(); diff --git a/core/src/main/java/org/polypheny/db/prepare/JavaTypeFactoryImpl.java b/core/src/main/java/org/polypheny/db/prepare/JavaTypeFactoryImpl.java index 84db214a54..5a6a875f62 100644 --- a/core/src/main/java/org/polypheny/db/prepare/JavaTypeFactoryImpl.java +++ b/core/src/main/java/org/polypheny/db/prepare/JavaTypeFactoryImpl.java @@ -149,7 +149,11 @@ public AlgDataType createType( Type type ) { } if ( type instanceof Types.ArrayType arrayType ) { final AlgDataType componentRelType = createType( arrayType.getComponentType() ); - return createArrayType( createTypeWithNullability( componentRelType, arrayType.componentIsNullable() ), arrayType.maximumCardinality() ); + if ( arrayType.componentIsNullable() ) { + return createArrayType( createTypeWithNullability( componentRelType, arrayType.componentIsNullable() ), arrayType.maximumCardinality() ); + } else { + return createVectorType( createTypeWithNullability( componentRelType, arrayType.componentIsNullable() ), arrayType.maximumCardinality() ); + } } if ( type instanceof Types.MapType mapType ) { final AlgDataType keyRelType = createType( mapType.getKeyType() ); diff --git a/core/src/main/java/org/polypheny/db/rex/RexBuilder.java b/core/src/main/java/org/polypheny/db/rex/RexBuilder.java index 94ecbf1f97..197b0df128 100644 --- a/core/src/main/java/org/polypheny/db/rex/RexBuilder.java +++ b/core/src/main/java/org/polypheny/db/rex/RexBuilder.java @@ -75,6 +75,7 @@ import org.polypheny.db.type.PolyType; import org.polypheny.db.type.PolyTypeFamily; import org.polypheny.db.type.PolyTypeUtil; +import org.polypheny.db.type.VectorType; import org.polypheny.db.type.entity.PolyBinary; import org.polypheny.db.type.entity.PolyBoolean; import org.polypheny.db.type.entity.PolyInterval; @@ -459,7 +460,7 @@ public RexNode makeCast( AlgDataType type, RexNode exp ) { */ public RexNode makeCast( AlgDataType type, RexNode exp, boolean matchNullability ) { // MV: This might be a bad idea. It would be better to implement cast support for array columns - if ( exp.getType().getPolyType() == PolyType.ARRAY ) { + if ( exp.getType().getPolyType() == PolyType.ARRAY && !(type instanceof VectorType) ) { return exp; } diff --git a/core/src/main/java/org/polypheny/db/type/PolyTypeFactoryImpl.java b/core/src/main/java/org/polypheny/db/type/PolyTypeFactoryImpl.java index a8ab561c8d..44b5cc64cf 100644 --- a/core/src/main/java/org/polypheny/db/type/PolyTypeFactoryImpl.java +++ b/core/src/main/java/org/polypheny/db/type/PolyTypeFactoryImpl.java @@ -43,6 +43,7 @@ import org.polypheny.db.algebra.type.AlgDataTypeField; import org.polypheny.db.algebra.type.AlgDataTypeSystem; import org.polypheny.db.nodes.IntervalQualifier; +import org.polypheny.db.type.VectorType.ElementType; import org.polypheny.db.util.Collation; import org.polypheny.db.util.Util; @@ -130,6 +131,20 @@ public AlgDataType createArrayType( AlgDataType elementType, long maxCardinality } + @Override + public AlgDataType createVectorType( AlgDataType elementType, long dimension ) { + ElementType kind = switch ( elementType.getPolyType() ) { + case FLOAT, REAL -> ElementType.FLOAT; + case DOUBLE -> ElementType.DOUBLE; + case INTEGER -> ElementType.INTEGER; + case BOOLEAN -> ElementType.BIT; + default -> throw new IllegalArgumentException( "Unsupported vector element type: " + elementType.getPolyType() ); + }; + VectorType newType = new VectorType( elementType, false, dimension, kind ); + return canonize( newType ); + } + + @Override public AlgDataType createMapType( AlgDataType keyType, AlgDataType valueType ) { MapPolyType newType = new MapPolyType( keyType, valueType, false ); @@ -222,6 +237,8 @@ public AlgDataType createTypeWithNullability( final AlgDataType type, final bool newType = basicPolyType.createWithNullability( nullable ); } else if ( type instanceof MapPolyType mapPolyType ) { newType = copyMapType( mapPolyType, nullable ); + } else if ( type instanceof VectorType vectorType ) { + newType = copyVectorType( vectorType, nullable ); } else if ( type instanceof ArrayType arrayType ) { newType = copyArrayType( arrayType, nullable ); } else if ( type instanceof MultisetPolyType multisetPolyType ) { @@ -508,6 +525,12 @@ private AlgDataType copyArrayType( ArrayType at, boolean nullable ) { } + private AlgDataType copyVectorType( VectorType vt, boolean nullable ) { + AlgDataType elementType = copyType( vt.getComponentType() ); + return new VectorType( elementType, nullable, vt.getVectorDimension(), vt.getVectorElementType() ); + } + + private AlgDataType copyMapType( MapPolyType mt, boolean nullable ) { AlgDataType keyType = copyType( mt.getKeyType() ); AlgDataType valueType = copyType( mt.getValueType() ); diff --git a/core/src/main/java/org/polypheny/db/type/VectorType.java b/core/src/main/java/org/polypheny/db/type/VectorType.java new file mode 100644 index 0000000000..f4d013e867 --- /dev/null +++ b/core/src/main/java/org/polypheny/db/type/VectorType.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.type; + +import lombok.Getter; +import org.polypheny.db.algebra.type.AlgDataType; + +/** + * Marker for Arrays of the form (dim, card) = (1, n) and entries in {@link ElementType}. + */ +public class VectorType extends ArrayType { + + public enum ElementType { + FLOAT, DOUBLE, INTEGER, BIT + } + + @Getter + private final ElementType vectorElementType; + + public VectorType( AlgDataType elementType, boolean isNullable, long + dimension, ElementType vectorElementType ) { + super( elementType, isNullable, dimension, 1 ); + this.vectorElementType = vectorElementType; + computeDigest(); + } + + public long getVectorDimension() { + return getCardinality(); + } + + + @Override + protected void generateTypeString( StringBuilder sb, boolean withDetail ) { + if ( withDetail ) { + sb.append( getComponentType().getFullTypeString() ); + } else { + sb.append( getComponentType().toString() ); + } + sb.append( " VECTOR" ); + if ( withDetail ) { + sb.append( String.format( "(%d)", getVectorDimension() ) ); + } + } + +} diff --git a/core/src/main/java/org/polypheny/db/type/entity/PolyValue.java b/core/src/main/java/org/polypheny/db/type/entity/PolyValue.java index 8aafe529da..22bbdee2ff 100644 --- a/core/src/main/java/org/polypheny/db/type/entity/PolyValue.java +++ b/core/src/main/java/org/polypheny/db/type/entity/PolyValue.java @@ -224,16 +224,16 @@ public PolyValue( PolyType type ) { public static Function1 getPolyToJava( AlgDataType type, boolean arrayAsList ) { return switch ( type.getPolyType() ) { - case VARCHAR, CHAR, TEXT -> o -> o.asString().value; - case INTEGER, TINYINT, SMALLINT -> o -> o.asNumber().IntValue(); - case FLOAT, REAL -> o -> o.asNumber().FloatValue(); - case DOUBLE -> o -> o.asNumber().DoubleValue(); - case BIGINT -> o -> o.asNumber().LongValue(); - case DECIMAL -> o -> o.asNumber().BigDecimalValue(); - case DATE -> o -> o.asDate().getDaysSinceEpoch(); - case TIME -> o -> o.asTime().getMillisOfDay(); - case TIMESTAMP -> o -> o.asTimestamp().millisSinceEpoch; - case BOOLEAN -> o -> o.asBoolean().value; + case VARCHAR, CHAR, TEXT -> o -> o == null || o.isNull() ? null : o.asString().value; + case INTEGER, TINYINT, SMALLINT -> o -> o == null || o.isNull() ? null : o.asNumber().IntValue(); + case FLOAT, REAL -> o -> o == null || o.isNull() ? null : o.asNumber().FloatValue(); + case DOUBLE -> o -> o == null || o.isNull() ? null : o.asNumber().DoubleValue(); + case BIGINT -> o -> o == null || o.isNull() ? null : o.asNumber().LongValue(); + case DECIMAL -> o -> o == null || o.isNull() ? null : o.asNumber().BigDecimalValue(); + case DATE -> o -> o == null || o.isNull() ? null : o.asDate().getDaysSinceEpoch(); + case TIME -> o -> o == null || o.isNull() ? null : o.asTime().getMillisOfDay(); + case TIMESTAMP -> o -> o == null || o.isNull() ? null : o.asTimestamp().millisSinceEpoch; + case BOOLEAN -> o -> o == null || o.isNull() ? null : o.asBoolean().value; case ARRAY -> { Function1 elTrans = getPolyToJava( getAndDecreaseArrayDimensionIfNecessary( (ArrayType) type ), arrayAsList ); yield o -> o == null || o.isNull() diff --git a/dbms/build.gradle b/dbms/build.gradle index 36eb2da9e1..9ee78a97ee 100644 --- a/dbms/build.gradle +++ b/dbms/build.gradle @@ -64,6 +64,7 @@ dependencies { testImplementation group: "org.hamcrest", name: "hamcrest-core", version: hamcrest_core_version // BSD 3-clause testImplementation group: "com.konghq", name: "unirest-java", version: unirest_version // MIT testImplementation group: "org.mongodb", name: "mongodb-driver-sync", version: mongodb_driver_sync_version // Apache 2.0 + testImplementation group: "org.postgresql", name: "postgresql", version: postgresql_version // BSD 2-clause // GIS testImplementation group: "org.locationtech.jts", name: "jts-core", version: jts_version // Eclipse Public License 2.0 && Eclipse Distribution License 1.0 (BSD-3 Clause) @@ -103,10 +104,10 @@ licensee { allowDependency('com.j256.simplemagic', 'simplemagic', '1.17') { because 'ISC license' } allowDependency('com.adobe.xmp', 'xmpcore', '6.0.6') { because 'BSD 3-Clause' } - allowDependency('org.bouncycastle', 'bcprov-jdk18on', '1.80') { because 'MIT license' } - allowDependency('org.bouncycastle', 'bcpkix-jdk18on', '1.80') { because 'MIT license' } - allowDependency('org.bouncycastle', 'bcutil-jdk18on', '1.80') { because 'MIT license' } - allowDependency('org.bouncycastle', 'bctls-jdk18on', '1.80') { because 'MIT license' } + allowDependency('org.bouncycastle', 'bcprov-jdk18on', bouncycastle_version) { because 'MIT license' } + allowDependency('org.bouncycastle', 'bcpkix-jdk18on', bouncycastle_version) { because 'MIT license' } + allowDependency('org.bouncycastle', 'bcutil-jdk18on', bouncycastle_version) { because 'MIT license' } + allowDependency('org.bouncycastle', 'bctls-jdk18on', bouncycastle_version) { because 'MIT license' } allowDependency('jakarta.xml.bind', 'jakarta.xml.bind-api', '2.3.3') { because 'Eclipse Distribution License 1.0' } allowDependency('org.codehaus.janino', 'janino', '3.0.11') { because 'BSD 3-Clause' } allowDependency('org.codehaus.janino', 'commons-compiler', '3.0.11') { because 'BSD 3-Clause' } diff --git a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java index 545315f7b2..e954dc9f07 100644 --- a/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java +++ b/dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java @@ -53,6 +53,7 @@ import org.polypheny.db.algebra.logical.relational.LogicalRelScan; import org.polypheny.db.algebra.logical.relational.LogicalRelViewScan; import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeFactory; import org.polypheny.db.algebra.type.AlgDataTypeField; import org.polypheny.db.algebra.type.DocumentType; import org.polypheny.db.catalog.Catalog; @@ -89,6 +90,7 @@ import org.polypheny.db.catalog.logistic.DataPlacementRole; import org.polypheny.db.catalog.logistic.EntityType; import org.polypheny.db.catalog.logistic.ForeignKeyOption; +import org.polypheny.db.catalog.logistic.IndexCategory; import org.polypheny.db.catalog.logistic.IndexType; import org.polypheny.db.catalog.logistic.NameGenerator; import org.polypheny.db.catalog.logistic.PartitionType; @@ -115,6 +117,7 @@ import org.polypheny.db.transaction.TransactionException; import org.polypheny.db.type.ArrayType; import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; import org.polypheny.db.type.entity.PolyValue; import org.polypheny.db.util.Pair; import org.polypheny.db.view.MaterializedViewManager; @@ -291,6 +294,7 @@ private void createRelationalSource( Transaction transaction, DataSource adap exportedColumn.dimension(), exportedColumn.cardinality(), exportedColumn.nullable(), + exportedColumn.elementsNullable(), Collation.getDefaultCollation() ); AllocationColumn allocationColumn = catalog.getAllocRel( namespace ).addColumn( @@ -457,6 +461,7 @@ public void addColumnToSourceTable( LogicalTable table, String columnPhysicalNam exportedColumn.dimension(), exportedColumn.cardinality(), exportedColumn.nullable(), + exportedColumn.elementsNullable(), Collation.getDefaultCollation() ); @@ -533,6 +538,7 @@ public void createColumn( String columnName, LogicalTable table, String beforeCo type.dimension(), type.cardinality(), nullable, + type.elementsNullable(), Collation.getDefaultCollation() ); @@ -595,10 +601,64 @@ public void createForeignKey( LogicalTable table, LogicalTable refTable, List columnNames, String indexName, boolean isUnique, DataStore location, Statement statement ) throws TransactionException { + public void createIndex( LogicalTable table, String indexMethodName, List columnNames, String indexName, boolean isUnique, DataStore location, Statement statement, Map options ) throws TransactionException { List columnIds = new ArrayList<>(); + boolean hasVectorColumn = columnNames.stream() + .map( name -> catalog.getSnapshot().rel().getColumn( table.id, name ).orElseThrow() ) + .anyMatch( col -> col.getAlgDataType( AlgDataTypeFactory.DEFAULT ) instanceof VectorType ); + + IndexCategory requestedCategory = IndexCategory.REGULAR; + if ( indexMethodName != null ) { + IndexMethodModel aim = IndexManager.getAvailableIndexMethods().stream() + .filter( m -> m.name().equals( indexMethodName ) ) + .findFirst() + .orElse( null ); + if ( aim == null && location != null ) { + aim = location.getAvailableIndexMethods().stream() + .filter( m -> m.name().equals( indexMethodName ) ) + .findFirst() + .orElse( null ); + } + if ( aim != null ) { + requestedCategory = aim.category(); + } else if ( hasVectorColumn ) { + requestedCategory = IndexCategory.VECTOR; + } + } + if ( requestedCategory == IndexCategory.VECTOR && location == null ) { + throw new GenericRuntimeException( "Vector indexes must be placed on a specific store. Use ON STORE ." ); + } + for ( String columnName : columnNames ) { - LogicalColumn logicalColumn = catalog.getSnapshot().rel().getColumn( table.id, columnName ).orElseThrow(); + LogicalColumn logicalColumn = catalog.getSnapshot().rel().getColumn( table.id, columnName + ).orElseThrow(); + AlgDataType colType = logicalColumn.getAlgDataType( AlgDataTypeFactory.DEFAULT ); + boolean isVectorColumn = colType instanceof VectorType; + if ( requestedCategory == IndexCategory.VECTOR && !isVectorColumn ) { + throw new GenericRuntimeException( "Index method '%s' can only be created on vector columns.", indexMethodName ); + } + if ( requestedCategory != IndexCategory.VECTOR && isVectorColumn ) { + throw new GenericRuntimeException( "Standard index methods cannot be created on vector columns. Please use 'hnsw' or 'ivfflat'." ); + } + if ( requestedCategory == IndexCategory.VECTOR && isVectorColumn && options != null ) { + String metric = options.get( "metric" ); + if ( metric != null ) { + VectorType.ElementType elemType = ((VectorType) colType).getVectorElementType(); + boolean isBitMetric = metric.equalsIgnoreCase( "HAMMING" ) || + metric.equalsIgnoreCase( "JACCARD" ); + boolean isBitVector = elemType == VectorType.ElementType.BIT; + if ( isBitMetric && !isBitVector ) { + throw new GenericRuntimeException( + "Metric '%s' is only valid for BIT vector columns, but column '%s' has element type %s.", + metric, columnName, elemType ); + } + if ( !isBitMetric && isBitVector ) { + throw new GenericRuntimeException( + "Metric '%s' is not valid for BIT vector columns. Use HAMMING or JACCARD.", + metric, columnName ); + } + } + } columnIds.add( logicalColumn.id ); } @@ -639,7 +699,7 @@ public void createIndex( LogicalTable table, String indexMethodName, List location, Statement statement, List columnIds, IndexType type ) { + private void addDataStoreIndex( LogicalTable table, String indexMethodName, String indexName, boolean isUnique, @NotNull DataStore location, Statement statement, List columnIds, IndexType type, Map options ) { List partitions = catalog.getSnapshot().alloc().getPartitionsFromLogical( table.id ); if ( partitions.size() != 1 ) { @@ -717,7 +777,8 @@ private void addDataStoreIndex( LogicalTable table, String indexMethodName, Stri methodDisplayName, location.getAdapterId(), type, - indexName ); + indexName, + options ); String physicalName = location.addIndex( statement.getPrepareContext(), @@ -775,7 +836,8 @@ public void createPolyphenyIndex( LogicalTable table, String indexMethodName, Li methodDisplayName, -1, type, - indexName ); + indexName, + null ); IndexManager.getInstance().addIndex( index, statement ); } @@ -1179,7 +1241,8 @@ public void setColumnType( LogicalTable table, String columnName, ColumnTypeInfo type.precision(), type.scale(), type.dimension(), - type.cardinality() ); + type.cardinality(), + type.elementsNullable() ); catalog.updateSnapshot(); for ( AllocationColumn allocationColumn : catalog.getSnapshot().alloc().getColumnFromLogical( logicalColumn.id ).orElseThrow() ) { statement.getTransaction().attachCommitAction( () -> { @@ -1754,6 +1817,7 @@ public void createView( String viewName, long namespaceId, AlgNode algNode, AlgC column.typeInformation().dimension(), column.typeInformation().cardinality(), column.typeInformation().nullable(), + column.typeInformation().elementsNullable(), column.collation() ); } @@ -2044,7 +2108,9 @@ private List getViewColumnInformation( List projectedC type.getScale(), alg.getType().getPolyType() == PolyType.ARRAY ? (int) ((ArrayType) alg.getType()).getDimension() : -1, alg.getType().getPolyType() == PolyType.ARRAY ? (int) ((ArrayType) alg.getType()).getCardinality() : -1, - alg.getType().isNullable() ), + alg.getType().isNullable(), + alg.getType().getComponentType() == null + || alg.getType().getComponentType().isNullable() ), Collation.getDefaultCollation(), null, position ) ); @@ -2063,7 +2129,8 @@ private List getViewColumnInformation( List projectedC -1, -1, -1, - false ), + false, + true ), Collation.getDefaultCollation(), null, position ) ); @@ -2486,7 +2553,8 @@ public void createTablePartition( PartitionInformation partitionInfo, ListReuses the existing PostgreSQL Store container, + * prepopulates it with {@code boolean[]} and {@code bit(n)} columns, attaches it + * as a Polypheny source, then asserts that the catalog reflects the correct internal types: + *
    + *
  • {@code boolean[]} -> plain {@code ARRAY}, cardinality {@code null} -> not VectorType
    • + *
  • {@code bit(5)} -> {@code ARRAY} with cardinality 5 -> is VectorType<BIT>(5)
    • + *
+ */ +@SuppressWarnings({ "SqlDialectInspection", "SqlNoDataSourceInspection" }) +@Slf4j +@Tag("adapter") +@EnabledIfSystemProperty(named = "store.default", matches = "postgresql") +public class PostgresqlSourceDiscoveryTest { + + private static final String SOURCE_ADAPTER = "pg_discovery_source"; + private static final String TABLE_NAME = "public.discovery_test"; + private static final String RAW_TABLE_NAME = "discovery_test"; + + private static boolean setupSucceeded = false; + + + @BeforeAll + static void start() throws SQLException { + //noinspection ResultOfMethodCallIgnored + TestHelper.getInstance(); + + Optional maybeStore = Catalog.getInstance().getSnapshot().getAdapters().stream() + .filter( ad -> ad.type == AdapterType.STORE && ad.adapterName.equalsIgnoreCase( "PostgreSQL" ) ) + .findFirst(); + + assertTrue( maybeStore.isPresent(), "PostgreSQL Store not found in Catalog." ); + + LogicalAdapter pgStore = maybeStore.get(); + Map settingsMap = pgStore.settings; + String host; + int port; + if ( pgStore.mode == DeployMode.DOCKER ) { + String deploymentId = settingsMap.get( "deploymentId" ); + DockerContainer container = DockerContainer.getContainerByUUID( deploymentId ) + .orElseThrow( () -> new RuntimeException( "Could not find docker container for PostgreSQL store" ) ); + DockerContainer.HostAndPort hp = container.connectToContainer( 5432 ); + host = hp.host(); + port = hp.port(); + } else { + host = settingsMap.get( "host" ); + port = Integer.parseInt( settingsMap.get( "port" ) ); + } + + String database = settingsMap.getOrDefault( "database", "postgres" ); + String username = settingsMap.getOrDefault( "username", "postgres" ); + String password = settingsMap.getOrDefault( "password", "polypheny" ); + + String jdbcUrl = String.format( "jdbc:postgresql://%s:%d/%s", host, port, database ); + try ( Connection conn = DriverManager.getConnection( jdbcUrl, username, password ); + Statement st = conn.createStatement() ) { + st.executeUpdate( "DROP TABLE IF EXISTS " + TABLE_NAME ); + st.executeUpdate( + "CREATE TABLE " + TABLE_NAME + " (" + + " id SERIAL PRIMARY KEY," + + " bool_array BOOLEAN[]," + + " bit_vector BIT(5)" + + ")" ); + } + + + String settings = String.format( + "'{ \"mode\": \"REMOTE\", \"host\": \"%s\", \"port\": \"%d\", \"database\": \"%s\", \"username\": \"%s\", \"password\": \"%s\", \"tables\": \"%s\", \"maxConnections\": \"25\", \"transactionIsolation\": \"SERIALIZABLE\" }'", + host, port, database, username, password, TABLE_NAME ); + try ( JdbcConnection jc = new JdbcConnection( true ); + Statement st = jc.getConnection().createStatement() ) { + st.executeUpdate( "ALTER ADAPTERS ADD \"" + SOURCE_ADAPTER + "\"" + + " USING 'PostgreSQL' AS 'Source' WITH " + settings ); + } + setupSucceeded = true; + } + + + @AfterAll + static void stop() { + if ( setupSucceeded ) { + try ( JdbcConnection jc = new JdbcConnection( true ); + Statement st = jc.getConnection().createStatement() ) { + st.executeUpdate( "ALTER ADAPTERS DROP \"" + SOURCE_ADAPTER + "\"" ); + } catch ( Exception e ) { + log.warn( "Could not drop source adapter during teardown", e ); + } + } + } + + + private List columns() { + LogicalTable table = Catalog.getInstance().getSnapshot().rel() + .getTable( "public", RAW_TABLE_NAME ).orElseThrow(); + return Catalog.getInstance().getSnapshot().rel().getColumns( table.id ); + } + + + @Test + void boolArrayDiscoveredAsPlainArrayNotVector() { + LogicalColumn col = columns().stream() + .filter( c -> c.name.equals( "bool_array" ) ) + .findFirst().orElseThrow(); + + assertEquals( PolyType.BOOLEAN, col.type ); + assertEquals( PolyType.ARRAY, col.collectionsType ); + // createArrayType must NOT promote this to VectorType iff cardinality is null + assertNull( col.cardinality, + "boolean[] must have null cardinality so it is not mistaken for a bitvector" ); + assertFalse( col.getAlgDataType( AlgDataTypeFactory.DEFAULT ) instanceof VectorType, + "boolean[] must not be promoted to VectorType" ); + } + + + @Test + void bitColumnDiscoveredAsVectorType() { + LogicalColumn col = columns().stream() + .filter( c -> c.name.equals( "bit_vector" ) ) + .findFirst().orElseThrow(); + + assertEquals( PolyType.BOOLEAN, col.type ); + assertEquals( PolyType.ARRAY, col.collectionsType ); + assertNotNull( col.cardinality, "bit(5) must have non-null cardinality" ); + assertEquals( 5, col.cardinality ); + assertEquals( 1, col.dimension ); + + assertInstanceOf( VectorType.class, col.getAlgDataType( AlgDataTypeFactory.DEFAULT ), "bit(5) must be promoted to VectorType" ); + VectorType vt = (VectorType) col.getAlgDataType( AlgDataTypeFactory.DEFAULT ); + assertEquals( 5, vt.getVectorDimension() ); + assertEquals( VectorType.ElementType.BIT, vt.getVectorElementType() ); + } + +} diff --git a/dbms/src/test/java/org/polypheny/db/catalog/CatalogTransactionTest.java b/dbms/src/test/java/org/polypheny/db/catalog/CatalogTransactionTest.java index 0fdcbb6446..88f546f060 100644 --- a/dbms/src/test/java/org/polypheny/db/catalog/CatalogTransactionTest.java +++ b/dbms/src/test/java/org/polypheny/db/catalog/CatalogTransactionTest.java @@ -56,11 +56,11 @@ public void simpleRollbackTest() { LogicalTable table = catalog.getLogicalRel( namespaceId ).addTable( "testTable", EntityType.ENTITY, true ); - catalog.getLogicalRel( namespaceId ).addColumn( "testCol1", table.id, 1, PolyType.BIGINT, null, null, null, null, null, false, null ); + catalog.getLogicalRel( namespaceId ).addColumn( "testCol1", table.id, 1, PolyType.BIGINT, null, null, null, null, null, false, true, null ); catalog.commit(); - LogicalColumn id = catalog.getLogicalRel( namespaceId ).addColumn( "testCol4", table.id, 2, PolyType.BIGINT, null, null, null, null, null, true, null ); + LogicalColumn id = catalog.getLogicalRel( namespaceId ).addColumn( "testCol4", table.id, 2, PolyType.BIGINT, null, null, null, null, null, true, true, null ); catalog.rollback(); assert (catalog.getSnapshot().rel().getColumn( id.id ).isEmpty()); @@ -77,13 +77,13 @@ public void rollbackTest() { LogicalTable table = catalog.getLogicalRel( namespaceId ).addTable( "testTable", EntityType.ENTITY, true ); - catalog.getLogicalRel( namespaceId ).addColumn( "testCol1", table.id, 1, PolyType.BIGINT, null, null, null, null, null, false, null ); - catalog.getLogicalRel( namespaceId ).addColumn( "testCol2", table.id, 2, PolyType.VARCHAR, null, 2646, 5, 2, 2, true, Collation.CASE_INSENSITIVE ); + catalog.getLogicalRel( namespaceId ).addColumn( "testCol1", table.id, 1, PolyType.BIGINT, null, null, null, null, null, false, true, null ); + catalog.getLogicalRel( namespaceId ).addColumn( "testCol2", table.id, 2, PolyType.VARCHAR, null, 2646, 5, 2, 2, true, true, Collation.CASE_INSENSITIVE ); // catalog.getLogicalRel( namespaceId ).createColumn( "testCol3", table.id, 3, PolyType.BIGINT, null,null, null, null, null, true, null ); 127 catalog.commit(); - LogicalColumn id = catalog.getLogicalRel( namespaceId ).addColumn( "testCol4", table.id, 3, PolyType.BIGINT, null, null, null, null, null, true, null ); + LogicalColumn id = catalog.getLogicalRel( namespaceId ).addColumn( "testCol4", table.id, 3, PolyType.BIGINT, null, null, null, null, null, true, true, null ); catalog.rollback(); assert (catalog.getSnapshot().rel().getColumn( id.id ).isEmpty()); diff --git a/dbms/src/test/java/org/polypheny/db/constraints/ArrayNotNullConstraintTest.java b/dbms/src/test/java/org/polypheny/db/constraints/ArrayNotNullConstraintTest.java new file mode 100644 index 0000000000..af62ec1c34 --- /dev/null +++ b/dbms/src/test/java/org/polypheny/db/constraints/ArrayNotNullConstraintTest.java @@ -0,0 +1,240 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.constraints; + +import com.google.common.collect.ImmutableList; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.TestHelper.JdbcConnection; +import org.polypheny.jdbc.PrismInterfaceServiceException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; + +/** +* A column declared as {@code REAL NOT NULL ARRAY(1,n)} or {@code BOOLEAN NOT NULL ARRAY(1,n)} +* is internally mapped to a VectorType. Inserting an array that contains a null element +* must be rejected; inserting a null for the column itself (i.e. a null array) is still +* allowed because the column is not declared NOT NULL at the column level. +*/ +@SuppressWarnings({ "SqlDialectInspection", "SqlNoDataSourceInspection" }) +@Slf4j +@Tag("adapter") +public class ArrayNotNullConstraintTest { + + @BeforeAll + public static void start() throws SQLException { + //noinspection ResultOfMethodCallIgnored + TestHelper.getInstance(); + } + + + //--------------------- REAL NOT NULL ARRAY(1,n) --------------------- + @Test + void realVectorInsertWithAllNonNullElementsSucceeds() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "CREATE TABLE vnn_real( id INTEGER NOT NULL, vec REAL NOT NULL ARRAY(1,3), PRIMARY KEY (id) )" ); + try { + statement.executeUpdate( "INSERT INTO vnn_real VALUES (1, ARRAY[1.0, 2.0, 3.0])" ); + connection.commit(); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT vec FROM vnn_real WHERE id = 1" ), + ImmutableList.of( new Object[]{ new Object[]{ 1.0f, 2.0f, 3.0f } } ) ); + } finally { + statement.executeUpdate( "DROP TABLE vnn_real" ); + connection.commit(); + } + } + } + } + + + @Test + void realVectorColumnLevelNullIsAllowed() throws SQLException { + // The column itself has no NOT NULL, so a null array value is permitted. + // Only element-level nulls are forbidden. + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "CREATE TABLE vnn_real_colnull( id INTEGER NOT NULL, vec REAL NOT NULL ARRAY(1,3), PRIMARY KEY (id) )" ); + try { + statement.executeUpdate( "INSERT INTO vnn_real_colnull VALUES (1, NULL)" ); + connection.commit(); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT vec FROM vnn_real_colnull WHERE id = 1" ), + ImmutableList.of( new Object[]{ null } ) + ); + } finally { + statement.executeUpdate( "DROP TABLE vnn_real_colnull" ); + connection.commit(); + } + } + } + } + + + @Test + void realVectorLiteralWithNullElementIsRejected() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "CREATE TABLE vnn_real_rej( id INTEGER NOT NULL, vec REAL NOT NULL ARRAY(1,3), PRIMARY KEY (id) )" ); + try { + Assertions.assertThrows( + PrismInterfaceServiceException.class, + () -> statement.executeUpdate( "INSERT INTO vnn_real_rej VALUES (1, ARRAY[1.0, NULL, 3.0])" ), + "Expected rejection of a null element in a NOT NULL REAL ARRAY" ); + } finally { + statement.executeUpdate( "DROP TABLE vnn_real_rej" ); + connection.commit(); + } + } + } + } + + + @Test + void realVectorPreparedStatementWithNullElementIsRejected() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "CREATE TABLE vnn_real_ps( id INTEGER NOT NULL, vec REAL NOT NULL ARRAY(1,3), PRIMARY KEY (id) )" ); + connection.commit(); + } + try ( PreparedStatement ps = connection.prepareStatement( "INSERT INTO vnn_real_ps VALUES (?, ?)" ) ) { + try { + ps.setInt( 1, 1 ); + ps.setArray( 2, connection.createArrayOf( "REAL", new Float[]{ 1.0f, null, 3.0f } ) ); + Assertions.assertThrows( + PrismInterfaceServiceException.class, + ps::executeUpdate, + "Expected rejection of a null element via PreparedStatement in a NOT NULL REAL ARRAY" + ); + } finally { + try ( Statement drop = connection.createStatement() ) { + drop.executeUpdate( "DROP TABLE vnn_real_ps" ); + connection.commit(); + } + } + } + } + } + + + //--------------------- BOOLEAN NOT NULL ARRAY(1,n) --------------------- + @Test + void booleanVectorInsertWithAllNonNullElementsSucceeds() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( + "CREATE TABLE vnn_bool( id INTEGER NOT NULL, vec BOOLEAN NOT NULL ARRAY(1,3), PRIMARY KEY (id) )" ); + try { + statement.executeUpdate( "INSERT INTO vnn_bool VALUES (1, ARRAY[TRUE, FALSE, TRUE])" ); + connection.commit(); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT vec FROM vnn_bool WHERE id = 1" ), + ImmutableList.of( new Object[]{ new Object[]{ true, false, true } } ) + ); + } finally { + statement.executeUpdate( "DROP TABLE vnn_bool" ); + connection.commit(); + } + } + } + } + + + @Test + void booleanVectorLiteralWithNullElementIsRejected() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( + "CREATE TABLE vnn_bool_rej( id INTEGER NOT NULL, vec BOOLEAN NOT NULL ARRAY(1,3), PRIMARY KEY (id) )" ); + try { + Assertions.assertThrows( + PrismInterfaceServiceException.class, + () -> statement.executeUpdate( "INSERT INTO vnn_bool_rej VALUES (1, ARRAY[TRUE, NULL, FALSE])" ), + "Expected rejection of a null element in a NOT NULL BOOLEAN ARRAY" + ); + } finally { + statement.executeUpdate( "DROP TABLE vnn_bool_rej" ); + connection.commit(); + } + } + } + } + + + @Test + void booleanVectorPreparedStatementWithNullElementIsRejected() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "CREATE TABLE vnn_bool_ps( id INTEGER NOT NULL, vec BOOLEAN NOT NULL ARRAY(1,3), PRIMARY KEY (id) )" ); + connection.commit(); + } + try ( PreparedStatement ps = connection.prepareStatement( "INSERT INTO vnn_bool_ps VALUES (?, ?)" ) ) { + try { + ps.setInt( 1, 1 ); + ps.setArray( 2, connection.createArrayOf( "BOOLEAN", new Boolean[]{ true, null, false } ) ); + Assertions.assertThrows( + PrismInterfaceServiceException.class, + ps::executeUpdate, + "Expected rejection of a null element via PreparedStatement in a NOT NULL BOOLEAN ARRAY" ); + } finally { + try ( Statement drop = connection.createStatement() ) { + drop.executeUpdate( "DROP TABLE vnn_bool_ps" ); + connection.commit(); + } + } + } + } + } + + + @Test + void regularNullableRealArrayAllowsNullElements() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "CREATE TABLE vnn_real_nullable( id INTEGER NOT NULL, vec REAL ARRAY(1,3), PRIMARY KEY (id) )" ); + try { + statement.executeUpdate( "INSERT INTO vnn_real_nullable VALUES (1, ARRAY[1.0, NULL, 3.0])" ); + connection.commit(); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id FROM vnn_real_nullable WHERE id = 1" ), + ImmutableList.of( new Object[]{ 1 } ) + ); + } finally { + statement.executeUpdate( "DROP TABLE vnn_real_nullable" ); + connection.commit(); + } + } + } + } + +} + diff --git a/dbms/src/test/java/org/polypheny/db/cypher/CypherVectorDistanceTest.java b/dbms/src/test/java/org/polypheny/db/cypher/CypherVectorDistanceTest.java new file mode 100644 index 0000000000..455d0d7898 --- /dev/null +++ b/dbms/src/test/java/org/polypheny/db/cypher/CypherVectorDistanceTest.java @@ -0,0 +1,139 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.cypher; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.polypheny.db.TestHelper.CypherConnection; +import org.polypheny.db.cypher.helper.TestLiteral; +import org.polypheny.db.webui.models.results.GraphResult; + + +public class CypherVectorDistanceTest extends CypherTestTemplate { + + private static final String NODE_A = "CREATE (:Item {name: 'a', embedding: [1.0,1.0]})"; + private static final String NODE_B = "CREATE (:Item {name: 'b', embedding: [2.0,2.0]})"; + private static final String NODE_C = "CREATE (:Item {name: 'c', embedding: [0.0,3.0]})"; + + @BeforeEach + public void reset() { + tearDown(); + createGraph(); + } + + + @Test + public void l2DistanceReturnsDouble() { + execute( NODE_A ); + GraphResult res = execute( + "MATCH (n:Item) " + + "RETURN vector_distance(n.embedding, [1.0, 1.0], 'L2') AS dist " + + "LIMIT 1" ); + assert res.getData().length == 1; + } + + + @Test + public void l2DistanceCorrectValues() { + execute( NODE_A ); + execute( NODE_B ); + execute( NODE_C ); + GraphResult res = execute( "MATCH (n:Item) RETURN n.name, vector_distance(n.embedding, [1.0, 1.0], 'L2') AS dist LIMIT 3" ); + assert containsRows( res, true, false, + Row.of( TestLiteral.from( "a" ), TestLiteral.from( "0.0" ) ), + Row.of( TestLiteral.from( "b" ), TestLiteral.from( "1.4142135623730951" ) ), + Row.of( TestLiteral.from( "c" ), TestLiteral.from( "2.23606797749979" ) ) + ); + } + + + @Test + public void l1DistanceCorrectValues() { + execute( NODE_A ); + execute( NODE_B ); + execute( NODE_C ); + GraphResult res = execute( "MATCH (n:Item) RETURN n.name, vector_distance(n.embedding, [1.0, 1.0], 'L1') AS dist LIMIT 3" ); + assert containsRows( res, true, false, + Row.of( TestLiteral.from( "a" ), TestLiteral.from( "0.0" ) ), + Row.of( TestLiteral.from( "b" ), TestLiteral.from( "2.0" ) ), + Row.of( TestLiteral.from( "c" ), TestLiteral.from( "3.0" ) ) + ); + } + + + @Test + public void cosineDistanceCorrectValues() { + execute( NODE_A ); + execute( NODE_B ); + execute( NODE_C ); + GraphResult res = execute( "MATCH (n:Item) WHERE vector_distance(n.embedding, [1.0, 1.0], 'COSINE') < 1e-10 RETURN n.name LIMIT 3" ); + assert res.getData().length == 2; + } + + + @Test + public void l2DistanceAsFilter() { + execute( NODE_A ); + execute( NODE_B ); + execute( NODE_C ); + GraphResult res = execute( "MATCH (n:Item) WHERE vector_distance(n.embedding, [1.0, 1.0], 'L2') < 2.0 RETURN n.name LIMIT 3" ); + assert res.getData().length == 2; + } + + + @Test + public void l2DistanceOrderByLimit() { + execute( NODE_A ); + execute( NODE_B ); + execute( NODE_C ); + GraphResult res = execute( + "MATCH (n:Item) " + + "RETURN n.name, vector_distance(n.embedding, [1.0, 1.0], 'L2') AS dist " + + "ORDER BY dist " + + "LIMIT 2" ); + assert res.getData().length == 2; + assert containsRows( res, true, true, + Row.of( TestLiteral.from( "a" ), TestLiteral.from( "0.0" ) ), + Row.of( TestLiteral.from( "b" ), TestLiteral.from( "1.4142135623730951" ) ) + ); + } + + + @Test + public void unknownMetricThrows() { + execute( NODE_A ); + GraphResult res = CypherConnection.executeGetResponse( + "MATCH (n:Item) " + + "RETURN vector_distance(n.embedding, [1.0, 1.0], 'UNKNOWN') AS dist" ); + assert res.getError() != null; + } + + + @Test + public void l2SquaredMetric() { + execute( NODE_A ); + execute( NODE_B ); + execute( NODE_C ); + GraphResult res = execute( "MATCH (n:Item) RETURN n.name, vector_distance(n.embedding, [1.0, 1.0], 'L2SQUARED') AS dist LIMIT 3" ); + assert containsRows( res, true, false, + Row.of( TestLiteral.from( "a" ), TestLiteral.from( "0.0" ) ), + Row.of( TestLiteral.from( "b" ), TestLiteral.from( "2.0" ) ), + Row.of( TestLiteral.from( "c" ), TestLiteral.from( "5.0" ) ) + ); + } + +} diff --git a/dbms/src/test/java/org/polypheny/db/jdbc/JdbcBooleanArrayTest.java b/dbms/src/test/java/org/polypheny/db/jdbc/JdbcBooleanArrayTest.java new file mode 100644 index 0000000000..9fa3ff1257 --- /dev/null +++ b/dbms/src/test/java/org/polypheny/db/jdbc/JdbcBooleanArrayTest.java @@ -0,0 +1,205 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.jdbc; + +import com.google.common.collect.ImmutableList; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; +import org.polypheny.db.TestHelper; +import org.polypheny.db.TestHelper.JdbcConnection; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; + + +@SuppressWarnings({ "SqlDialectInspection", "SqlNoDataSourceInspection" }) +@Slf4j +@Tag("adapter") +@EnabledIfSystemProperty(named = "store.default", matches = "postgresql|mongoDB|hsqldb|monetdb|neo4j") +public class JdbcBooleanArrayTest { + + @BeforeAll + public static void start() throws SQLException { + //noinspection ResultOfMethodCallIgnored + TestHelper.getInstance(); + addTestData(); + } + + + private static void addTestData() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( + "CREATE TABLE booleanarraytest( id INTEGER NOT NULL, bvec BOOLEAN ARRAY(1,3), PRIMARY KEY (id) )" ); + statement.executeUpdate( "INSERT INTO booleanarraytest VALUES (1, ARRAY[TRUE, FALSE, TRUE])" ); + statement.executeUpdate( "INSERT INTO booleanarraytest VALUES (2, ARRAY[FALSE, FALSE, FALSE])" ); + statement.executeUpdate( "INSERT INTO booleanarraytest VALUES (3, ARRAY[TRUE, TRUE, TRUE])" ); + statement.executeUpdate( "INSERT INTO booleanarraytest VALUES (4, NULL)" ); + connection.commit(); + } + } + } + + + @AfterAll + public static void stop() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( true ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "DROP TABLE booleanarraytest" ); + } + } + } + + + @Test + void nullBooleanArrayIsInsertedAndReadBack() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + TestHelper.checkResultSet( + statement.executeQuery( "SELECT COUNT(id) FROM booleanarraytest WHERE bvec IS NULL" ), + ImmutableList.of( new Object[]{ 1L } ) + ); + } + } + } + + + @Test + void verifyDataIntegrity() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + TestHelper.checkResultSet( + statement.executeQuery( "SELECT bvec FROM booleanarraytest WHERE id = 1" ), + ImmutableList.of( new Object[]{ new Object[]{ true, false, true } } ) + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT bvec FROM booleanarraytest WHERE id = 2" ), + ImmutableList.of( new Object[]{ new Object[]{ false, false, false } } ) + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT bvec FROM booleanarraytest WHERE id = 4" ), + ImmutableList.of( new Object[]{ null } ) + ); + } + } + } + + + @Test + void singleElementBooleanArray() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( + "CREATE TABLE booleanarray1test( id INTEGER NOT NULL, bvec BOOLEAN ARRAY(1,1), PRIMARY KEY (id) )" ); + statement.executeUpdate( "INSERT INTO booleanarray1test VALUES (1, ARRAY[TRUE])" ); + statement.executeUpdate( "INSERT INTO booleanarray1test VALUES (2, ARRAY[FALSE])" ); + connection.commit(); + } + } + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + try { + TestHelper.checkResultSet( + statement.executeQuery( "SELECT bvec FROM booleanarray1test ORDER BY id" ), + ImmutableList.of( + new Object[]{ new Object[]{ true } }, + new Object[]{ new Object[]{ false } } + ) + ); + } finally { + statement.executeUpdate( "DROP TABLE booleanarray1test" ); + } + } + } + } + + + @Test + void insertWithPreparedStatement() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( false ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( java.sql.PreparedStatement ps = connection.prepareStatement( "INSERT INTO booleanarraytest(id, bvec) VALUES (?, ?)" ) ) { + ps.setInt( 1, 10 ); + ps.setArray( 2, connection.createArrayOf( "BOOLEAN", new Boolean[]{ true, true, false } ) ); + ps.executeUpdate(); + connection.commit(); + } + } + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + try { + TestHelper.checkResultSet( + statement.executeQuery( "SELECT bvec FROM booleanarraytest WHERE id = 10" ), + ImmutableList.of( new Object[]{ new Object[]{ true, true, false } } ) + ); + } finally { + statement.executeUpdate( "DELETE FROM booleanarraytest WHERE id = 10" ); + connection.commit(); + } + } + } + } + + + @Test + void updateWithPreparedStatement() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( false ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( java.sql.PreparedStatement ps = connection.prepareStatement( "INSERT INTO booleanarraytest(id, bvec) VALUES (?, ?)" ) ) { + ps.setInt( 1, 11 ); + ps.setArray( 2, connection.createArrayOf( "BOOLEAN", new Boolean[]{ false, false, false } ) ); + ps.executeUpdate(); + connection.commit(); + } + } + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( false ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( java.sql.PreparedStatement ps = connection.prepareStatement( "UPDATE booleanarraytest SET bvec = ? WHERE id = ?" ) ) { + ps.setArray( 1, connection.createArrayOf( "BOOLEAN", new Boolean[]{ true, false, true } ) ); + ps.setInt( 2, 11 ); + ps.executeUpdate(); + connection.commit(); + } + } + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + try { + TestHelper.checkResultSet( + statement.executeQuery( "SELECT bvec FROM booleanarraytest WHERE id = 11" ), + ImmutableList.of( new Object[]{ new Object[]{ true, false, true } } ) + ); + } finally { + statement.executeUpdate( "DELETE FROM booleanarraytest WHERE id = 11" ); + connection.commit(); + } + } + } + } + +} diff --git a/dbms/src/test/java/org/polypheny/db/mql/MqlVectorDistanceTest.java b/dbms/src/test/java/org/polypheny/db/mql/MqlVectorDistanceTest.java new file mode 100644 index 0000000000..05af8aa572 --- /dev/null +++ b/dbms/src/test/java/org/polypheny/db/mql/MqlVectorDistanceTest.java @@ -0,0 +1,130 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.mql; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.polypheny.db.TestHelper.MongoConnection; +import org.polypheny.db.webui.models.results.DocResult; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +@Tag("adapter") +public class MqlVectorDistanceTest extends MqlTestTemplate { + + @BeforeEach + public void insertData() { + insert( "{\"name\": \"a\", \"embedding\": [1.0, 1.0]}" ); + insert( "{\"name\": \"b\", \"embedding\": [2.0, 2.0]}" ); + insert( "{\"name\": \"c\", \"embedding\": [0.0, 3.0]}" ); + } + + + @Test + public void l2VectorSearchReturnsResults() { + DocResult res = aggregate( + "{\"$vectorSearch\": {" + + "\"path\": \"embedding\", \"queryVector\": [1.0, 1.0], " + + "\"metric\": \"L2\", \"numCandidates\": 10, \"limit\": 3}}" + ); + assert res.getData().length == 3; + } + + + @Test + public void l2VectorSearchOrderedByDistance() { + DocResult res = aggregate( + "{\"$vectorSearch\": {" + + "\"path\": \"embedding\", \"queryVector\": [1.0, 1.0], " + + "\"metric\": \"L2\", \"numCandidates\": 10, \"limit\": 3}}" + ); + assert res.getData()[0].contains( "\"a\"" ); + assert res.getData()[1].contains( "\"b\"" ); + assert res.getData()[2].contains( "\"c\"" ); + } + + + @Test + public void l1VectorSearchOrderedByDistance() { + DocResult res = aggregate( + "{\"$vectorSearch\": {" + + "\"path\": \"embedding\", \"queryVector\": [1.0, 1.0], " + + "\"metric\": \"L1\", \"numCandidates\": 10, \"limit\": 3}}" + ); + assert res.getData()[0].contains( "\"a\"" ); + } + + + @Test + public void l2VectorSearchLimit() { + DocResult res = aggregate( + "{\"$vectorSearch\": {" + + "\"path\": \"embedding\", \"queryVector\": [1.0, 1.0], " + + "\"metric\": \"L2\", \"numCandidates\": 10, \"limit\": 1}}" + ); + assert res.getData().length == 1; + assert res.getData()[0].contains( "\"a\"" ); + } + + + @Test + public void l2SquaredVectorSearch() { + DocResult res = aggregate( + "{\"$vectorSearch\": {" + + "\"path\": \"embedding\", \"queryVector\": [1.0, 1.0], " + + "\"metric\": \"L2SQUARED\", \"numCandidates\": 10, \"limit\": 3}}" + ); + assert res.getData()[0].contains( "\"a\"" ); + } + + + @Test + public void chiSquaredVectorSearch() { + DocResult res = aggregate( + "{\"$vectorSearch\": {" + + "\"path\": \"embedding\", \"queryVector\": [1.0, 1.0], " + + "\"metric\": \"CHISQUARED\", \"numCandidates\": 10, \"limit\": 3}}" + ); + assert res.getData().length == 3; + assert res.getData()[0].contains( "\"a\"" ); + } + + + @Test + public void cosineVectorSearch() { + DocResult res = aggregate( + "{\"$vectorSearch\": {" + + "\"path\": \"embedding\", \"queryVector\": [1.0, 1.0], " + + "\"metric\": \"COSINE\", \"numCandidates\": 10, \"limit\": 3}}" + ); + assert res.getData().length == 3; + } + + + @Test + public void unsupportedMetricThrows() { + assertThrows( RuntimeException.class, () -> + MongoConnection.executeGetResponse( + "db.test.aggregate([{\"$vectorSearch\": {" + + "\"path\":\"embedding\", \"queryVector\": [1.0, 1.0], " + + "\"metric\": \"UNKNOWN\", \"numCandidates\": 10, \"limit\": 3}}])" + ) + ); + } + +} diff --git a/dbms/src/test/java/org/polypheny/db/sql/fun/SqlPgvectorOperatorTest.java b/dbms/src/test/java/org/polypheny/db/sql/fun/SqlPgvectorOperatorTest.java new file mode 100644 index 0000000000..4bf054d7a1 --- /dev/null +++ b/dbms/src/test/java/org/polypheny/db/sql/fun/SqlPgvectorOperatorTest.java @@ -0,0 +1,432 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.sql.fun; + + +import com.google.common.collect.ImmutableList; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.TestHelper.JdbcConnection; + + +@SuppressWarnings({ "SqlDialectInspection", "SqlNoDataSourceInspection" }) +@Slf4j +@Tag("adapter") +public class SqlPgvectorOperatorTest { + + @BeforeAll + public static void start() throws SQLException { + //noinspection ResultOfMethodCallIgnored + TestHelper.getInstance(); + addTestData(); + } + + + private static void addTestData() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( false ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "CREATE TABLE pgvecrealtest( id INTEGER NOT NULL, myarray REAL NOT NULL ARRAY(1,2), PRIMARY KEY (id) )" ); + statement.executeUpdate( "INSERT INTO pgvecrealtest VALUES (1, ARRAY[1.0, 1.0])" ); + statement.executeUpdate( "INSERT INTO pgvecrealtest VALUES (2, ARRAY[2.0, 2.0])" ); + statement.executeUpdate( "INSERT INTO pgvecrealtest VALUES (3, ARRAY[0.0, 3.0])" ); + + statement.executeUpdate( "CREATE TABLE pgvecbooltest( id INTEGER NOT NULL, myarray BOOLEAN NOT NULL ARRAY(1,3), PRIMARY KEY (id) )" ); + statement.executeUpdate( "INSERT INTO pgvecbooltest VALUES (1, ARRAY[true, true, true])" ); + statement.executeUpdate( "INSERT INTO pgvecbooltest VALUES (2, ARRAY[true, false, true])" ); + statement.executeUpdate( "INSERT INTO pgvecbooltest VALUES (3, ARRAY[false, false, false])" ); + connection.commit(); + } + } + } + + + @AfterAll + public static void stop() throws SQLException { + try ( JdbcConnection jdbcConnection = new JdbcConnection( true ) ) { + Connection connection = jdbcConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + statement.executeUpdate( "DROP TABLE pgvecrealtest" ); + statement.executeUpdate( "DROP TABLE pgvecbooltest" ); + + } + } + } + + + // --------------- L2 operator (<->) --------------- + @Test + public void l2OperatorTest() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 1.4142135623730951 }, + new Object[]{ 3, 2.23606797749979 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <-> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + } + } + } + + + @Test + public void l2OperatorReversedTest() throws SQLException { + // Operator is symmetric: literal on the left must give the same result. + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 1.4142135623730951 }, + new Object[]{ 3, 2.23606797749979 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, ARRAY[1.0, 1.0] <-> myarray AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + } + } + } + + + @Test + public void l2EquivalenceTest() throws SQLException { + // <-> must produce identical results to distance(..., 'L2'). + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 1.4142135623730951 }, + new Object[]{ 3, 2.23606797749979 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, distance(myarray, ARRAY[1.0, 1.0], 'L2') AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <-> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + } + } + } + + + @Test + public void knnTopKL2Test() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 1.4142135623730951 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <-> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY dist LIMIT 2" ), + expected + ); + } + } + } + + + @Test + public void filterL2Test() throws SQLException { + // Rows 1 (dist 0.0) and 2 (dist 1.414) are within L2 distance 2.0 of [1,1]. + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + TestHelper.checkResultSet( + statement.executeQuery( "SELECT COUNT(id) FROM pgvecrealtest WHERE myarray <-> ARRAY[1.0, 1.0] < 2.0" ), + ImmutableList.of( new Object[]{ 2L } ) + ); + } + } + } + + + @Test + public void crossJoinKnnTest() throws SQLException { + // Find the 2 rows in the table nearest to the vector of row id=1 via cross join. + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 1.4142135623730951 } + ); + TestHelper.checkResultSet( + statement.executeQuery( + "SELECT a.id, a.myarray <-> b.myarray AS dist " + + "FROM pgvecrealtest a, (SELECT myarray FROM pgvecrealtest WHERE id = 1) b " + + "ORDER BY dist LIMIT 2" ), + expected + ); + } + } + } + + + // --------------- L1 operator (<+>) --------------- + @Test + public void l1OperatorTest() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 2.0 }, + new Object[]{ 3, 3.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <+> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + } + } + } + + + @Test + public void knnTopKL1Test() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 2.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <+> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY dist LIMIT 2" ), + expected + ); + } + } + } + + + @Test + public void filterL1Test() throws SQLException { + // Rows 1 (dist 0.0) and 2 (dist 2.0) are within L1 distance 2.5 of [1,1]. + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + TestHelper.checkResultSet( + statement.executeQuery( "SELECT COUNT(id) FROM pgvecrealtest WHERE myarray <+> ARRAY[1.0, 1.0] < 2.5" ), + ImmutableList.of( new Object[]{ 2L } ) + ); + } + } + } + + + @Test + public void l1EquivalenceTest() throws SQLException { + // <+> must produce identical results to distance(..., 'L1'). + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 2.0 }, + new Object[]{ 3, 3.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, distance(myarray, ARRAY[1.0, 1.0], 'L1') AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <+> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + } + } + } + + + // --------------- Cosine operator (<=>) --------------- + @Test + public void cosOperatorTest() throws SQLException { + // cosDistance([2,2],[1,1]) = 0 (same direction); cosDistance([0,3],[1,1]) = 1 - 1/sqrt(2) + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 0.0 }, + new Object[]{ 2, 0.0 }, + new Object[]{ 3, 1.0 - 1.0 / Math.sqrt( 2 ) } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <=> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + } + } + } + + + // --------------- Hamming operator (<~>) --------------- + @Test + public void hammingOperatorTest() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 1.0 }, + new Object[]{ 2, 2.0 }, + new Object[]{ 3, 2.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <~> ARRAY[true, true, false] AS dist FROM pgvecbooltest ORDER BY id" ), + expected + ); + } + } + } + + @Test + public void hammingEquivalenceTest() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 1.0 }, + new Object[]{ 2, 2.0 }, + new Object[]{ 3, 2.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, hamming_distance(myarray, ARRAY[true, true, false]) AS dist FROM pgvecbooltest ORDER BY id" ), + expected + ); + } + } + } + + // --------------- INNER_PRODUCT operator (<#>) --------------- + @Test + public void innerProductOperatorTest() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, -2.0 }, + new Object[]{ 2, -4.0 }, + new Object[]{ 3, -3.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <#> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + } + } + } + + + @Test + public void innerProductEquivalenceTest() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, -2.0 }, + new Object[]{ 2, -4.0 }, + new Object[]{ 3, -3.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, distance(myarray, ARRAY[1.0, 1.0], 'INNER_PRODUCT') AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, inner_product_distance(myarray, ARRAY[1.0, 1.0]) AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <#> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY id" ), + expected + ); + } + } + } + + + @Test + public void knnTopKInnerProductTest() throws SQLException { + // Ordering ASC by inner_product_distance finds highest inner product first: row 2 (-4), then row 3 (-3). + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 2, -4.0 }, + new Object[]{ 3, -3.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <#> ARRAY[1.0, 1.0] AS dist FROM pgvecrealtest ORDER BY dist LIMIT 2" ), + expected + ); + } + } + } + + + @Test + public void filterInnerProductTest() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + TestHelper.checkResultSet( + statement.executeQuery( "SELECT COUNT(id) FROM pgvecrealtest WHERE myarray <#> ARRAY[1.0, 1.0] < -2.5" ), + ImmutableList.of( new Object[]{ 2L } ) + ); + } + } + } + + + // --------------- Jaccard operator (<%>) --------------- + @Test + public void jaccardOperatorTest() throws SQLException { + try ( JdbcConnection polyphenyDbConnection = new JdbcConnection( true ) ) { + Connection connection = polyphenyDbConnection.getConnection(); + try ( Statement statement = connection.createStatement() ) { + List expected = ImmutableList.of( + new Object[]{ 1, 1.0 - (2.0 / 3.0) }, + new Object[]{ 2, 1.0 - (1.0 / 3.0) }, + new Object[]{ 3, 1.0 } + ); + TestHelper.checkResultSet( + statement.executeQuery( "SELECT id, myarray <%> ARRAY[true, true, false] AS dist FROM pgvecbooltest ORDER BY id" ), + expected + ); + } + } + } + +} diff --git a/dbms/src/test/java/org/polypheny/db/sql/view/ComplexViewTest.java b/dbms/src/test/java/org/polypheny/db/sql/view/ComplexViewTest.java index fbf431d751..9b032ca3e7 100644 --- a/dbms/src/test/java/org/polypheny/db/sql/view/ComplexViewTest.java +++ b/dbms/src/test/java/org/polypheny/db/sql/view/ComplexViewTest.java @@ -229,16 +229,7 @@ public class ComplexViewTest { + "'fast'" + ")"; - private final static Object[] ORDERS_TEST_DATA = new Object[]{ - 1, - 1, - "A", - new BigDecimal( "65.15" ), - Date.valueOf( "2020-07-03" ), - "orderPriority", - "clerk", - 1, - "fast" }; + private static Object[] ORDERS_TEST_DATA; public final static String LINEITEM_TABLE = "CREATE TABLE lineitem ( " + "l_orderkey INTEGER NOT NULL," @@ -278,38 +269,16 @@ public class ComplexViewTest { + "'shipingComment'" + ")"; - private final static Object[] LINEITEM_TEST_DATA = new Object[]{ - 1, - 1, - 1, - 1, - new BigDecimal( "20.15" ), - new BigDecimal( "50.15" ), - new BigDecimal( "20.15" ), - new BigDecimal( "10.15" ), - "R", - "L", - Date.valueOf( "2020-07-03" ), - Date.valueOf( "2020-07-03" ), - Date.valueOf( "2020-09-03" ), - "shipingstruct", - "mode", - "shipingComment" }; + private static Object[] LINEITEM_TEST_DATA; - private final static Object[] date_TEST_DATA = new Object[]{ - Date.valueOf( "2020-07-03" ) }; + private static Object[] date_TEST_DATA; private final static Object[] decimal_TEST_DATA = new Object[]{ new BigDecimal( "65.15" ) }; - private final static Object[] decimalDate_TEST_DATA = new Object[]{ - new BigDecimal( "65.15" ), - Date.valueOf( "2020-07-03" ) }; + private static Object[] decimalDate_TEST_DATA; - private final static Object[] decimalDateInt_TEST_DATA = new Object[]{ - new BigDecimal( "65.15" ), - Date.valueOf( "2020-07-03" ), - 1 }; + private static Object[] decimalDateInt_TEST_DATA; private final static Object[] q1_TEST_DATA = new Object[]{ "R", @@ -336,11 +305,7 @@ public class ComplexViewTest { 1L, 0 }; - private final static Object[] q3_TEST_DATA = new Object[]{ - 1, - new BigDecimal( "-960.3725" ), - Date.valueOf( "2020-07-03" ), - 1 }; + private static Object[] q3_TEST_DATA; private final static Object[] q4_TEST_DATA = new Object[]{ "orderPriority", @@ -408,6 +373,67 @@ public static void start() { // Ensures that Polypheny-DB is running //noinspection ResultOfMethodCallIgnored TestHelper.getInstance(); + initTimeZoneDependentQueries(); + } + + + /** + * Initializes test data arrays that rely on {@link java.sql.Date} objects. + *

+ * This initialization is deliberately deferred until after the test environment + * is set up. The {@code Date.valueOf(...)} method resolves midnight using the JVM's + * default time zone. Because {@code TestHelper.getInstance()} overrides the application + * time zone to UTC, statically initializing these arrays would capture the host machine's + * local time zone before the UTC override occurs, resulting in 1-day date shifts. + */ + private static void initTimeZoneDependentQueries() { + + ORDERS_TEST_DATA = new Object[]{ + 1, + 1, + "A", + new BigDecimal( "65.15" ), + Date.valueOf( "2020-07-03" ), + "orderPriority", + "clerk", + 1, + "fast" }; + + LINEITEM_TEST_DATA = new Object[]{ + 1, + 1, + 1, + 1, + new BigDecimal( "20.15" ), + new BigDecimal( "50.15" ), + new BigDecimal( "20.15" ), + new BigDecimal( "10.15" ), + "R", + "L", + Date.valueOf( "2020-07-03" ), + Date.valueOf( "2020-07-03" ), + Date.valueOf( "2020-09-03" ), + "shipingstruct", + "mode", + "shipingComment" }; + + date_TEST_DATA = new Object[]{ + Date.valueOf( "2020-07-03" ) }; + + decimalDate_TEST_DATA = new Object[]{ + new BigDecimal( "65.15" ), + Date.valueOf( "2020-07-03" ) }; + + decimalDateInt_TEST_DATA = new Object[]{ + new BigDecimal( "65.15" ), + Date.valueOf( "2020-07-03" ), + 1 }; + + q3_TEST_DATA = new Object[]{ + 1, + new BigDecimal( "-960.3725" ), + Date.valueOf( "2020-07-03" ), + 1 }; } @@ -1741,7 +1767,7 @@ o_orderkey IN ( o_totalprice DESC, o_orderdate LIMIT 100""" ), - ImmutableList.of( new Object[]{ "CName", 1, 1, Date.valueOf( "2020-07-03" ), 65.15, 20.15 } ) + ImmutableList.of( new Object[]{ "CName", 1, 1, Date.valueOf( "2020-07-03" ) , 65.15, 20.15 } ) ); connection.commit(); diff --git a/gradle.properties b/gradle.properties index c2dea07078..e1ef2f3fe6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -26,7 +26,7 @@ org.gradle.jvmargs = -Xmx6g -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF- # Dependency versions activej_serializer_version = 6.0-beta2 airline_version = 2.9.0 -bouncycastle_version = 1.80 +bouncycastle_version = 1.80.2 calcite_linq4j_version = 1.36.0 commons_codec_version = 1.16.0 commons_collections_version = 4.4 @@ -78,6 +78,7 @@ polypheny_jdbc_driver_version = 2.2 polypheny_ui_version = 2.0-SNAPSHOT postgresql_version = 42.2.19 postgis_version = 2024.1.0 +pgvector_version = 0.1.6 proj4j_version = 1.3.0 pf4j_version = 3.12.0 prism_api_version = 1.9 diff --git a/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvSource.java b/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvSource.java index bee19c1662..441c71a93f 100644 --- a/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvSource.java +++ b/plugins/csv-adapter/src/main/java/org/polypheny/db/adapter/csv/CsvSource.java @@ -262,6 +262,7 @@ public Map> getExportedColumns() { null, null, false, + true, fileName, physicalTableName, name, diff --git a/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/expression/CypherFunctionInvocation.java b/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/expression/CypherFunctionInvocation.java index 0b2d227292..eca4c60d1b 100644 --- a/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/expression/CypherFunctionInvocation.java +++ b/plugins/cypher-language/src/main/java/org/polypheny/db/cypher/expression/CypherFunctionInvocation.java @@ -22,7 +22,15 @@ import lombok.Getter; import org.polypheny.db.algebra.operators.OperatorName; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; +import org.polypheny.db.cypher.cypher2alg.CypherToAlgConverter.CypherContext; +import org.polypheny.db.cypher.cypher2alg.CypherToAlgConverter.RexType; +import org.polypheny.db.languages.OperatorRegistry; import org.polypheny.db.languages.ParserPos; +import org.polypheny.db.nodes.Operator; +import org.polypheny.db.rex.RexLiteral; +import org.polypheny.db.rex.RexNode; +import org.polypheny.db.type.entity.PolyString; +import org.polypheny.db.util.Pair; @Getter public class CypherFunctionInvocation extends CypherExpression { @@ -49,4 +57,50 @@ public CypherFunctionInvocation( ParserPos beforePos, ParserPos namePos, List getRex( CypherContext context, RexType type ) { + if ( this.op == OperatorName.VECTOR_DISTANCE ) { + return getVectorDistanceRex( context, type ); + } + return super.getRex( context, type ); + } + + + private Pair getVectorDistanceRex( CypherContext context, RexType type ) { + if ( arguments.size() != 3 ) { + throw new GenericRuntimeException( "vector_distance requires exactly 3 arguments" ); + } + + RexNode v1 = arguments.get( 0 ).getRex( context, type ).right; + RexNode v2 = arguments.get( 1 ).getRex( context, type ).right; + + RexNode metricRex = arguments.get( 2 ).getRex( context, type ).right; + if ( !(metricRex instanceof RexLiteral metricLit) ) { + throw new GenericRuntimeException( "vector_distance metric must be a string literal" ); + } + String metric = metricLit.value.asString().value.toUpperCase( Locale.ROOT ); + + OperatorName namedOp = switch ( metric ) { + case "L1" -> OperatorName.L1_DISTANCE; + case "L2" -> OperatorName.L2_DISTANCE; + case "COSINE" -> OperatorName.COS_DISTANCE; + case "HAMMING" -> OperatorName.HAMMING_DISTANCE; + case "JACCARD" -> OperatorName.JACCARD_DISTANCE; + case "INNER_PRODUCT" -> OperatorName.INNER_PRODUCT_DISTANCE; + // parameterized version + case "CHISQUARED", "L2SQUARED" -> OperatorName.DISTANCE; + default -> throw new GenericRuntimeException( "Unknown distance metric: ", metric ); + }; + Operator operator = OperatorRegistry.get( namedOp ); + + if ( namedOp == OperatorName.DISTANCE ) { + return Pair.of( PolyString.of( namedOp.name() ), context.rexBuilder.makeCall( operator, List.of( v1, v2, metricRex ) ) ); + } + + return Pair.of( PolyString.of( namedOp.name() ), context.rexBuilder.makeCall( operator, List.of( v1, v2 ) ) ); + } + + + } diff --git a/plugins/ethereum-adapter/src/main/java/org/polypheny/db/adapter/ethereum/EthereumPlugin.java b/plugins/ethereum-adapter/src/main/java/org/polypheny/db/adapter/ethereum/EthereumPlugin.java index 18ea5093f7..058f431928 100644 --- a/plugins/ethereum-adapter/src/main/java/org/polypheny/db/adapter/ethereum/EthereumPlugin.java +++ b/plugins/ethereum-adapter/src/main/java/org/polypheny/db/adapter/ethereum/EthereumPlugin.java @@ -187,6 +187,7 @@ public Map> getExportedColumns() { dimension, cardinality, false, + true, "public", "block", blockCol, @@ -208,6 +209,7 @@ public Map> getExportedColumns() { dimension, cardinality, false, + true, "public", "transaction", transactCol, diff --git a/plugins/excel-adapter/src/main/java/org/polypheny/db/adapter/excel/ExcelSource.java b/plugins/excel-adapter/src/main/java/org/polypheny/db/adapter/excel/ExcelSource.java index 0163bfd9ca..e070f724b9 100644 --- a/plugins/excel-adapter/src/main/java/org/polypheny/db/adapter/excel/ExcelSource.java +++ b/plugins/excel-adapter/src/main/java/org/polypheny/db/adapter/excel/ExcelSource.java @@ -324,6 +324,7 @@ public Map> getExportedColumns() { dimension, cardinality, false, + true, fileName, physicalTableName, name, diff --git a/plugins/file-adapter/src/main/java/org/polypheny/db/adapter/file/source/Qfs.java b/plugins/file-adapter/src/main/java/org/polypheny/db/adapter/file/source/Qfs.java index fe4c7b1881..79f0d03e1b 100644 --- a/plugins/file-adapter/src/main/java/org/polypheny/db/adapter/file/source/Qfs.java +++ b/plugins/file-adapter/src/main/java/org/polypheny/db/adapter/file/source/Qfs.java @@ -243,6 +243,7 @@ public Map> getExportedColumns() { null, null, false, + true, physSchemaName, physTableName, "path", @@ -259,6 +260,7 @@ public Map> getExportedColumns() { null, null, false, + true, physSchemaName, physTableName, "name", @@ -275,6 +277,7 @@ public Map> getExportedColumns() { null, null, true, + true, physSchemaName, physTableName, "size", @@ -291,6 +294,7 @@ public Map> getExportedColumns() { null, null, false, + true, physSchemaName, physTableName, "file", diff --git a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java index 83a056a65d..77a424b02d 100644 --- a/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java +++ b/plugins/google-sheet-adapter/src/main/java/org/polypheny/db/adapter/googlesheet/GoogleSheetSource.java @@ -296,6 +296,7 @@ public Map> getExportedColumns() { null, null, false, + true, "public", tableName, col.toString(), diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcRules.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcRules.java index 7e9b97bb04..86fdc6a2fa 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcRules.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcRules.java @@ -109,6 +109,8 @@ import org.polypheny.db.sql.language.fun.SqlItemOperator; import org.polypheny.db.tools.AlgBuilderFactory; import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.PolyTypeUtil; +import org.polypheny.db.type.VectorType; import org.polypheny.db.util.ImmutableBitSet; import org.polypheny.db.util.Pair; import org.polypheny.db.util.Quadruple; @@ -246,9 +248,17 @@ public AlgNode convert( Join join, boolean convertInputTraits ) { } newInputs.add( input ); } - if ( convertInputTraits && !canJoinOnCondition( join.getCondition() ) ) { - return null; + if ( convertInputTraits ) { + if ( join.getCondition().isAlwaysTrue() ) { + if ( !out.dialect.supportsVector() || !hasVectorColumn( newInputs.get( 0 ) ) + || !hasVectorColumn( newInputs.get( 1 ) ) ) { + return null; + } + } else if ( !canJoinOnCondition( join.getCondition() ) ) { + return null; + } } + if ( containsAggregateSubquery( join.getLeft() ) || containsAggregateSubquery( join.getRight() ) ) { return null; } @@ -268,6 +278,12 @@ public AlgNode convert( Join join, boolean convertInputTraits ) { } + private boolean hasVectorColumn( AlgNode input ) { + return input.getTupleType().getFields().stream() + .anyMatch( f -> f.getType() instanceof VectorType ); + } + + private boolean containsAggregateSubquery( AlgNode input ) { return input instanceof Aggregate || (input instanceof AlgSubset subset && subset.getOriginal() instanceof Aggregate); } @@ -501,7 +517,7 @@ private static boolean supports( JdbcConvention out, Project project ) { return (out.dialect.supportsWindowFunctions() || !RexOver.containsOver( project.getProjects(), null )) && !userDefinedFunctionInProject( project ) - && !knnFunctionInProject( project ) + && (!knnFunctionInProject( project ) || supportsKnnFunctionInProject( out.dialect, project )) && !multimediaFunctionInProject( project ) && !contains( project, List.of( OperatorName.INITCAP ) ) && (!geoFunctionInProject( project ) || supportsGeoFunction( out.dialect, project )) @@ -599,6 +615,18 @@ private static boolean itemOperatorInProject( Project project ) { } + private static boolean supportsKnnFunctionInProject( SqlDialect dialect, Project project ) { + CheckingKnnFunctionSupportVisitor visitor = new CheckingKnnFunctionSupportVisitor( dialect ); + for ( RexNode node : project.getChildExps() ) { + node.accept( visitor ); + if ( visitor.supportsKnnFunction() ) { + return true; + } + } + return false; + } + + @Override public AlgNode convert( AlgNode alg ) { final Project project = (Project) alg; @@ -680,7 +708,7 @@ public JdbcFilterRule( JdbcConvention out, AlgBuilderFactory algBuilderFactory ) filter -> ( !userDefinedFunctionInFilter( filter ) && !containUnsupportedArray( filter, out ) - && !knnFunctionInFilter( filter ) + && (!knnFunctionInFilter( filter ) || supportsKnnFunctionInFilter( out.dialect, filter )) && !multimediaFunctionInFilter( filter ) && (!geoFunctionInFilter( filter ) || supportsGeoFunctionInFilter( out.dialect, filter )) && !DocumentRules.containsJson( filter ) @@ -720,6 +748,18 @@ private static boolean knnFunctionInFilter( Filter filter ) { } + private static boolean supportsKnnFunctionInFilter( SqlDialect dialect, Filter filter ) { + CheckingKnnFunctionSupportVisitor visitor = new CheckingKnnFunctionSupportVisitor( dialect ); + for ( RexNode node : filter.getChildExps() ) { + node.accept( visitor ); + if ( visitor.supportsKnnFunction() ) { + return true; + } + } + return false; + } + + private static boolean multimediaFunctionInFilter( Filter filter ) { CheckingMultimediaFunctionVisitor visitor = new CheckingMultimediaFunctionVisitor(); for ( RexNode node : filter.getChildExps() ) { @@ -770,7 +810,8 @@ private static boolean itemOperatorInFilter( Filter filter ) { private static boolean isStringComparableArrayType( Filter filter ) { for ( AlgDataTypeField dataTypeField : filter.getTupleType().getFields() ) { - if ( dataTypeField.getType().getPolyType() == PolyType.ARRAY ) { + if ( dataTypeField.getType().getPolyType() == PolyType.ARRAY + && !(dataTypeField.getType() instanceof VectorType) ) { switch ( dataTypeField.getType().getComponentType().getPolyType() ) { case BOOLEAN: case TINYINT: @@ -1523,6 +1564,51 @@ public Void visitCall( RexCall call ) { } + private static class CheckingKnnFunctionSupportVisitor extends RexVisitorImpl { + private boolean supportsKnnFunction = false; + private SqlDialect dialect; + + + CheckingKnnFunctionSupportVisitor( SqlDialect dialect ) { + super(true); + this.dialect = dialect; + } + + + public boolean supportsKnnFunction() { + return supportsKnnFunction; + } + + + @Override + public Void visitCall(RexCall call) { + Operator operator = call.getOperator(); + if (operator instanceof SqlFunction sqlFunction + && sqlFunction.getFunctionCategory().isKnn() + && dialect.supportedKnnFunctions().contains(sqlFunction.getOperatorName()) + && call.operands.size() >= 2) { + AlgDataType t1 = call.operands.get(0).getType(); + AlgDataType t2 = call.operands.get(1).getType(); + if (t1 instanceof VectorType vectorType + && isCompatibleQueryVector( t2 ) + && dialect.vectorPushdownTypeIsPresent( vectorType.getVectorElementType() )) { + supportsKnnFunction = true; + } + } + return super.visitCall(call); + } + + + private static boolean isCompatibleQueryVector( AlgDataType t2 ) { + if ( t2 instanceof VectorType ) return true; + if ( t2.getPolyType() != PolyType.ARRAY ) return false; + AlgDataType comp = t2.getComponentType(); + return comp != null && (PolyTypeUtil.isNumeric( comp ) || comp.getPolyType() == PolyType.BOOLEAN ); + } + + } + + private static class CheckingItemOperatorVisitor extends RexVisitorImpl { private boolean containsItemOperator = false; diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcToEnumerableConverter.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcToEnumerableConverter.java index 1cde74115b..67c5cbeb17 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcToEnumerableConverter.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/JdbcToEnumerableConverter.java @@ -44,6 +44,7 @@ import java.util.Calendar; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.TimeZone; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -87,6 +88,7 @@ import org.polypheny.db.sql.language.util.SqlString; import org.polypheny.db.type.ArrayType; import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; import org.polypheny.db.type.entity.PolyBinary; import org.polypheny.db.type.entity.PolyBoolean; import org.polypheny.db.type.entity.PolyDefaults; @@ -352,6 +354,14 @@ private void generateGet( @NonNull private static Expression getPreprocessArrayExpression( ParameterExpression resultSet_, int i, SqlDialect dialect, AlgDataType fieldType ) { + Optional arrayRetrieval = dialect.getCustomArrayRetrievalExpression( resultSet_, i, fieldType ); + if ( fieldType instanceof VectorType && arrayRetrieval.isPresent() ) { + Expression parsed = arrayRetrieval.get(); + return Expressions.condition( + Expressions.call( resultSet_, "wasNull" ), + Expressions.constant( null ), + parsed ); + } if ( (dialect.supportsArrays() && (fieldType.unwrapOrThrow( ArrayType.class ).getDimension() == 1 || dialect.supportsNestedArrays())) ) { ParameterExpression argument = Expressions.parameter( Object.class ); diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java index 8f16f52b53..2f398bc207 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/ResultSetEnumerable.java @@ -47,6 +47,7 @@ import java.util.Calendar; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.TimeZone; import lombok.extern.slf4j.Slf4j; import org.apache.calcite.linq4j.AbstractEnumerable; @@ -61,8 +62,10 @@ import org.polypheny.db.adapter.jdbc.connection.ConnectionHandler; import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; +import org.polypheny.db.sql.language.SqlDialect; import org.polypheny.db.sql.language.validate.SqlType; import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; import org.polypheny.db.type.entity.PolyValue; import org.polypheny.db.type.entity.numerical.PolyLong; import org.polypheny.db.type.entity.temporal.PolyDate; @@ -254,6 +257,7 @@ private static void setDynamicParam( PreparedStatement preparedStatement, int i, preparedStatement.setNull( i, Types.NULL ); return; } + SqlDialect dialect = connectionHandler.getDialect(); switch ( type.getPolyType() ) { case BIGINT: @@ -290,7 +294,7 @@ private static void setDynamicParam( PreparedStatement preparedStatement, int i, preparedStatement.setTime( i, value.asTime().asSqlTime(), Calendar.getInstance( TimeZone.getTimeZone( "UTC" ) ) ); break; case TIMESTAMP: - if ( connectionHandler.getDialect().handlesUtcIncorrectly() ) { + if ( dialect.handlesUtcIncorrectly() ) { preparedStatement.setTimestamp( i, PolyTimestamp.of( value.asTimestamp().millisSinceEpoch + OFFSET ).asSqlTimestamp() ); } else { preparedStatement.setTimestamp( i, value.asTimestamp().asSqlTimestamp(), Calendar.getInstance( TimeZone.getTimeZone( "UTC" ) ) ); @@ -301,7 +305,16 @@ private static void setDynamicParam( PreparedStatement preparedStatement, int i, handleBinary( preparedStatement, i, value, connectionHandler ); break; case ARRAY: - if ( (type.getComponentType().getPolyType() == PolyType.ARRAY && connectionHandler.getDialect().supportsNestedArrays()) || (type.getComponentType().getPolyType() != PolyType.ARRAY) && connectionHandler.getDialect().supportsArrays() ) { + Optional vectorType = type.unwrap( VectorType.class ); + if ( vectorType.isPresent() && dialect.vectorPushdownTypeIsPresent( vectorType.get().getVectorElementType() ) ) { + Object dbObj = dialect.getVectorDbObject( vectorType.get().getVectorElementType(), value.asList() ); + if ( dbObj != null ) { + preparedStatement.setObject( i, dbObj ); + break; + } + } + + if ( (type.getComponentType().getPolyType() == PolyType.ARRAY && dialect.supportsNestedArrays()) || (type.getComponentType().getPolyType() != PolyType.ARRAY) && dialect.supportsArrays() ) { Array array = getArray( value, type, connectionHandler ); preparedStatement.setArray( i, array ); array.free(); // according to documentation this is advised to not hog the memory diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/connection/TransactionalConnectionFactory.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/connection/TransactionalConnectionFactory.java index 2a1c88bacf..3198037208 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/connection/TransactionalConnectionFactory.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/connection/TransactionalConnectionFactory.java @@ -93,7 +93,7 @@ private TransactionalConnectionHandler getFreeTransactionHandler() throws Connec if ( getNumActive() + getNumIdle() < maxConnections ) { log.debug( "Creating a new transaction handler. Current freeInstances-Size: {}", freeInstances.size() ); try { - handler = new TransactionalConnectionHandler( dataSource.getConnection(), dialect ); + handler = createNewHandler( dataSource.getConnection(), dialect ); } catch ( SQLException e ) { throw new ConnectionHandlerException( "Caught exception while creating connection handler", e ); } @@ -114,6 +114,16 @@ private TransactionalConnectionHandler getFreeTransactionHandler() throws Connec } + private TransactionalConnectionHandler createNewHandler( Connection connection, SqlDialect dialect ) throws ConnectionHandlerException { + try { + dialect.initializeConnection( connection ); + } catch ( SQLException e ) { + throw new ConnectionHandlerException( "Failed to initialize dialect connection", e ); + } + return new TransactionalConnectionHandler( connection, dialect ); + } + + @Override public int getMaxTotal() { return maxConnections; diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/rel2sql/AlgToSqlConverter.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/rel2sql/AlgToSqlConverter.java index b7d1340037..d2d99c36a9 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/rel2sql/AlgToSqlConverter.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/rel2sql/AlgToSqlConverter.java @@ -395,6 +395,7 @@ public Result visit( Values e ) { public Result visit( Sort e ) { Result x = visitChild( 0, e.getInput() ); Builder builder = x.builder( e, false, Clause.ORDER_BY ); + boolean setExplicitSelect = false; if ( stack.size() != 1 && builder.select.getSqlSelectList() == null ) { // Generates explicit column names instead of start(*) for non-root ORDER BY to avoid ambiguity. final List selectList = Expressions.list(); @@ -402,6 +403,7 @@ public Result visit( Sort e ) { addSelect( selectList, builder.context.field( field.getIndex() ), e.getTupleType() ); } builder.select.setSelectList( new SqlNodeList( selectList, POS ) ); + setExplicitSelect = true; } List orderByList = Expressions.list(); for ( AlgFieldCollation field : e.getCollation().getFieldCollations() ) { @@ -410,6 +412,8 @@ public Result visit( Sort e ) { if ( !orderByList.isEmpty() ) { builder.setOrderBy( new SqlNodeList( orderByList, POS ) ); x = builder.result(); + } else if ( setExplicitSelect ) { + x = builder.result(); } if ( e.fetch != null ) { builder = x.builder( e, false, Clause.FETCH ); diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/rel2sql/SqlImplementor.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/rel2sql/SqlImplementor.java index 3b7b1c02db..de9cfbd5de 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/rel2sql/SqlImplementor.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/rel2sql/SqlImplementor.java @@ -114,6 +114,7 @@ import org.polypheny.db.type.IntervalPolyType; import org.polypheny.db.type.PolyType; import org.polypheny.db.type.PolyTypeFamily; +import org.polypheny.db.type.VectorType; import org.polypheny.db.type.entity.PolyInterval; import org.polypheny.db.type.entity.PolyValue; import org.polypheny.db.util.Util; @@ -534,6 +535,10 @@ public SqlNode toSql( RexProgram program, RexNode rex ) { case BINARY: return SqlBinaryStringLiteral.createBinaryString( literal.value.asBinary(), POS ); case ARRAY: + if ( literal.getType() instanceof VectorType vectorType ) { + SqlNode vectorNode = dialect.getVectorLiteral( vectorType, literal.getValue().asList(), POS ); + if ( vectorNode != null ) return vectorNode; + } if ( dialect.supportsNestedArrays() ) { List array = literal.getValue().asList(); return SqlLiteral.createArray( array, literal.getType(), POS ); diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java index 84b704ff85..f053944efd 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/sources/AbstractJdbcSource.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Optional; import java.util.stream.Collectors; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; @@ -53,10 +54,12 @@ import org.polypheny.db.plugins.PolyPluginManager; import org.polypheny.db.prepare.Context; import org.polypheny.db.schema.Namespace; +import org.polypheny.db.sql.language.SqlDbFeature; import org.polypheny.db.sql.language.SqlDialect; import org.polypheny.db.transaction.PUID; import org.polypheny.db.transaction.PolyXid; import org.polypheny.db.type.PolyType; +import javax.annotation.Nullable; @Slf4j @@ -241,14 +244,19 @@ public Map> getExportedColumns() { primaryKeyColumns.add( row.getString( "COLUMN_NAME" ) ); } } + Map cardinalities = fetchColumnMetadata( connection, schemaPattern, tableName ); try ( ResultSet row = dbmd.getColumns( settings.get( "database" ), schemaPattern, tableName, "%" ) ) { List list = new ArrayList<>(); while ( row.next() ) { - PolyType type = PolyType.getNameForJdbcType( row.getInt( "DATA_TYPE" ) ); - Integer length = null; - Integer scale = null; - Integer dimension = null; - Integer cardinality = null; + int jdbcDataType = row.getInt( "DATA_TYPE" ); + String typeName = row.getString( "TYPE_NAME" ); + log.debug( "PolyType integer read: {}", row.getInt( "DATA_TYPE" ) ); + log.debug( "PolyType name read: {}", typeName ); + PolyType type; + PolyType collectionsType = null; + Integer length = null, scale = null, dimension = null, cardinality = null; + type = PolyType.getNameForJdbcType( jdbcDataType ); + if ( isNativeVectorType( typeName ) ) type = PolyType.OTHER; switch ( type ) { case BOOLEAN: case TINYINT: @@ -286,18 +294,34 @@ public Map> getExportedColumns() { type = PolyType.VARBINARY; length = row.getInt( "COLUMN_SIZE" ); break; + case ARRAY: + case OTHER: + Optional nativeType = resolveNativeColumnType( cardinalities, typeName, row ); + if ( nativeType.isPresent() ){ + ColumnTypeInfo info = nativeType.get(); + type = info.type; + collectionsType = info.collectionType; + length = info.length; + scale = info.scale; + dimension = info.dimension; + cardinality = info.cardinality; + } + break; + default: throw new GenericRuntimeException( "Unsupported data type: " + type.getName() ); } + String colName = row.getString( "COLUMN_NAME" ).toLowerCase(); list.add( new ExportedColumn( - row.getString( "COLUMN_NAME" ).toLowerCase(), + colName, type, - null, + collectionsType, length, scale, dimension, cardinality, row.getString( "IS_NULLABLE" ).equalsIgnoreCase( "YES" ), + !isNativeVectorType( typeName ), requiresSchema() ? row.getString( "TABLE_SCHEM" ) : row.getString( "TABLE_CAT" ), row.getString( "TABLE_NAME" ), row.getString( "COLUMN_NAME" ), @@ -367,4 +391,59 @@ public interface Exclude { } + + /** + * Resolve database-specific column type names that cannot be identified by JDBC. + * + * @param typeName {@code TYPE_NAME} from e.g. {@link DatabaseMetaData#getColumns} + * @param columnRow {@link ResultSet} as current row of {@link + * java.sql.DatabaseMetaData#getColumns} + * @return {@link ColumnTypeInfo} + * @throws SQLException + */ + protected Optional resolveNativeColumnType( Map metadata, String typeName, ResultSet columnRow ) throws SQLException { + return Optional.empty(); + } + + + protected boolean isNativeVectorType( String typeName ) { + return false; + } + + + /** + * + * @return Map of the form {attribute name -> ColumnMetadata(dims, typeMod)} + */ + protected Map fetchColumnMetadata( Connection conn, String schema, String table ) throws SQLException { + return Map.of(); + } + + + public record ColumnTypeInfo( + PolyType type, + @Nullable PolyType collectionType, + @Nullable Integer length, + @Nullable Integer scale, + @Nullable Integer dimension, + @Nullable Integer cardinality ) {} + + + /** + * Raw PostgreSQL catalog metadata for a single column that is either + * a typed collection (array, vector) or carries a type modifier. + * + * @param arrayDimensions value of {@code pg_attribute.attndims}; 0 for non-arrays + * @param typeModifier value of {@code pg_attribute.atttypmod} if > 0, else null + */ + public record CollectionMetadata( int arrayDimensions, @Nullable Integer typeModifier ) {} + + + @Override + public List getActiveFeatureNames() { + return dialect.getSupportedFeatures().stream() + .map( SqlDbFeature::displayName ) + .collect( Collectors.toList() ); + } + } diff --git a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/stores/AbstractJdbcStore.java b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/stores/AbstractJdbcStore.java index 0d279457b7..1421b2ca5a 100644 --- a/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/stores/AbstractJdbcStore.java +++ b/plugins/jdbc-adapter-framework/src/main/java/org/polypheny/db/adapter/jdbc/stores/AbstractJdbcStore.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; @@ -35,6 +36,8 @@ import org.polypheny.db.adapter.jdbc.JdbcUtils; import org.polypheny.db.adapter.jdbc.connection.ConnectionFactory; import org.polypheny.db.adapter.jdbc.connection.ConnectionHandlerException; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeFactory; import org.polypheny.db.catalog.catalogs.RelAdapterCatalog; import org.polypheny.db.catalog.entity.allocation.AllocationCollection; import org.polypheny.db.catalog.entity.allocation.AllocationGraph; @@ -51,10 +54,12 @@ import org.polypheny.db.prepare.Context; import org.polypheny.db.runtime.PolyphenyDbException; import org.polypheny.db.schema.Namespace; +import org.polypheny.db.sql.language.SqlDbFeature; import org.polypheny.db.sql.language.SqlDialect; import org.polypheny.db.sql.language.SqlLiteral; import org.polypheny.db.transaction.PolyXid; import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; @Slf4j @@ -94,10 +99,9 @@ public AbstractJdbcStore( // Register the JDBC Pool Size as information in the information manager and enable it registerJdbcInformation(); - + registerFeatures(); // Create udfs createUdfs(); - this.delegate = new RelationalModifyDelegate( this, adapterCatalog ); } @@ -139,6 +143,11 @@ public void createUdfs() { } + public void registerFeatures() { + + } + + @Override public Namespace getCurrentNamespace() { return currentJdbcSchema; @@ -252,9 +261,16 @@ protected StringBuilder buildAddColumnQuery( PhysicalTable table, PhysicalColumn } - protected void createColumnDefinition( PhysicalColumn column, StringBuilder builder ) { + protected String getColumnDefinitionString( PhysicalColumn column ) { + StringBuilder builder = new StringBuilder(); boolean supportsThisArray = column.collectionsType == PolyType.ARRAY && column.dimension != null && this.dialect.supportsArrays() && (this.dialect.supportsNestedArrays() || column.dimension == 1); - if ( supportsThisArray ) { + AlgDataType algType = column.getAlgDataType( AlgDataTypeFactory.DEFAULT ); + if ( algType instanceof VectorType vectorType && dialect.vectorPushdownTypeIsPresent( vectorType.getVectorElementType() ) ) { + builder.append( dialect.getTypeString( vectorType.getVectorElementType() ) ) + .append( "(" ) + .append( column.cardinality != null && column.cardinality > 0 ? column.cardinality : "" ) + .append( ")" ); + } else if ( supportsThisArray ) { // Returns e.g. TEXT if arrays are not supported builder.append( getTypeString( column.type ) ).append( " " ).append( getTypeString( PolyType.ARRAY ).repeat( column.dimension ) ); } else if ( column.collectionsType == PolyType.MAP ) { @@ -280,6 +296,11 @@ protected void createColumnDefinition( PhysicalColumn column, StringBuilder buil builder.append( " " ).append( getTypeString( column.collectionsType ) ); } } + return builder.toString().trim(); + } + + protected void createColumnDefinition( PhysicalColumn column, StringBuilder builder ) { + builder.append( getColumnDefinitionString( column ) ); } @@ -323,15 +344,7 @@ public void updateColumnType( Context context, long allocId, LogicalColumn newCo .append( "." ) .append( dialect.quoteIdentifier( physicalTable.name ) ); builder.append( " ALTER COLUMN " ).append( dialect.quoteIdentifier( column.name ) ); - builder.append( " " ).append( getTypeString( column.type ) ); - if ( column.length != null && doesTypeUseLength( column.type ) ) { - builder.append( "(" ); - builder.append( column.length ); - if ( column.scale != null ) { - builder.append( "," ).append( column.scale ); - } - builder.append( ")" ); - } + builder.append( " " ).append( getColumnDefinitionString( column ) ); executeUpdate( builder, context ); updateNativePhysical( allocId ); @@ -517,6 +530,14 @@ protected String getPhysicalIndexName( long physicalId, long indexId ) { public abstract String getDefaultPhysicalSchemaName(); + @Override + public List getActiveFeatureNames() { + return dialect.getSupportedFeatures().stream() + .map( SqlDbFeature::displayName ) + .collect( Collectors.toList() ); + } + + @SuppressWarnings("unused") public interface Exclude { diff --git a/plugins/mongodb-adapter/src/main/java/org/polypheny/db/adapter/mongodb/rules/MongoRules.java b/plugins/mongodb-adapter/src/main/java/org/polypheny/db/adapter/mongodb/rules/MongoRules.java index 91b6ac5077..9a425ddb75 100644 --- a/plugins/mongodb-adapter/src/main/java/org/polypheny/db/adapter/mongodb/rules/MongoRules.java +++ b/plugins/mongodb-adapter/src/main/java/org/polypheny/db/adapter/mongodb/rules/MongoRules.java @@ -67,6 +67,7 @@ import org.polypheny.db.rex.RexLiteral; import org.polypheny.db.rex.RexNameRef; import org.polypheny.db.rex.RexNode; +import org.polypheny.db.rex.RexShuttle; import org.polypheny.db.rex.RexVisitorImpl; import org.polypheny.db.schema.document.DocumentRules; import org.polypheny.db.schema.types.ModifiableTable; @@ -431,13 +432,14 @@ public AlgNode convert( AlgNode alg ) { private static boolean containsIncompatible( SingleAlg alg ) { MongoExcludeVisitor visitor = new MongoExcludeVisitor(); - for ( RexNode node : alg.getChildExps() ) { - node.accept( visitor ); - if ( visitor.isContainsIncompatible() ) { - return true; + alg.accept( new RexShuttle() { + @Override + public RexNode visitCall( RexCall call ) { + call.accept( visitor ); + return call; } - } - return false; + } ); + return visitor.isContainsIncompatible(); } @@ -474,6 +476,12 @@ public Void visitCall( RexCall call ) { || operator.getOperatorName() == OperatorName.SUBSTRING || operator.getOperatorName() == OperatorName.FLOOR || operator.getOperatorName() == OperatorName.DISTANCE + || operator.getOperatorName() == OperatorName.L1_DISTANCE + || operator.getOperatorName() == OperatorName.L2_DISTANCE + || operator.getOperatorName() == OperatorName.INNER_PRODUCT_DISTANCE + || operator.getOperatorName() == OperatorName.COS_DISTANCE + || operator.getOperatorName() == OperatorName.HAMMING_DISTANCE + || operator.getOperatorName() == OperatorName.JACCARD_DISTANCE || (operator.getOperatorName() == OperatorName.CAST && call.operands.get( 0 ).getType().getPolyType() == PolyType.DATE) || operator instanceof SqlDatetimeSubtractionOperator || operator instanceof SqlDatetimePlusOperator ) { diff --git a/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java b/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java index edc8372c9f..a2a9ac391e 100644 --- a/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java +++ b/plugins/mql-language/src/main/java/org/polypheny/db/languages/mql2alg/MqlToAlgConverter.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.function.BiFunction; import java.util.stream.Collectors; +import io.activej.common.tuple.Tuple2; import org.bson.BsonArray; import org.bson.BsonBoolean; import org.bson.BsonDocument; @@ -37,6 +38,7 @@ import org.bson.BsonRegularExpression; import org.bson.BsonString; import org.bson.BsonValue; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.polypheny.db.algebra.AlgCollation; import org.polypheny.db.algebra.AlgCollations; @@ -96,6 +98,8 @@ import org.polypheny.db.schema.document.DocumentUtil; import org.polypheny.db.schema.document.DocumentUtil.UpdateOperation; import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; +import org.polypheny.db.type.VectorType.ElementType; import org.polypheny.db.type.entity.PolyBoolean; import org.polypheny.db.type.entity.PolyList; import org.polypheny.db.type.entity.PolyString; @@ -103,6 +107,7 @@ import org.polypheny.db.type.entity.document.PolyDocument; import org.polypheny.db.type.entity.numerical.PolyBigDecimal; import org.polypheny.db.type.entity.numerical.PolyDouble; +import org.polypheny.db.type.entity.numerical.PolyFloat; import org.polypheny.db.type.entity.numerical.PolyInteger; import org.polypheny.db.util.BsonUtil; import org.polypheny.db.util.DateString; @@ -779,6 +784,9 @@ private AlgNode convertAggregate( MqlAggregate query, AlgDataType rowType, AlgNo case "$replaceWith": node = combineReplaceRoot( value.asDocument().get( "$replaceWith" ), node, true ); break; + case "$vectorSearch": + node = convertVectorSearch( value.asDocument().get( "$vectorSearch" ) , rowType, node ); + break; // todo dl add more pipeline statements default: throw new IllegalStateException( "Unexpected value: " + ((BsonDocument) value).getFirstKey() ); @@ -792,6 +800,186 @@ private AlgNode convertAggregate( MqlAggregate query, AlgDataType rowType, AlgNo } + private AlgNode convertVectorSearch( BsonValue vectorSearchVal, AlgDataType rowType, AlgNode node ) { + BsonDocument vectorSearch = vectorSearchVal.asDocument(); + if ( ! vectorSearch.containsKey( "path" ) ) { + throw new GenericRuntimeException( "$vectorSearch requires a 'path' field specifying the vector column." ); + } + if ( !vectorSearch.containsKey( "queryVector" ) || !vectorSearch.get( "queryVector" ).isArray() ) { + throw new GenericRuntimeException( "$vectorSearch requires a 'queryVector' field of type Array." ); + } + if ( !vectorSearch.containsKey( "limit" ) || !vectorSearch.get( "limit" ).isNumber() ) { + throw new GenericRuntimeException( "$vectorSearch requires a 'limit' field of type number." ); + } + if ( !vectorSearch.containsKey( "metric" ) ) { + throw new GenericRuntimeException( "$vectorSearch requires a 'metric' field." ); + } + + String path = vectorSearch.getString( "path" ).getValue(); + BsonArray queryVector = vectorSearch.getArray( "queryVector" ); + int limit = vectorSearch.getNumber( "limit" ).intValue(); + String metric = vectorSearch.getString( "metric" ).getValue(); + + Tuple2 distanceOpName = getOperatorName( metric ); + Operator distanceOperator = OperatorRegistry.get( distanceOpName.value1() ); + + if ( vectorSearch.containsKey( "filter" ) ) { + RexNode filterNode = translateDocument( vectorSearch.getDocument( "filter" ), rowType, null ); + node = LogicalDocumentFilter.create( node, filterNode ); + } + + RexNode pathRef; + AlgDataType pathType = null; + for ( AlgDataTypeField field : rowType.getFields() ) { + if ( field.getName().equals( path ) ) { + pathType = field.getType(); + break; + } + } + + if ( pathType != null ) { + pathRef = RexNameRef.create( Collections.singletonList( path ), null, pathType ); + } else { + pathRef = getIdentifier( path, rowType ); + pathType = pathRef.getType(); + } + + AlgDataType queryLiteralType; + RexNode queryVectorRef; + if ( pathType instanceof VectorType vectorType ) { + queryLiteralType = cluster.getTypeFactory().createVectorType( vectorType.getComponentType(), vectorType.getVectorDimension() ); + if ( queryVector.size() != vectorType.getVectorDimension() ) { + throw new GenericRuntimeException( String.format( + "$vectorSearch 'queryVector' has %d elements but column '%s' has dimension %d.", + queryVector.size(), path, vectorType.getVectorDimension() ) ); + } + List polyValues = new ArrayList<>(); + ElementType elementType = vectorType.getVectorElementType(); + + boolean isBinaryMetric = metric.equals( "HAMMING" ) || metric.equals( "JACCARD" ); + boolean isBinaryVector = elementType == ElementType.BIT; + if ( isBinaryMetric && !isBinaryVector ) { + throw new GenericRuntimeException( String.format( + "Metric '%s' requires a BIT vector column, but column '%s' has element type %s.", + metric, path, elementType ) ); + } + if ( !isBinaryMetric && isBinaryVector ) { + throw new GenericRuntimeException( String.format( + "Metric '%s' requires a numeric vector column, but column '%s' has element type BIT.", + metric, path ) ); + } + + for ( BsonValue value : queryVector ) { + switch ( elementType ) { + case BIT -> { + if ( !value.isBoolean() && !value.isNumber() ) { + throw new GenericRuntimeException( String.format( + "$vectorSearch 'queryVector' element '%s' cannot be interpreted as a bit (expected boolean or 0/1 integer) for column '%s'.", + value, path ) ); + } + boolean b = value.isBoolean() ? value.asBoolean().getValue() : (value.isNumber() && value.asNumber().intValue() != 0); + polyValues.add( new PolyBoolean( b ) ); + } + case INTEGER -> { + if ( !value.isNumber() ) { + throw new GenericRuntimeException( String.format( + "$vectorSearch 'queryVector' element '%s' is not numeric, but column '%s' has element type INTEGER.", + value, path ) ); + } + int i = value.isNumber() ? value.asNumber().intValue() : 0; + polyValues.add( PolyInteger.of( i ) ); + } + case FLOAT -> { + if ( !value.isNumber() ) { + throw new GenericRuntimeException( String.format( + "$vectorSearch 'queryVector' element '%s' is not numeric, but column '%s' has element type FLOAT.", + value, path ) ); + } + float f = value.isNumber() ? (float) value.asNumber().doubleValue() : 0.0f; + polyValues.add( PolyFloat.of( f ) ); + } + } + } + queryVectorRef = builder.makeArray( queryLiteralType, polyValues ); + pathRef = builder.makeCast( queryLiteralType, pathRef ); + + } else { + AlgDataType nullableAny = cluster.getTypeFactory().createTypeWithNullability( + cluster.getTypeFactory().createPolyType( PolyType.ANY ), + true ); + List arr = convertArray( path, queryVector, true, rowType, "queryVector must be an array" ); + queryLiteralType = cluster.getTypeFactory().createArrayType( nullableAny, arr.size() ); + queryVectorRef = DocumentUtil.getArray( arr, queryLiteralType ); + + AlgDataType dynamicArrayType = cluster.getTypeFactory().createArrayType( nullableAny, -1 ); + pathRef = builder.makeCast( dynamicArrayType, pathRef ); + } + + AlgDataType distanceType = cluster.getTypeFactory().createPolyType( PolyType.DOUBLE ); + RexNode distanceCall; + if ( distanceOpName.value2() ) { + // value2 of the tuple == true -> parameterized Distance Operator + // e.g. (e.g. DISTANCE(path, queryVector, 'L2SQUARED')) + RexNode metricLiteral = convertLiteral( new BsonString( metric ) ); + distanceCall = builder.makeCall( distanceType, distanceOperator, Arrays.asList( pathRef, queryVectorRef, metricLiteral ) ); + } else { + // unparameterized version + distanceCall = builder.makeCall( distanceType, distanceOperator, Arrays.asList( pathRef, queryVectorRef ) ); + } + + List projects = new ArrayList<>(); + List projectNames = new ArrayList<>(); + for ( AlgDataTypeField field : node.getTupleType().getFields() ) { + projects.add( builder.makeInputRef( node, field.getIndex() ) ); + projectNames.add( field.getName() ); + } + projects.add( distanceCall ); + String scoreName = "$vectorSearchScore"; + projectNames.add( scoreName ); + + node = LogicalDocumentProject.create( node, projects, projectNames ); + + List names = Collections.singletonList( scoreName ); + List dirs = Collections.singletonList( Direction.ASCENDING ); + List projectionNodes = Collections.singletonList( builder.makeInputRef( node, projects.size() - 1 ) ); + RexNode fetchLimit = convertLiteral( new BsonInt32( limit ) ); + + return LogicalDocumentSort.create( + node, + AlgCollations.of( generateCollation( dirs, names, projectNames ) ), + projectionNodes, + null, + fetchLimit ); + } + + + /** + *

The returned {@link Tuple2} consists of an OperatorName and a boolean flag.

+ *

The flag {@code parameterizedDistance} indicates if the standard {@code DISTANCE(, , [, ])} operator is used or a special unparameterized version.

+ *

e.g. {@code L1_DISTANCE(, )} is the unparameterized version for the standard {@code DISTANCE} with metric='L1'.

+ */ + private static @NotNull Tuple2 getOperatorName( String metric ) { + OperatorName distanceOpName; + boolean parameterizedDistance = false; + switch ( metric ) { + case "L1" -> distanceOpName = OperatorName.L1_DISTANCE; + case "L2" -> distanceOpName = OperatorName.L2_DISTANCE; + case "INNER_PRODUCT" -> distanceOpName = OperatorName.INNER_PRODUCT_DISTANCE; + case "COSINE" -> distanceOpName = OperatorName.COS_DISTANCE; + case "HAMMING" -> distanceOpName = OperatorName.HAMMING_DISTANCE; + case "JACCARD" -> distanceOpName = OperatorName.JACCARD_DISTANCE; + case "L2SQUARED", "CHISQUARED" -> { + distanceOpName = OperatorName.DISTANCE; + parameterizedDistance = true; + } + default -> throw new GenericRuntimeException( String.format( + "Unsupported metric '%s' in $vectorSearch. Supported metrics are: L1, L2, IP, L2SQUARED, CHISQUARED, COSINE, HAMMING, JACCARD.", + metric ) ); + } + return new Tuple2<>( distanceOpName, parameterizedDistance ); + } + + /** * Translates the $replaceRoot or $replaceWith stage of the aggregation pipeline * diff --git a/plugins/neo4j-adapter/src/main/java/org/polypheny/db/adapter/neo4j/rules/NeoGraphRules.java b/plugins/neo4j-adapter/src/main/java/org/polypheny/db/adapter/neo4j/rules/NeoGraphRules.java index 7af752d599..650bdda9a3 100644 --- a/plugins/neo4j-adapter/src/main/java/org/polypheny/db/adapter/neo4j/rules/NeoGraphRules.java +++ b/plugins/neo4j-adapter/src/main/java/org/polypheny/db/adapter/neo4j/rules/NeoGraphRules.java @@ -112,6 +112,13 @@ static boolean supports( LpgProject r ) { } + static boolean supports( LpgFilter r ) { + NeoSupportVisitor visitor = new NeoSupportVisitor(); + r.getCondition().accept( visitor ); + return visitor.isSupports(); + } + + class NeoGraphProjectRule extends NeoConverterRule { public static NeoGraphProjectRule INSTANCE = new NeoGraphProjectRule( LpgProject.class, NeoGraphRules::supports, "NeoGraphProjectRule" ); @@ -138,7 +145,7 @@ public AlgNode convert( AlgNode alg ) { class NeoGraphFilterRule extends NeoConverterRule { - public static NeoGraphFilterRule INSTANCE = new NeoGraphFilterRule( LpgFilter.class, r -> true, "NeoGraphFilterRule" ); + public static NeoGraphFilterRule INSTANCE = new NeoGraphFilterRule( LpgFilter.class, NeoGraphRules::supports, "NeoGraphFilterRule" ); private NeoGraphFilterRule( Class clazz, Predicate supports, String description ) { diff --git a/plugins/neo4j-adapter/src/main/java/org/polypheny/db/adapter/neo4j/util/NeoUtil.java b/plugins/neo4j-adapter/src/main/java/org/polypheny/db/adapter/neo4j/util/NeoUtil.java index a2e16464f1..af256ebaa8 100644 --- a/plugins/neo4j-adapter/src/main/java/org/polypheny/db/adapter/neo4j/util/NeoUtil.java +++ b/plugins/neo4j-adapter/src/main/java/org/polypheny/db/adapter/neo4j/util/NeoUtil.java @@ -431,7 +431,7 @@ static Object fixParameterValue( PolyValue value, NestedPolyType type, boolean i } } if ( value.isList() ) { - if ( isNested ) { + if ( isNested || value.asList().stream().anyMatch( e -> e == null || e.isNull() )) { return value.toTypedJson(); } return value.asList().value.stream().map( e -> fixParameterValue( e, type.asList().types.get( 0 ), true ) ).toList(); diff --git a/plugins/postgres-adapter/build.gradle b/plugins/postgres-adapter/build.gradle index e87572c823..cfeeb30a7e 100644 --- a/plugins/postgres-adapter/build.gradle +++ b/plugins/postgres-adapter/build.gradle @@ -8,10 +8,15 @@ dependencies { implementation group: "net.postgis", name: "postgis-jdbc", version: postgis_version implementation group: "org.postgresql", name: "postgresql", version: postgresql_version // BSD 2-clause + implementation group: "com.pgvector", name: "pgvector", version: pgvector_version // MIT License // --- Test Compile --- + testImplementation project(":core") + testImplementation project(":plugins:sql-language") testImplementation project(path: ":core", configuration: "tests") testImplementation project(path: ":plugins:sql-language", configuration: "tests") + testImplementation project(":plugins:jdbc-adapter-framework") + testImplementation group: "org.mockito", name: "mockito-core", version: mockito_core_version } diff --git a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/PostgresqlSqlDialect.java b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/PostgresqlSqlDialect.java index 5fd94fa109..d5fe7a46be 100644 --- a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/PostgresqlSqlDialect.java +++ b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/PostgresqlSqlDialect.java @@ -21,9 +21,13 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import com.pgvector.PGbit; +import com.pgvector.PGvector; +import lombok.extern.slf4j.Slf4j; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.ParameterExpression; +import org.polypheny.db.adapter.postgres.source.PostgresqlFeature; import org.polypheny.db.algebra.constant.FunctionCategory; import org.polypheny.db.algebra.constant.Kind; import org.polypheny.db.algebra.constant.NullCollation; @@ -31,11 +35,13 @@ import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.algebra.type.AlgDataTypeSystem; import org.polypheny.db.algebra.type.AlgDataTypeSystemImpl; +import org.polypheny.db.languages.OperatorRegistry; import org.polypheny.db.languages.ParserPos; import org.polypheny.db.nodes.TimeUnitRange; import org.polypheny.db.sql.language.SqlBasicCall; import org.polypheny.db.sql.language.SqlCall; import org.polypheny.db.sql.language.SqlDataTypeSpec; +import org.polypheny.db.sql.language.SqlDbFeature; import org.polypheny.db.sql.language.SqlDialect; import org.polypheny.db.sql.language.SqlFunction; import org.polypheny.db.sql.language.SqlIdentifier; @@ -46,10 +52,14 @@ import org.polypheny.db.sql.language.fun.SqlFloorFunction; import org.polypheny.db.sql.language.validate.SqlType; import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; +import org.polypheny.db.type.VectorType.ElementType; +import org.polypheny.db.type.entity.PolyList; +import org.polypheny.db.type.entity.PolyValue; import org.polypheny.db.type.entity.spatial.PolyGeometry; import org.polypheny.db.type.inference.ReturnTypes; - +@Slf4j /** * A SqlDialect implementation for the PostgreSQL database. */ @@ -79,6 +89,13 @@ public int getMaxPrecision( PolyType typeName ) { .withDataTypeSystem( POSTGRESQL_TYPE_SYSTEM ) ); + public PostgresqlSqlDialect() { + this( EMPTY_CONTEXT + .withNullCollation( NullCollation.HIGH ) + .withIdentifierQuoteString( "\"" ) + .withDataTypeSystem( POSTGRESQL_TYPE_SYSTEM )) ; + } + /** * Creates a PostgresqlSqlDialect. */ @@ -107,29 +124,53 @@ public boolean supportsArrays() { @Override public List supportedGeoFunctions() { - return ImmutableList.of( OperatorName.ST_GEOMFROMTEXT, OperatorName.ST_TRANSFORM, OperatorName.ST_EQUALS, - OperatorName.ST_ISSIMPLE, OperatorName.ST_ISCLOSED, OperatorName.ST_ISEMPTY, OperatorName.ST_ISRING, - OperatorName.ST_NUMPOINTS, OperatorName.ST_DIMENSION, OperatorName.ST_LENGTH, OperatorName.ST_AREA, - OperatorName.ST_ENVELOPE, OperatorName.ST_BOUNDARY, OperatorName.ST_CONVEXHULL, OperatorName.ST_CENTROID, - OperatorName.ST_CENTROID, OperatorName.ST_DISJOINT, OperatorName.ST_TOUCHES, OperatorName.ST_INTERSECTS, - OperatorName.ST_CROSSES, OperatorName.ST_WITHIN, OperatorName.ST_CONTAINS, OperatorName.ST_OVERLAPS, - OperatorName.ST_COVERS, OperatorName.ST_COVEREDBY, OperatorName.ST_RELATE, - OperatorName.ST_INTERSECTION, OperatorName.ST_UNION, OperatorName.ST_DIFFERENCE, OperatorName.ST_SYMDIFFERENCE, - OperatorName.ST_X, OperatorName.ST_Y, OperatorName.ST_Z, OperatorName.ST_STARTPOINT, OperatorName.ST_ENDPOINT, - OperatorName.ST_EXTERIORRING, OperatorName.ST_NUMINTERIORRING, OperatorName.ST_INTERIORRINGN, - OperatorName.ST_NUMGEOMETRIES, OperatorName.ST_GEOMETRYN ); + if ( supportsPostGIS() ) { + return ImmutableList.of( OperatorName.ST_GEOMFROMTEXT, OperatorName.ST_TRANSFORM, OperatorName.ST_EQUALS, + OperatorName.ST_ISSIMPLE, OperatorName.ST_ISCLOSED, OperatorName.ST_ISEMPTY, OperatorName.ST_ISRING, + OperatorName.ST_NUMPOINTS, OperatorName.ST_DIMENSION, OperatorName.ST_LENGTH, OperatorName.ST_AREA, + OperatorName.ST_ENVELOPE, OperatorName.ST_BOUNDARY, OperatorName.ST_CONVEXHULL, OperatorName.ST_CENTROID, + OperatorName.ST_CENTROID, OperatorName.ST_DISJOINT, OperatorName.ST_TOUCHES, OperatorName.ST_INTERSECTS, + OperatorName.ST_CROSSES, OperatorName.ST_WITHIN, OperatorName.ST_CONTAINS, OperatorName.ST_OVERLAPS, + OperatorName.ST_COVERS, OperatorName.ST_COVEREDBY, OperatorName.ST_RELATE, + OperatorName.ST_INTERSECTION, OperatorName.ST_UNION, OperatorName.ST_DIFFERENCE, OperatorName.ST_SYMDIFFERENCE, + OperatorName.ST_X, OperatorName.ST_Y, OperatorName.ST_Z, OperatorName.ST_STARTPOINT, OperatorName.ST_ENDPOINT, + OperatorName.ST_EXTERIORRING, OperatorName.ST_NUMINTERIORRING, OperatorName.ST_INTERIORRINGN, + OperatorName.ST_NUMGEOMETRIES, OperatorName.ST_GEOMETRYN ); + } else { + return ImmutableList.of(); + } } @Override public boolean supportsGeoJson() { - return true; + return supportsFeature( PostgresqlFeature.POSTGIS ); } @Override public boolean supportsPostGIS() { - return true; + return supportsFeature( PostgresqlFeature.POSTGIS ); + } + + + @Override + public List supportedKnnFunctions() { + return supportsVector() ? + ImmutableList.of( + OperatorName.L1_DISTANCE, + OperatorName.L2_DISTANCE, + OperatorName.COS_DISTANCE, + OperatorName.INNER_PRODUCT_DISTANCE, + OperatorName.HAMMING_DISTANCE, + OperatorName.JACCARD_DISTANCE ) + : ImmutableList.of(); + } + + + @Override + public boolean supportsVector() { + return supportsFeature( PostgresqlFeature.PGVECTOR ); } @@ -158,6 +199,14 @@ public Expression handleRetrieval( AlgDataType fieldType, Expression child, Para @Override public SqlNode getCastSpec( AlgDataType type ) { + if ( type instanceof VectorType vectorType + && vectorPushdownTypeIsPresent( vectorType.getVectorElementType() )) { + + String typeName = "_" + getTypeString( vectorType.getVectorElementType() ); + return new SqlDataTypeSpec( new SqlIdentifier( typeName, ParserPos.ZERO ), + (int) vectorType.getVectorDimension(), -1, null, null, ParserPos.ZERO ); + } + String castSpec; switch ( type.getPolyType() ) { case TINYINT: @@ -279,10 +328,144 @@ public void unparseCall( SqlWriter writer, SqlCall call, int leftPrec, int right super.unparseCall( writer, call, leftPrec, rightPrec ); } break; - + case L1_DISTANCE: + PostgresqlVectorHelper.unparseAsPgVector( writer, call.operand( 0 ), leftPrec, rightPrec ); + writer.print( " <+> " ); + PostgresqlVectorHelper.unparseAsPgVector( writer, call.operand( 1 ), leftPrec, rightPrec ); + break; + case L2_DISTANCE: + PostgresqlVectorHelper.unparseAsPgVector( writer, call.operand( 0 ), leftPrec, rightPrec ); + writer.print( " <-> " ); + PostgresqlVectorHelper.unparseAsPgVector( writer, call.operand( 1 ), leftPrec, rightPrec ); + break; + case COS_DISTANCE: + PostgresqlVectorHelper.unparseAsPgVector( writer, call.operand( 0 ), leftPrec, rightPrec ); + writer.print( " <=> " ); + PostgresqlVectorHelper.unparseAsPgVector( writer, call.operand( 1 ), leftPrec, rightPrec ); + break; + case HAMMING_DISTANCE: + PostgresqlVectorHelper.unparse( writer, call.operand( 0 ), leftPrec, rightPrec ); + writer.print( " <~> " ); + PostgresqlVectorHelper.unparse( writer, call.operand( 1 ), leftPrec, rightPrec ); + break; + case JACCARD_DISTANCE: + PostgresqlVectorHelper.unparse( writer, call.operand( 0 ), leftPrec, rightPrec ); + writer.print( " <%> " ); + PostgresqlVectorHelper.unparse( writer, call.operand( 1 ), leftPrec, rightPrec ); + break; + case INNER_PRODUCT_DISTANCE: + PostgresqlVectorHelper.unparseAsPgVector( writer, call.operand( 0 ), leftPrec, rightPrec ); + writer.print( " <#> " ); + PostgresqlVectorHelper.unparseAsPgVector( writer, call.operand( 1 ), leftPrec, rightPrec ); + break; default: super.unparseCall( writer, call, leftPrec, rightPrec ); } } + + /** + * Bypasses the default {@code getArray()} path because the PostgreSQL driver returns a PGobject + * instead of a standard java.sql.Array for pgvector columns. + */ + @Override + public Optional getCustomArrayRetrievalExpression( ParameterExpression resultSet, int i, AlgDataType fieldType ) { + if ( fieldType.getPolyType() != PolyType.ARRAY || !(fieldType instanceof VectorType vectorType)) { + return Optional.empty(); + } + if ( vectorType.getVectorElementType() == ElementType.BIT ) { + Expression object = Expressions.call( resultSet, "getString", Expressions.constant( i + 1 ) ); + return Optional.of( Expressions.call( PostgresqlVectorHelper.class, "parseVector", object ) ); + } + if ( !supportsVector() ) { + return Optional.empty(); + } + + Expression object = Expressions.call( resultSet, "getObject", Expressions.constant( i + 1 ) ); + return Optional.of( Expressions.call( PostgresqlVectorHelper.class, "parseVector", object ) ); + } + + + @Override + public boolean supportsFeature( SqlDbFeature feature ) { + return supportedFeatures.contains( feature ); + } + + + @Override + public void initializeConnection( java.sql.Connection conn ) throws java.sql.SQLException { + PGbit.registerType( conn ); + if ( supportsVector() ) { + PGvector.registerTypes( conn ); + } + if ( supportsPostGIS() ) { + org.postgresql.PGConnection pgConn = conn.unwrap( org.postgresql.PGConnection.class ); + pgConn.addDataType( "geometry", net.postgis.jdbc.PGgeometry.class ); + } + + } + + + @Override + public boolean vectorPushdownTypeIsPresent( VectorType.ElementType vectorType ) { + return switch ( vectorType ) { + case FLOAT -> supportsVector(); + case BIT -> true; + default -> false; + }; + } + + + @Override + public Object getVectorDbObject( VectorType.ElementType vectorType, PolyList vectorAsList ) { + return switch ( vectorType ) { + case FLOAT -> { + float[] fa = new float[vectorAsList.size()]; + for ( int i = 0; i < vectorAsList.size(); ++i ) fa[i] = vectorAsList.get( i ).asNumber().floatValue(); + yield new PGvector( fa ); + } + case BIT -> { + boolean[] ba = new boolean[vectorAsList.size()]; + for ( int i = 0; i < vectorAsList.size(); ++i ) { + PolyValue val = vectorAsList.get( i ); + ba[i] = (val != null && !val.isNull() && val.asBoolean().getValue() != null + && val.asBoolean().getValue()); + } + yield new PGbit( ba ); + } + case DOUBLE, INTEGER -> null; + }; + } + + + @Override + public String getTypeString( VectorType.ElementType vectorType ) { + return switch ( vectorType ) { + case FLOAT -> "vector"; + case BIT -> "bit"; + case DOUBLE, INTEGER -> throw new UnsupportedOperationException("Vectors of type " + vectorType + + " are not supported by PG and do therefore not have a dedicated type string"); + }; + } + + + @Override + public SqlNode getVectorLiteral( VectorType vectorType, PolyList vectorAsList, ParserPos pos ) { + if ( vectorType.getVectorElementType() == ElementType.BIT ) { + StringBuilder sb = new StringBuilder(); + for ( PolyValue val : vectorAsList ) { + if ( val == null || val.isNull() || val.asBoolean().getValue() == null ) { + throw new RuntimeException( "Vector cannot contain null elements." ); + } + sb.append( (val.asBoolean().getValue() ? "1" : "0") ); + } + return (SqlNode) OperatorRegistry.get( OperatorName.CAST ).createCall( + pos, + SqlLiteral.createCharString( sb.toString(), pos ), + getCastSpec( vectorType ) + ); + } + return null; + } + } diff --git a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/PostgresqlVectorHelper.java b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/PostgresqlVectorHelper.java new file mode 100644 index 0000000000..633f8037bb --- /dev/null +++ b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/PostgresqlVectorHelper.java @@ -0,0 +1,107 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.postgres; + +import com.pgvector.PGbit; +import com.pgvector.PGhalfvec; +import com.pgvector.PGsparsevec; +import com.pgvector.PGvector; +import lombok.extern.slf4j.Slf4j; +import org.polypheny.db.algebra.constant.Kind; +import org.polypheny.db.sql.language.SqlCall; +import org.polypheny.db.sql.language.SqlDynamicParam; +import org.polypheny.db.sql.language.SqlNode; +import org.polypheny.db.sql.language.SqlWriter; +import org.polypheny.db.type.entity.PolyBoolean; +import org.polypheny.db.type.entity.PolyValue; +import org.polypheny.db.type.entity.numerical.PolyFloat; +import java.util.ArrayList; +import java.util.List; + +@Slf4j +public class PostgresqlVectorHelper { + + private PostgresqlVectorHelper() {} + + + public static void unparseAsPgVector( SqlWriter writer, SqlNode operand, int leftPrec, int rightPrec ) { + if ( operand instanceof SqlCall castCall && castCall.getKind() == Kind.CAST ) { + operand = castCall.operand( 0 ); + } + operand.unparse( writer, leftPrec, rightPrec ); + if ( operand instanceof SqlDynamicParam ) { + writer.print( "::float4[]::vector " ); + } else { + writer.print( "::vector " ); + } + } + + + public static void unparse( SqlWriter writer, SqlNode operand, int leftPrec, int rightPrec ) { + if ( operand instanceof SqlCall castCall && castCall.getKind() == Kind.CAST ) { + operand = castCall.operand( 0 ); + } + operand.unparse( writer, leftPrec, rightPrec ); + } + + + /** + * + * @param dbObject database Object that represents a vector. + * @return {@code List} representation of the vector. + * + *

+ * Possible PolyValue objects: + *

    + *
  • {@link PolyFloat},
    • + *
  • {@link PolyBoolean}
    • + *
+ *

+ */ + public static List parseVector( Object dbObject ) { + float[] vector = null; + boolean[] bitvector = null; + if ( dbObject instanceof PGvector vec ) { + vector = vec.toArray(); + } else if ( dbObject instanceof PGhalfvec vec ) { + vector = vec.toArray(); + } else if ( dbObject instanceof PGsparsevec vec ) { + vector = vec.toArray(); + } else if ( dbObject instanceof PGbit vec ) { + bitvector = vec.toArray(); + } else if ( dbObject instanceof String s ) { + bitvector = new boolean[s.length()]; + for ( int j = 0; j < s.length(); ++j ) { + bitvector[j] = s.charAt( j ) == '1'; + } + } + if ( vector != null ) { + List list = new ArrayList<>( vector.length ); + for ( float f : vector ) list.add( PolyFloat.of( f ) ); + return list; + } + if ( bitvector != null ) { + List list = new ArrayList<>( bitvector.length ); + for ( boolean b : bitvector ) list.add( PolyBoolean.of( b ) ); + return list; + } + log.warn( "Was not able to parse the vector object." ); + return null; + } + + +} diff --git a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlCatalogQueries.java b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlCatalogQueries.java new file mode 100644 index 0000000000..f58c813a52 --- /dev/null +++ b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlCatalogQueries.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.postgres.source; + +/** + * Utility class holding raw SQL queries used to interrogate the PostgreSQL system catalog. + */ +public final class PostgresqlCatalogQueries { + + /** + SQL to query postgres system catalog attribute modifier count. + a.attnum > 0: filters out hidden system columns with attnum < 0 + a.attisdropped: marked but not yet removed columns + a.atttypmod > 0: attribute type modifier was used i.e. vector(atttymod), otherwise atttypmod = -1 + */ + public static final String SQL_COLUMN_TYPE_MODIFIERS_AND_ATTR_DIMENSIONS = """ + SELECT a.attname, a.atttypmod, a.attndims + FROM pg_attribute a + JOIN pg_class c ON a.attrelid = c.oid + JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.relname = ? AND n.nspname = ? + AND a.attnum > 0 AND NOT a.attisdropped + AND (a.attndims > 0 OR a.atttypmod > 0) + """; + + + /** + * Retrieves a list of all currently installed and active extensions in the database. + */ + public static final String SQL_INSTALLED_EXTENSIONS = """ + SELECT extname + FROM pg_extension + WHERE extname = ANY(?) + """; +} diff --git a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlFeature.java b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlFeature.java new file mode 100644 index 0000000000..ee05429feb --- /dev/null +++ b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlFeature.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.postgres.source; + +import org.polypheny.db.sql.language.SqlDbFeature; +import org.polypheny.db.sql.language.SqlDialect; +import java.util.function.Predicate; + +public enum PostgresqlFeature implements SqlDbFeature { + + PGVECTOR( "vector", "pgvector", SqlDialect::supportsVector ), + + POSTGIS( "postgis", "PostGIS", SqlDialect::supportsPostGIS ); + + /** + * Name as it appears in {@code pg_extension.extname}. + */ + private final String name; + private final String displayName; + private final Predicate supportCheck; + + PostgresqlFeature( String name, String displayName, Predicate supportCheck ) { + this.name = name; + this.displayName = displayName; + this.supportCheck = supportCheck; + } + + + @Override + public String featureName() { + return name; + } + + + @Override + public String displayName() { + return displayName; + } + + + @Override + public boolean isSupported( SqlDialect dialect ) { + return this.supportCheck.test( dialect ); + } + + + @Override + public String getFeatureRegistrationQuery() { + return "CREATE EXTENSION IF NOT EXISTS \"" + this.name + "\""; + } +} diff --git a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlSource.java b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlSource.java index 5ca93ba4ba..eda841abaf 100644 --- a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlSource.java +++ b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/source/PostgresqlSource.java @@ -17,9 +17,19 @@ package org.polypheny.db.adapter.postgres.source; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.polypheny.db.adapter.DeployMode; import org.polypheny.db.adapter.RelationalDataSource; @@ -27,8 +37,17 @@ import org.polypheny.db.adapter.annotations.AdapterSettingInteger; import org.polypheny.db.adapter.annotations.AdapterSettingList; import org.polypheny.db.adapter.annotations.AdapterSettingString; +import org.polypheny.db.adapter.jdbc.connection.ConnectionHandler; +import org.polypheny.db.adapter.jdbc.connection.ConnectionHandlerException; import org.polypheny.db.adapter.jdbc.sources.AbstractJdbcSource; import org.polypheny.db.adapter.postgres.PostgresqlSqlDialect; +import org.polypheny.db.sql.language.SqlDbFeature; +import org.polypheny.db.transaction.PUID; +import org.polypheny.db.transaction.PolyXid; +import org.polypheny.db.type.PolyType; + +import static org.polypheny.db.adapter.postgres.source.PostgresqlCatalogQueries.SQL_COLUMN_TYPE_MODIFIERS_AND_ATTR_DIMENSIONS; +import static org.polypheny.db.adapter.postgres.source.PostgresqlCatalogQueries.SQL_INSTALLED_EXTENSIONS; @Slf4j @@ -62,8 +81,19 @@ public PostgresqlSource( final long storeId, final String uniqueName, final Map< settings, mode, "org.postgresql.Driver", - PostgresqlSqlDialect.DEFAULT, + new PostgresqlSqlDialect(), false ); + try { + PolyXid xid = PolyXid.generateLocalTransactionIdentifier( PUID.EMPTY_PUID, PUID.EMPTY_PUID ); + ConnectionHandler connectionHandler = connectionFactory.getOrCreateConnectionHandler( xid ); + try ( Statement statement = connectionHandler.getStatement() ) { + Connection connection = statement.getConnection(); + Set features = detectFeatures( connection ); + dialect.addSupportedFeatures( features ); + } + } catch ( SQLException | ConnectionHandlerException e) { + log.error( "Could not query feature information.", e ); + } } @@ -101,4 +131,101 @@ public RelationalDataSource asRelationalDataSource() { return this; } + + /** + * {@inheritDoc} + * + *

Handled type names: + *

    + *
  • {@code vector, halfvec} - pgvector float4 and float2 vector, mapped to {@code + ARRAY}
    • + *
  • {@code bit} - bitvectors mappte to {@code ARRAY}
    • + *
  • {@code _float4} - PostgreSQL float4 array, mapped to {@code + ARRAY}
    • + *
  • {@code _float8} - PostgreSQL float8 array, mapped to {@code + ARRAY}
    • + *
  • {@code _int4} - PostgreSQL int4 array, mapped to {@code + ARRAY}
    • + *
  • {@code _int8} - PostgreSQL int8 array, mapped to {@code + ARRAY}
    • + *
+ *

Note: PostgreSQL has no enforced array size limits. We therefore only detect the specified (but not enforced) dimensions.

+ * @see PostgreSQL Arrays Documentation + */ + @Override + protected Optional resolveNativeColumnType( Map metadata, String typeName, ResultSet columnRow ) throws SQLException { + CollectionMetadata meta = metadata.get( columnRow.getString( "COLUMN_NAME" ).toLowerCase() ); + return switch ( typeName ) { + case "vector", "halfvec", "sparsevec" -> Optional.of( new ColumnTypeInfo( PolyType.REAL, PolyType.ARRAY, + null, null, 1, meta != null ? meta.typeModifier() : null) ); + case "bit" -> Optional.of( new ColumnTypeInfo( PolyType.BOOLEAN, PolyType.ARRAY, + null, null, 1, meta != null ? meta.typeModifier() : null) ); + case "_float4" -> Optional.of( new ColumnTypeInfo( PolyType.REAL, PolyType.ARRAY, + null, null, arrayDim( meta ), null ) ); + case "_float8" -> Optional.of( new ColumnTypeInfo( PolyType.DOUBLE, PolyType.ARRAY, + null, null, arrayDim( meta ), null ) ); + case "_int4" -> Optional.of( new ColumnTypeInfo( PolyType.INTEGER, PolyType.ARRAY, + null, null, arrayDim( meta ), null ) ); + case "_int8" -> Optional.of( new ColumnTypeInfo( PolyType.BIGINT, PolyType.ARRAY, + null, null, arrayDim( meta ), null ) ); + case "_bool" -> Optional.of( new ColumnTypeInfo( PolyType.BOOLEAN, PolyType.ARRAY, + null, null, arrayDim( meta ), null ) ); + default -> Optional.empty(); + }; + } + + + private int arrayDim( CollectionMetadata meta ) { + return (meta != null && meta.arrayDimensions() > 0) ? meta.arrayDimensions() : -1; + } + + + @Override + public boolean isNativeVectorType( String typeName ) { + return (typeName.equals( "vector" ) + || typeName.equals( "bit" )) + || typeName.equals( "halfvec" ) + || typeName.equals( "sparsevec" ); + } + + + @Override + protected Map fetchColumnMetadata( Connection conn, String schema, String table ) throws SQLException { + Map result = new HashMap<>(); + try ( PreparedStatement ps = conn.prepareStatement( SQL_COLUMN_TYPE_MODIFIERS_AND_ATTR_DIMENSIONS ) ) { + ps.setString( 1, table ); + ps.setString( 2, schema ); + try ( ResultSet rs = ps.executeQuery() ) { + while ( rs.next() ) { + String col = rs.getString( "attname" ); + int dims = rs.getInt( "attndims" ); + int rawMod = rs.getInt( "atttypmod" ); + Integer typeMod = rs.wasNull() ? null : rawMod; + result.put( col, new CollectionMetadata( dims, typeMod ) ); + log.debug( "Column metadata: {} -> dims={}, typeMod={}", col, dims, typeMod ); + } + } + } + return result; + } + + + public static Set detectFeatures( Connection conn ) throws SQLException { + Set found = EnumSet.noneOf( PostgresqlFeature.class ); + PreparedStatement ps = conn.prepareStatement( SQL_INSTALLED_EXTENSIONS ); + String[] featureNames = Arrays.stream( PostgresqlFeature.values() ) + .map( PostgresqlFeature::featureName ) + .toArray( String[]::new ); + ps.setArray( 1, conn.createArrayOf( "text", featureNames ) ); + ResultSet rs = ps.executeQuery(); + while ( rs.next() ) { + String name = rs.getString( 1 ); + Arrays.stream( PostgresqlFeature.values() ) + .filter( f -> f.featureName().equals( name ) ) + .findFirst() + .ifPresent( found::add ); + } + return Collections.unmodifiableSet( found ); + } + } diff --git a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/store/PostgresqlImageVariant.java b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/store/PostgresqlImageVariant.java new file mode 100644 index 0000000000..076469701c --- /dev/null +++ b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/store/PostgresqlImageVariant.java @@ -0,0 +1,41 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.postgres.store; + +import org.polypheny.db.adapter.postgres.source.PostgresqlFeature; +import org.polypheny.db.sql.language.SqlDbFeature; +import java.util.Set; + +public enum PostgresqlImageVariant { + + DEFAULT ( "polypheny/postgres:17-debian", Set.of() ), + + PGVECTOR ( "polypheny/postgres-pgvector:17-debian", Set.of( PostgresqlFeature.PGVECTOR ) ), + + POSTGIS ( "polypheny/postgres-postgis:17-debian", Set.of( PostgresqlFeature.POSTGIS ) ), + + PGVECTOR_POSTGIS( "polypheny/postgres-pgvector-postgis:17-debian", Set.of( PostgresqlFeature.PGVECTOR, PostgresqlFeature.POSTGIS ) ); + + public final String imageName; + public final Set features; + + PostgresqlImageVariant( String imageName, Set features ) { + this.imageName = imageName; + this.features = features; + } + +} diff --git a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/store/PostgresqlStore.java b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/store/PostgresqlStore.java index 7ef188cc5e..6669aac92d 100644 --- a/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/store/PostgresqlStore.java +++ b/plugins/postgres-adapter/src/main/java/org/polypheny/db/adapter/postgres/store/PostgresqlStore.java @@ -22,6 +22,7 @@ import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import lombok.Getter; @@ -30,7 +31,9 @@ import org.polypheny.db.adapter.DeployMode; import org.polypheny.db.adapter.DeployMode.DeploySetting; import org.polypheny.db.adapter.annotations.AdapterProperties; +import org.polypheny.db.adapter.annotations.AdapterSettingsPreset; import org.polypheny.db.adapter.annotations.AdapterSettingInteger; +import org.polypheny.db.adapter.annotations.AdapterSettingList; import org.polypheny.db.adapter.annotations.AdapterSettingString; import org.polypheny.db.adapter.jdbc.connection.ConnectionFactory; import org.polypheny.db.adapter.jdbc.connection.ConnectionHandler; @@ -38,6 +41,10 @@ import org.polypheny.db.adapter.jdbc.connection.TransactionalConnectionFactory; import org.polypheny.db.adapter.jdbc.stores.AbstractJdbcStore; import org.polypheny.db.adapter.postgres.PostgresqlSqlDialect; +import org.polypheny.db.adapter.postgres.source.PostgresqlFeature; +import org.polypheny.db.adapter.postgres.source.PostgresqlSource; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeFactory; import org.polypheny.db.catalog.entity.allocation.AllocationTable; import org.polypheny.db.catalog.entity.logical.LogicalColumn; import org.polypheny.db.catalog.entity.logical.LogicalIndex; @@ -46,6 +53,7 @@ import org.polypheny.db.catalog.entity.physical.PhysicalEntity; import org.polypheny.db.catalog.entity.physical.PhysicalTable; import org.polypheny.db.catalog.exceptions.GenericRuntimeException; +import org.polypheny.db.catalog.logistic.IndexCategory; import org.polypheny.db.docker.DockerContainer; import org.polypheny.db.docker.DockerContainer.HostAndPort; import org.polypheny.db.docker.DockerInstance; @@ -57,6 +65,7 @@ import org.polypheny.db.transaction.PolyXid; import org.polypheny.db.type.PolyType; import org.polypheny.db.type.PolyTypeFamily; +import org.polypheny.db.type.VectorType; import org.polypheny.db.util.PasswordGenerator; @@ -78,6 +87,26 @@ description = "Password to be used for authenticating at the remote instance.", appliesTo = DeploySetting.REMOTE) @AdapterSettingInteger(name = "maxConnections", defaultValue = 25, position = 6, description = "Maximum number of concurrent JDBC connections.") +@AdapterSettingList( + name = "imageVariant", + options = { "Default", "pgvector", "PostGIS", "pgvector & PostGIS" }, + defaultValue = "pgvector & PostGIS", + position = 7, + description = "PostgreSQL Docker image variant to deploy.", + appliesTo = DeploySetting.DOCKER +) +@AdapterSettingsPreset( + name = "Minimal PostgreSQL", + description = "Plain Docker image, no extensions", + mode = DeployMode.DOCKER, + settings = { @AdapterSettingsPreset.Setting(name = "imageVariant", value = "Default") } +) +@AdapterSettingsPreset( + name = "Full PostgreSQL", + description = "Docker image with pgvector & PostGIS extensions installed", + mode = DeployMode.DOCKER, + settings = { @AdapterSettingsPreset.Setting(name = "imageVariant", value = "pgvector & PostGIS") } +) public class PostgresqlStore extends AbstractJdbcStore { @@ -89,7 +118,7 @@ public class PostgresqlStore extends AbstractJdbcStore { public PostgresqlStore( final long storeId, final String uniqueName, final Map settings, final DeployMode mode ) { - super( storeId, uniqueName, settings, mode, PostgresqlSqlDialect.DEFAULT, true ); + super( storeId, uniqueName, settings, mode, new PostgresqlSqlDialect(), true ); } @@ -102,16 +131,17 @@ public ConnectionFactory deployDocker( int instanceId ) { database = "postgres"; username = "postgres"; + PostgresqlImageVariant variant = PostgresqlImageVariant.valueOf( + settings.getOrDefault( "imageVariant", PostgresqlImageVariant.PGVECTOR_POSTGIS.name() ).toUpperCase().replace( " & ", "_" ) ); if ( settings.getOrDefault( "deploymentId", "" ).isEmpty() ) { if ( settings.getOrDefault( "password", "polypheny" ).equals( "polypheny" ) ) { settings.put( "password", PasswordGenerator.generatePassword() ); updateSettings( settings ); } - DockerInstance instance = DockerManager.getInstance().getInstanceById( instanceId ) .orElseThrow( () -> new GenericRuntimeException( "No docker instance with id " + instanceId ) ); try { - container = instance.newBuilder( "polypheny/postgres:latest", getUniqueName() ) + container = instance.newBuilder( variant.imageName, getUniqueName() ) .withEnvironmentVariable( "POSTGRES_PASSWORD", settings.get( "password" ) ) .createAndStart(); } catch ( IOException e ) { @@ -135,6 +165,7 @@ public ConnectionFactory deployDocker( int instanceId ) { throw new GenericRuntimeException( "Could not connect to container" ); } } + dialect.addSupportedFeatures( variant.features ); return createConnectionFactory(); } @@ -149,7 +180,19 @@ protected ConnectionFactory deployRemote() { if ( !testConnection() ) { throw new GenericRuntimeException( "Unable to connect" ); } - return createConnectionFactory(); + ConnectionFactory factory = createConnectionFactory(); + try { + PolyXid xid = PolyXid.generateLocalTransactionIdentifier( PUID.EMPTY_PUID, PUID.EMPTY_PUID ); + ConnectionHandler handler = factory.getOrCreateConnectionHandler( xid ); + try ( java.sql.Statement statement = handler.getStatement() ) { + java.sql.Connection connection = statement.getConnection(); + java.util.Set features = PostgresqlSource.detectFeatures( connection ); + dialect.addSupportedFeatures( features ); + } + } catch ( ConnectionHandlerException | SQLException e ) { + log.error( "Could not query feature information on remote PostgreSQL store.", e ); + } + return factory; } @@ -187,8 +230,45 @@ public void createUdfs() { } + /** + *

Generally the docker images already have the extension registered. + * This is merely used as a safeguard.

+ */ + @Override + public void registerFeatures() { + PolyXid xid = PolyXid.generateLocalTransactionIdentifier( PUID.randomPUID( Type.CONNECTION ), PUID.randomPUID( Type.CONNECTION ) ); + try { + ConnectionHandler ch = connectionFactory.getOrCreateConnectionHandler( xid ); + for ( PostgresqlFeature f : PostgresqlFeature.values() ) { + if ( f.isSupported( dialect ) ) { + ch.executeUpdate( f.getFeatureRegistrationQuery() ); + } + } + ch.commit(); + } catch ( ConnectionHandlerException | SQLException e ) { + log.error( "Error while registering features (CREATE EXTENSION) on Postgres", e ); + } + } + + @Override public void updateColumnType( Context context, long allocId, LogicalColumn newCol ) { + PhysicalColumn old = adapterCatalog.getColumn( newCol.id, allocId ); + AlgDataType oldAlg = old.getAlgDataType( AlgDataTypeFactory.DEFAULT ); + AlgDataType newAlg = newCol.getAlgDataType( AlgDataTypeFactory.DEFAULT ); + + if ( oldAlg instanceof VectorType oldVec + && newAlg instanceof VectorType newVec + && dialect.vectorPushdownTypeIsPresent( oldVec.getVectorElementType() ) + && dialect.vectorPushdownTypeIsPresent( newVec.getVectorElementType() ) + && oldVec.getVectorElementType() != VectorType.ElementType.BIT + && oldVec.getCardinality() != newVec.getCardinality() ) { + throw new GenericRuntimeException( + "Cannot change dimension of vector(%d) to vector(%d) on PostgreSQL. " + + "pgvector does not support resizing float vector dimensions. " + + "Drop and recreate the column.", + oldVec.getCardinality(), newVec.getCardinality() ); + } PhysicalColumn column = adapterCatalog.updateColumnType( allocId, newCol ); PhysicalTable physicalTable = adapterCatalog.fromAllocation( allocId ); @@ -199,26 +279,35 @@ public void updateColumnType( Context context, long allocId, LogicalColumn newCo .append( "." ) .append( dialect.quoteIdentifier( physicalTable.name ) ); builder.append( " ALTER COLUMN " ).append( dialect.quoteIdentifier( column.name ) ); - builder.append( " TYPE " ).append( getTypeString( column.type ) ); - if ( column.collectionsType != null ) { - builder.append( " " ).append( column.collectionsType ); - } - if ( column.length != null && doesTypeUseLength( column.type ) ) { - builder.append( "(" ); - builder.append( column.length ); - if ( column.scale != null ) { - builder.append( "," ).append( column.scale ); + + AlgDataType algType = column.getAlgDataType( AlgDataTypeFactory.DEFAULT ); + String typeString; + if ( algType instanceof VectorType vectorType && dialect.vectorPushdownTypeIsPresent( vectorType.getVectorElementType() ) ) { + typeString = dialect.getTypeString( vectorType.getVectorElementType() ) + + "(" + (column.cardinality != null && column.cardinality > 0 ? column.cardinality : "") + ")"; + } else { + StringBuilder typeBuilder = new StringBuilder(); + typeBuilder.append( getTypeString( column.type ) ); + if ( column.collectionsType != null ) { + typeBuilder.append( " " ).append( column.collectionsType ); } - builder.append( ")" ); + if ( column.length != null && doesTypeUseLength( column.type ) ) { + typeBuilder.append( "(" ); + typeBuilder.append( column.length ); + if ( column.scale != null ) { + typeBuilder.append( "," ).append( column.scale ); + } + typeBuilder.append( ")" ); + } + typeString = typeBuilder.toString(); } + + builder.append( " TYPE " ).append( typeString ); builder.append( " USING " ) .append( dialect.quoteIdentifier( column.name ) ) .append( "::" ) - .append( getTypeString( column.type ) ); + .append( typeString ); - if ( column.collectionsType != null ) { - builder.append( " " ).append( column.collectionsType ); - } executeUpdate( builder, context ); updateNativePhysical( allocId ); @@ -261,6 +350,14 @@ public String addIndex( Context context, LogicalIndex index, AllocationTable all case "brin": builder.append( "brin" ); break; + case "hnsw": + builder.append( "hnsw " ); + break; + case "ivfflat": + builder.append( "ivfflat " ); + break; + default: + throw new GenericRuntimeException( "Unknown index method: " + index.method ); } builder.append( "(" ); @@ -272,7 +369,43 @@ public String addIndex( Context context, LogicalIndex index, AllocationTable all first = false; builder.append( dialect.quoteIdentifier( getPhysicalColumnName( columnId ) ) ).append( " " ); } + String metric = index.options.getOrDefault( "metric", "L2" ).toUpperCase(); + if ( index.method.equals( "hnsw" ) ) { + String operatorClass = switch ( metric ) { + case "L1" -> "vector_l1_ops"; + case "L2" -> "vector_l2_ops"; + case "COSINE" -> "vector_cosine_ops"; + case "INNER_PRODUCT" -> "vector_ip_ops"; + case "HAMMING" -> "bit_hamming_ops"; + case "JACCARD" -> "bit_jaccard_ops"; + default -> throw new GenericRuntimeException( "Unsupported distance metric for pgvector HNSW indexes: " + metric); + }; + builder.append( " " ).append( operatorClass ); + } else if ( index.method.equals( "ivfflat" ) ) { + String operatorClass = switch ( metric ) { + case "L2" -> "vector_l2_ops"; + case "COSINE" -> "vector_cosine_ops"; + case "INNER_PRODUCT" -> "vector_ip_ops"; + case "HAMMING" -> "bit_hamming_ops"; + default -> throw new GenericRuntimeException( "Unsupported distance metric for pgvector HNSW indexes: " + metric); + }; + builder.append( " " ).append( operatorClass ); + } builder.append( ")" ); + if ( index.method.equals( "hnsw" ) ) { + List params = new ArrayList<>(); + if ( index.options.containsKey( "m" ) ) { + params.add( "m = " + Integer.parseInt( index.options.get( "m" ) ) ); + } + if ( index.options.containsKey( "ef_construction" ) ) { + params.add( "ef_construction = " + Integer.parseInt( index.options.get( "ef_construction" ) ) ); + } + if ( !params.isEmpty() ) { + builder.append( " WITH (" ).append( String.join( ", ", params ) ).append( ")" ); + } + } else if ( index.method.equals( "ivfflat" ) && index.options.containsKey( "lists" ) ) { + builder.append( " WITH (lists = " ).append( Integer.parseInt( index.options.get( "lists" ) ) ).append( ")" ); + } executeUpdate( builder, context ); @@ -283,22 +416,77 @@ public String addIndex( Context context, LogicalIndex index, AllocationTable all @Override public void dropIndex( Context context, LogicalIndex index, long allocId ) { PhysicalTable table = adapterCatalog.fromAllocation( allocId ); + String physicalIndexName = getPhysicalIndexName( table.id, index.id ); StringBuilder builder = new StringBuilder(); builder.append( "DROP INDEX " ); - builder.append( dialect.quoteIdentifier( index.physicalName + "_" + table.id ) ); + builder.append( dialect.quoteIdentifier( physicalIndexName ) ); executeUpdate( builder, context ); } @Override public List getAvailableIndexMethods() { - return ImmutableList.of( - new IndexMethodModel( "btree", "B-TREE" ), - new IndexMethodModel( "hash", "HASH" ), - new IndexMethodModel( "gin", "GIN (Generalized Inverted Index)" ), - new IndexMethodModel( "brin", "BRIN (Block Range index)" ) + List methods = new ArrayList<>( List.of( + new IndexMethodModel( "btree", "B-TREE" ), + new IndexMethodModel( "hash", "HASH" ), + new IndexMethodModel( "gin", "GIN (Generalized Inverted Index)" ), + new IndexMethodModel( "brin", "BRIN (Block Range index)" ) ) ); + + if ( dialect.supportsVector() ) { + List hnswParams = List.of( + new IndexParameterModel( + "metric", + "Distance Metric", + "ENUM", + List.of( "L1", "L2", "COSINE", "INNER_PRODUCT", "JACCARD", "HAMMING" ), + "L2" ), + new IndexParameterModel( + "m", + "Max number of connections per layer (m)", + "INTEGER", + null, + "16" + ), + new IndexParameterModel( + "ef_construction", + "Size of the dynamic candidate list for constructing the graph (efConstruction)", + "INTEGER", + null, + "64" + ) + ); + List ivfflatParams = List.of( + new IndexParameterModel( + "metric", + "Distance Metric", + "ENUM", + List.of( "L2", "COSINE", "INNER_PRODUCT", "HAMMING"), + "L2" + ), + new IndexParameterModel( + "lists", + "Number of lists the vectors are divided into", + "INTEGER", + null, + "100" + ) + ); + methods.add( new IndexMethodModel( + "hnsw", + "HNSW (Hierarchical Navigable Small World)", + IndexCategory.VECTOR, + hnswParams + ) ); + methods.add( new IndexMethodModel( + "ivfflat", + "IVFFlat (Inverted File Flat)", + IndexCategory.VECTOR, + ivfflatParams + ) ); + } + return ImmutableList.copyOf( methods ); } @@ -336,7 +524,10 @@ protected String getTypeString( PolyType type ) { case DECIMAL -> "DECIMAL"; case VARCHAR -> "VARCHAR"; case JSON, TEXT -> "TEXT"; - case GEOMETRY -> "GEOMETRY"; + case GEOMETRY -> { + if ( !dialect.supportsPostGIS() ) throw new GenericRuntimeException( "GEOMETRY type requires PostGIS" ); + yield "GEOMETRY"; + } case DATE -> "DATE"; case TIME -> "TIME"; case TIMESTAMP -> "TIMESTAMP"; diff --git a/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/dialect/PostgresqlSqlDialectTest.java b/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/dialect/PostgresqlSqlDialectTest.java new file mode 100644 index 0000000000..aef62c70e2 --- /dev/null +++ b/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/dialect/PostgresqlSqlDialectTest.java @@ -0,0 +1,230 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.postgres.dialect; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.polypheny.db.adapter.postgres.source.PostgresqlFeature.PGVECTOR; + +import java.sql.Array; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Optional; +import java.util.Set; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.ParameterExpression; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.polypheny.db.adapter.postgres.PostgresqlSqlDialect; +import org.polypheny.db.adapter.postgres.source.PostgresqlFeature; +import org.polypheny.db.adapter.postgres.source.PostgresqlSource; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.algebra.type.AlgDataTypeFactory; +import org.polypheny.db.sql.language.SqlDbFeature; +import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; +import org.polypheny.db.util.PolyphenyHomeDirManager; +import org.polypheny.db.util.RunMode; + +public class PostgresqlSqlDialectTest { + + @BeforeAll + static void init() { + if ( PolyphenyHomeDirManager.getMode() == null ) { + PolyphenyHomeDirManager.setModeAndGetInstance( RunMode.TEST ); + } + } + + + // ---- detectFeatures ---------------------------------------------------------------- + + @Test + void returnsEmptyWhenNoExtensionInstalled() throws SQLException { + Connection conn = mockConnection( false ); + assertTrue( PostgresqlSource.detectFeatures( conn ).isEmpty() ); + } + + + @Test + void detectsPgVectorExtensionWhenPresent() throws SQLException { + Connection conn = mockConnectionWithExtensions( "vector" ); + Set features = PostgresqlSource.detectFeatures( conn ); + assertTrue( features.contains( PGVECTOR ) ); + assertFalse( features.contains( PostgresqlFeature.POSTGIS ) ); + } + + + @Test + void detectsPostgisExtensionWhenPresent() throws SQLException { + Connection conn = mockConnectionWithExtensions( "postgis" ); + Set features = PostgresqlSource.detectFeatures( conn ); + assertTrue( features.contains( PostgresqlFeature.POSTGIS ) ); + assertFalse( features.contains( PGVECTOR ) ); + } + + + @Test + void detectsAllFeaturesWhenBothPresent() throws SQLException { + Connection conn = mockConnectionWithExtensions( "postgis", "vector" ); + Set features = PostgresqlSource.detectFeatures( conn ); + assertTrue( features.contains( PostgresqlFeature.POSTGIS ) ); + assertTrue( features.contains( PGVECTOR ) ); + } + + + @Test + void detectedFeaturesSetIsImmutable() throws SQLException { + Set features = PostgresqlSource.detectFeatures( mockConnection( false ) ); + assertThrows( UnsupportedOperationException.class, () -> features.add( PGVECTOR ) ); + } + + + @Test + void dialectReflectsDetectedFeatures() throws SQLException { + Connection conn = mockConnectionWithExtensions( "vector" ); + PostgresqlSqlDialect d = new PostgresqlSqlDialect(); + d.addSupportedFeatures( PostgresqlSource.detectFeatures( conn ) ); + assertTrue( d.supportsVector() ); + } + + + // ---- getCustomArrayRetrievalExpression ---------------------------------------------------------------- + + @Test + void bitVectorAlwaysUsesGetString() { + // bit(n) is a native PostgreSQL type - no pgvector required + PostgresqlSqlDialect dialect = new PostgresqlSqlDialect(); + AlgDataType bitVec = bitVectorType( 3 ); + ParameterExpression rs = Expressions.parameter( ResultSet.class, "rs" ); + + Optional expr = dialect.getCustomArrayRetrievalExpression( rs, 0, bitVec ); + + assertTrue( expr.isPresent() ); + assertTrue( expr.get().toString().contains( "getString" ) ); + assertFalse( expr.get().toString().contains( "getObject" ) ); + } + + + @Test + void floatVectorReturnsEmptyWithoutPgvector() { + PostgresqlSqlDialect dialect = new PostgresqlSqlDialect(); + ParameterExpression rs = Expressions.parameter( ResultSet.class, "rs" ); + + Optional expr = dialect.getCustomArrayRetrievalExpression( rs, 0, floatVectorType( 3 ) ); + + assertTrue( expr.isEmpty() ); + } + + + @Test + void floatVectorUsesGetObjectWithPgvector() { + PostgresqlSqlDialect dialect = new PostgresqlSqlDialect(); + dialect.addSupportedFeatures( Set.of( PGVECTOR ) ); + ParameterExpression rs = Expressions.parameter( ResultSet.class, "rs" ); + + Optional expr = dialect.getCustomArrayRetrievalExpression( rs, 0, floatVectorType( 3 ) ); + + assertTrue( expr.isPresent() ); + assertTrue( expr.get().toString().contains( "getObject" ) ); + assertFalse( expr.get().toString().contains( "getString" ) ); + } + + + @Test + void nonArrayTypeReturnsEmpty() { + PostgresqlSqlDialect dialect = new PostgresqlSqlDialect(); + dialect.addSupportedFeatures( Set.of( PGVECTOR ) ); + AlgDataType intType = AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.INTEGER ); + ParameterExpression rs = Expressions.parameter( ResultSet.class, "rs" ); + + assertTrue( dialect.getCustomArrayRetrievalExpression( rs, 0, intType ).isEmpty() ); + } + + + // ---- vectorPushdownTypeIsPresent ---------------------------------------------------------------- + + @Test + void bitPushdownAlwaysPresent() { + // bit(n) is native PostgreSQL - pushdown does not depend on pgvector + PostgresqlSqlDialect dialect = new PostgresqlSqlDialect(); + assertTrue( dialect.vectorPushdownTypeIsPresent( VectorType.ElementType.BIT ) ); + } + + + @Test + void floatPushdownRequiresPgvector() { + PostgresqlSqlDialect dialect = new PostgresqlSqlDialect(); + assertFalse( dialect.vectorPushdownTypeIsPresent( VectorType.ElementType.FLOAT ) ); + + dialect.addSupportedFeatures( Set.of( PGVECTOR ) ); + assertTrue( dialect.vectorPushdownTypeIsPresent( VectorType.ElementType.FLOAT ) ); + } + + + // ---- helpers ---------------------------------------------------------------- + + private static AlgDataType bitVectorType( int dim ) { + return AlgDataTypeFactory.DEFAULT.createVectorType( + AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.BOOLEAN ), dim ); + } + + + private static AlgDataType floatVectorType( int dim ) { + return AlgDataTypeFactory.DEFAULT.createVectorType( + AlgDataTypeFactory.DEFAULT.createPolyType( PolyType.REAL ), dim ); + } + + + private static Connection mockConnection( boolean hasRows ) throws SQLException { + Connection conn = mock( Connection.class ); + PreparedStatement ps = mock( PreparedStatement.class ); + ResultSet rs = mock( ResultSet.class ); + Array arr = mock( Array.class ); + when( conn.prepareStatement( any() ) ).thenReturn( ps ); + when( conn.createArrayOf( eq( "text" ), any() ) ).thenReturn( arr ); + when( ps.executeQuery() ).thenReturn( rs ); + when( rs.next() ).thenReturn( hasRows, false ); + return conn; + } + + + private static Connection mockConnectionWithExtensions( String... extensions ) throws SQLException { + Connection conn = mock( Connection.class ); + PreparedStatement ps = mock( PreparedStatement.class ); + ResultSet rs = mock( ResultSet.class ); + Array arr = mock( Array.class ); + when( conn.prepareStatement( any() ) ).thenReturn( ps ); + when( conn.createArrayOf( eq( "text" ), any() ) ).thenReturn( arr ); + when( ps.executeQuery() ).thenReturn( rs ); + Boolean[] hasNext = new Boolean[extensions.length + 1]; + for ( int i = 0; i < extensions.length; i++ ) hasNext[i] = true; + hasNext[extensions.length] = false; + when( rs.next() ).thenReturn( hasNext[0], java.util.Arrays.copyOfRange( hasNext, 1, hasNext.length ) ); + String[] rest = java.util.Arrays.copyOfRange( extensions, 1, extensions.length ); + when( rs.getString( 1 ) ).thenReturn( extensions[0], rest ); + return conn; + } + +} diff --git a/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/dialect/PostgresqlVectorHelperTest.java b/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/dialect/PostgresqlVectorHelperTest.java new file mode 100644 index 0000000000..8b8d3e7d9d --- /dev/null +++ b/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/dialect/PostgresqlVectorHelperTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.postgres.dialect; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.pgvector.PGbit; +import com.pgvector.PGhalfvec; +import com.pgvector.PGsparsevec; +import com.pgvector.PGvector; +import java.sql.SQLException; +import java.util.List; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.polypheny.db.adapter.postgres.PostgresqlVectorHelper; +import org.polypheny.db.type.entity.PolyBoolean; +import org.polypheny.db.type.entity.PolyValue; +import org.polypheny.db.type.entity.numerical.PolyFloat; +import org.polypheny.db.util.PolyphenyHomeDirManager; +import org.polypheny.db.util.RunMode; +import org.postgresql.jdbc.PgArray; + +public class PostgresqlVectorHelperTest { + + @BeforeAll + static void init() { + if ( PolyphenyHomeDirManager.getMode() == null ) { + PolyphenyHomeDirManager.setModeAndGetInstance( RunMode.TEST ); + } + } + + + // ---- float vectors ---------------------------------------------------------------- + + @Test + void parsesVectorCorrectly() { + List result = PostgresqlVectorHelper.parseVector( new PGvector( new float[]{ 1f, 2.5f, 3f } ) ); + assertNotNull( result ); + assertEquals( 3, result.size() ); + assertFloatValues( result, 1f, 2.5f, 3f ); + } + + + @Test + void parsesNegativeVectorCorrectly() { + List result = PostgresqlVectorHelper.parseVector( new PGvector( new float[]{ -1f, -2.5f, -3f } ) ); + assertNotNull( result ); + assertFloatValues( result, -1f, -2.5f, -3f ); + } + + + @Test + void parsesSingleEntryVector() { + List result = PostgresqlVectorHelper.parseVector( new PGvector( new float[]{ -1f } ) ); + assertNotNull( result ); + assertEquals( 1, result.size() ); + assertFloatValues( result, -1f ); + } + + + @Test + void parsesEmptyFloatVectorReturnsEmptyList() { + List result = PostgresqlVectorHelper.parseVector( new PGvector( new float[]{} ) ); + assertNotNull( result ); + assertEquals( 0, result.size() ); + } + + + // ---- halfvec ---------------------------------------------------------------- + + @Test + void parsesHalfvecCorrectly() { + List result = PostgresqlVectorHelper.parseVector( new PGhalfvec( new float[]{ 1f, 2.5f, 3f } ) ); + assertNotNull( result ); + assertFloatValues( result, 1f, 2.5f, 3f ); + } + + + @Test + void parsesNegativeHalfvecCorrectly() { + List result = PostgresqlVectorHelper.parseVector( new PGhalfvec( new float[]{ -1f, -2.5f, -3f } ) ); + assertNotNull( result ); + assertFloatValues( result, -1f, -2.5f, -3f ); + } + + + // ---- sparsevec ---------------------------------------------------------------- + + @Test + void parsesSparsevecCorrectly() { + List result = PostgresqlVectorHelper.parseVector( new PGsparsevec( new float[]{ 1f, 0f, 2.5f } ) ); + assertNotNull( result ); + assertEquals( 3, result.size() ); + assertFloatValues( result, 1f, 0f, 2.5f ); + } + + + // ---- bitvector ---------------------------------------------------------------- + + @Test + void parsesBitVectorCorrectly() { + List result = PostgresqlVectorHelper.parseVector( new PGbit( new boolean[]{ true, false, false } ) ); + assertNotNull( result ); + assertEquals( 3, result.size() ); + assertInstanceOf( PolyBoolean.class, result.get( 0 ) ); + assertEquals( true, result.get( 0 ).asBoolean().getValue() ); + assertEquals( false, result.get( 1 ).asBoolean().getValue() ); + assertEquals( false, result.get( 2 ).asBoolean().getValue() ); + } + + + @Test + void parsesEmptyBitVectorReturnsEmptyList() { + List result = PostgresqlVectorHelper.parseVector( new PGbit( new boolean[]{} ) ); + assertNotNull( result ); + assertEquals( 0, result.size() ); + } + + + @Test + void parsesBitVectorFromStringRepresentation() { + // PostgreSQL's getString() on a bit(n) column returns e.g. "101" + List result = PostgresqlVectorHelper.parseVector( "101" ); + assertNotNull( result ); + assertEquals( 3, result.size() ); + assertEquals( true, result.get( 0 ).asBoolean().getValue() ); + assertEquals( false, result.get( 1 ).asBoolean().getValue() ); + assertEquals( true, result.get( 2 ).asBoolean().getValue() ); + } + + + @Test + void parsesAllZeroStringBitVector() { + List result = PostgresqlVectorHelper.parseVector( "000" ); + assertNotNull( result ); + assertEquals( 3, result.size() ); + result.forEach( v -> assertEquals( false, v.asBoolean().getValue() ) ); + } + + + @Test + void nullObjectReturnsNull() { + assertNull( PostgresqlVectorHelper.parseVector( (Object) null ) ); + } + + + @Test + void unknownObjectTypeReturnsNull() throws SQLException { + assertNull( PostgresqlVectorHelper.parseVector( new PgArray( null, 0, "" ) ) ); + } + + // ---- helpers ---------------------------------------------------------------- + + private static void assertFloatValues( List result, float... expected ) { + assertEquals( expected.length, result.size() ); + for ( int i = 0; i < expected.length; i++ ) { + assertInstanceOf( PolyFloat.class, result.get( i ) ); + assertEquals( expected[i], ((PolyFloat) result.get( i )).floatValue() ); + } + } + +} diff --git a/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/source/PostgresqlSourceTest.java b/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/source/PostgresqlSourceTest.java new file mode 100644 index 0000000000..6327914e2f --- /dev/null +++ b/plugins/postgres-adapter/src/test/java/org/polypheny/db/adapter/postgres/source/PostgresqlSourceTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.adapter.postgres.source; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; + +import org.junit.jupiter.api.Test; + +public class PostgresqlSourceTest { + + private final PostgresqlSource source = mock( PostgresqlSource.class, CALLS_REAL_METHODS ); + + + @Test + void bitRequiresNativeResolution() { + assertTrue( source.isNativeVectorType( "bit" ) ); + } + + + @Test + void vectorRequiresNativeResolution() { + assertTrue( source.isNativeVectorType( "vector" ) ); + } + + + @Test + void halfvecRequiresNativeResolution() { + assertTrue( source.isNativeVectorType( "halfvec" ) ); + } + + + @Test + void sparsevecRequiresNativeResolution() { + assertTrue( source.isNativeVectorType( "sparsevec" ) ); + } + + + @Test + void regularTypesDoNotRequireNativeResolution() { + assertFalse( source.isNativeVectorType( "float4" ) ); + assertFalse( source.isNativeVectorType( "_float4" ) ); + assertFalse( source.isNativeVectorType( "_float8" ) ); + assertFalse( source.isNativeVectorType( "_int4" ) ); + assertFalse( source.isNativeVectorType( "_int8" ) ); + assertFalse( source.isNativeVectorType( "_bool" ) ); + assertFalse( source.isNativeVectorType( "int4" ) ); + assertFalse( source.isNativeVectorType( "varchar" ) ); + assertFalse( source.isNativeVectorType( "boolean" ) ); + } + +} diff --git a/plugins/prism-interface/build.gradle b/plugins/prism-interface/build.gradle index 244efa1278..114759a698 100644 --- a/plugins/prism-interface/build.gradle +++ b/plugins/prism-interface/build.gradle @@ -36,6 +36,9 @@ dependencies { testImplementation project(path: ':core', configuration: 'tests') testImplementation project(path: ':core') testImplementation project(path: ':dbms') + testImplementation(group: "org.polypheny", name: "polypheny-jdbc-driver", version: polypheny_jdbc_driver_version) { + exclude(group: "com.fasterxml.jackson.core") + } // Apache 2.0 } diff --git a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedIndexedStatement.java b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedIndexedStatement.java index ec5eb97051..b2c21f868a 100644 --- a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedIndexedStatement.java +++ b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedIndexedStatement.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.stream.IntStream; import lombok.Getter; import lombok.Setter; @@ -31,6 +32,7 @@ import org.polypheny.db.transaction.Statement; import org.polypheny.db.transaction.Transaction; import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; import org.polypheny.db.type.entity.PolyValue; import org.polypheny.prism.ParameterMeta; import org.polypheny.prism.StatementResult; @@ -98,58 +100,6 @@ public StatementResult execute( List values, List para } - private AlgDataType deriveType( JavaTypeFactory typeFactory, AlgDataType parameterMeta ) { - PolyType type = parameterMeta.getPolyType(); - return switch ( type ) { - case DECIMAL -> { - if ( parameterMeta.getPrecision() >= 0 && parameterMeta.getScale() >= 0 ) { - yield typeFactory.createPolyType( PolyType.DECIMAL, parameterMeta.getPrecision(), parameterMeta.getScale() ); - } else if ( parameterMeta.getPrecision() >= 0 ) { - yield typeFactory.createPolyType( PolyType.DECIMAL, parameterMeta.getPrecision() ); - } - yield typeFactory.createPolyType( PolyType.DECIMAL ); - } - case VARCHAR -> { - if ( parameterMeta.getPrecision() > 0 ) { - yield typeFactory.createPolyType( PolyType.VARCHAR, parameterMeta.getPrecision() ); - } - yield typeFactory.createPolyType( PolyType.VARCHAR ); - } - case CHAR -> { - if ( parameterMeta.getPrecision() > 0 ) { - yield typeFactory.createPolyType( PolyType.CHAR, parameterMeta.getPrecision() ); - } - yield typeFactory.createPolyType( PolyType.CHAR ); - } - case TIME -> { - if ( parameterMeta.getPrecision() >= 0 ) { - yield typeFactory.createPolyType( PolyType.TIME, parameterMeta.getPrecision() ); - } - yield typeFactory.createPolyType( PolyType.TIME ); - } - case TIMESTAMP -> { - if ( parameterMeta.getPrecision() >= 0 ) { - yield typeFactory.createPolyType( PolyType.TIMESTAMP, parameterMeta.getPrecision() ); - } - yield typeFactory.createPolyType( PolyType.TIMESTAMP ); - } - case BINARY -> { - if ( parameterMeta.getPrecision() > 0 ) { - yield typeFactory.createPolyType( PolyType.BINARY, parameterMeta.getPrecision() ); - } - yield typeFactory.createPolyType( PolyType.BINARY ); - } - case VARBINARY -> { - if ( parameterMeta.getPrecision() > 0 ) { - yield typeFactory.createPolyType( PolyType.VARBINARY, parameterMeta.getPrecision() ); - } - yield typeFactory.createPolyType( PolyType.VARBINARY ); - } - default -> typeFactory.createPolyType( type ); - }; - } - - @Override public void close() { if ( statement != null ) { diff --git a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedNamedStatement.java b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedNamedStatement.java index 9dadddf7c0..cc5f9a8364 100644 --- a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedNamedStatement.java +++ b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedNamedStatement.java @@ -21,6 +21,7 @@ import lombok.Getter; import lombok.Setter; import org.polypheny.db.PolyImplementation; +import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.catalog.entity.logical.LogicalNamespace; import org.polypheny.db.languages.QueryLanguage; import org.polypheny.db.prisminterface.NamedValueProcessor; @@ -67,7 +68,10 @@ public StatementResult execute( Map values, int fetchSize ) t } List valueList = namedValueProcessor.transformValueMap( values ); for ( int i = 0; i < valueList.size(); i++ ) { - statement.getDataContext().addParameterValues( i, PolyValue.deriveType( valueList.get( i ), this.statement.getDataContext().getTypeFactory() ), List.of( valueList.get( i ) ) ); + AlgDataType type = (parameterPolyTypes != null && i < parameterPolyTypes.size()) + ? deriveType( statement.getDataContext().getTypeFactory(), parameterPolyTypes.get( i ) ) + : PolyValue.deriveType( valueList.get( i ), this.statement.getDataContext().getTypeFactory() ); + statement.getDataContext().addParameterValues( i, type, List.of( valueList.get( i ) ) ); } StatementProcessor.implement( this ); return StatementProcessor.executeAndGetResult( this, fetchSize ); diff --git a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedStatement.java b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedStatement.java index 548bf52813..13a191358c 100644 --- a/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedStatement.java +++ b/plugins/prism-interface/src/main/java/org/polypheny/db/prisminterface/statements/PIPreparedStatement.java @@ -17,19 +17,26 @@ package org.polypheny.db.prisminterface.statements; import java.util.List; +import java.util.Optional; +import lombok.Getter; import lombok.Setter; import org.jetbrains.annotations.NotNull; +import org.polypheny.db.adapter.java.JavaTypeFactory; import org.polypheny.db.algebra.type.AlgDataType; import org.polypheny.db.catalog.entity.logical.LogicalNamespace; import org.polypheny.db.languages.QueryLanguage; import org.polypheny.db.prisminterface.PIClient; import org.polypheny.db.prisminterface.statementProcessing.StatementProcessor; +import org.polypheny.db.type.ArrayType; +import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.VectorType; import org.polypheny.prism.ParameterMeta; @Setter public abstract class PIPreparedStatement extends PIStatement implements Signaturizable { protected List parameterMetas; + @Getter protected List parameterPolyTypes; @@ -50,4 +57,71 @@ protected PIPreparedStatement( } + protected AlgDataType deriveType( JavaTypeFactory typeFactory, AlgDataType parameterMeta ) { + PolyType type = parameterMeta.getPolyType(); + return switch ( type ) { + case DECIMAL -> { + if ( parameterMeta.getPrecision() >= 0 && parameterMeta.getScale() >= 0 ) { + yield typeFactory.createPolyType( PolyType.DECIMAL, parameterMeta.getPrecision(), parameterMeta.getScale() ); + } else if ( parameterMeta.getPrecision() >= 0 ) { + yield typeFactory.createPolyType( PolyType.DECIMAL, parameterMeta.getPrecision() ); + } + yield typeFactory.createPolyType( PolyType.DECIMAL ); + } + case VARCHAR -> { + if ( parameterMeta.getPrecision() > 0 ) { + yield typeFactory.createPolyType( PolyType.VARCHAR, parameterMeta.getPrecision() ); + } + yield typeFactory.createPolyType( PolyType.VARCHAR ); + } + case CHAR -> { + if ( parameterMeta.getPrecision() > 0 ) { + yield typeFactory.createPolyType( PolyType.CHAR, parameterMeta.getPrecision() ); + } + yield typeFactory.createPolyType( PolyType.CHAR ); + } + case TIME -> { + if ( parameterMeta.getPrecision() >= 0 ) { + yield typeFactory.createPolyType( PolyType.TIME, parameterMeta.getPrecision() ); + } + yield typeFactory.createPolyType( PolyType.TIME ); + } + case TIMESTAMP -> { + if ( parameterMeta.getPrecision() >= 0 ) { + yield typeFactory.createPolyType( PolyType.TIMESTAMP, parameterMeta.getPrecision() ); + } + yield typeFactory.createPolyType( PolyType.TIMESTAMP ); + } + case BINARY -> { + if ( parameterMeta.getPrecision() > 0 ) { + yield typeFactory.createPolyType( PolyType.BINARY, parameterMeta.getPrecision() ); + } + yield typeFactory.createPolyType( PolyType.BINARY ); + } + case VARBINARY -> { + if ( parameterMeta.getPrecision() > 0 ) { + yield typeFactory.createPolyType( PolyType.VARBINARY, parameterMeta.getPrecision() ); + } + yield typeFactory.createPolyType( PolyType.VARBINARY ); + } + case ARRAY -> { + Optional vt = parameterMeta.unwrap( VectorType.class ); + if ( vt.isPresent() ) { + yield vt.get(); + } + Optional at = parameterMeta.unwrap( ArrayType.class ); + if ( at.isPresent() ) { + yield typeFactory.createArrayType( + typeFactory.createPolyType( + at.get().getComponentType().getPolyType() ), + at.get().getCardinality(), + at.get().getDimension() ); + } + yield typeFactory.createPolyType( type ); + } + default -> typeFactory.createPolyType( type ); + }; + } + + } diff --git a/plugins/prism-interface/src/test/java/org/polypheny/db/prisminterface/PreparedIndexedStatementTest.java b/plugins/prism-interface/src/test/java/org/polypheny/db/prisminterface/PreparedIndexedStatementTest.java new file mode 100644 index 0000000000..e53d66d5d9 --- /dev/null +++ b/plugins/prism-interface/src/test/java/org/polypheny/db/prisminterface/PreparedIndexedStatementTest.java @@ -0,0 +1,189 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.prisminterface; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.TestHelper.JdbcConnection; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.entity.LogicalUser; +import org.polypheny.db.catalog.entity.logical.LogicalNamespace; +import org.polypheny.db.languages.QueryLanguage; +import org.polypheny.db.prisminterface.statementProcessing.StatementProcessor; +import org.polypheny.db.prisminterface.statements.PIPreparedIndexedStatement; +import org.polypheny.db.transaction.TransactionManager; +import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.entity.PolyBoolean; +import org.polypheny.db.type.entity.PolyList; +import org.polypheny.db.type.entity.PolyValue; +import org.polypheny.db.type.entity.numerical.PolyInteger; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@SuppressWarnings( "SqlNoDataSourceInspection" ) +@Tag( "adapter" ) +public class PreparedIndexedStatementTest { + + private static final String TABLE = "pis_test"; + private static TransactionManager transactionManager; + private PIClient client; + + + @BeforeAll + public static void init() { + //noinspection ResultOfMethodCallIgnored + TestHelper.getInstance(); + transactionManager = TestHelper.getInstance().getTransactionManager(); + } + + + @BeforeEach + public void start() throws SQLException { + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement stmt = conn.getConnection().createStatement() ) { + stmt.execute( "CREATE TABLE " + TABLE + " (id INTEGER PRIMARY KEY, v BOOLEAN ARRAY(1,3))" ); + } + } + LogicalUser user = Catalog.snapshot().getUser( Catalog.USER_NAME ).orElseThrow(); + LogicalNamespace namespace = Catalog.snapshot().getNamespace( Catalog.DEFAULT_NAMESPACE_NAME ).orElseThrow(); + MonitoringPage monitoringPage = new MonitoringPage( "test-pis-" + System.nanoTime(), + "Test PI Client" ); + client = new PIClient( "test-uuid-pis", user, transactionManager, namespace, monitoringPage, true ); + } + + + @AfterEach + public void stop() throws SQLException { + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement stmt = conn.getConnection().createStatement() ) { + stmt.execute( "DROP TABLE IF EXISTS " + TABLE ); + } + } + } + + + private PIPreparedIndexedStatement prepareInsert( int stmtId ) throws + Exception { + LogicalNamespace namespace = Catalog.snapshot().getNamespace( + Catalog.DEFAULT_NAMESPACE_NAME ).orElseThrow(); + PIPreparedIndexedStatement stmt = new PIPreparedIndexedStatement( + stmtId, client, QueryLanguage.from( "sql" ), namespace, + "INSERT INTO " + TABLE + " (id, v) VALUES (?, ?)" + ); + StatementProcessor.prepare( stmt ); + return stmt; + } + + + @Test + public void parameterPolyTypesArePopulatedAfterPrepare() throws Exception { + PIPreparedIndexedStatement stmt = prepareInsert( 1 ); + assertNotNull( stmt.getParameterPolyTypes() ); + assertEquals( 2, stmt.getParameterPolyTypes().size() ); + } + + + @Test + public void arrayParameterTypeIsPreservedFromPlan() throws Exception { + PIPreparedIndexedStatement stmt = prepareInsert( 2 ); + AlgDataType arrayType = stmt.getParameterPolyTypes().get( 1 ); + assertEquals( PolyType.ARRAY, arrayType.getPolyType() ); + assertNotEquals( PolyType.ANY, arrayType.getComponentType().getPolyType(), + "component type must not be erased to ANY" ); + assertEquals( PolyType.BOOLEAN, arrayType.getComponentType().getPolyType() ); + } + + + @Test + public void indexedStatementInsertsRowAndReadsBack() throws Exception { + PIPreparedIndexedStatement stmt = prepareInsert( 3 ); + List values = List.of( + PolyInteger.of( 42 ), + PolyList.ofElements( PolyBoolean.TRUE, PolyBoolean.FALSE, PolyBoolean.TRUE ) ); + org.polypheny.prism.StatementResult result = stmt.execute( values, stmt.getParameterMetas(), 100 ); + assertEquals( 1, result.getScalar() ); + + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement s = conn.getConnection().createStatement() ) { + ResultSet rs = s.executeQuery( "SELECT id FROM " + TABLE + " WHERE id = 42" ); + assertTrue( rs.next() ); + assertEquals( 42, rs.getInt( 1 ) ); + } + } + } + + + @Test + public void indexedStatementCanBeExecutedMultipleTimesWithDifferentValues() throws Exception { + PIPreparedIndexedStatement stmt = prepareInsert( 4 ); + for ( int i = 1; i <= 3; i++ ) { + List values = List.of( + PolyInteger.of( i ), + PolyList.ofElements( PolyBoolean.of( i % 2 == 0 ), PolyBoolean.FALSE, PolyBoolean.TRUE ) ); + org.polypheny.prism.StatementResult result = stmt.execute( values, stmt.getParameterMetas(), 100 ); + assertEquals( 1, result.getScalar() ); + } + + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement s = conn.getConnection().createStatement() ) { + ResultSet rs = s.executeQuery( "SELECT COUNT(*) FROM " + TABLE ); + assertTrue( rs.next() ); + assertEquals( 3, rs.getInt( 1 ) ); + } + } + } + + + @Test + public void executeBatchInsertsAllRows() throws Exception { + PIPreparedIndexedStatement stmt = prepareInsert( 5 ); + // outer list = columns, inner list = rows + List> batch = List.of( + // id column + List.of( PolyInteger.of( 10 ), PolyInteger.of( 20 ), + PolyInteger.of( 30 ) ), + // val column + List.of( + PolyList.ofElements( PolyBoolean.TRUE, PolyBoolean.FALSE, PolyBoolean.TRUE ), + PolyList.ofElements( PolyBoolean.FALSE, PolyBoolean.FALSE, PolyBoolean.TRUE ), + PolyList.ofElements( PolyBoolean.TRUE, PolyBoolean.TRUE, PolyBoolean.FALSE ) ) ); + List counts = stmt.executeBatch( batch ); + assertEquals( 1, counts.size() ); + assertEquals( 3L, counts.get( 0 ) ); + + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement s = conn.getConnection().createStatement() ) { + ResultSet rs = s.executeQuery( "SELECT COUNT(*) FROM " + TABLE ); + assertTrue( rs.next() ); + assertEquals( 3, rs.getInt( 1 ) ); + } + } + } + +} diff --git a/plugins/prism-interface/src/test/java/org/polypheny/db/prisminterface/PreparedNamedStatementTest.java b/plugins/prism-interface/src/test/java/org/polypheny/db/prisminterface/PreparedNamedStatementTest.java new file mode 100644 index 0000000000..a8acc59584 --- /dev/null +++ b/plugins/prism-interface/src/test/java/org/polypheny/db/prisminterface/PreparedNamedStatementTest.java @@ -0,0 +1,159 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.prisminterface; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.polypheny.db.TestHelper; +import org.polypheny.db.TestHelper.JdbcConnection; +import org.polypheny.db.algebra.type.AlgDataType; +import org.polypheny.db.catalog.Catalog; +import org.polypheny.db.catalog.entity.LogicalUser; +import org.polypheny.db.catalog.entity.logical.LogicalNamespace; +import org.polypheny.db.languages.QueryLanguage; +import org.polypheny.db.prisminterface.statementProcessing.StatementProcessor; +import org.polypheny.db.prisminterface.statements.PIPreparedNamedStatement; +import org.polypheny.db.transaction.TransactionManager; +import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.entity.PolyBoolean; +import org.polypheny.db.type.entity.PolyList; +import org.polypheny.db.type.entity.PolyValue; +import org.polypheny.db.type.entity.numerical.PolyInteger; +import org.polypheny.prism.StatementResult; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@SuppressWarnings( "SqlNoDataSourceInspection" ) +@Tag( "adapter" ) +public class PreparedNamedStatementTest { + private static final String TABLE = "pns_test"; + private static TransactionManager transactionManager; + private PIClient client; + + + @BeforeAll + public static void init() { + //noinspection ResultOfMethodCallIgnored + TestHelper.getInstance(); + transactionManager = TestHelper.getInstance().getTransactionManager(); + } + + + @BeforeEach + public void start() throws SQLException { + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement stmt = conn.getConnection().createStatement() ) { + stmt.execute( "CREATE TABLE " + TABLE + " (id INTEGER PRIMARY KEY, v BOOLEAN ARRAY(1,3))" ); + } + } + LogicalUser user = Catalog.snapshot().getUser( Catalog.USER_NAME ).orElseThrow(); + LogicalNamespace namespace = Catalog.snapshot().getNamespace( Catalog.DEFAULT_NAMESPACE_NAME ).orElseThrow(); + MonitoringPage monitoringPage = new MonitoringPage( "test-pns-" + System.nanoTime(), "Test PI Client" ); + client = new PIClient( "test-uuid-pns", user, transactionManager, namespace, monitoringPage, true ); + } + + + @AfterEach + public void stop() throws SQLException { + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement stmt = conn.getConnection().createStatement() ) { + stmt.execute( "DROP TABLE IF EXISTS " + TABLE ); + } + } + } + + + private PIPreparedNamedStatement prepareInsert( int stmtId ) { + LogicalNamespace namespace = Catalog.snapshot().getNamespace( Catalog.DEFAULT_NAMESPACE_NAME ).orElseThrow(); + PIPreparedNamedStatement stmt = new PIPreparedNamedStatement( stmtId, client, QueryLanguage.from( "sql" ), namespace, + "INSERT INTO " + TABLE + " (id, v) VALUES (:id, :v)" ); + StatementProcessor.prepare( stmt ); + return stmt; + } + + + @Test + public void parameterPolyTypesArePopulatedAfterPrepare() { + PIPreparedNamedStatement stmt = prepareInsert( 1 ); + assertNotNull( stmt.getParameterPolyTypes() ); + assertEquals( 2, stmt.getParameterPolyTypes().size() ); + } + + + @Test + public void arrayParameterTypeIsPreservedFromPlan() { + PIPreparedNamedStatement stmt = prepareInsert( 2 ); + AlgDataType arrayType = stmt.getParameterPolyTypes().get( 1 ); + assertEquals( PolyType.ARRAY, arrayType.getPolyType() ); + assertNotEquals( PolyType.ANY, arrayType.getComponentType().getPolyType(), + "component type must not be erased to ANY by value-based type derivation" ); + assertEquals( PolyType.BOOLEAN, arrayType.getComponentType().getPolyType() ); + } + + + @Test + public void namedStatementInsertsRowAndReadsBack() throws Exception { + PIPreparedNamedStatement stmt = prepareInsert( 3 ); + Map values = Map.of( + "id", PolyInteger.of( 42 ), + "v", PolyList.ofElements( PolyBoolean.TRUE, PolyBoolean.FALSE, + PolyBoolean.TRUE ) ); + StatementResult result = stmt.execute( values, 100 ); + assertEquals( 1, result.getScalar() ); + + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement s = conn.getConnection().createStatement() ) { + ResultSet rs = s.executeQuery( "SELECT id FROM " + TABLE + " WHERE id = 42" ); + assertTrue( rs.next() ); + assertEquals( 42, rs.getInt( 1 ) ); + } + } + } + + + @Test + public void namedStatementCanBeExecutedMultipleTimesWithDifferentValues() throws Exception { + PIPreparedNamedStatement stmt = prepareInsert( 4 ); + for ( int i = 1; i <= 3; i++ ) { + Map values = Map.of( + "id", PolyInteger.of( i ), + "v", PolyList.ofElements( PolyBoolean.of( i % 2 == 0 ), + PolyBoolean.FALSE, PolyBoolean.TRUE ) ); + StatementResult result = stmt.execute( values, 100 ); + assertEquals( 1, result.getScalar() ); + } + + try ( JdbcConnection conn = new JdbcConnection( true ) ) { + try ( Statement s = conn.getConnection().createStatement() ) { + ResultSet rs = s.executeQuery( "SELECT COUNT(*) FROM " + TABLE ); + assertTrue( rs.next() ); + assertEquals( 3, rs.getInt( 1 ) ); + } + } + } + +} diff --git a/plugins/sql-language/src/main/codegen/Parser.jj b/plugins/sql-language/src/main/codegen/Parser.jj index 62e3cef494..01b2372c36 100644 --- a/plugins/sql-language/src/main/codegen/Parser.jj +++ b/plugins/sql-language/src/main/codegen/Parser.jj @@ -1162,7 +1162,9 @@ SqlAlterMaterializedView SqlAlterMaterializedView(Span s) : final SqlIdentifier indexMethod; final boolean unique; final SqlIdentifier storeName; - + Map indexOptions; + String optKey; + String optVal; } { @@ -1214,8 +1216,22 @@ SqlAlterMaterializedView SqlAlterMaterializedView(Span s) : | { storeName = null; } ) + ( + + { indexOptions = new HashMap(); } + optKey = IndexOptionValue() optVal = IndexOptionValue() + { indexOptions.put( optKey, optVal ); } + ( + + optKey = IndexOptionValue() optVal = IndexOptionValue() + { indexOptions.put( optKey, optVal ); } + )* + + | + { indexOptions = null; } + ) { - return new SqlAlterMaterializedViewAddIndex(s.end(this), materializedview, columnList, unique, indexMethod, indexName, storeName); + return new SqlAlterMaterializedViewAddIndex(s.end(this), materializedview, columnList, unique, indexMethod, indexName, storeName, indexOptions); } | @@ -1250,6 +1266,9 @@ SqlAlterTable SqlAlterTable(Span s) : final SqlIdentifier store; final SqlIdentifier indexName; final SqlIdentifier indexMethod; + Map indexOptions; + String optKey; + String optVal; final SqlIdentifier storeName; final String onUpdate; final String onDelete; @@ -1597,8 +1616,22 @@ SqlAlterTable SqlAlterTable(Span s) : | { storeName = null; } ) + ( + + { indexOptions = new HashMap(); } + optKey = IndexOptionValue() optVal = IndexOptionValue() + { indexOptions.put( optKey, optVal ); } + ( + + optKey = IndexOptionValue() optVal = IndexOptionValue() + { indexOptions.put( optKey, optVal ); } + )* + + | + { indexOptions = null; } + ) { - return new SqlAlterTableAddIndex(s.end(this), entity, columnList, unique, indexMethod, indexName, storeName); + return new SqlAlterTableAddIndex(s.end(this), entity, columnList, unique, indexMethod, indexName, storeName, indexOptions); } | @@ -1715,6 +1748,26 @@ SqlAlterTable SqlAlterTable(Span s) : ) } + +/** Parses a string literal, identifier, or integer for use as a WITH option key or value. */ +String IndexOptionValue() : + { + SqlIdentifier id; + SqlLiteral lit; + SqlNode node; + } + { + id = SimpleIdentifier() + { return id.getSimple(); } + | + lit = NumericLiteral() + { return lit.toValue(); } + | + node = StringLiteral() + { return ((SqlLiteral) node).toValue(); } + } + + /** * Parses the MODIFY COLUMN part of an ALTER TABLE statement. */ @@ -5851,6 +5904,7 @@ SqlDataTypeSpec DataType() : int dimension = -1; int cardinality = -1; String charSetName = null; + boolean elementsNullable = true; final Span s; } { @@ -5870,6 +5924,10 @@ SqlDataTypeSpec DataType() : charSetName = Identifier() ] + [ + LOOKAHEAD( ( | ) ) + { elementsNullable=false; } + ] [ //e.g. ARRAY(dimension, cardinality) collectionTypeName = CollectionsTypeName() @@ -5893,6 +5951,7 @@ SqlDataTypeSpec DataType() : dimension, cardinality, charSetName, + elementsNullable, s.end(collectionTypeName)); } return new SqlDataTypeSpec( @@ -5901,6 +5960,7 @@ SqlDataTypeSpec DataType() : scale, charSetName, null, + elementsNullable, s.end(this)); } } @@ -7415,6 +7475,12 @@ SqlBinaryOperator BinaryRowOperator() : | { return OperatorRegistry.get( OperatorName.SUCCEEDS, SqlBinaryOperator.class ); } | { return OperatorRegistry.get( OperatorName.IMMEDIATELY_PRECEDES, SqlBinaryOperator.class ); } | { return OperatorRegistry.get( OperatorName.IMMEDIATELY_SUCCEEDS, SqlBinaryOperator.class ); } +| { return OperatorRegistry.get( OperatorName.PGVECTOR_L2, SqlBinaryOperator.class ); } +| { return OperatorRegistry.get( OperatorName.PGVECTOR_L1, SqlBinaryOperator.class ); } +| { return OperatorRegistry.get( OperatorName.PGVECTOR_COS, SqlBinaryOperator.class ); } +| { return OperatorRegistry.get( OperatorName.PGVECTOR_HAMMING, SqlBinaryOperator.class ); } +| { return OperatorRegistry.get( OperatorName.PGVECTOR_JACCARD, SqlBinaryOperator.class ); } +| { return OperatorRegistry.get( OperatorName.PGVECTOR_INNER_PRODUCT, SqlBinaryOperator.class ); } | op = BinaryMultisetOperator() { return op; } } @@ -8624,6 +8690,12 @@ void NonReservedKeyWord2of3() : | < SLASH: "/" > | < PERCENT_REMAINDER: "%" > | < CONCAT: "||" > +| < L2_DIST_OP: "<->" > +| < L1_DIST_OP: "<+>" > +| < COS_DIST_OP: "<=>" > +| < HAMMING_DIST_OP: "<~>" > +| < JACCARD_DIST_OP: "<%>" > +| < INNER_PRODUCT_DIST_OP: "<#>" > | < NAMED_ARGUMENT_ASSIGNMENT: "=>" > | < DOUBLE_PERIOD: ".." > | < QUOTE: "'" > diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/SqlLanguagePlugin.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/SqlLanguagePlugin.java index e387aafca2..75f862895b 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/SqlLanguagePlugin.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/SqlLanguagePlugin.java @@ -129,6 +129,7 @@ import org.polypheny.db.sql.language.fun.SqlMultisetQueryConstructor; import org.polypheny.db.sql.language.fun.SqlMultisetSetOperator; import org.polypheny.db.sql.language.fun.SqlMultisetValueConstructor; +import org.polypheny.db.sql.language.fun.SqlNamedDistanceFunction; import org.polypheny.db.sql.language.fun.SqlNewOperator; import org.polypheny.db.sql.language.fun.SqlNthValueAggFunction; import org.polypheny.db.sql.language.fun.SqlNtileAggFunction; @@ -1526,6 +1527,18 @@ public OperandCountRange getOperandCountRange() { */ register( OperatorName.DISTANCE, new SqlDistanceFunction() ); + /* + * distance functions without additional parameters + */ + register( OperatorName.L1_DISTANCE, new SqlNamedDistanceFunction( "L1_DISTANCE", Kind.L1_DISTANCE, FunctionCategory.L1_DISTANCE, SqlNamedDistanceFunction.TWO_NUMERIC_ARRAYS ) ); + register( OperatorName.L2_DISTANCE, new SqlNamedDistanceFunction( "L2_DISTANCE", Kind.L2_DISTANCE, FunctionCategory.L2_DISTANCE, SqlNamedDistanceFunction.TWO_NUMERIC_ARRAYS ) ); + register( OperatorName.COS_DISTANCE, new SqlNamedDistanceFunction( "COS_DISTANCE", Kind.COS_DISTANCE, FunctionCategory.COS_DISTANCE, SqlNamedDistanceFunction.TWO_NUMERIC_ARRAYS ) ); + register( OperatorName.HAMMING_DISTANCE, new SqlNamedDistanceFunction( "HAMMING_DISTANCE", Kind.HAMMING_DISTANCE, FunctionCategory.HAMMING_DISTANCE, SqlNamedDistanceFunction.TWO_BOOLEAN_ARRAYS ) ); + register( OperatorName.JACCARD_DISTANCE, new SqlNamedDistanceFunction( "JACCARD_DISTANCE", Kind.JACCARD_DISTANCE, FunctionCategory.JACCARD_DISTANCE, SqlNamedDistanceFunction.TWO_BOOLEAN_ARRAYS ) ); + register( OperatorName.INNER_PRODUCT_DISTANCE, new SqlNamedDistanceFunction( "INNER_PRODUCT_DISTANCE", Kind.INNER_PRODUCT_DISTANCE, FunctionCategory.INNER_PRODUCT_DISTANCE, SqlNamedDistanceFunction.TWO_NUMERIC_ARRAYS ) ); + + + /* * Get metadata of multimedia files */ @@ -2432,6 +2445,34 @@ public List getAuxiliaryFunctions() { null, null ) ); + //------------------------------------------------------------ + // PostgreSQL pgvector OPERATORS + //------------------------------------------------------------ + /* + Note on chosen precedence: + AND=24, comparisons =/ = 30, + - = 40, * / = 60 are already existing values. + We therefore use precedence 36 with left-associativity. + */ + register( OperatorName.PGVECTOR_L2, new SqlBinaryOperator( + "<->", Kind.L2_DISTANCE, 36, true, + ReturnTypes.DOUBLE, null, SqlNamedDistanceFunction.TWO_NUMERIC_ARRAYS ) ); + register( OperatorName.PGVECTOR_L1, new SqlBinaryOperator( + "<+>", Kind.L1_DISTANCE, 36, true, + ReturnTypes.DOUBLE, null, SqlNamedDistanceFunction.TWO_NUMERIC_ARRAYS ) ); + register( OperatorName.PGVECTOR_COS, new SqlBinaryOperator( + "<=>", Kind.COS_DISTANCE, 36, true, + ReturnTypes.DOUBLE, null, SqlNamedDistanceFunction.TWO_NUMERIC_ARRAYS ) ); + register( OperatorName.PGVECTOR_HAMMING, new SqlBinaryOperator( + "<~>", Kind.HAMMING_DISTANCE, 36, true, + ReturnTypes.DOUBLE, null, SqlNamedDistanceFunction.TWO_BOOLEAN_ARRAYS ) ); + register( OperatorName.PGVECTOR_JACCARD, new SqlBinaryOperator( + "<%>", Kind.JACCARD_DISTANCE, 36, true, + ReturnTypes.DOUBLE, null, SqlNamedDistanceFunction.TWO_BOOLEAN_ARRAYS ) ); + register( OperatorName.PGVECTOR_INNER_PRODUCT, new SqlBinaryOperator( + "<#>", Kind.INNER_PRODUCT_DISTANCE, 36, true, + ReturnTypes.DOUBLE, null, SqlNamedDistanceFunction.TWO_NUMERIC_ARRAYS ) ); + + /* * Operator to quantify patterns within {@code MATCH_RECOGNIZE}. * diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDataTypeSpec.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDataTypeSpec.java index daca411461..5c89df686c 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDataTypeSpec.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDataTypeSpec.java @@ -28,6 +28,7 @@ import org.polypheny.db.nodes.DataTypeSpec; import org.polypheny.db.nodes.Node; import org.polypheny.db.nodes.NodeVisitor; +import org.polypheny.db.sql.language.SqlWriter.FrameTypeEnum; import org.polypheny.db.sql.language.validate.SqlValidator; import org.polypheny.db.sql.language.validate.SqlValidatorScope; import org.polypheny.db.type.PolyType; @@ -75,6 +76,13 @@ public class SqlDataTypeSpec extends SqlNode implements DataTypeSpec { private final Boolean nullable; + /** + *

Whether element are allowed to have null values.

+ *

This is only meaningful for collection types.

+ */ + private final Boolean elementsNullable; + + /** * Creates a type specification representing a regular, non-collection type. */ @@ -105,6 +113,57 @@ public SqlDataTypeSpec( } + /** + * Creates a type specification representing a collection type. (Used by Parser.jj) + */ + public SqlDataTypeSpec( + SqlIdentifier collectionsTypeName, + SqlIdentifier typeName, + int precision, + int scale, + int dimension, + int cardinality, + String charSetName, + boolean elementsNullable, + ParserPos pos ) { + this( collectionsTypeName, typeName, typeName, precision, scale, dimension, cardinality, charSetName, null, null, elementsNullable, pos ); + } + + + /** + * Creates a type specification representing a regular, non-collection type. (Used by Parser.jj) + */ + public SqlDataTypeSpec( + final SqlIdentifier typeName, + int precision, + int scale, + String charSetName, + TimeZone timeZone, + Boolean elementsNullable, + ParserPos pos ) { + this( null, typeName, typeName, precision, scale, -1, -1, charSetName, timeZone, null, elementsNullable, pos ); + } + + + /** + * Creates a type specification representing a regular, non-collection type. (Used for collections with elementsNullable) + */ + public SqlDataTypeSpec( + final SqlIdentifier collectionsTypeName, + final SqlIdentifier typeName, + int precision, + int scale, + int dimension, + int cardinality, + String charSetName, + TimeZone timeZone, + Boolean nullable, + Boolean elementsNullable, + ParserPos pos ) { + this( collectionsTypeName, typeName, null, precision, scale, dimension, cardinality, charSetName, timeZone, nullable, elementsNullable, pos ); + } + + /** * Creates a type specification that has no base type. */ @@ -119,7 +178,7 @@ public SqlDataTypeSpec( TimeZone timeZone, Boolean nullable, ParserPos pos ) { - this( collectionsTypeName, typeName, typeName, precision, scale, dimension, cardinality, charSetName, timeZone, nullable, pos ); + this( collectionsTypeName, typeName, typeName, precision, scale, dimension, cardinality, charSetName, timeZone, nullable, true, pos ); } @@ -137,6 +196,7 @@ public SqlDataTypeSpec( String charSetName, TimeZone timeZone, Boolean nullable, + Boolean elementsNullable, ParserPos pos ) { super( pos ); this.collectionsTypeName = collectionsTypeName; @@ -149,14 +209,15 @@ public SqlDataTypeSpec( this.charSetName = charSetName; this.timeZone = timeZone; this.nullable = nullable; + this.elementsNullable = elementsNullable; } @Override public SqlNode clone( ParserPos pos ) { return (collectionsTypeName != null) - ? new SqlDataTypeSpec( collectionsTypeName, typeName, precision, scale, dimension, cardinality, charSetName, pos ) - : new SqlDataTypeSpec( typeName, precision, scale, charSetName, timeZone, pos ); + ? new SqlDataTypeSpec( collectionsTypeName, typeName, precision, scale, dimension, cardinality, charSetName, elementsNullable, pos ) + : new SqlDataTypeSpec( typeName, precision, scale, charSetName, timeZone, elementsNullable, pos ); } @@ -195,7 +256,7 @@ public SqlDataTypeSpec withNullable( Boolean nullable ) { if ( Objects.equals( nullable, this.nullable ) ) { return this; } - return new SqlDataTypeSpec( collectionsTypeName, typeName, precision, scale, dimension, cardinality, charSetName, timeZone, nullable, getPos() ); + return new SqlDataTypeSpec( collectionsTypeName, typeName, precision, scale, dimension, cardinality, charSetName, timeZone, nullable, elementsNullable, getPos() ); } @@ -255,9 +316,25 @@ public void unparse( SqlWriter writer, int leftPrec, int rightPrec ) { // We're generating a type for an alien system. For example, UNSIGNED is a built-in type in MySQL. // (Need a more elegant way than '_' of flagging this.) writer.keyword( name.substring( 1 ) ); + unparsePrecision( writer ); } else { // else we have a user defined type typeName.unparse( writer, leftPrec, rightPrec ); + unparsePrecision( writer ); + } + } + + + private void unparsePrecision( SqlWriter writer ) { + if ( precision >= 0 ) { + final SqlWriter.Frame frame = writer.startList( + FrameTypeEnum.FUN_CALL, "(", ")" ); + writer.print( precision ); + if ( scale >= 0 ) { + writer.sep(",", true); + writer.print( scale ); + } + writer.endList( frame ); } } @@ -377,6 +454,9 @@ public AlgDataType deriveType( AlgDataTypeFactory typeFactory, boolean nullable } type = typeFactory.createTypeWithCharsetAndCollation( type, charset, collation ); } + if ( elementsNullable != null ) { + type = typeFactory.createTypeWithNullability( type, elementsNullable ); + } if ( null != collectionsTypeName ) { final String collectionName = collectionsTypeName.getSimple(); @@ -384,7 +464,16 @@ public AlgDataType deriveType( AlgDataTypeFactory typeFactory, boolean nullable type = switch ( collectionsPolyType ) { case MULTISET -> typeFactory.createMultisetType( type, cardinality ); - case ARRAY -> typeFactory.createArrayType( type, cardinality, dimension ); + case ARRAY -> { + if ( !type.isNullable() && (type.getPolyType() == PolyType.FLOAT + || type.getPolyType() == PolyType.REAL + || type.getPolyType() == PolyType.BOOLEAN) + && dimension == 1 && cardinality > 0 ) { + yield typeFactory.createVectorType( type, cardinality ); + } else { + yield typeFactory.createArrayType( type, cardinality, dimension ); + } + } default -> throw Util.unexpected( collectionsPolyType ); }; } diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDbFeature.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDbFeature.java new file mode 100644 index 0000000000..2f56561174 --- /dev/null +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDbFeature.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.sql.language; + +/** + * Represents a named feature of a SQL database that may or may not be available + * on a given instance. + * + *

Examples of such features include database extensions (e.g. pgvector, PostGIS + * in PostgreSQL), plugins (e.g. in MySQL), or any other optional capability that + * is not guaranteed to be present on every instance of a database. + * + *

Each SQL adapter defines its own enum implementing this interface, enumerating + * the features it knows how to detect. The detected set is passed to the + * {@link SqlDialect}, which exposes individual capabilities + * through methods such as {@link SqlDialect#supportsVector()} via + * {@link SqlDialect#supportsFeature(SqlDbFeature)}. + */ +public interface SqlDbFeature { + + /** + * Returns the name by which this feature is identified in the database's + * own catalog or metadata system. + * + * @return the feature's catalog name, never {@code null} + */ + String featureName(); + + /** + * Returns the human-readable name of this feature, intended for display in + * user interfaces. + * + * @return the feature's display name, never {@code null} + */ + String displayName(); + + boolean isSupported( SqlDialect dialect ); + + /** + * Creates a query than can be run in order to register the feature in the database. + * In PostgreSQL this is e.g. {@code CREATE EXTENSION IF NOT EXISTS }. + * Preferably the query should be idempotent. + */ + String getFeatureRegistrationQuery(); +} diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDialect.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDialect.java index e1b094880c..6c0187eda7 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDialect.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/SqlDialect.java @@ -21,9 +21,11 @@ import java.sql.ResultSet; import java.sql.Timestamp; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Optional; +import java.util.Set; import java.util.regex.Pattern; import lombok.NonNull; import lombok.Value; @@ -57,8 +59,12 @@ import org.polypheny.db.type.BasicPolyType; import org.polypheny.db.type.PolyType; import org.polypheny.db.type.PolyTypeFactoryImpl; +import org.polypheny.db.type.VectorType; +import org.polypheny.db.type.VectorType.ElementType; import org.polypheny.db.type.entity.PolyBinary; +import org.polypheny.db.type.entity.PolyList; import org.polypheny.db.type.entity.PolyString; +import org.polypheny.db.type.entity.PolyValue; import org.polypheny.db.type.entity.category.PolyBlob; import org.polypheny.db.type.entity.spatial.PolyGeometry; import org.polypheny.db.util.temporal.DateTimeUtils; @@ -93,13 +99,16 @@ public class SqlDialect { @NonNull AlgDataTypeSystem dataTypeSystem; + @NonNull + protected Set supportedFeatures = new java.util.HashSet<>(); + /** * Creates a SqlDialect. * * @param context All the information necessary to create a dialect */ - public SqlDialect( Context context ) { + public SqlDialect( Context context) { this.name = context.name; this.nullCollation = context.nullCollation; this.dataTypeSystem = context.dataTypeSystem; @@ -695,11 +704,40 @@ public List supportedGeoFunctions() { } + public List supportedKnnFunctions() { + return List.of(); + } + + + public boolean supportsVector() { + return false; + } + + public boolean supportsComplexBinary() { return true; } + /** + * Returns whether this dialect supports a specific database feature (e.g., pgvector for PostgreSQL). + * Override in dialect to use. + */ + public boolean supportsFeature( SqlDbFeature feature ) { + return false; + } + + + public void addSupportedFeatures( java.util.Set features ) { + supportedFeatures.addAll( features ); + } + + + public Set getSupportedFeatures() { + return Collections.unmodifiableSet( supportedFeatures ); + } + + public Expression handleRetrieval( AlgDataType fieldType, Expression child, ParameterExpression resultSet_, int index ) { final String methodName = fieldType.isNullable() ? "ofNullable" : "of"; return switch ( fieldType.getPolyType() ) { @@ -808,6 +846,19 @@ public static String replace( String s, String find, String replace ) { } + /** + * Override this to provide a custom Linq4j Expression when the JDBC driver does not + * support standard {@code java.sql.Array} retrieval for a specific column type. + * + *

If {@link Optional#empty()} is returned, the caller falls back to the default + * {@code getArray()} path. Any returned expression must evaluate to a + * {@code Collection} suitable for {@link PolyList#of}. + */ + public Optional getCustomArrayRetrievalExpression(ParameterExpression resultSet, int i, AlgDataType fieldType) { + return Optional.empty(); + } + + /** * Whether this JDBC driver needs you to pass a Calendar object to methods such as {@link ResultSet#getTimestamp(int, java.util.Calendar)}. */ @@ -838,4 +889,53 @@ public record Context( } + /** + * Dialect specific setup on new connection, e.g. register certain non-standard types. + */ + public void initializeConnection( java.sql.Connection conn ) throws java.sql.SQLException { + + } + + + /** + * Returns whether the underlying database supports a vector with the given {@link ElementType}. + */ + public boolean vectorPushdownTypeIsPresent( VectorType.ElementType vectorType ) { + return false; + } + + + /** + *

Takes a vector type, i.e. the {@link ElementType} of a vector and returns the database specific object for that vector type.

+ *

Each dialect needs to overwrite this method, if providing dedicated vectors requiring non-standard vector database objects, in order to implement correct handling.

+ *

It is strongly advised to first check {@link SqlDialect#vectorPushdownTypeIsPresent(ElementType)} since otherwise the object will be null.

+ * @param vectorType The type of the vector elements. + * @param vectorAsList A {@link PolyList} representing the vector values. + * @return The database specific representation of the vector. Can be {@code null}. + */ + public Object getVectorDbObject( VectorType.ElementType vectorType, PolyList vectorAsList ) { + return null; + } + + + /** + *

Takes a vector type, i.e. the {@link ElementType} and returns the type string of the vector type.

+ *

Each dialect needs to overwrite this method, if providing dedicated vectors with non-standard type definitions, in order to implement correct handling.

+ *

It is strongly advised to first check {@link SqlDialect#vectorPushdownTypeIsPresent(ElementType)} since otherwise the string will be null.

+ * @param vectorType the element type of the vector indicating the vector type + * @return type string of the vector type + */ + public String getTypeString( VectorType.ElementType vectorType ) { + return null; + } + + + /** + * Returns a SqlNode that represents a vector literal for this dialect. + * Returns null if the dialect does not have a special syntax for vectors. + */ + public SqlNode getVectorLiteral( VectorType vectorType, PolyList vectorAsList, ParserPos pos) { + return null; + } + } diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/ddl/altermaterializedview/SqlAlterMaterializedViewAddIndex.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/ddl/altermaterializedview/SqlAlterMaterializedViewAddIndex.java index 77a8713eac..56cbceddab 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/ddl/altermaterializedview/SqlAlterMaterializedViewAddIndex.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/ddl/altermaterializedview/SqlAlterMaterializedViewAddIndex.java @@ -53,6 +53,7 @@ public class SqlAlterMaterializedViewAddIndex extends SqlAlterMaterializedView { SqlNodeList columnList; boolean unique; SqlIdentifier storeName; + Map options; public SqlAlterMaterializedViewAddIndex( @@ -62,7 +63,8 @@ public SqlAlterMaterializedViewAddIndex( boolean unique, SqlIdentifier indexMethod, SqlIdentifier indexName, - SqlIdentifier storeName ) { + SqlIdentifier storeName, + Map options ) { super( pos ); this.table = Objects.requireNonNull( table ); this.columnList = Objects.requireNonNull( columnList ); @@ -70,6 +72,7 @@ public SqlAlterMaterializedViewAddIndex( this.indexName = indexName; this.indexMethod = indexMethod; this.storeName = storeName; + this.options = options; } @@ -107,6 +110,20 @@ public void unparse( SqlWriter writer, int leftPrec, int rightPrec ) { writer.keyword( "STORE" ); storeName.unparse( writer, leftPrec, rightPrec ); } + if ( options != null && !options.isEmpty() ) { + writer.keyword( "WITH" ); + writer.print( "(" ); + boolean first = true; + for ( Map.Entry e : options.entrySet() ) { + if ( !first ) writer.print( "," ); + writer.identifier( e.getKey() ); + writer.print( "=" ); + writer.literal( e.getValue() ); + first = false; + } + writer.print( ")" ); + } + } @@ -140,7 +157,8 @@ public void execute( Context context, Statement statement, ParsedQueryContext pa indexName.getSimple(), unique, storeInstance, - statement ); + statement, + options ); } } diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/ddl/altertable/SqlAlterTableAddIndex.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/ddl/altertable/SqlAlterTableAddIndex.java index 21ffba6951..306d72bd3e 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/ddl/altertable/SqlAlterTableAddIndex.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/ddl/altertable/SqlAlterTableAddIndex.java @@ -55,6 +55,7 @@ public class SqlAlterTableAddIndex extends SqlAlterTable { private final SqlNodeList columnList; private final boolean unique; private final SqlIdentifier storeName; + private final Map options; public SqlAlterTableAddIndex( @@ -64,7 +65,8 @@ public SqlAlterTableAddIndex( boolean unique, SqlIdentifier indexMethod, SqlIdentifier indexName, - SqlIdentifier storeName ) { + SqlIdentifier storeName, + Map options ) { super( pos ); this.table = Objects.requireNonNull( table ); this.columnList = Objects.requireNonNull( columnList ); @@ -72,6 +74,7 @@ public SqlAlterTableAddIndex( this.indexName = indexName; this.indexMethod = indexMethod; this.storeName = storeName; + this.options = options; } @@ -109,6 +112,19 @@ public void unparse( SqlWriter writer, int leftPrec, int rightPrec ) { writer.keyword( "STORE" ); storeName.unparse( writer, leftPrec, rightPrec ); } + if ( options != null && !options.isEmpty() ) { + writer.keyword( "WITH" ); + writer.print( "(" ); + boolean first = true; + for ( Map.Entry e : options.entrySet() ) { + if ( !first ) writer.print( "," ); + writer.identifier( e.getKey() ); + writer.print( "=" ); + writer.literal( e.getValue() ); + first = false; + } + writer.print( ")" ); + } } @@ -147,7 +163,8 @@ public void execute( Context context, Statement statement, ParsedQueryContext pa indexName.getSimple(), unique, storeInstance, - statement ); + statement, + options ); } } diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/fun/SqlNamedDistanceFunction.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/fun/SqlNamedDistanceFunction.java new file mode 100644 index 0000000000..63bcf6846f --- /dev/null +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/fun/SqlNamedDistanceFunction.java @@ -0,0 +1,194 @@ +/* + * Copyright 2019-2026 The Polypheny Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.polypheny.db.sql.language.fun; + +import org.polypheny.db.algebra.constant.FunctionCategory; +import org.polypheny.db.algebra.constant.Kind; +import org.polypheny.db.nodes.CallBinding; +import org.polypheny.db.nodes.Operator; +import org.polypheny.db.sql.language.SqlFunction; +import org.polypheny.db.type.OperandCountRange; +import org.polypheny.db.type.PolyOperandCountRanges; +import org.polypheny.db.type.PolyType; +import org.polypheny.db.type.PolyTypeUtil; +import org.polypheny.db.type.checker.PolyOperandTypeChecker; +import org.polypheny.db.type.inference.ReturnTypes; +import org.polypheny.db.util.CoreUtil; + +import static org.polypheny.db.util.Static.RESOURCE; + +/** + * Represents an {@link SqlDistanceFunction} function that is not parameterized anymore. + */ +public class SqlNamedDistanceFunction extends SqlFunction { + + + public SqlNamedDistanceFunction( String name, Kind kind, FunctionCategory functionCategory, PolyOperandTypeChecker opreandTypeChecker ) { + super( name, + kind, + ReturnTypes.DOUBLE, + null, + opreandTypeChecker, + functionCategory ); + } + + + @Override + public String getSignatureTemplate( int operandsCount ) { + if ( operandsCount == 3) return "{0}({1}, {2})"; + throw new AssertionError(); + } + + + public static final PolyOperandTypeChecker TWO_NUMERIC_ARRAYS = new PolyOperandTypeChecker() { + + /** + * This method is similar to {@link SqlDistanceFunction#getOperandTypeChecker()#checkOperandTypes(CallBinding, boolean)}. + */ + @Override + public boolean checkOperandTypes( CallBinding callBinding, boolean throwOnFailure ) { + + // Make sure the first argument is not null + if ( CoreUtil.isNullLiteral( callBinding.operand( 0 ), false ) ) { + if ( throwOnFailure ) { + throw callBinding.getValidator().newValidationError( callBinding.operand( 0 ), RESOURCE.nullIllegal() ); + } else { + return false; + } + } + // Make sure the first argument is an array of numeric values + if ( !PolyTypeUtil.isArray( callBinding.getOperandType( 0 ) ) + || !PolyTypeUtil.isNumeric( callBinding.getOperandType( 0 ).getComponentType() ) ) { + if ( throwOnFailure ) { + throw callBinding.newValidationSignatureError(); + } else { + return false; + } + } + // Make sure the second argument is not null + if ( CoreUtil.isNullLiteral( callBinding.operand( 1 ), false ) ) { + if ( throwOnFailure ) { + throw callBinding.getValidator().newValidationError( callBinding.operand( 1 ), RESOURCE.nullIllegal() ); + } else { + return false; + } + } + // Make sure the second argument is an array of numeric values + if ( !PolyTypeUtil.isArray( callBinding.getOperandType( 1 ) ) + || !PolyTypeUtil.isNumeric( callBinding.getOperandType( 1 ).getComponentType() ) ) { + if ( throwOnFailure ) { + throw callBinding.newValidationSignatureError(); + } else { + return false; + } + } + return true; + } + + @Override + public OperandCountRange getOperandCountRange() { + return PolyOperandCountRanges.of( 2 ); + } + + + @Override + public String getAllowedSignatures( Operator op, String opName ) { + return "'" + opName + "(, )'"; + } + + + @Override + public Consistency getConsistency() { + return Consistency.NONE; + } + + + @Override + public boolean isOptional( int i ) { + return false; + } + }; + + + public static final PolyOperandTypeChecker TWO_BOOLEAN_ARRAYS = new PolyOperandTypeChecker() { + + @Override + public boolean checkOperandTypes( CallBinding callBinding, boolean throwOnFailure ) { + + // Make sure the first argument is not null + if ( CoreUtil.isNullLiteral( callBinding.operand( 0 ), false ) ) { + if ( throwOnFailure ) { + throw callBinding.getValidator().newValidationError( callBinding.operand( 0 ), RESOURCE.nullIllegal() ); + } else { + return false; + } + } + // Make sure the first argument is an array of numeric values + if ( !PolyTypeUtil.isArray( callBinding.getOperandType( 0 ) ) + || (callBinding.getOperandType( 0 ).getComponentType().getPolyType() != PolyType.BOOLEAN) ) { + if ( throwOnFailure ) { + throw callBinding.newValidationSignatureError(); + } else { + return false; + } + } + // Make sure the second argument is not null + if ( CoreUtil.isNullLiteral( callBinding.operand( 1 ), false ) ) { + if ( throwOnFailure ) { + throw callBinding.getValidator().newValidationError( callBinding.operand( 1 ), RESOURCE.nullIllegal() ); + } else { + return false; + } + } + // Make sure the second argument is an array of numeric values + if ( !PolyTypeUtil.isArray( callBinding.getOperandType( 1 ) ) + || (callBinding.getOperandType( 1 ).getComponentType().getPolyType() != PolyType.BOOLEAN) ) { + if ( throwOnFailure ) { + throw callBinding.newValidationSignatureError(); + } else { + return false; + } + } + return true; + } + + + @Override + public OperandCountRange getOperandCountRange() { + return PolyOperandCountRanges.of( 2 ); + } + + + @Override + public String getAllowedSignatures( Operator op, String opName ) { + return "'" + opName + "(, )'"; + } + + + @Override + public Consistency getConsistency() { + return Consistency.NONE; + } + + + @Override + public boolean isOptional( int i ) { + return false; + } + }; + +} diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/util/SqlTypeUtil.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/util/SqlTypeUtil.java index 5def854e18..ed85badb84 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/util/SqlTypeUtil.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/util/SqlTypeUtil.java @@ -186,6 +186,34 @@ public static DataTypeSpec createDataTypeSpec( boolean nullable, ParserPos zero ) { + return createDataTypeSpec( + typeIdentifier, + componentTypeIdentifier, + precision, + scale, + dimension, + cardinality, + charSetName, + o, + nullable, + true, + zero ); + } + + + public static DataTypeSpec createDataTypeSpec( + Identifier typeIdentifier, + Identifier componentTypeIdentifier, + int precision, + int scale, + int dimension, + int cardinality, + String charSetName, + TimeZone o, + boolean nullable, + Boolean elementsNullable, + ParserPos zero ) { + return new SqlDataTypeSpec( (SqlIdentifier) typeIdentifier, (SqlIdentifier) componentTypeIdentifier, @@ -196,6 +224,7 @@ public static DataTypeSpec createDataTypeSpec( charSetName, o, nullable, + elementsNullable, zero ); } diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/validate/SqlValidatorImpl.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/validate/SqlValidatorImpl.java index 5f068a9566..7568e853ed 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/validate/SqlValidatorImpl.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/language/validate/SqlValidatorImpl.java @@ -4315,6 +4315,16 @@ protected void validateValues( SqlCall node, AlgDataType targetRowType, final Sq if ( !pair.right.getType().isNullable() && CoreUtil.isNullLiteral( pair.left, false ) ) { throw newValidationError( node, RESOURCE.columnNotNullable( pair.right.getName() ) ); } + if ( pair.right.getType().getPolyType() == PolyType.ARRAY + && !pair.right.getType().getComponentType().isNullable() + && pair.left instanceof SqlCall arrayCall + && arrayCall.getKind() == Kind.ARRAY_VALUE_CONSTRUCTOR ) { + for ( Node element : arrayCall.getOperandList() ) { + if ( CoreUtil.isNullLiteral( element, true ) ) { + throw newValidationError( node, RESOURCE.columnNotNullable( pair.right.getName() ) ); + } + } + } } } } diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/sql2alg/SqlToAlgConverter.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/sql2alg/SqlToAlgConverter.java index 0138b280cd..d1f5c37833 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/sql2alg/SqlToAlgConverter.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/sql2alg/SqlToAlgConverter.java @@ -225,6 +225,7 @@ import org.polypheny.db.tools.AlgBuilder; import org.polypheny.db.type.PolyType; import org.polypheny.db.type.PolyTypeUtil; +import org.polypheny.db.type.VectorType; import org.polypheny.db.type.entity.PolyString; import org.polypheny.db.type.entity.PolyValue; import org.polypheny.db.type.inference.PolyReturnTypeInference; @@ -2831,6 +2832,10 @@ protected AlgNode convertColumnList( final SqlInsert call, AlgNode source ) { // bare nulls are dangerous in the wrong hands sourceExps.set( i, castNullLiteralIfNeeded( sourceExps.get( i ), field.getType() ) ); + } else if ( field.getType() instanceof VectorType ){ + // Assume an insert ARRAY[1,2,3]: We do not know if the column type where the insert should go is a VectorType (i.e. REAL NOT NULL ARRAY(1,n) w/ n > 0 and all elements non-null) or an ArrayType. + // We therefore cast to target column. Checks if types are compatible are already done therefore "instanceof" is enough. + sourceExps.set( i, rexBuilder.makeCast( field.getType(), sourceExps.get( i ) ) ); } } diff --git a/plugins/sql-language/src/main/java/org/polypheny/db/sql/sql2alg/StandardConvertletTable.java b/plugins/sql-language/src/main/java/org/polypheny/db/sql/sql2alg/StandardConvertletTable.java index 1ca88ab91c..7136f8abbe 100644 --- a/plugins/sql-language/src/main/java/org/polypheny/db/sql/sql2alg/StandardConvertletTable.java +++ b/plugins/sql-language/src/main/java/org/polypheny/db/sql/sql2alg/StandardConvertletTable.java @@ -23,6 +23,7 @@ import java.math.RoundingMode; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Objects; import org.polypheny.db.algebra.constant.FunctionCategory; import org.polypheny.db.algebra.constant.Kind; @@ -78,6 +79,7 @@ import org.polypheny.db.type.PolyType; import org.polypheny.db.type.PolyTypeFamily; import org.polypheny.db.type.PolyTypeUtil; +import org.polypheny.db.type.VectorType; import org.polypheny.db.type.checker.PolyOperandTypeChecker; import org.polypheny.db.type.entity.PolyList; import org.polypheny.db.type.entity.PolyValue; @@ -251,6 +253,94 @@ private StandardConvertletTable() { registerOp( OperatorRegistry.get( OperatorName.TIMESTAMP_ADD ), new TimestampAddConvertlet() ); registerOp( OperatorRegistry.get( OperatorName.TIMESTAMP_DIFF ), new TimestampDiffConvertlet() ); + /* Register special internal versions of distance functions */ + registerOp( OperatorRegistry.get( OperatorName.DISTANCE ), (cx, call) -> { + List operands = call.getSqlOperandList(); + if ( operands.size() == 3 && operands.get( 2 ) instanceof SqlLiteral metric ) { + OperatorName name = switch ( metric.toValue().toUpperCase( Locale.ROOT ) ) { + case "L1" -> OperatorName.L1_DISTANCE; + case "L2" -> OperatorName.L2_DISTANCE; + case "COSINE" -> OperatorName.COS_DISTANCE; + case "HAMMING" -> OperatorName.HAMMING_DISTANCE; + case "JACCARD" -> OperatorName.JACCARD_DISTANCE; + case "INNER_PRODUCT" -> OperatorName.INNER_PRODUCT_DISTANCE; + default -> null; + }; + if ( name != null ) { + RexBuilder rb = cx.getRexBuilder(); + RexNode arg0 = cx.convertExpression( operands.get( 0 ) ); + RexNode arg1 = cx.convertExpression( operands.get( 1 ) ); + RexNode c0 = coerceQueryVector( rb, arg0, arg1.getType() ); + RexNode c1 = coerceQueryVector( rb, arg1, arg0.getType() ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return rb.makeCall( returnType, OperatorRegistry.get( name ), ImmutableList.of( c0, c1 ) ); + } + } + return convertCall( cx, call ); + }); + + /* pgvector binary operator to non-parameterized distance functions */ + registerOp( OperatorRegistry.get( OperatorName.PGVECTOR_L1 ), (cx, call) -> { + RexNode arg0 = cx.convertExpression( call.operand( 0 ) ); + RexNode arg1 = cx.convertExpression( call.operand( 1 ) ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return cx.getRexBuilder().makeCall( returnType, OperatorRegistry.get( OperatorName.L1_DISTANCE ), ImmutableList.of( arg0, arg1 ) ); + } ); + registerOp( OperatorRegistry.get( OperatorName.PGVECTOR_L2 ), (cx, call) -> { + RexNode arg0 = cx.convertExpression( call.operand( 0 ) ); + RexNode arg1 = cx.convertExpression( call.operand( 1 ) ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return cx.getRexBuilder().makeCall( returnType, OperatorRegistry.get( OperatorName.L2_DISTANCE ), ImmutableList.of( arg0, arg1 ) ); + } ); + registerOp( OperatorRegistry.get( OperatorName.PGVECTOR_COS ), (cx, call) -> { + RexNode arg0 = cx.convertExpression( call.operand( 0 ) ); + RexNode arg1 = cx.convertExpression( call.operand( 1 ) ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return cx.getRexBuilder().makeCall( returnType, OperatorRegistry.get( OperatorName.COS_DISTANCE ), ImmutableList.of( arg0, arg1 ) ); + } ); + registerOp( OperatorRegistry.get( OperatorName.PGVECTOR_HAMMING ), (cx, call) -> { + RexBuilder rb = cx.getRexBuilder(); + RexNode arg0 = cx.convertExpression( call.operand( 0 ) ); + RexNode arg1 = cx.convertExpression( call.operand( 1 ) ); + RexNode c0 = coerceQueryVector( rb, arg0, arg1.getType() ); + RexNode c1 = coerceQueryVector( rb, arg1, arg0.getType() ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return rb.makeCall( returnType, OperatorRegistry.get( OperatorName.HAMMING_DISTANCE ), ImmutableList.of( c0, c1 ) ); + } ); + registerOp( OperatorRegistry.get( OperatorName.PGVECTOR_JACCARD ), (cx, call) -> { + RexBuilder rb = cx.getRexBuilder(); + RexNode arg0 = cx.convertExpression( call.operand( 0 ) ); + RexNode arg1 = cx.convertExpression( call.operand( 1 ) ); + RexNode c0 = coerceQueryVector( rb, arg0, arg1.getType() ); + RexNode c1 = coerceQueryVector( rb, arg1, arg0.getType() ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return rb.makeCall( returnType, OperatorRegistry.get( OperatorName.JACCARD_DISTANCE ), ImmutableList.of( c0, c1 ) ); + } ); + registerOp( OperatorRegistry.get( OperatorName.PGVECTOR_INNER_PRODUCT ), (cx, call) -> { + RexNode arg0 = cx.convertExpression( call.operand( 0 ) ); + RexNode arg1 = cx.convertExpression( call.operand( 1 ) ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return cx.getRexBuilder().makeCall( returnType, OperatorRegistry.get( OperatorName.INNER_PRODUCT_DISTANCE ), ImmutableList.of( arg0, arg1 ) ); + } ); + registerOp( OperatorRegistry.get( OperatorName.HAMMING_DISTANCE ), (cx, call) -> { + RexBuilder rb = cx.getRexBuilder(); + RexNode arg0 = cx.convertExpression( call.operand( 0 ) ); + RexNode arg1 = cx.convertExpression( call.operand( 1 ) ); + RexNode c0 = coerceQueryVector( rb, arg0, arg1.getType() ); + RexNode c1 = coerceQueryVector( rb, arg1, arg0.getType() ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return rb.makeCall( returnType, OperatorRegistry.get( OperatorName.HAMMING_DISTANCE ), ImmutableList.of( c0, c1 ) ); + } ); + registerOp( OperatorRegistry.get( OperatorName.JACCARD_DISTANCE ), (cx, call) -> { + RexBuilder rb = cx.getRexBuilder(); + RexNode arg0 = cx.convertExpression( call.operand( 0 ) ); + RexNode arg1 = cx.convertExpression( call.operand( 1 ) ); + RexNode c0 = coerceQueryVector( rb, arg0, arg1.getType() ); + RexNode c1 = coerceQueryVector( rb, arg1, arg0.getType() ); + AlgDataType returnType = cx.getValidator().getValidatedNodeType( call ); + return rb.makeCall( returnType, OperatorRegistry.get( OperatorName.JACCARD_DISTANCE ), ImmutableList.of( c0, c1 ) ); + } ); + // Convert "element()" to "$element_slice()", if the expression is a multiset of scalars. if ( false ) { registerOp( @@ -312,6 +402,43 @@ private static RexNode divideInt( RexBuilder rexBuilder, RexNode a0, RexNode a1 } + /** + * When a distance call compares a stored vector column against a plain array query vector (e.g. + * {@code ARRAY[true, false, true]}), retypes that array operand to the column's {@link VectorType} so the + * query vector is carried as a vector all the way down to the adapter. This is required for bit (boolean) + * vectors: Postgres has no {@code boolean[] -> bit} cast, so the query vector must reach the JDBC layer as a + * {@code VectorType} parameter. + */ + private static RexNode coerceQueryVector( RexBuilder rb, RexNode operand, AlgDataType otherType ) { + if ( !(otherType instanceof VectorType vectorType) + || vectorType.getVectorElementType() != VectorType.ElementType.BIT + || operand.getType() instanceof VectorType + || !isCandidateQueryVector( operand.getType() ) ) { + return operand; + } + if ( operand instanceof RexCall arrayCall && arrayCall.getKind() == Kind.ARRAY_VALUE_CONSTRUCTOR ) { + return rb.makeCall( vectorType, arrayCall.getOperator(), arrayCall.getOperands() ); + } + if ( operand instanceof RexLiteral literal ) { + return rb.makeLiteral( literal.value, vectorType, literal.getPolyType() ); + } + return rb.makeCast( vectorType, operand ); + } + + + /** + * A query vector operand that may be coerced to a {@link VectorType}: a non-vector array whose elements are + * numeric or boolean. + */ + private static boolean isCandidateQueryVector( AlgDataType type ) { + if ( type.getPolyType() != PolyType.ARRAY ) { + return false; + } + AlgDataType comp = type.getComponentType(); + return comp != null && (PolyTypeUtil.isNumeric( comp ) || comp.getPolyType() == PolyType.BOOLEAN); + } + + private RexNode plus( RexBuilder rexBuilder, RexNode a0, RexNode a1 ) { return rexBuilder.makeCall( OperatorRegistry.get( OperatorName.PLUS ), a0, a1 ); } diff --git a/plugins/sql-language/src/test/java/org/polypheny/db/sql/SqlLanguageDependent.java b/plugins/sql-language/src/test/java/org/polypheny/db/sql/SqlLanguageDependent.java index 6e5d38317b..51ee21dfa1 100644 --- a/plugins/sql-language/src/test/java/org/polypheny/db/sql/SqlLanguageDependent.java +++ b/plugins/sql-language/src/test/java/org/polypheny/db/sql/SqlLanguageDependent.java @@ -107,9 +107,9 @@ private static void createHrSchema( TestHelper testHelper ) throws TransactionEx DdlManager manager = DdlManager.getInstance(); List columns = List.of( - new FieldInformation( "deptno", new ColumnTypeInformation( PolyType.INTEGER, null, null, null, null, null, false ), null, null, 0 ), - new FieldInformation( "name", new ColumnTypeInformation( PolyType.VARCHAR, null, 20, null, null, null, false ), null, null, 1 ), - new FieldInformation( "loc", new ColumnTypeInformation( PolyType.VARCHAR, null, 50, null, null, null, true ), null, null, 2 ) + new FieldInformation( "deptno", new ColumnTypeInformation( PolyType.INTEGER, null, null, null, null, null, false, true ), null, null, 0 ), + new FieldInformation( "name", new ColumnTypeInformation( PolyType.VARCHAR, null, 20, null, null, null, false, true ), null, null, 1 ), + new FieldInformation( "loc", new ColumnTypeInformation( PolyType.VARCHAR, null, 50, null, null, null, true, true ), null, null, 2 ) ); List constraints = List.of( @@ -133,9 +133,9 @@ private static void createTestSchema( TestHelper testHelper ) { DdlManager manager = DdlManager.getInstance(); List columns = List.of( - new FieldInformation( "deptno", new ColumnTypeInformation( PolyType.INTEGER, null, null, null, null, null, false ), null, null, 0 ), - new FieldInformation( "name", new ColumnTypeInformation( PolyType.VARCHAR, null, 20, null, null, null, false ), null, null, 1 ), - new FieldInformation( "loc", new ColumnTypeInformation( PolyType.VARCHAR, null, 50, null, null, null, true ), null, null, 2 ) + new FieldInformation( "deptno", new ColumnTypeInformation( PolyType.INTEGER, null, null, null, null, null, false, true ), null, null, 0 ), + new FieldInformation( "name", new ColumnTypeInformation( PolyType.VARCHAR, null, 20, null, null, null, false, true ), null, null, 1 ), + new FieldInformation( "loc", new ColumnTypeInformation( PolyType.VARCHAR, null, 50, null, null, null, true, true ), null, null, 2 ) ); List constraints = List.of( @@ -146,13 +146,13 @@ private static void createTestSchema( TestHelper testHelper ) { // "CREATE TABLE employee( empid BIGINT NOT NULL, ename VARCHAR(20), job VARCHAR(10), mgr INTEGER, hiredate DATE, salary DECIMAL(7,2), commission DECIMAL(7,2), deptno INTEGER NOT NULL, PRIMARY KEY (empid)) " columns = List.of( - new FieldInformation( "empid", new ColumnTypeInformation( PolyType.BIGINT, null, null, null, null, null, false ), null, null, 0 ), - new FieldInformation( "ename", new ColumnTypeInformation( PolyType.VARCHAR, null, 20, null, null, null, true ), null, null, 1 ), - new FieldInformation( "job", new ColumnTypeInformation( PolyType.VARCHAR, null, 10, null, null, null, true ), null, null, 2 ), - new FieldInformation( "mgr", new ColumnTypeInformation( PolyType.INTEGER, null, null, null, null, null, true ), null, null, 3 ), - new FieldInformation( "hiredate", new ColumnTypeInformation( PolyType.DATE, null, null, null, null, null, true ), null, null, 4 ), - new FieldInformation( "salary", new ColumnTypeInformation( PolyType.DECIMAL, null, null, 7, 2, null, true ), null, null, 5 ), - new FieldInformation( "deptno", new ColumnTypeInformation( PolyType.INTEGER, null, null, null, null, null, true ), null, null, 6 ) + new FieldInformation( "empid", new ColumnTypeInformation( PolyType.BIGINT, null, null, null, null, null, false, true ), null, null, 0 ), + new FieldInformation( "ename", new ColumnTypeInformation( PolyType.VARCHAR, null, 20, null, null, null, true, true ), null, null, 1 ), + new FieldInformation( "job", new ColumnTypeInformation( PolyType.VARCHAR, null, 10, null, null, null, true, true ), null, null, 2 ), + new FieldInformation( "mgr", new ColumnTypeInformation( PolyType.INTEGER, null, null, null, null, null, true, true ), null, null, 3 ), + new FieldInformation( "hiredate", new ColumnTypeInformation( PolyType.DATE, null, null, null, null, null, true, true ), null, null, 4 ), + new FieldInformation( "salary", new ColumnTypeInformation( PolyType.DECIMAL, null, null, 7, 2, null, true, true ), null, null, 5 ), + new FieldInformation( "deptno", new ColumnTypeInformation( PolyType.INTEGER, null, null, null, null, null, true, true ), null, null, 6 ) ); constraints = List.of( @@ -165,7 +165,7 @@ private static void createTestSchema( TestHelper testHelper ) { long id = manager.createNamespace( "customer", DataModel.RELATIONAL, true, false, false, transaction.createStatement() ); columns = List.of( - new FieldInformation( "fname", new ColumnTypeInformation( PolyType.VARCHAR, null, 50, null, null, null, false ), null, null, 0 ) + new FieldInformation( "fname", new ColumnTypeInformation( PolyType.VARCHAR, null, 50, null, null, null, false, true ), null, null, 0 ) ); constraints = List.of( diff --git a/plugins/sql-language/src/test/java/org/polypheny/db/sql/language/parser/SqlParserTest.java b/plugins/sql-language/src/test/java/org/polypheny/db/sql/language/parser/SqlParserTest.java index 9f217f3b13..f850f264cf 100644 --- a/plugins/sql-language/src/test/java/org/polypheny/db/sql/language/parser/SqlParserTest.java +++ b/plugins/sql-language/src/test/java/org/polypheny/db/sql/language/parser/SqlParserTest.java @@ -4416,7 +4416,7 @@ public void testDateTimeCast() { // "CAST(2001-12-21)"); checkExp( "CAST('2001-12-21' AS DATE)", "CAST('2001-12-21' AS DATE)" ); checkExp( "CAST(12 AS DATE)", "CAST(12 AS DATE)" ); - checkFails( "CAST('2000-12-21' AS DATE ^NOT^ NULL)", "(?s).*Encountered \"NOT\" at line 1, column 27.*" ); + checkFails( "CAST('2000-12-21' AS DATE ^NOT^ NULL)", "(?s).*Encountered \"NOT NULL \\)\" at line 1, column 27.*" ); checkFails( "CAST('foo' as ^1^)", "(?s).*Encountered \"1\" at line 1, column 15.*" ); checkExp( "Cast(DATE '2004-12-21' AS VARCHAR(10))", "CAST(DATE '2004-12-21' AS VARCHAR(10))" ); } diff --git a/plugins/workflow-engine/src/main/java/org/polypheny/db/workflow/dag/settings/CastValue.java b/plugins/workflow-engine/src/main/java/org/polypheny/db/workflow/dag/settings/CastValue.java index 65cb91fe35..61aa64a037 100644 --- a/plugins/workflow-engine/src/main/java/org/polypheny/db/workflow/dag/settings/CastValue.java +++ b/plugins/workflow-engine/src/main/java/org/polypheny/db/workflow/dag/settings/CastValue.java @@ -200,7 +200,8 @@ public void buildType() { collectionsType, cardinality, dimension, - nullable ); + nullable, + true ); converter = PolyValue.getConverter( type ); nullValue = PolyValue.getNull( PolyValue.classFrom( type ) ); diff --git a/plugins/workflow-engine/src/main/java/org/polypheny/db/workflow/engine/storage/StorageManagerImpl.java b/plugins/workflow-engine/src/main/java/org/polypheny/db/workflow/engine/storage/StorageManagerImpl.java index a0509b19a0..b656002835 100644 --- a/plugins/workflow-engine/src/main/java/org/polypheny/db/workflow/engine/storage/StorageManagerImpl.java +++ b/plugins/workflow-engine/src/main/java/org/polypheny/db/workflow/engine/storage/StorageManagerImpl.java @@ -574,7 +574,8 @@ private static ColumnTypeInformation getColTypeInfo( AlgDataTypeField field ) { type.getScale(), isArray ? (int) ((ArrayType) field.getType()).getDimension() : -1, isArray ? (int) ((ArrayType) field.getType()).getCardinality() : -1, - field.getType().isNullable() ); + field.getType().isNullable(), + field.getType().getComponentType() == null || field.getType().getComponentType().isNullable() ); } diff --git a/webui/src/main/java/org/polypheny/db/webui/Crud.java b/webui/src/main/java/org/polypheny/db/webui/Crud.java index d49c899bde..475d037fce 100644 --- a/webui/src/main/java/org/polypheny/db/webui/Crud.java +++ b/webui/src/main/java/org/polypheny/db/webui/Crud.java @@ -48,6 +48,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -322,6 +323,7 @@ RelationalResult getTable( final UIRequest request ) { .dataType( logicalColumn.type.getName() ) .collectionsType( collectionsType ) .nullable( logicalColumn.nullable ) + .elementsNullable( logicalColumn.elementsNullable ) .precision( logicalColumn.length ) .scale( logicalColumn.scale ) .dimension( logicalColumn.dimension ) @@ -974,6 +976,7 @@ void getColumns( final Context ctx ) { .dataType( logicalColumn.type.getName() ) .collectionsType( collectionsType ) .nullable( logicalColumn.nullable ) + .elementsNullable( logicalColumn.elementsNullable ) .precision( logicalColumn.length ) .scale( logicalColumn.scale ) .dimension( logicalColumn.dimension ) @@ -1039,6 +1042,7 @@ void getDataSourceColumns( final Context ctx ) { .name( col.name ) .dataType( col.type.getName() ) .collectionsType( col.collectionsType == null ? "" : col.collectionsType.getName() ).nullable( col.nullable ) + .elementsNullable( col.elementsNullable ) .precision( col.length ) .scale( col.scale ) .dimension( col.dimension ) @@ -1167,6 +1171,7 @@ void updateColumn( final Context ctx ) { UiColumnDefinition oldColumn = request.oldColumn; UiColumnDefinition newColumn = request.newColumn; + String elementsNullable = newColumn.elementsNullable ? "" : " NOT NULL"; List queries = new ArrayList<>(); StringBuilder sBuilder = new StringBuilder(); @@ -1205,6 +1210,7 @@ void updateColumn( final Context ctx ) { } //collectionType if ( newColumn.collectionsType != null && !newColumn.collectionsType.isEmpty() ) { + query = query + elementsNullable; query = query + " " + request.newColumn.collectionsType; int dimension = newColumn.dimension == null ? -1 : newColumn.dimension; int cardinality = newColumn.cardinality == null ? -1 : newColumn.cardinality; @@ -1305,6 +1311,9 @@ void addColumn( final Context ctx ) { query = query + ")"; } if ( request.newColumn.collectionsType != null && !request.newColumn.collectionsType.isEmpty() ) { + if ( !request.newColumn.elementsNullable ) { + query = query + " NOT NULL"; + } query = query + " " + request.newColumn.collectionsType; int dimension = request.newColumn.dimension == null ? -1 : request.newColumn.dimension; int cardinality = request.newColumn.cardinality == null ? -1 : request.newColumn.cardinality; @@ -1543,18 +1552,23 @@ void getIndexes( final Context ctx ) { LogicalTable table = getLogicalTable( namespaceTable.left.name, namespaceTable.right.name ); List logicalIndices = Catalog.snapshot().rel().getIndexes( table.id, false ); - UiColumnDefinition[] header = { - UiColumnDefinition.builder().name( "Name" ).build(), - UiColumnDefinition.builder().name( "Columns" ).build(), - UiColumnDefinition.builder().name( "Location" ).build(), - UiColumnDefinition.builder().name( "Method" ).build(), - UiColumnDefinition.builder().name( "Type" ).build() }; + // Only show the parameters column if at least one index actually defines options (e.g. a vector index) + boolean showParameters = logicalIndices.stream().anyMatch( idx -> idx.options != null && !idx.options.isEmpty() ); + + List header = new ArrayList<>(); + header.add( UiColumnDefinition.builder().name( "Name" ).build() ); + header.add( UiColumnDefinition.builder().name( "Columns" ).build() ); + header.add( UiColumnDefinition.builder().name( "Location" ).build() ); + header.add( UiColumnDefinition.builder().name( "Method" ).build() ); + if ( showParameters ) { + header.add( UiColumnDefinition.builder().name( "Parameters" ).build() ); + } + header.add( UiColumnDefinition.builder().name( "Type" ).build() ); List data = new ArrayList<>(); // Get explicit indexes for ( LogicalIndex logicalIndex : logicalIndices ) { - String[] arr = new String[5]; String storeUniqueName; if ( logicalIndex.location < 0 ) { // a polystore index @@ -1562,12 +1576,16 @@ void getIndexes( final Context ctx ) { } else { storeUniqueName = Catalog.snapshot().getAdapter( logicalIndex.location ).orElseThrow().uniqueName; } - arr[0] = logicalIndex.name; - arr[1] = String.join( ", ", logicalIndex.key.getFieldNames() ); - arr[2] = storeUniqueName; - arr[3] = logicalIndex.methodDisplayName; - arr[4] = logicalIndex.type.name(); - data.add( arr ); + List row = new ArrayList<>(); + row.add( logicalIndex.name ); + row.add( String.join( ", ", logicalIndex.key.getFieldNames() ) ); + row.add( storeUniqueName ); + row.add( logicalIndex.methodDisplayName ); + if ( showParameters ) { + row.add( formatIndexOptions( logicalIndex.options ) ); + } + row.add( logicalIndex.type.name() ); + data.add( row.toArray( new String[0] ) ); } // Get functional indexes @@ -1581,17 +1599,41 @@ void getIndexes( final Context ctx ) { break; } for ( FunctionalIndexInfo fif : store.getFunctionalIndexes( table ) ) { - String[] arr = new String[5]; - arr[0] = ""; - arr[1] = String.join( ", ", fif.getColumnNames() ); - arr[2] = store.getUniqueName(); - arr[3] = fif.methodDisplayName(); - arr[4] = "FUNCTIONAL"; - data.add( arr ); + List row = new ArrayList<>(); + row.add( "" ); + row.add( String.join( ", ", fif.getColumnNames() ) ); + row.add( store.getUniqueName() ); + row.add( fif.methodDisplayName() ); + if ( showParameters ) { + row.add( "" ); + } + row.add( "FUNCTIONAL" ); + data.add( row.toArray( new String[0] ) ); } } - ctx.json( RelationalResult.builder().header( header ).data( data.toArray( new String[0][2] ) ).build() ); + ctx.json( RelationalResult.builder().header( header.toArray( new UiColumnDefinition[0] ) ).data( data.toArray( new String[0][] ) ).build() ); + } + + + /** + * Formats the options of an index for display, rendering well-known keys in a stable, readable order. + */ + private static String formatIndexOptions( Map options ) { + if ( options == null || options.isEmpty() ) { + return ""; + } + StringJoiner joiner = new StringJoiner( ", " ); + Map remaining = new LinkedHashMap<>( options ); + for ( String key : List.of( "metric", "m", "ef_construction", "lists" ) ) { + String value = remaining.remove( key ); + if ( value != null ) { + joiner.add( key + "=" + value ); + } + } + // Append any remaining, less common options + remaining.forEach( ( key, value ) -> joiner.add( key + "=" + value ) ); + return joiner.toString(); } @@ -1635,11 +1677,18 @@ void createIndex( final Context ctx ) { } String onStore = String.format( "ON STORE \"%s\"", store ); - String query = String.format( "ALTER TABLE %s ADD INDEX \"%s\" ON %s USING \"%s\" %s", tableId, index.getName(), colJoiner, index.getMethod(), onStore ); + StringBuilder query = new StringBuilder( String.format( "ALTER TABLE %s ADD INDEX \"%s\" ON %s USING \"%s\" %s", tableId, index.getName(), colJoiner, index.getMethod(), onStore ) ); + if ( index.options != null && !index.options.isEmpty() ) { + StringJoiner withJoiner = new StringJoiner( ", ", " WITH (", ")" ); + for ( Map.Entry e : index.options.entrySet() ) { + withJoiner.add( e.getKey() + "=" + e.getValue() ); + } + query.append( withJoiner ); + } QueryLanguage language = QueryLanguage.from( "sql" ); Result res = LanguageCrud.anyQueryResult( QueryContext.builder() - .query( query ) + .query( query.toString() ) .language( language ) .origin( ORIGIN ) .transactionManager( transactionManager ) diff --git a/webui/src/main/java/org/polypheny/db/webui/models/AdapterTemplateModel.java b/webui/src/main/java/org/polypheny/db/webui/models/AdapterTemplateModel.java index 0b01bfdc9a..cc46710e6f 100644 --- a/webui/src/main/java/org/polypheny/db/webui/models/AdapterTemplateModel.java +++ b/webui/src/main/java/org/polypheny/db/webui/models/AdapterTemplateModel.java @@ -18,8 +18,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.jetbrains.annotations.NotNull; import org.polypheny.db.adapter.AbstractAdapterSetting; import org.polypheny.db.adapter.AbstractAdapterSetting.AdapterSettingType; @@ -27,12 +29,13 @@ import org.polypheny.db.adapter.BindableAbstractAdapterSettingsList; import org.polypheny.db.adapter.DeployMode; import org.polypheny.db.adapter.DeployMode.DeploySetting; +import org.polypheny.db.adapter.annotations.AdapterSettingsPreset; import org.polypheny.db.adapter.java.AdapterTemplate; import org.polypheny.db.catalog.entity.LogicalAdapter.AdapterType; import org.polypheny.db.config.ConfigDocker; import org.polypheny.db.config.RuntimeConfig; -public record AdapterTemplateModel( @JsonProperty String adapterName, @JsonProperty AdapterType adapterType, @JsonProperty List settings, @JsonProperty String description, @JsonProperty List modes ) { +public record AdapterTemplateModel( @JsonProperty String adapterName, @JsonProperty AdapterType adapterType, @JsonProperty List settings, @JsonProperty String description, @JsonProperty List modes, @JsonProperty List presets ) { public AdapterTemplateModel( @@ -40,12 +43,14 @@ public AdapterTemplateModel( @NotNull AdapterType adapterType, @NotNull List settings, @NotNull String description, - @NotNull List modes ) { + @NotNull List modes, + @NotNull List presets ) { this.adapterName = adapterName; this.adapterType = adapterType; this.settings = settings; this.description = description; this.modes = modes; + this.presets = presets; } @@ -67,7 +72,26 @@ public static AdapterTemplateModel from( AdapterTemplate template ) { template.adapterType, settings, template.description, - template.modes ); + template.modes, + template.presets.stream().map( AdapterPresetModel::from ).toList() ); + } + + + public record AdapterPresetModel( + @JsonProperty String name, + @JsonProperty String description, + @JsonProperty DeployMode mode, + @JsonProperty Map settings + ) { + + public static AdapterPresetModel from( AdapterSettingsPreset preset ) { + return new AdapterPresetModel( + preset.name(), + preset.description(), + preset.mode(), + Arrays.stream( preset.settings() ).collect( Collectors.toMap( AdapterSettingsPreset.Setting::name, AdapterSettingsPreset.Setting::value ) ) ); + } + } diff --git a/webui/src/main/java/org/polypheny/db/webui/models/IndexAdapterModel.java b/webui/src/main/java/org/polypheny/db/webui/models/IndexAdapterModel.java index 841e16cc8d..6ac901e73c 100644 --- a/webui/src/main/java/org/polypheny/db/webui/models/IndexAdapterModel.java +++ b/webui/src/main/java/org/polypheny/db/webui/models/IndexAdapterModel.java @@ -16,12 +16,14 @@ package org.polypheny.db.webui.models; +import java.util.ArrayList; import java.util.List; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Value; import org.jetbrains.annotations.Nullable; import org.polypheny.db.adapter.DataStore; +import org.polypheny.db.adapter.DataStore.IndexParameterModel; import org.polypheny.db.webui.models.catalog.IdEntity; @EqualsAndHashCode(callSuper = true) @@ -47,10 +49,19 @@ public static class IndexMethodModel { public String name; public String displayName; + public String category; + public List parameters; + + public IndexMethodModel() {} public static IndexMethodModel from( DataStore.IndexMethodModel index ) { - return new IndexMethodModel( index.name(), index.displayName() ); + IndexMethodModel model = new IndexMethodModel(); + model.name = index.name(); + model.displayName = index.displayName(); + model.category = index.category().name(); + model.parameters = index.parameters() == null ? new ArrayList<>() : index.parameters(); + return model; } } diff --git a/webui/src/main/java/org/polypheny/db/webui/models/IndexModel.java b/webui/src/main/java/org/polypheny/db/webui/models/IndexModel.java index 5d4bdea874..32ba9b3b34 100644 --- a/webui/src/main/java/org/polypheny/db/webui/models/IndexModel.java +++ b/webui/src/main/java/org/polypheny/db/webui/models/IndexModel.java @@ -19,7 +19,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.HashMap; import java.util.List; +import java.util.Map; import lombok.Getter; import lombok.Value; @@ -37,6 +39,7 @@ public class IndexModel { public String storeUniqueName; public String method; public List columnIds; + public Map options; @JsonCreator @@ -45,13 +48,15 @@ public IndexModel( @JsonProperty("entityId") final Long entityId, @JsonProperty("name") final String name, @JsonProperty("method") final String method, - @JsonProperty("columnIds") final List columnIds ) { + @JsonProperty("columnIds") final List columnIds, + @JsonProperty("options") Map options ) { this.namespaceId = namespaceId; this.entityId = entityId; this.name = name; this.method = method; this.columnIds = columnIds; this.storeUniqueName = null; + this.options = options == null ? new HashMap<>() : options; } diff --git a/webui/src/main/java/org/polypheny/db/webui/models/catalog/AdapterModel.java b/webui/src/main/java/org/polypheny/db/webui/models/catalog/AdapterModel.java index 2d957e8e25..90fa1f199a 100644 --- a/webui/src/main/java/org/polypheny/db/webui/models/catalog/AdapterModel.java +++ b/webui/src/main/java/org/polypheny/db/webui/models/catalog/AdapterModel.java @@ -61,6 +61,9 @@ public class AdapterModel extends IdEntity { @JsonProperty public boolean dataReadOnly; + @JsonProperty + public List features; + public AdapterModel( @JsonProperty("id") @Nullable Long id, @@ -71,7 +74,8 @@ public AdapterModel( @JsonProperty("mode") DeployMode mode, @JsonProperty("indexMethods") List indexMethods, @JsonProperty("persistent") boolean persistent, - @JsonProperty("dataReadOnly") boolean dataReadOnly ) { + @JsonProperty("dataReadOnly") boolean dataReadOnly, + @JsonProperty("features") List features ) { super( id, name ); this.adapterName = adapterName; this.type = type; @@ -80,6 +84,7 @@ public AdapterModel( this.indexMethods = indexMethods; this.persistent = persistent; this.dataReadOnly = dataReadOnly; + this.features = features; } @@ -97,7 +102,8 @@ public static AdapterModel from( LogicalAdapter adapter ) { adapter.mode, adapter.type == AdapterType.STORE ? ((DataStore) dataStore).getAvailableIndexMethods() : List.of(), adapter.type == AdapterType.STORE && ((DataStore) dataStore).isPersistent(), - adapter.type == AdapterType.SOURCE && ((DataSource) dataStore).isDataReadOnly() + adapter.type == AdapterType.SOURCE && ((DataSource) dataStore).isDataReadOnly(), + dataStore.getActiveFeatureNames() ) ).orElse( null ); } diff --git a/webui/src/main/java/org/polypheny/db/webui/models/catalog/UiColumnDefinition.java b/webui/src/main/java/org/polypheny/db/webui/models/catalog/UiColumnDefinition.java index 422be82d6d..cfa1119abd 100644 --- a/webui/src/main/java/org/polypheny/db/webui/models/catalog/UiColumnDefinition.java +++ b/webui/src/main/java/org/polypheny/db/webui/models/catalog/UiColumnDefinition.java @@ -63,6 +63,8 @@ public class UiColumnDefinition extends FieldDefinition { @JsonProperty @Nullable public String collectionsType; + @JsonProperty + public Boolean elementsNullable; //for data source columns @JsonProperty diff --git a/webui/src/main/java/org/polypheny/db/webui/models/catalog/schema/ColumnModel.java b/webui/src/main/java/org/polypheny/db/webui/models/catalog/schema/ColumnModel.java index d1b101d0fe..817636b1f2 100644 --- a/webui/src/main/java/org/polypheny/db/webui/models/catalog/schema/ColumnModel.java +++ b/webui/src/main/java/org/polypheny/db/webui/models/catalog/schema/ColumnModel.java @@ -58,6 +58,9 @@ public class ColumnModel extends FieldModel { @JsonProperty public boolean nullable; + @JsonProperty + public boolean elementsNullable; + @JsonProperty public int position; @@ -74,10 +77,12 @@ public ColumnModel( @JsonProperty("dimension") Integer dimension, @JsonProperty("cardinality") Integer cardinality, @JsonProperty("nullable") boolean nullable, + @JsonProperty("elementsNullable") boolean elementsNullable, @JsonProperty("position") int position ) { super( id, name, tableId ); this.type = type; this.nullable = nullable; + this.elementsNullable = elementsNullable; this.position = position; this.collectionsType = collectionsType; this.precision = precision; @@ -101,6 +106,7 @@ public static ColumnModel from( LogicalColumn column ) { column.dimension, column.cardinality, column.nullable, + column.elementsNullable, column.position ); }