parquet-adapter

config/src/main/java/org/polypheny/db/util/PolyphenyHomeDirManager.java

@@ -103,6 +103,9 @@ private PolyphenyHomeDirManager() {
 
     public static PolyphenyHomeDirManager setModeAndGetInstance( RunMode mode ) {
         if ( PolyphenyHomeDirManager.mode != null ) {
+            if ( PolyphenyHomeDirManager.mode == mode ) {
+                return PolyphenyHomeDirManager.getInstance();
+            }
             throw new RuntimeException( "Could not set the mode." );
         }
         PolyphenyHomeDirManager.mode = mode;

core/src/main/java/org/polypheny/db/adapter/statistics/AdapterStatisticsProvider.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2019-2026 The Polypheny Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.polypheny.db.adapter.statistics;
+
+
+import java.util.Optional;
+import org.polypheny.db.catalog.entity.logical.LogicalColumn;
+
+
+/**
+ * Interface for metadata-backed statistics.
+ */
+public interface AdapterStatisticsProvider {
+
+    default Optional<ProvidedEntityStatistics> getEntityStatistics( long logicalEntityId ) {
+        return Optional.empty();
+    }
+
+
+    default Optional<ProvidedColumnStatistics> getColumnStatistics( LogicalColumn column, int uniqueValueLimit ) {
+        return Optional.empty();
+    }
+
+}

core/src/main/java/org/polypheny/db/adapter/statistics/ProvidedColumnStatistics.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019-2026 The Polypheny Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.polypheny.db.adapter.statistics;
+
+
+import java.util.List;
+import javax.annotation.Nullable;
+import org.polypheny.db.type.entity.PolyValue;
+
+
+/**
+ * Column statistics supplied directly by an adapter.
+ *
+ * @param count estimated values count, null if unavailable
+ * @param min minimum value, null if unavailable
+ * @param max maximum value, null if unavailable
+ * @param uniqueValues bounded unique values, empty if unavailable
+ * @param full whether the unique value set is known to be incomplete
+ */
+public record ProvidedColumnStatistics(
+        @Nullable Long count,
+        @Nullable PolyValue min,
+        @Nullable PolyValue max,
+        List<PolyValue> uniqueValues,
+        boolean full ) {
+
+    public ProvidedColumnStatistics {
+        uniqueValues = uniqueValues == null ? List.of() : List.copyOf( uniqueValues );
+    }
+
+}

core/src/main/java/org/polypheny/db/adapter/statistics/ProvidedEntityStatistics.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright 2019-2026 The Polypheny Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.polypheny.db.adapter.statistics;
+
+
+import javax.annotation.Nullable;
+
+
+/**
+ * Entity statistics supplied directly by an adapter.
+ *
+ * @param rowCount row count, or {@code null} if unavailable
+ */
+public record ProvidedEntityStatistics(
+        @Nullable Long rowCount ) {
+
+}

dbms/src/main/java/org/polypheny/db/ddl/DdlManagerImpl.java

@@ -215,12 +215,13 @@ public void createStore( String uniqueName, String adapterName, AdapterType adap
     public void createSource( Transaction transaction, String uniqueName, String adapterName, long namespace, AdapterType adapterType, Map<String, String> config, DeployMode mode ) {
         uniqueName = uniqueName.toLowerCase();
         DataSource<?> adapter = (DataSource<?>) AdapterManager.getInstance().addAdapter( catalog, adapterName, uniqueName, adapterType, mode, config );
-        namespace = adapter.getCurrentNamespace() == null ? namespace : adapter.getCurrentNamespace().getId(); // TODO: clean implementation. Sources should either create their own namespace or there should be default namespaces for different models.
+        long sourceNamespace = adapter.getCurrentNamespace() == null ? namespace : adapter.getCurrentNamespace().getId(); // TODO: clean implementation. Sources should either create their own namespace or there should be default namespaces for different models.
+        // Support more than one data model per adapter
         if ( adapter.supportsRelational() ) {
-            createRelationalSource( transaction, adapter, namespace );
+            createRelationalSource( transaction, adapter, resolveSourceNamespace( uniqueName, sourceNamespace, DataModel.RELATIONAL ) );
         }
         if ( adapter.supportsDocument() ) {
-            createDocumentSource( adapter, namespace );
+            createDocumentSource( adapter, resolveSourceNamespace( uniqueName, sourceNamespace, DataModel.DOCUMENT ) );
         }
         if ( adapter.supportsGraph() ) {
             // TODO: implement graph source creation
@@ -230,6 +231,29 @@ public void createSource( Transaction transaction, String uniqueName, String ada
     }
 
 
+    /**
+     * Support more than one data model per adapter
+     * if there is existing namespace for the provided adapter model (RELATIONAL\DOCUMENT) - use it
+     * otherwise create appropriate namespace
+     * @param uniqueName - adapter name
+     * @param namespaceId - namesapce id of adapter to check
+     * @param model - RELATIONAL\DOCUMENT
+     * @return namespace id
+     */
+    private long resolveSourceNamespace( String uniqueName, long namespaceId, DataModel model ) {
+        LogicalNamespace namespace = catalog.getSnapshot().getNamespace( namespaceId ).orElseThrow();
+        if ( namespace.dataModel == model ) {
+            return namespaceId;
+        }
+
+        String sourceNamespaceName = uniqueName + "_" + model.name().toLowerCase();
+        return catalog.getSnapshot().getNamespace( sourceNamespaceName )
+                .filter( ns -> ns.dataModel == model )
+                .map( ns -> ns.id )
+                .orElseGet( () -> createNamespace( sourceNamespaceName, model, true, false, false, null ) );
+    }
+
+
     private void createDocumentSource( DataSource<?> adapter, long namespace ) {
         List<ExportedDocument> exportedCollections;
         try {

dbms/src/main/java/org/polypheny/db/processing/AbstractQueryProcessor.java

@@ -82,6 +82,7 @@
 import org.polypheny.db.catalog.entity.physical.PhysicalEntity;
 import org.polypheny.db.catalog.exceptions.GenericRuntimeException;
 import org.polypheny.db.catalog.logistic.DataModel;
+import org.polypheny.db.catalog.logistic.PartitionType;
 import org.polypheny.db.config.RuntimeConfig;
 import org.polypheny.db.information.InformationPolyAlg.PlanType;
 import org.polypheny.db.interpreter.BindableConvention;
@@ -104,6 +105,8 @@
 import org.polypheny.db.processing.shuttles.QueryParameterizer;
 import org.polypheny.db.processing.util.Plan;
 import org.polypheny.db.processing.util.ProposedImplementations;
+import org.polypheny.db.partition.PartitionManagerFactory;
+import org.polypheny.db.partition.properties.PartitionProperty;
 import org.polypheny.db.rex.RexBuilder;
 import org.polypheny.db.rex.RexDynamicParam;
 import org.polypheny.db.rex.RexIndexRef;
@@ -1204,7 +1207,7 @@ private LogicalQueryInformation analyzeQueryAndPrepareMonitoring( Statement stat
 
         // Get partitions of logical information
 
-        Map<Long, List<Long>> accessedPartitions = extractPartitions( logicalRoot.alg.getEntities() );
+        Map<Long, List<Long>> accessedPartitions = extractPartitions( logicalRoot.alg.getEntities(), analyzer.partitionValueFilterPerScan );
 
         // Build queryClass from query-name and partitions.
         String queryHash = analyzer.getQueryName() + accessedPartitions;
@@ -1227,11 +1230,11 @@ private LogicalQueryInformation analyzeQueryAndPrepareMonitoring( Statement stat
     }
 
 
-    private Map<Long, List<Long>> extractPartitions( Set<Entity> entities ) {
+    private Map<Long, List<Long>> extractPartitions( Set<Entity> entities, Map<Long, Set<String>> partitionValueFilterPerScan ) {
         Map<Long, List<Long>> map = new HashMap<>();
         for ( Entity entity : entities ) {
             if ( entity.isLogical() ) {
-                map.computeIfAbsent( entity.getId(), k -> new ArrayList<>() ).addAll( Catalog.snapshot().alloc().getPartitionsFromLogical( entity.getId() ).stream().map( p -> p.id ).toList() );
+                map.computeIfAbsent( entity.getId(), k -> new ArrayList<>() ).addAll( extractPartitionsForLogicalEntity( entity, partitionValueFilterPerScan ) );
             } else if ( entity.isAllocation() ) {
                 map.computeIfAbsent( ((AllocationEntity) entity).getLogicalId(), k -> new ArrayList<>() ).add( ((AllocationEntity) entity).getPartitionId() );
             } else if ( entity.isPhysical() ) {
@@ -1242,6 +1245,32 @@ private Map<Long, List<Long>> extractPartitions( Set<Entity> entities ) {
     }
 
 
+    private List<Long> extractPartitionsForLogicalEntity( Entity entity, Map<Long, Set<String>> partitionValueFilterPerScan ) {
+        List<Long> allPartitions = Catalog.snapshot().alloc().getPartitionsFromLogical( entity.getId() ).stream().map( p -> p.id ).toList();
+        Set<String> partitionValues = partitionValueFilterPerScan.get( entity.getId() );
+        if ( partitionValues == null || partitionValues.isEmpty() ) {
+            return allPartitions;
+        }
+
+        PartitionProperty property = Catalog.snapshot().alloc().getPartitionProperty( entity.getId() ).orElse( null );
+        if ( property == null || property.partitionType == PartitionType.NONE ) {
+            return allPartitions;
+        }
+
+        LogicalTable table = Catalog.snapshot().rel().getTable( entity.getId() ).orElse( null );
+        if ( table == null ) {
+            return allPartitions;
+        }
+
+        List<Long> selectedPartitions = partitionValues.stream()
+                .map( value -> PartitionManagerFactory.getInstance().getPartitionManager( property.partitionType ).getTargetPartitionId( table, property, value ) )
+                .filter( partitionId -> partitionId >= 0 )
+                .distinct()
+                .toList();
+        return selectedPartitions.isEmpty() ? allPartitions : selectedPartitions;
+    }
+
+
     private void prepareMonitoring( Statement statement, AlgRoot logicalRoot, boolean isAnalyze, boolean isSubquery, LogicalQueryInformation queryInformation ) {
 
         // Initialize Monitoring

gradle.properties

@@ -91,3 +91,5 @@ typesafe_config_version = 1.2.1
 unirest_version = 3.14.5
 web3j_version = 5.0.0
 zip4j_version = 2.11.5
+parquet_version = 1.14.4
+hadoop_version = 3.4.1