The per-operator reconciliation below ({@link
+ * #checkParallelismPreconditions(OperatorState, ExecutionJobVertex)}) adopts each operator's
+ * recorded maximum parallelism onto the shared vertex, so when operators disagree the vertex is
+ * left with whichever value is reconciled last. Any keyed operator on the vertex is then
+ * restored under that value rather than its own, remapping its state through an incompatible
+ * {@code hash % maxParallelism} layout. Operators sharing a vertex normally record its single
+ * maximum parallelism and therefore agree; they can only differ here if the chaining topology
+ * regrouped them since the checkpoint. This regrouping was permitted for graph construction but
+ * never validated on restore. A disagreeing operator need not be keyed itself: its recorded
+ * value can win the reconciliation and misroute another operator's keyed state, so all
+ * operators are compared. Vertices without keyed state are unaffected, since maximum
+ * parallelism only governs keyed-state routing.
+ */
+ private static void checkMaxParallelismAgreement(TaskStateAssignment taskStateAssignment) {
+ OperatorID referenceOperator = null;
+ int referenceMaxParallelism = -1;
+ OperatorID conflictingOperator = null;
+ int conflictingMaxParallelism = -1;
+ boolean vertexHasKeyedState = false;
+
+ for (Map.Entry entry : taskStateAssignment.oldState.entrySet()) {
+ final OperatorState operatorState = entry.getValue();
+ vertexHasKeyedState |= hasKeyedState(operatorState);
+
+ if (referenceOperator == null) {
+ referenceOperator = entry.getKey();
+ referenceMaxParallelism = operatorState.getMaxParallelism();
+ } else if (conflictingOperator == null
+ && operatorState.getMaxParallelism() != referenceMaxParallelism) {
+ conflictingOperator = entry.getKey();
+ conflictingMaxParallelism = operatorState.getMaxParallelism();
+ }
+ }
+
+ if (vertexHasKeyedState && conflictingOperator != null) {
+ throw new IllegalStateException(
+ "The state for the execution job vertex "
+ + taskStateAssignment.executionJobVertex.getJobVertexId()
+ + " can not be restored. Operators "
+ + referenceOperator
+ + " and "
+ + conflictingOperator
+ + " are chained into the same keyed vertex but recorded different"
+ + " maximum parallelism in the checkpoint ("
+ + referenceMaxParallelism
+ + " and "
+ + conflictingMaxParallelism
+ + "). Restoring would remap keyed state through an incompatible"
+ + " key-group layout. This is currently not supported.");
+ }
+ }
+
+ private static boolean hasKeyedState(OperatorState operatorState) {
+ for (OperatorSubtaskState subtaskState : operatorState.getStates()) {
+ if (!subtaskState.getManagedKeyedState().isEmpty()
+ || !subtaskState.getRawKeyedState().isEmpty()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private void reDistributeKeyedStates(
List keyGroupPartitions, TaskStateAssignment stateAssignment) {
stateAssignment.oldState.forEach(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandler.java
index ca01ff37bd3696..8f1eeb24feca0b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandler.java
@@ -73,28 +73,156 @@ void recover(Info info, int oldSubtaskIndex, BufferWithContext bufferWi
throws IOException, InterruptedException;
}
-class InputChannelRecoveredStateHandler
+/**
+ * Abstract base for all input-channel recovery handlers. Holds the channel mapping logic shared by
+ * both variants (no-filtering, filtering).
+ *
+ *
Subclasses implement {@link #recover} according to their specific recovery mode and override
+ * {@link #closeInternal()} to release mode-specific resources.
+ *
+ *
Use the static {@link #create} factory to obtain the correct concrete subclass.
+ */
+abstract class AbstractInputChannelRecoveredStateHandler
implements RecoveredChannelStateHandler {
- private final InputGate[] inputGates;
- private final InflightDataRescalingDescriptor channelMapping;
+ final InputGate[] inputGates;
+ final InflightDataRescalingDescriptor channelMapping;
+ final Map rescaledChannels = new HashMap<>();
+ final Map oldToNewMappings = new HashMap<>();
- private final Map rescaledChannels = new HashMap<>();
- private final Map oldToNewMappings = new HashMap<>();
+ AbstractInputChannelRecoveredStateHandler(
+ InputGate[] inputGates, InflightDataRescalingDescriptor channelMapping) {
+ this.inputGates = inputGates;
+ this.channelMapping = channelMapping;
+ }
/**
- * Optional filtering handler for filtering recovered buffers. When non-null, filtering is
- * performed during recovery in the channel-state-unspilling thread.
+ * Factory that selects the correct subclass based on {@code checkpointingDuringRecoveryEnabled}
+ * and whether a {@code filteringHandler} is present.
+ *
+ *
*/
- @Nullable private final ChannelStateFilteringHandler filteringHandler;
+ static AbstractInputChannelRecoveredStateHandler create(
+ InputGate[] inputGates,
+ InflightDataRescalingDescriptor channelMapping,
+ boolean checkpointingDuringRecoveryEnabled,
+ @Nullable ChannelStateFilteringHandler filteringHandler,
+ int memorySegmentSize) {
+ if (!checkpointingDuringRecoveryEnabled) {
+ return new NoSpillingHandler(inputGates, channelMapping);
+ }
+ // FLINK-38544 transitional: the flag-on path still uses the in-memory handlers until the
+ // spilling backend lands.
+ if (filteringHandler == null) {
+ return new NoSpillingHandler(inputGates, channelMapping);
+ }
+ return new FilteringHandler(
+ inputGates, channelMapping, filteringHandler, memorySegmentSize);
+ }
+
+ /** Default buffer allocation from the network buffer pool, used by non-filtering modes. */
+ @Override
+ public BufferWithContext getBuffer(InputChannelInfo channelInfo)
+ throws IOException, InterruptedException {
+ RecoveredInputChannel channel = getMappedChannels(channelInfo);
+ Buffer buffer = channel.requestBufferBlocking();
+ return new BufferWithContext<>(wrap(buffer), buffer);
+ }
+
+ @Override
+ public void close() throws IOException {
+ // note that we need to finish all RecoveredInputChannels, not just those with state
+ for (final InputGate inputGate : inputGates) {
+ inputGate.finishReadRecoveredState();
+ }
+ closeInternal();
+ }
+
+ /** Hook for subclasses to release their own resources. Called by {@link #close()}. */
+ void closeInternal() throws IOException {}
+
+ RecoveredInputChannel getMappedChannels(InputChannelInfo channelInfo) {
+ return rescaledChannels.computeIfAbsent(channelInfo, this::calculateMapping);
+ }
+
+ @Nonnull
+ private RecoveredInputChannel calculateMapping(InputChannelInfo info) {
+ final RescaleMappings oldToNewMapping =
+ oldToNewMappings.computeIfAbsent(
+ info.getGateIdx(), idx -> channelMapping.getChannelMapping(idx).invert());
+ int[] mappedIndexes = oldToNewMapping.getMappedIndexes(info.getInputChannelIdx());
+ checkState(
+ mappedIndexes.length == 1,
+ "One buffer is only distributed to one target InputChannel since "
+ + "one buffer is expected to be processed once by the same task.");
+ return getChannel(info.getGateIdx(), mappedIndexes[0]);
+ }
+
+ private RecoveredInputChannel getChannel(int gateIndex, int subPartitionIndex) {
+ final InputChannel inputChannel = inputGates[gateIndex].getChannel(subPartitionIndex);
+ if (!(inputChannel instanceof RecoveredInputChannel)) {
+ throw new IllegalStateException(
+ "Cannot restore state to a non-recovered input channel: " + inputChannel);
+ }
+ return (RecoveredInputChannel) inputChannel;
+ }
+}
+
+/**
+ * Recovery handler for the case where checkpointing during recovery is disabled. Delivers recovered
+ * buffers directly into the input channel via {@code onRecoveredStateBuffer}, with no spill file.
+ */
+class NoSpillingHandler extends AbstractInputChannelRecoveredStateHandler {
+
+ NoSpillingHandler(InputGate[] inputGates, InflightDataRescalingDescriptor channelMapping) {
+ super(inputGates, channelMapping);
+ }
+
+ @Override
+ public void recover(
+ InputChannelInfo channelInfo,
+ int oldSubtaskIndex,
+ BufferWithContext bufferWithContext)
+ throws IOException, InterruptedException {
+ Buffer buffer = bufferWithContext.context;
+ try {
+ if (buffer.readableBytes() > 0) {
+ RecoveredInputChannel channel = getMappedChannels(channelInfo);
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(
+ new SubtaskConnectionDescriptor(
+ oldSubtaskIndex, channelInfo.getInputChannelIdx()),
+ false));
+ channel.onRecoveredStateBuffer(buffer.retainBuffer());
+ }
+ } finally {
+ buffer.recycleBuffer();
+ }
+ }
+}
+
+/**
+ * Recovery handler for the case where checkpointing during recovery is enabled and a filtering
+ * handler is present. Uses a reusable heap-backed pre-filter buffer (isolated from the Network
+ * Buffer Pool), filters recovered buffers through {@link
+ * ChannelStateFilteringHandler#filterAndRewrite}, and delivers the filtered buffers directly into
+ * the input channel via {@code onRecoveredStateBuffer}.
+ */
+class FilteringHandler extends AbstractInputChannelRecoveredStateHandler {
+
+ private final ChannelStateFilteringHandler filteringHandler;
/** Network buffer memory segment size in bytes. Used to size the reusable pre-filter buffer. */
private final int memorySegmentSize;
/**
* Reusable heap memory segment backing the pre-filter buffer in filtering mode. Lazily
- * allocated on the first {@link #getPreFilterBuffer} call, reused for every subsequent call,
- * and freed in {@link #close()}.
+ * allocated on the first {@link #getBuffer} call, reused for every subsequent call, and freed
+ * in {@link #closeInternal()}.
*
*
Reuse is safe because at most one pre-filter buffer is in flight per task at any moment.
* This invariant is enforced at runtime by {@link #preFilterBufferInUse}.
@@ -108,39 +236,26 @@ class InputChannelRecoveredStateHandler
*/
private boolean preFilterBufferInUse;
- InputChannelRecoveredStateHandler(
+ FilteringHandler(
InputGate[] inputGates,
InflightDataRescalingDescriptor channelMapping,
- @Nullable ChannelStateFilteringHandler filteringHandler,
+ ChannelStateFilteringHandler filteringHandler,
int memorySegmentSize) {
- this.inputGates = inputGates;
- this.channelMapping = channelMapping;
+ super(inputGates, channelMapping);
this.filteringHandler = filteringHandler;
checkArgument(
memorySegmentSize > 0, "memorySegmentSize must be positive: %s", memorySegmentSize);
this.memorySegmentSize = memorySegmentSize;
}
- @Override
- public BufferWithContext getBuffer(InputChannelInfo channelInfo)
- throws IOException, InterruptedException {
- if (filteringHandler != null) {
- return getPreFilterBuffer();
- }
- // Non-filtering mode: use existing network buffer pool allocation.
- RecoveredInputChannel channel = getMappedChannels(channelInfo);
- Buffer buffer = channel.requestBufferBlocking();
- return new BufferWithContext<>(wrap(buffer), buffer);
- }
-
/**
* Allocates a pre-filter buffer from a reusable heap segment (isolated from the Network Buffer
* Pool) in filtering mode.
*
*
Memory management: a single {@link MemorySegment} per task is lazily allocated on first
* invocation and reused across every subsequent call. The custom {@link BufferRecycler} does
- * not free the segment — it only flips {@link #preFilterBufferInUse} back to {@code false} so
- * the next call can reuse it. The segment itself is freed in {@link #close()}.
+ * not free the segment; it only flips {@link #preFilterBufferInUse} back to {@code false} so
+ * the next call can reuse it. The segment itself is freed in {@link #closeInternal()}.
*
*
Runtime invariant check: the one-at-a-time invariant on pre-filter buffers is guaranteed
* by Flink's serial recovery loop and the deserializer's ownership contract. This method
@@ -148,7 +263,8 @@ public BufferWithContext getBuffer(InputChannelInfo channelInfo)
* recycled, it throws {@link IllegalStateException} so any future regression fails loudly
* instead of silently corrupting memory.
*/
- private BufferWithContext getPreFilterBuffer() {
+ @Override
+ public BufferWithContext getBuffer(InputChannelInfo channelInfo) {
checkState(
!preFilterBufferInUse,
"Previous pre-filter buffer has not been recycled. This violates the "
@@ -165,17 +281,6 @@ private BufferWithContext getPreFilterBuffer() {
return new BufferWithContext<>(wrap(buffer), buffer);
}
- @VisibleForTesting
- boolean isPreFilterBufferInUse() {
- return preFilterBufferInUse;
- }
-
- @VisibleForTesting
- @Nullable
- MemorySegment getPreFilterSegmentForTesting() {
- return preFilterSegment;
- }
-
@Override
public void recover(
InputChannelInfo channelInfo,
@@ -186,18 +291,7 @@ public void recover(
try {
if (buffer.readableBytes() > 0) {
RecoveredInputChannel channel = getMappedChannels(channelInfo);
-
- if (filteringHandler != null) {
- recoverWithFiltering(
- channel, channelInfo, oldSubtaskIndex, buffer.retainBuffer());
- } else {
- channel.onRecoveredStateBuffer(
- EventSerializer.toBuffer(
- new SubtaskConnectionDescriptor(
- oldSubtaskIndex, channelInfo.getInputChannelIdx()),
- false));
- channel.onRecoveredStateBuffer(buffer.retainBuffer());
- }
+ recoverWithFiltering(channel, channelInfo, oldSubtaskIndex, buffer.retainBuffer());
}
} finally {
buffer.recycleBuffer();
@@ -232,44 +326,25 @@ private void recoverWithFiltering(
}
}
+ @VisibleForTesting
+ boolean isPreFilterBufferInUse() {
+ return preFilterBufferInUse;
+ }
+
+ @VisibleForTesting
+ @Nullable
+ MemorySegment getPreFilterSegmentForTesting() {
+ return preFilterSegment;
+ }
+
@Override
- public void close() throws IOException {
- // note that we need to finish all RecoveredInputChannels, not just those with state
- for (final InputGate inputGate : inputGates) {
- inputGate.finishReadRecoveredState();
- }
+ void closeInternal() throws IOException {
if (preFilterSegment != null) {
preFilterSegment.free();
preFilterSegment = null;
preFilterBufferInUse = false;
}
}
-
- private RecoveredInputChannel getChannel(int gateIndex, int subPartitionIndex) {
- final InputChannel inputChannel = inputGates[gateIndex].getChannel(subPartitionIndex);
- if (!(inputChannel instanceof RecoveredInputChannel)) {
- throw new IllegalStateException(
- "Cannot restore state to a non-recovered input channel: " + inputChannel);
- }
- return (RecoveredInputChannel) inputChannel;
- }
-
- private RecoveredInputChannel getMappedChannels(InputChannelInfo channelInfo) {
- return rescaledChannels.computeIfAbsent(channelInfo, this::calculateMapping);
- }
-
- @Nonnull
- private RecoveredInputChannel calculateMapping(InputChannelInfo info) {
- final RescaleMappings oldToNewMapping =
- oldToNewMappings.computeIfAbsent(
- info.getGateIdx(), idx -> channelMapping.getChannelMapping(idx).invert());
- int[] mappedIndexes = oldToNewMapping.getMappedIndexes(info.getInputChannelIdx());
- checkState(
- mappedIndexes.length == 1,
- "One buffer is only distributed to one target InputChannel since "
- + "one buffer is expected to be processed once by the same task.");
- return getChannel(info.getGateIdx(), mappedIndexes[0]);
- }
}
class ResultSubpartitionRecoveredStateHandler
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveryCheckpointBarrier.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveryCheckpointBarrier.java
new file mode 100644
index 00000000000000..984bfd42312f65
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/RecoveryCheckpointBarrier.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint.channel;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.runtime.event.RuntimeEvent;
+
+/** Task-local event marking the recovery-state cut for a recovery checkpoint. */
+@Internal
+public final class RecoveryCheckpointBarrier extends RuntimeEvent {
+
+ private final long checkpointId;
+
+ public RecoveryCheckpointBarrier(long checkpointId) {
+ this.checkpointId = checkpointId;
+ }
+
+ public long getCheckpointId() {
+ return checkpointId;
+ }
+
+ @Override
+ public void write(DataOutputView out) {
+ throw new UnsupportedOperationException(
+ "RecoveryCheckpointBarrier must be serialized via EventSerializer's dedicated"
+ + " type-tag path, not reflective write().");
+ }
+
+ @Override
+ public void read(DataInputView in) {
+ throw new UnsupportedOperationException(
+ "RecoveryCheckpointBarrier must be deserialized via EventSerializer's dedicated"
+ + " type-tag path, not reflective read().");
+ }
+
+ @Override
+ public int hashCode() {
+ return Long.hashCode(checkpointId);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other != null
+ && other.getClass() == RecoveryCheckpointBarrier.class
+ && ((RecoveryCheckpointBarrier) other).checkpointId == this.checkpointId;
+ }
+
+ @Override
+ public String toString() {
+ return "RecoveryCheckpointBarrier(" + checkpointId + ")";
+ }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/SequentialChannelStateReaderImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/SequentialChannelStateReaderImpl.java
index c52572e52faecb..3ebad6eb3a2a2b 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/SequentialChannelStateReaderImpl.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/channel/SequentialChannelStateReaderImpl.java
@@ -70,10 +70,11 @@ public void readInputData(InputGate[] inputGates, RecordFilterContext filterCont
: null;
try (ChannelStateFilteringHandler ignored = filteringHandler;
- InputChannelRecoveredStateHandler stateHandler =
- new InputChannelRecoveredStateHandler(
+ AbstractInputChannelRecoveredStateHandler stateHandler =
+ AbstractInputChannelRecoveredStateHandler.create(
inputGates,
taskStateSnapshot.getInputRescalingDescriptor(),
+ filterContext.isCheckpointingDuringRecoveryEnabled(),
filteringHandler,
filterContext.getMemorySegmentSize())) {
read(
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
index a704e3290fdb48..430cd9ee76da4f 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java
@@ -1708,7 +1708,11 @@ private boolean transitionState(
initializingOrRunningFuture.complete(null);
} else if (targetState.isTerminal()) {
if (preCompletionAction != null) {
- preCompletionAction.run();
+ try {
+ preCompletionAction.run();
+ } catch (Exception e) {
+ LOG.error("Error while executing pre-completion action.", e);
+ }
}
// complete the terminal state future
terminalStateFuture.complete(targetState);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializer.java
index e73d9168cb273f..5711d1640f9094 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializer.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/serialization/EventSerializer.java
@@ -27,6 +27,7 @@
import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.checkpoint.SavepointType;
import org.apache.flink.runtime.checkpoint.SnapshotType;
+import org.apache.flink.runtime.checkpoint.channel.RecoveryCheckpointBarrier;
import org.apache.flink.runtime.event.AbstractEvent;
import org.apache.flink.runtime.event.WatermarkEvent;
import org.apache.flink.runtime.io.network.api.CancelCheckpointMarker;
@@ -43,6 +44,7 @@
import org.apache.flink.runtime.io.network.buffer.BufferConsumer;
import org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler;
import org.apache.flink.runtime.io.network.buffer.NetworkBuffer;
+import org.apache.flink.runtime.io.network.partition.consumer.EndOfFetchedChannelStateEvent;
import org.apache.flink.runtime.io.network.partition.consumer.EndOfInputChannelStateEvent;
import org.apache.flink.runtime.io.network.partition.consumer.EndOfOutputChannelStateEvent;
import org.apache.flink.runtime.state.CheckpointStorageLocationReference;
@@ -87,6 +89,10 @@ public class EventSerializer {
private static final int END_OF_INPUT_CHANNEL_STATE_EVENT = 12;
+ private static final int RECOVERY_CHECKPOINT_BARRIER_EVENT = 13;
+
+ private static final int END_OF_FETCHED_CHANNEL_STATE_EVENT = 14;
+
private static final byte CHECKPOINT_TYPE_CHECKPOINT = 0;
private static final byte CHECKPOINT_TYPE_SAVEPOINT = 1;
@@ -116,6 +122,8 @@ public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOExcepti
return ByteBuffer.wrap(new byte[] {0, 0, 0, END_OF_OUTPUT_CHANNEL_STATE_EVENT});
} else if (eventClass == EndOfInputChannelStateEvent.class) {
return ByteBuffer.wrap(new byte[] {0, 0, 0, END_OF_INPUT_CHANNEL_STATE_EVENT});
+ } else if (eventClass == EndOfFetchedChannelStateEvent.class) {
+ return ByteBuffer.wrap(new byte[] {0, 0, 0, END_OF_FETCHED_CHANNEL_STATE_EVENT});
} else if (eventClass == EndOfData.class) {
return ByteBuffer.wrap(
new byte[] {
@@ -162,6 +170,13 @@ public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOExcepti
buf.putInt(0, RECOVERY_METADATA);
buf.putInt(4, recoveryMetadata.getFinalBufferSubpartitionId());
return buf;
+ } else if (eventClass == RecoveryCheckpointBarrier.class) {
+ RecoveryCheckpointBarrier barrier = (RecoveryCheckpointBarrier) event;
+
+ ByteBuffer buf = ByteBuffer.allocate(12);
+ buf.putInt(0, RECOVERY_CHECKPOINT_BARRIER_EVENT);
+ buf.putLong(4, barrier.getCheckpointId());
+ return buf;
} else if (eventClass == WatermarkEvent.class) {
try {
final DataOutputSerializer serializer = new DataOutputSerializer(128);
@@ -206,6 +221,8 @@ public static AbstractEvent fromSerializedEvent(ByteBuffer buffer, ClassLoader c
return EndOfOutputChannelStateEvent.INSTANCE;
} else if (type == END_OF_INPUT_CHANNEL_STATE_EVENT) {
return EndOfInputChannelStateEvent.INSTANCE;
+ } else if (type == END_OF_FETCHED_CHANNEL_STATE_EVENT) {
+ return EndOfFetchedChannelStateEvent.INSTANCE;
} else if (type == END_OF_USER_RECORDS_EVENT) {
return new EndOfData(StopMode.values()[buffer.get()]);
} else if (type == CANCEL_CHECKPOINT_MARKER_EVENT) {
@@ -222,6 +239,9 @@ public static AbstractEvent fromSerializedEvent(ByteBuffer buffer, ClassLoader c
} else if (type == RECOVERY_METADATA) {
int subpartitionId = buffer.getInt();
return new RecoveryMetadata(subpartitionId);
+ } else if (type == RECOVERY_CHECKPOINT_BARRIER_EVENT) {
+ long checkpointId = buffer.getLong();
+ return new RecoveryCheckpointBarrier(checkpointId);
} else if (type == GENERALIZED_WATERMARK_EVENT) {
final DataInputDeserializer deserializer = new DataInputDeserializer(buffer);
WatermarkEvent watermarkEvent = new WatermarkEvent();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/logger/NetworkActionsLogger.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/logger/NetworkActionsLogger.java
index 204f3d3c074b6c..137334ae14fc35 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/logger/NetworkActionsLogger.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/logger/NetworkActionsLogger.java
@@ -98,13 +98,13 @@ public static void traceRecover(
public static void tracePersist(
String action, Buffer buffer, Object channelInfo, long checkpointId) {
+ tracePersist(action, buffer.toDebugString(INCLUDE_HASH), channelInfo, checkpointId);
+ }
+
+ public static void tracePersist(
+ String action, Object persisted, Object channelInfo, long checkpointId) {
if (LOG.isTraceEnabled()) {
- LOG.trace(
- "{} {}, checkpoint {} @ {}",
- action,
- buffer.toDebugString(INCLUDE_HASH),
- checkpointId,
- channelInfo);
+ LOG.trace("{} {}, checkpoint {} @ {}", action, persisted, checkpointId, channelInfo);
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java
index ae57b39b08d3c5..1ac2687bf39463 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReader.java
@@ -81,12 +81,17 @@ class CreditBasedSequenceNumberingViewReader
private int numCreditsAvailable;
CreditBasedSequenceNumberingViewReader(
- InputChannelID receiverId, int initialCredit, PartitionRequestQueue requestQueue) {
+ InputChannelID receiverId,
+ int initialCredit,
+ boolean needsRecovery,
+ PartitionRequestQueue requestQueue) {
checkArgument(initialCredit >= 0, "Must be non-negative.");
this.receiverId = receiverId;
this.initialCredit = initialCredit;
- this.numCreditsAvailable = initialCredit;
+ // During spill recovery, exclusive buffers are on loan to the recovery drain; real credit
+ // is announced only after recovery completes.
+ this.numCreditsAvailable = needsRecovery ? 0 : initialCredit;
this.requestQueue = requestQueue;
this.subpartitionId = -1;
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyMessage.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyMessage.java
index ebe596e01f6b5a..6747e139a034c3 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyMessage.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyMessage.java
@@ -583,15 +583,19 @@ static class PartitionRequest extends NettyMessage {
final int credit;
+ final boolean needsRecovery;
+
PartitionRequest(
ResultPartitionID partitionId,
ResultSubpartitionIndexSet queueIndexSet,
InputChannelID receiverId,
- int credit) {
+ int credit,
+ boolean needsRecovery) {
this.partitionId = checkNotNull(partitionId);
this.queueIndexSet = queueIndexSet;
this.receiverId = checkNotNull(receiverId);
this.credit = credit;
+ this.needsRecovery = needsRecovery;
}
@Override
@@ -604,6 +608,7 @@ void write(ChannelOutboundInvoker out, ChannelPromise promise, ByteBufAllocator
queueIndexSet.writeTo(bb);
receiverId.writeTo(bb);
bb.writeInt(credit);
+ bb.writeBoolean(needsRecovery);
};
writeToChannel(
@@ -616,7 +621,8 @@ void write(ChannelOutboundInvoker out, ChannelPromise promise, ByteBufAllocator
+ ExecutionAttemptID.getByteBufLength()
+ ResultSubpartitionIndexSet.getByteBufLength(queueIndexSet)
+ InputChannelID.getByteBufLength()
- + Integer.BYTES);
+ + Integer.BYTES
+ + Byte.BYTES);
}
static PartitionRequest readFrom(ByteBuf buffer) {
@@ -628,8 +634,10 @@ static PartitionRequest readFrom(ByteBuf buffer) {
ResultSubpartitionIndexSet.fromByteBuf(buffer);
InputChannelID receiverId = InputChannelID.fromByteBuf(buffer);
int credit = buffer.readInt();
+ boolean needsRecovery = buffer.readBoolean();
- return new PartitionRequest(partitionId, queueIndexSet, receiverId, credit);
+ return new PartitionRequest(
+ partitionId, queueIndexSet, receiverId, credit, needsRecovery);
}
@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyPartitionRequestClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyPartitionRequestClient.java
index 7cc52a234e09a9..737e3de72690df 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyPartitionRequestClient.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyPartitionRequestClient.java
@@ -129,7 +129,8 @@ public void requestSubpartition(
partitionId,
subpartitionIndexSet,
inputChannel.getInputChannelId(),
- inputChannel.getInitialCredit());
+ inputChannel.getInitialCredit(),
+ inputChannel.needsRecovery());
final ChannelFutureListener listener =
future -> {
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
index f93651b9f3e8d2..6f2cd6b13ad419 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandler.java
@@ -85,7 +85,10 @@ protected void channelRead0(ChannelHandlerContext ctx, NettyMessage msg) throws
NetworkSequenceViewReader reader;
reader =
new CreditBasedSequenceNumberingViewReader(
- request.receiverId, request.credit, outboundQueue);
+ request.receiverId,
+ request.credit,
+ request.needsRecovery,
+ outboundQueue);
reader.requestSubpartitionViewOrRegisterListener(
partitionProvider, request.partitionId, request.queueIndexSet);
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferManager.java
index db38025def9e3d..1eed2a0284fd0a 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferManager.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/BufferManager.java
@@ -70,13 +70,24 @@ public class BufferManager implements BufferListener, BufferRecycler {
@GuardedBy("bufferQueue")
private int numRequiredBuffers;
+ /**
+ * Gates credit announcements while a recovery drain borrows this channel's buffers. Kept under
+ * {@code bufferQueue} to avoid inverting the queue/recovered-buffer lock order.
+ */
+ @GuardedBy("bufferQueue")
+ private boolean notifyAvailable;
+
public BufferManager(
- MemorySegmentProvider globalPool, InputChannel inputChannel, int numRequiredBuffers) {
+ MemorySegmentProvider globalPool,
+ InputChannel inputChannel,
+ int numRequiredBuffers,
+ boolean notifyInitiallyEnabled) {
this.globalPool = checkNotNull(globalPool);
this.inputChannel = checkNotNull(inputChannel);
checkArgument(numRequiredBuffers >= 0);
this.numRequiredBuffers = numRequiredBuffers;
+ this.notifyAvailable = notifyInitiallyEnabled;
}
// ------------------------------------------------------------------------
@@ -158,23 +169,22 @@ void requestExclusiveBuffers(int numExclusiveBuffers) throws IOException {
/**
* Requests floating buffers from the buffer pool based on the given required amount, and
- * returns the actual requested amount. If the required amount is not fully satisfied, it will
- * register as a listener.
+ * returns the number of buffers that may be announced to the producer as credit. During
+ * recovery, requested buffers are queued but announced only by {@link #enableNotify()}.
*/
int requestFloatingBuffers(int numRequired) {
- int numRequestedBuffers = 0;
synchronized (bufferQueue) {
// Similar to notifyBufferAvailable(), make sure that we never add a buffer after
// channel
// released all buffers via releaseAllResources().
if (inputChannel.isReleased()) {
- return numRequestedBuffers;
+ return 0;
}
numRequiredBuffers = numRequired;
- numRequestedBuffers = tryRequestBuffers();
+ int numRequestedBuffers = tryRequestBuffers();
+ return notifyAvailable ? numRequestedBuffers : 0;
}
- return numRequestedBuffers;
}
private int tryRequestBuffers() {
@@ -209,6 +219,7 @@ private int tryRequestBuffers() {
@Override
public void recycle(MemorySegment segment) {
@Nullable Buffer releasedFloatingBuffer = null;
+ boolean announceCredit = false;
synchronized (bufferQueue) {
try {
// Similar to notifyBufferAvailable(), make sure that we never add a buffer
@@ -226,11 +237,12 @@ public void recycle(MemorySegment segment) {
} finally {
bufferQueue.notifyAll();
}
+ announceCredit = releasedFloatingBuffer == null && notifyAvailable;
}
if (releasedFloatingBuffer != null) {
releasedFloatingBuffer.recycleBuffer();
- } else {
+ } else if (announceCredit) {
try {
inputChannel.notifyBufferAvailable(1);
} catch (Throwable t) {
@@ -344,6 +356,9 @@ public boolean notifyBufferAvailable(Buffer buffer) {
isBufferUsed = true;
numBuffers += 1 + tryRequestBuffers();
bufferQueue.notifyAll();
+ if (!notifyAvailable) {
+ numBuffers = 0;
+ }
}
inputChannel.notifyBufferAvailable(numBuffers);
@@ -359,6 +374,19 @@ public void notifyBufferDestroyed() {
// Nothing to do actually.
}
+ /**
+ * Opens the recovery credit gate and announces the queued buffers atomically with respect to
+ * concurrent recycle/floating-buffer callbacks.
+ */
+ void enableNotify() throws IOException {
+ int available;
+ synchronized (bufferQueue) {
+ notifyAvailable = true;
+ available = bufferQueue.getAvailableBufferSize();
+ }
+ inputChannel.notifyBufferAvailable(available);
+ }
+
// ------------------------------------------------------------------------
// Getter properties
// ------------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/EndOfFetchedChannelStateEvent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/EndOfFetchedChannelStateEvent.java
new file mode 100644
index 00000000000000..9900a0cdc660df
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/EndOfFetchedChannelStateEvent.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.io.network.partition.consumer;
+
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.runtime.event.RuntimeEvent;
+
+/**
+ * Marks the tail of recovered buffers that the spill drain pushed into a {@link
+ * RecoverableInputChannel}. The consume path polls this sentinel to learn the exact moment all
+ * recovered buffers have been consumed; it is never delivered to the operator. It is distinct from
+ * {@link EndOfInputChannelStateEvent} (which terminates the {@link RecoveredInputChannel} read
+ * stream) so the two recovery handoffs cannot be confused.
+ */
+public class EndOfFetchedChannelStateEvent extends RuntimeEvent {
+
+ /** The singleton instance of this event. */
+ public static final EndOfFetchedChannelStateEvent INSTANCE =
+ new EndOfFetchedChannelStateEvent();
+
+ // ------------------------------------------------------------------------
+
+ // not instantiable
+ private EndOfFetchedChannelStateEvent() {}
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void write(DataOutputView out) {
+ throw new UnsupportedOperationException(
+ "EndOfFetchedChannelStateEvent must be serialized via EventSerializer's dedicated"
+ + " type-tag path, not reflective write().");
+ }
+
+ @Override
+ public void read(DataInputView in) {
+ throw new UnsupportedOperationException(
+ "EndOfFetchedChannelStateEvent must be deserialized via EventSerializer's dedicated"
+ + " type-tag path, not reflective read().");
+ }
+
+ // ------------------------------------------------------------------------
+
+ @Override
+ public int hashCode() {
+ return 20250814;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj != null && obj.getClass() == EndOfFetchedChannelStateEvent.class;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannel.java
index e2969c675ec07a..d96f99e40cbe6c 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannel.java
@@ -33,6 +33,7 @@
import java.io.IOException;
import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicReference;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -146,6 +147,13 @@ public ResultSubpartitionIndexSet getConsumedSubpartitionIndexSet() {
return consumedSubpartitionIndexSet;
}
+ /**
+ * Completes once this channel has consumed all of its recovered state: it never had state to
+ * recover, all recovered data was consumed, or the channel was released. Implementations that
+ * never participate in recovery must return an already completed future.
+ */
+ public abstract CompletableFuture getStateConsumedFuture();
+
/**
* After sending a {@link org.apache.flink.runtime.io.network.api.CheckpointBarrier} of
* exactly-once mode, the upstream will be blocked and become unavailable. This method tries to
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGate.java
index dd744bae330ff3..2348c33ae98a44 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGate.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGate.java
@@ -141,6 +141,14 @@ public abstract void acknowledgeAllRecordsProcessed(InputChannelInfo channelInfo
/** Returns the channel of this gate. */
public abstract InputChannel getChannel(int channelIndex);
+ /**
+ * Returns the channel identified by {@code channelInfo}. Unlike {@link #getChannel(int)}, whose
+ * index is gate-global, this resolves through the full {@code (gateIdx, inputChannelIdx)} pair,
+ * so it stays correct for {@link UnionInputGate} where the global index differs from a member
+ * gate's local channel index.
+ */
+ public abstract InputChannel getChannel(InputChannelInfo channelInfo);
+
/** Returns the channel infos of this gate. */
public List getChannelInfos() {
return IntStream.range(0, getNumberOfInputChannels())
@@ -190,6 +198,15 @@ public String toString() {
public abstract void requestPartitions() throws IOException;
+ /**
+ * Requests the partitions. {@code needsRecovery} controls whether converted physical channels
+ * start in recovery (i.e. with no credit / no floating buffers until the recovered channel
+ * state has been drained). The default implementation ignores the flag.
+ */
+ public void requestPartitions(boolean needsRecovery) throws IOException {
+ requestPartitions();
+ }
+
public abstract CompletableFuture getStateConsumedFuture();
/**
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannel.java
index f7787c85c6cf75..9a7aa4b96dfc15 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannel.java
@@ -21,11 +21,15 @@
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.metrics.Counter;
import org.apache.flink.runtime.checkpoint.CheckpointException;
+import org.apache.flink.runtime.checkpoint.CheckpointFailureReason;
import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter;
+import org.apache.flink.runtime.checkpoint.channel.RecoveryCheckpointBarrier;
+import org.apache.flink.runtime.event.AbstractEvent;
import org.apache.flink.runtime.event.TaskEvent;
import org.apache.flink.runtime.execution.CancelTaskException;
import org.apache.flink.runtime.io.network.TaskEventPublisher;
import org.apache.flink.runtime.io.network.api.CheckpointBarrier;
+import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.io.network.buffer.CompositeBuffer;
import org.apache.flink.runtime.io.network.buffer.FileRegionBuffer;
@@ -43,21 +47,27 @@
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import javax.annotation.concurrent.GuardedBy;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Deque;
+import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.Timer;
import java.util.TimerTask;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkState;
/** An input channel, which requests a local subpartition. */
-public class LocalInputChannel extends InputChannel implements BufferAvailabilityListener {
+public class LocalInputChannel extends InputChannel
+ implements BufferAvailabilityListener, RecoverableInputChannel {
private static final Logger LOG = LoggerFactory.getLogger(LocalInputChannel.class);
@@ -81,12 +91,53 @@ public class LocalInputChannel extends InputChannel implements BufferAvailabilit
private final Deque toBeConsumedBuffers = new ArrayDeque<>();
/**
- * Flag indicating whether there is a pending priority event (e.g., checkpoint barrier) in the
- * subpartitionView that should be consumed before toBeConsumedBuffers. This is set by {@link
- * #notifyPriorityEvent} and checked in {@link #getNextBuffer()}.
+ * Buffers delivered from {@code RecoveredInputChannel}, kept separately from {@link
+ * #toBeConsumedBuffers} so that recovery semantics (priority event interleaving, checkpoint
+ * inflight persistence) do not leak into the FullyFilledBuffer split path. Holds recovered
+ * buffers plus the {@code RecoveryCheckpointBarrier} and {@code EndOfFetchedChannelStateEvent}
+ * sentinels. The deque object is its own monitor; {@link #inRecovery} and {@link
+ * #recoverySequenceNumber} are guarded by it too.
+ */
+ private final Deque recoveredBuffers = new ArrayDeque<>();
+
+ /**
+ * Whether the channel is still replaying recovered state. Starts {@code false} for channels
+ * that do not need recovery and is flipped to {@code false} the moment the consume path polls
+ * the {@code EndOfFetchedChannelStateEvent} sentinel appended after the last recovered buffer
+ * (see {@link #onRecoveredStateConsumed()}). While {@code true} the consume path serves
+ * recovered buffers and does not poll ordinary upstream data.
+ */
+ @GuardedBy("recoveredBuffers")
+ private boolean inRecovery;
+
+ private final CompletableFuture stateConsumedFuture = new CompletableFuture<>();
+
+ /**
+ * Sequence number assigned to recovered buffers, starting at {@link Integer#MIN_VALUE},
+ * consistent with {@link RecoveredInputChannel}.
+ */
+ private int recoverySequenceNumber = Integer.MIN_VALUE;
+
+ @Nullable private final BufferManager bufferManager;
+
+ private final int networkBuffersPerChannel;
+
+ private final boolean needsRecovery;
+
+ /**
+ * Whether a priority event (e.g., checkpoint barrier) is pending in {@code subpartitionView}
+ * and must be consumed before {@code recoveredBuffers}. Volatile because it is written by the
+ * network thread and read by the task thread.
*/
private volatile boolean hasPendingPriorityEvent = false;
+ /**
+ * One-shot latch that opens once the upstream subpartition view is registered (signalled by
+ * {@link #requestSubpartition} or by {@link #releaseAllResources()}). Recovery-side awaiters
+ * block on it before handing off.
+ */
+ private final CountDownLatch upstreamReady;
+
public LocalInputChannel(
SingleInputGate inputGate,
int channelIndex,
@@ -99,7 +150,41 @@ public LocalInputChannel(
Counter numBytesIn,
Counter numBuffersIn,
ChannelStateWriter stateWriter,
- ArrayDeque initialRecoveredBuffers) {
+ int networkBuffersPerChannel,
+ boolean needsRecovery) {
+ this(
+ inputGate,
+ channelIndex,
+ partitionId,
+ consumedSubpartitionIndexSet,
+ partitionManager,
+ taskEventPublisher,
+ initialBackoff,
+ maxBackoff,
+ numBytesIn,
+ numBuffersIn,
+ stateWriter,
+ networkBuffersPerChannel,
+ needsRecovery,
+ new CountDownLatch(1));
+ }
+
+ @VisibleForTesting
+ LocalInputChannel(
+ SingleInputGate inputGate,
+ int channelIndex,
+ ResultPartitionID partitionId,
+ ResultSubpartitionIndexSet consumedSubpartitionIndexSet,
+ ResultPartitionManager partitionManager,
+ TaskEventPublisher taskEventPublisher,
+ int initialBackoff,
+ int maxBackoff,
+ Counter numBytesIn,
+ Counter numBuffersIn,
+ ChannelStateWriter stateWriter,
+ int networkBuffersPerChannel,
+ boolean needsRecovery,
+ CountDownLatch upstreamReady) {
super(
inputGate,
@@ -113,48 +198,247 @@ public LocalInputChannel(
this.partitionManager = checkNotNull(partitionManager);
this.taskEventPublisher = checkNotNull(taskEventPublisher);
- this.channelStatePersister = new ChannelStatePersister(stateWriter, getChannelInfo());
-
- // Migrate recovered buffers from RecoveredInputChannel if provided.
- // These buffers have been filtered but not yet consumed by the Task.
- if (!initialRecoveredBuffers.isEmpty()) {
- final int expectedCount = initialRecoveredBuffers.size();
- // Sequence number starts at Integer.MIN_VALUE, consistent with RecoveredInputChannel.
- int seqNum = Integer.MIN_VALUE;
- while (!initialRecoveredBuffers.isEmpty()) {
- Buffer buffer = initialRecoveredBuffers.poll();
- // Determine next data type based on the next buffer in the queue
- Buffer.DataType nextDataType =
- initialRecoveredBuffers.isEmpty()
- ? Buffer.DataType.NONE
- : initialRecoveredBuffers.peek().getDataType();
- // buffersInBacklog is set to 0 as these are recovered buffers
- BufferAndBacklog bufferAndBacklog =
- new BufferAndBacklog(buffer, 0, nextDataType, seqNum++);
- toBeConsumedBuffers.add(bufferAndBacklog);
+ this.channelStatePersister =
+ new ChannelStatePersister(checkNotNull(stateWriter), getChannelInfo());
+ this.inRecovery = needsRecovery;
+ this.bufferManager =
+ needsRecovery
+ ? new BufferManager(inputGate.getMemorySegmentProvider(), this, 0, true)
+ : null;
+ this.networkBuffersPerChannel = networkBuffersPerChannel;
+ this.needsRecovery = needsRecovery;
+ this.upstreamReady = checkNotNull(upstreamReady);
+ if (!needsRecovery) {
+ stateConsumedFuture.complete(null);
+ }
+ }
+
+ @Override
+ void setup() throws IOException {
+ if (needsRecovery && networkBuffersPerChannel > 0) {
+ bufferManager.requestExclusiveBuffers(networkBuffersPerChannel);
+ }
+ }
+
+ // ------------------------------------------------------------------------
+ // RecoverableInputChannel implementation
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void onRecoveredStateBuffer(Buffer buffer) {
+ boolean wasEmpty;
+ synchronized (recoveredBuffers) {
+ if (isReleased) {
+ buffer.recycleBuffer();
+ return;
}
- checkState(
- toBeConsumedBuffers.size() == expectedCount,
- "Buffer migration failed: expected %s buffers but got %s",
- expectedCount,
- toBeConsumedBuffers.size());
+ // Migrate recovered buffers from RecoveredInputChannel. These buffers have been
+ // filtered but not yet consumed by the Task.
+ wasEmpty = offerRecoveredBuffer(buffer);
+ }
+ if (wasEmpty) {
+ notifyChannelNonEmpty();
+ }
+ }
+
+ @Override
+ public void finishRecoveredBufferDelivery() throws IOException, InterruptedException {
+ upstreamReady.await();
+ boolean wasEmpty;
+ synchronized (recoveredBuffers) {
+ // A release may have opened the latch instead of the subpartition view; bail out so we
+ // never append to a queue that releaseAllResources() already cleared.
+ if (isReleased) {
+ return;
+ }
+ checkState(inRecovery, "Recovery delivery already finished.");
+ // Append the sentinel after the last recovered buffer. The consume path flips out of
+ // recovery only once it polls this sentinel, guaranteeing all recovered buffers are
+ // consumed first.
+ wasEmpty =
+ offerRecoveredBuffer(
+ EventSerializer.toBuffer(
+ EndOfFetchedChannelStateEvent.INSTANCE, false));
+ }
+ if (wasEmpty) {
+ notifyChannelNonEmpty();
+ }
+ }
+
+ @Override
+ public Buffer requestRecoveryBufferBlocking() throws InterruptedException, IOException {
+ checkState(
+ bufferManager != null,
+ "requestRecoveryBufferBlocking called on a Local channel constructed with"
+ + " needsRecovery=false");
+ upstreamReady.await();
+ // If a release opened the latch instead of the subpartition view, requestBufferBlocking()
+ // detects the released channel and throws CancelTaskException.
+ return bufferManager.requestBufferBlocking();
+ }
+
+ @Override
+ public void insertRecoveryCheckpointBarrierIfInRecovery(long checkpointId) throws IOException {
+ boolean wasEmpty = false;
+ synchronized (recoveredBuffers) {
+ if (!isReleased && inRecovery) {
+ wasEmpty =
+ offerRecoveredBuffer(
+ EventSerializer.toBuffer(
+ new RecoveryCheckpointBarrier(checkpointId), false));
+ }
+ }
+ if (wasEmpty) {
+ notifyChannelNonEmpty();
}
}
+ /**
+ * Flips out of recovery the moment the consume path polls the {@code
+ * EndOfFetchedChannelStateEvent} sentinel, i.e. once all recovered buffers have been consumed.
+ * Live upstream data may flow again afterwards.
+ */
+ @Override
+ public void onRecoveredStateConsumed() {
+ synchronized (recoveredBuffers) {
+ checkState(inRecovery, "Recovery already finished.");
+ inRecovery = false;
+ }
+ notifyChannelNonEmpty();
+ stateConsumedFuture.complete(null);
+ }
+
+ @Override
+ public CompletableFuture getStateConsumedFuture() {
+ return stateConsumedFuture;
+ }
+
+ /**
+ * Appends a recovered buffer (or {@code RecoveryCheckpointBarrier} / {@code
+ * EndOfFetchedChannelStateEvent} sentinel) to {@link #recoveredBuffers}.
+ *
+ * @return {@code true} iff {@link #recoveredBuffers} transitioned from empty to non-empty.
+ */
+ private boolean offerRecoveredBuffer(Buffer buffer) {
+ assert Thread.holdsLock(recoveredBuffers);
+ checkState(inRecovery, "Push into recovered buffers after recovery finished.");
+ boolean wasEmpty = recoveredBuffers.isEmpty();
+ recoveredBuffers.add(buffer);
+ return wasEmpty;
+ }
+
+ private int nextRecoverySequenceNumber() {
+ assert Thread.holdsLock(recoveredBuffers);
+ return recoverySequenceNumber++;
+ }
+
+ /**
+ * Walks {@link #recoveredBuffers} up to the {@link RecoveryCheckpointBarrier} sentinel matching
+ * {@code checkpointId}, retaining each pre-barrier recovered data buffer and removing the
+ * sentinel. A barrier for an earlier (subsumed) checkpoint encountered on the way is logged and
+ * dropped; a barrier for a later checkpoint is an ordering violation.
+ *
+ * @throws IOException if a barrier for a later checkpoint is encountered before {@code
+ * checkpointId}, or if no sentinel matching {@code checkpointId} is found (the snapshot
+ * protocol guarantees one must be present while the channel is in recovery).
+ */
+ private List collectPreRecoveryBarrier(long checkpointId) throws IOException {
+ assert Thread.holdsLock(recoveredBuffers);
+ List retained = new ArrayList<>();
+ try {
+ Iterator it = recoveredBuffers.iterator();
+ while (it.hasNext()) {
+ Buffer b = it.next();
+ RecoveryCheckpointBarrier barrier = asRecoveryCheckpointBarrier(b);
+ if (barrier != null) {
+ long barrierId = barrier.getCheckpointId();
+ if (barrierId == checkpointId) {
+ it.remove();
+ b.recycleBuffer();
+ return retained;
+ }
+ if (barrierId > checkpointId) {
+ throw new IOException(
+ "Found RecoveryCheckpointBarrier for a later checkpoint "
+ + barrierId
+ + " before the target checkpoint "
+ + checkpointId
+ + " in recoveredBuffers for channel "
+ + getChannelInfo());
+ }
+ // barrierId < checkpointId: the checkpoint was subsumed; drop its stale
+ // barrier and keep scanning for the target.
+ LOG.warn(
+ "Discarding subsumed RecoveryCheckpointBarrier for checkpoint {} while "
+ + "collecting checkpoint {} on channel {}.",
+ barrierId,
+ checkpointId,
+ getChannelInfo());
+ it.remove();
+ b.recycleBuffer();
+ continue;
+ }
+ if (b.isBuffer()) {
+ retained.add(b.retainBuffer());
+ }
+ }
+ } catch (IOException e) {
+ releaseRetainedBuffers(retained);
+ throw e;
+ }
+ releaseRetainedBuffers(retained);
+ throw new IOException(
+ "Missing RecoveryCheckpointBarrier for checkpoint "
+ + checkpointId
+ + " in recoveredBuffers for channel "
+ + getChannelInfo());
+ }
+
+ private static void releaseRetainedBuffers(List retained) {
+ for (Buffer buffer : retained) {
+ buffer.recycleBuffer();
+ }
+ }
+
+ @Nullable
+ private static RecoveryCheckpointBarrier asRecoveryCheckpointBarrier(Buffer b)
+ throws IOException {
+ if (b.isBuffer()) {
+ return null;
+ }
+ AbstractEvent event =
+ EventSerializer.fromBuffer(b, RecoveryCheckpointBarrier.class.getClassLoader());
+ b.setReaderIndex(0);
+ return event instanceof RecoveryCheckpointBarrier
+ ? (RecoveryCheckpointBarrier) event
+ : null;
+ }
+
// ------------------------------------------------------------------------
// Consume
// ------------------------------------------------------------------------
+ @Override
public void checkpointStarted(CheckpointBarrier barrier) throws CheckpointException {
- // Collect inflight buffers from toBeConsumedBuffers to be persisted.
- // These are buffers that have not been consumed yet when the checkpoint barrier arrives.
- List inflightBuffers = new ArrayList<>();
- for (BufferAndBacklog bufferAndBacklog : toBeConsumedBuffers) {
- if (bufferAndBacklog.buffer().isBuffer()) {
- inflightBuffers.add(bufferAndBacklog.buffer().retainBuffer());
+ try {
+ List toPersist;
+ synchronized (recoveredBuffers) {
+ if (inRecovery) {
+ // Collect inflight buffers from recoveredBuffers to be persisted. These are
+ // recovered buffers that have not been consumed yet when the checkpoint barrier
+ // arrives.
+ toPersist = collectPreRecoveryBarrier(barrier.getId());
+ } else {
+ toPersist = Collections.emptyList();
+ }
}
+ channelStatePersister.startPersisting(barrier.getId(), toPersist);
+ } catch (IOException e) {
+ throw new CheckpointException(
+ "Failed to extract recovered buffers for checkpoint " + barrier.getId(),
+ CheckpointFailureReason.CHECKPOINT_DECLINED,
+ e);
}
- channelStatePersister.startPersisting(barrier.getId(), inflightBuffers);
}
public void checkpointStopped(long checkpointId) {
@@ -163,6 +447,8 @@ public void checkpointStopped(long checkpointId) {
@Override
protected void requestSubpartitions() throws IOException {
+ checkState(toBeConsumedBuffers.isEmpty());
+
boolean retriggerRequest = false;
boolean notifyDataAvailable = false;
@@ -196,6 +482,7 @@ protected void requestSubpartitions() throws IOException {
this.subpartitionView = null;
} else {
notifyDataAvailable = true;
+ upstreamReady.countDown();
}
} catch (PartitionNotFoundException notFound) {
if (increaseBackoff()) {
@@ -272,8 +559,35 @@ protected int peekNextBufferSubpartitionIdInternal() throws IOException {
public Optional getNextBuffer() throws IOException {
checkError();
+ // Read inRecovery and poll the recovered buffer under a single lock acquisition to avoid
+ // grabbing the monitor twice on the hot path.
+ boolean inRecovery;
+ Buffer recoveredBuf = null;
+ synchronized (recoveredBuffers) {
+ inRecovery = this.inRecovery;
+ if (inRecovery && !hasPendingPriorityEvent && !recoveredBuffers.isEmpty()) {
+ recoveredBuf = recoveredBuffers.poll();
+ }
+ }
+
+ if (inRecovery) {
+ // Always return an already-polled recovered buffer first: hasPendingPriorityEvent may
+ // be flipped to true by a concurrent notifyPriorityEvent() after the poll, and
+ // re-reading
+ // it here would otherwise drop this buffer. A pending priority event is served on the
+ // next getNextBuffer() call instead.
+ if (recoveredBuf != null) {
+ return wrapRecoveredBufferAsAvailability(recoveredBuf);
+ }
+ if (hasPendingPriorityEvent) {
+ return pullPriorityFromSubpartitionView();
+ }
+ // Drain not finished yet; block normal upstream data until delivery completes.
+ return Optional.empty();
+ }
+
if (!toBeConsumedBuffers.isEmpty()) {
- return getNextRecoveredBuffer();
+ return getBufferAndAvailability(toBeConsumedBuffers.removeFirst());
}
ResultSubpartitionView subpartitionView = this.subpartitionView;
@@ -329,79 +643,102 @@ public Optional getNextBuffer() throws IOException {
seq++));
}
- return Optional.of(getBufferAndAvailability(toBeConsumedBuffers.removeFirst()));
+ return getBufferAndAvailability(toBeConsumedBuffers.removeFirst());
}
- BufferAndAvailability bufferAndAvailability = getBufferAndAvailability(next);
- channelStatePersister.checkForBarrier(bufferAndAvailability.buffer());
- channelStatePersister.maybePersist(bufferAndAvailability.buffer());
- return Optional.of(bufferAndAvailability);
+ return getBufferAndAvailability(next);
}
- /**
- * Consumes the next buffer from toBeConsumedBuffers (recovered buffers), handling pending
- * priority events and dynamic availability detection for the last recovered buffer.
- */
- private Optional getNextRecoveredBuffer() throws IOException {
- // If there is a pending priority event (e.g., unaligned checkpoint barrier), fetch it
- // from subpartitionView first, skipping toBeConsumedBuffers. This ensures priority
- // events are processed immediately even when there are pending recovered buffers.
- if (hasPendingPriorityEvent) {
- checkState(subpartitionView != null, "No subpartition view available");
- BufferAndBacklog next = subpartitionView.getNextBuffer();
- checkState(
- next != null && next.buffer().getDataType().hasPriority(),
- "Expected priority event, but got %s",
- next == null ? "null" : next.buffer().getDataType());
-
- // Check for barrier to update channel state persister.
- // Note: maybePersist is not needed for barriers as they are not regular data buffers.
- channelStatePersister.checkForBarrier(next.buffer());
-
- Buffer.DataType expectedNextDataType = next.getNextDataType();
- if (!expectedNextDataType.hasPriority()) {
- // Reset hasPendingPriorityEvent to false if no more priority event
- hasPendingPriorityEvent = false;
- if (!toBeConsumedBuffers.isEmpty()) {
- // Correct nextDataType: if toBeConsumedBuffers is not empty, the actual next
- // element to consume is from toBeConsumedBuffers, not from subpartitionView
- expectedNextDataType = toBeConsumedBuffers.peek().buffer().getDataType();
- }
- }
+ private Optional pullPriorityFromSubpartitionView() throws IOException {
+ // If there is a pending priority event (e.g., unaligned checkpoint barrier), fetch it from
+ // subpartitionView first, skipping recoveredBuffers. This ensures priority events are
+ // processed immediately even when there are pending recovered buffers.
+ checkState(subpartitionView != null, "No subpartition view available");
+ BufferAndBacklog next = subpartitionView.getNextBuffer();
+ checkState(
+ next != null && next.buffer().getDataType().hasPriority(),
+ "Expected priority event, but got %s",
+ next == null ? "null" : next.buffer().getDataType());
+
+ // Check for barrier to update channel state persister. Note: maybePersist is not needed for
+ // barriers as they are not regular data buffers.
+ channelStatePersister.checkForBarrier(next.buffer());
+
+ Buffer.DataType expectedNextDataType = next.getNextDataType();
+ if (!expectedNextDataType.hasPriority()) {
+ // Reset hasPendingPriorityEvent to false if no more priority event.
+ hasPendingPriorityEvent = false;
+ // Correct nextDataType: if recoveredBuffers is not empty, the actual next element to
+ // consume is from recoveredBuffers, not from subpartitionView.
+ expectedNextDataType = peekNextDataType(next.getNextDataType());
+ }
- return Optional.of(
- getBufferAndAvailability(
- new BufferAndBacklog(
- next.buffer(),
- next.buffersInBacklog(),
- expectedNextDataType,
- next.getSequenceNumber())));
- }
-
- BufferAndBacklog next = toBeConsumedBuffers.removeFirst();
-
- // If this is the last recovered buffer and nextDataType is NONE,
- // dynamically check if subpartitionView has data available.
- // The last buffer's nextDataType was preset to NONE during construction,
- // but subpartitionView may already have data available.
- if (toBeConsumedBuffers.isEmpty()
- && next.getNextDataType() == Buffer.DataType.NONE
- && subpartitionView != null) {
- ResultSubpartitionView.AvailabilityWithBacklog availability =
- subpartitionView.getAvailabilityAndBacklog(true);
- if (availability.isAvailable()) {
- next =
- new BufferAndBacklog(
- next.buffer(),
- availability.getBacklog(),
- Buffer.DataType.DATA_BUFFER,
- next.getSequenceNumber());
+ return Optional.of(
+ new BufferAndAvailability(
+ next.buffer(),
+ expectedNextDataType,
+ next.buffersInBacklog(),
+ next.getSequenceNumber()));
+ }
+
+ private Optional wrapRecoveredBufferAsAvailability(Buffer buf)
+ throws IOException {
+ if (buf instanceof FileRegionBuffer) {
+ buf = ((FileRegionBuffer) buf).readInto(inputGate.getUnpooledSegment());
+ }
+ if (buf instanceof CompositeBuffer) {
+ buf = ((CompositeBuffer) buf).getFullBufferData(inputGate.getUnpooledSegment());
+ }
+
+ numBytesIn.inc(buf.readableBytes());
+ numBuffersIn.inc();
+
+ ResultSubpartitionView view = subpartitionView;
+ Buffer.DataType upstreamProbe;
+ if (view != null && view.getAvailabilityAndBacklog(true).isAvailable()) {
+ upstreamProbe = Buffer.DataType.DATA_BUFFER;
+ } else {
+ upstreamProbe = Buffer.DataType.NONE;
+ }
+
+ int sequenceNumber;
+ synchronized (recoveredBuffers) {
+ Buffer.DataType nextDataType = peekNextDataType(upstreamProbe);
+ sequenceNumber = nextRecoverySequenceNumber();
+ NetworkActionsLogger.traceInput(
+ "LocalInputChannel#getNextBuffer",
+ buf,
+ inputGate.getOwningTaskName(),
+ channelInfo,
+ channelStatePersister,
+ sequenceNumber);
+ // buffersInBacklog is set to 0 as these are recovered buffers.
+ return Optional.of(new BufferAndAvailability(buf, nextDataType, 0, sequenceNumber));
+ }
+ }
+
+ private Buffer.DataType peekNextDataType(Buffer.DataType nextDataTypeOnUpstream) {
+ synchronized (recoveredBuffers) {
+ if (!recoveredBuffers.isEmpty()) {
+ return recoveredBuffers.peek().getDataType();
+ }
+ if (inRecovery) {
+ // If this is the last currently available recovered buffer, hide upstream data
+ // until the EndOfFetchedChannelStateEvent sentinel flips the channel out of
+ // recovery. The last buffer's nextDataType is effectively NONE while the drain can
+ // still append more recovered buffers.
+ return Buffer.DataType.NONE;
}
}
- return Optional.of(getBufferAndAvailability(next));
+ // If this is the last recovered buffer after delivery finished, dynamically check if
+ // subpartitionView has data available. The last buffer's nextDataType may have been NONE
+ // while recovered data was still being delivered, but subpartitionView may already have
+ // data
+ // available now.
+ return nextDataTypeOnUpstream;
}
- private BufferAndAvailability getBufferAndAvailability(BufferAndBacklog next)
+ private Optional getBufferAndAvailability(BufferAndBacklog next)
throws IOException {
Buffer buffer = next.buffer();
if (buffer instanceof FileRegionBuffer) {
@@ -414,6 +751,8 @@ private BufferAndAvailability getBufferAndAvailability(BufferAndBacklog next)
numBytesIn.inc(buffer.readableBytes());
numBuffersIn.inc();
+ channelStatePersister.checkForBarrier(buffer);
+ channelStatePersister.maybePersist(buffer);
NetworkActionsLogger.traceInput(
"LocalInputChannel#getNextBuffer",
buffer,
@@ -421,8 +760,12 @@ private BufferAndAvailability getBufferAndAvailability(BufferAndBacklog next)
channelInfo,
channelStatePersister,
next.getSequenceNumber());
- return new BufferAndAvailability(
- buffer, next.getNextDataType(), next.buffersInBacklog(), next.getSequenceNumber());
+ return Optional.of(
+ new BufferAndAvailability(
+ buffer,
+ next.getNextDataType(),
+ next.buffersInBacklog(),
+ next.getSequenceNumber()));
}
@Override
@@ -433,7 +776,7 @@ public void notifyDataAvailable(ResultSubpartitionView view) {
@Override
public void notifyPriorityEvent(int prioritySequenceNumber) {
// Set flag so that getNextBuffer() knows to fetch priority event from subpartitionView
- // before consuming toBeConsumedBuffers.
+ // before consuming recoveredBuffers.
hasPendingPriorityEvent = true;
super.notifyPriorityEvent(prioritySequenceNumber);
}
@@ -504,18 +847,35 @@ void releaseAllResources() throws IOException {
if (!isReleased) {
isReleased = true;
+ // Unblock any thread awaiting upstreamReady (drain still in flight) so it falls
+ // through and observes the released state instead of deadlocking.
+ upstreamReady.countDown();
+
+ // Recovery will never be consumed on a released channel; unblock anyone gating on it.
+ stateConsumedFuture.completeExceptionally(new CancelTaskException("Channel released."));
+
ResultSubpartitionView view = subpartitionView;
if (view != null) {
view.releaseAllResources();
subpartitionView = null;
}
- // Release any remaining buffers in toBeConsumedBuffers to avoid memory leak.
- // These may be recovered buffers or partial buffers from FullyFilledBuffer.
+ // Release any remaining buffers in recoveredBuffers (migrated recovered buffers not yet
+ // consumed) and toBeConsumedBuffers (FullyFilledBuffer partial splits) to avoid memory
+ // leak.
+ synchronized (recoveredBuffers) {
+ for (Buffer buffer : recoveredBuffers) {
+ buffer.recycleBuffer();
+ }
+ recoveredBuffers.clear();
+ }
for (BufferAndBacklog bufferAndBacklog : toBeConsumedBuffers) {
bufferAndBacklog.buffer().recycleBuffer();
}
toBeConsumedBuffers.clear();
+ if (bufferManager != null) {
+ bufferManager.releaseAllBuffers(new ArrayDeque<>());
+ }
}
}
@@ -532,14 +892,16 @@ void announceBufferSize(int newBufferSize) {
@Override
int getBuffersInUseCount() {
ResultSubpartitionView view = this.subpartitionView;
- return toBeConsumedBuffers.size() + (view == null ? 0 : view.getNumberOfQueuedBuffers());
+ return recoveredBuffers.size()
+ + toBeConsumedBuffers.size()
+ + (view == null ? 0 : view.getNumberOfQueuedBuffers());
}
@Override
public int unsynchronizedGetNumberOfQueuedBuffers() {
ResultSubpartitionView view = subpartitionView;
- int count = toBeConsumedBuffers.size();
+ int count = recoveredBuffers.size() + toBeConsumedBuffers.size();
if (view != null) {
count += view.unsynchronizedGetNumberOfQueuedBuffers();
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalRecoveredInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalRecoveredInputChannel.java
index bdde2244f38ef6..de058bd3723ce5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalRecoveredInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/LocalRecoveredInputChannel.java
@@ -19,14 +19,11 @@
package org.apache.flink.runtime.io.network.partition.consumer;
import org.apache.flink.runtime.io.network.TaskEventPublisher;
-import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.io.network.metrics.InputChannelMetrics;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.io.network.partition.ResultSubpartitionIndexSet;
-import java.util.ArrayDeque;
-
import static org.apache.flink.util.Preconditions.checkNotNull;
/**
@@ -64,7 +61,7 @@ public class LocalRecoveredInputChannel extends RecoveredInputChannel {
}
@Override
- protected InputChannel toInputChannelInternal(ArrayDeque remainingBuffers) {
+ protected InputChannel toInputChannelInternal(boolean needsRecovery) {
return new LocalInputChannel(
inputGate,
getChannelIndex(),
@@ -77,6 +74,7 @@ protected InputChannel toInputChannelInternal(ArrayDeque remainingBuffer
numBytesIn,
numBuffersIn,
channelStateWriter,
- remainingBuffers);
+ networkBuffersPerChannel,
+ needsRecovery);
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RecoverableInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RecoverableInputChannel.java
new file mode 100644
index 00000000000000..40a4ab27a0908d
--- /dev/null
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RecoverableInputChannel.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.io.network.partition.consumer;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.runtime.checkpoint.channel.InputChannelInfo;
+import org.apache.flink.runtime.io.network.buffer.Buffer;
+
+import java.io.IOException;
+
+/** Physical input channel that can receive recovered buffers pushed by the spill drain. */
+@Internal
+public interface RecoverableInputChannel {
+
+ InputChannelInfo getChannelInfo();
+
+ /**
+ * Appends a recovered buffer or recovery-checkpoint sentinel. Released channels recycle the
+ * buffer silently.
+ */
+ void onRecoveredStateBuffer(Buffer buffer);
+
+ /**
+ * Marks producer-side recovery delivery complete. Implementations wait for upstream readiness
+ * before flipping this state so channels without spill entries still observe the same handoff.
+ */
+ void finishRecoveredBufferDelivery() throws IOException, InterruptedException;
+
+ /**
+ * Inserts a {@code RecoveryCheckpointBarrier} for {@code checkpointId} into this channel's
+ * recovery queue if the channel is still in recovery.
+ */
+ void insertRecoveryCheckpointBarrierIfInRecovery(long checkpointId) throws IOException;
+
+ /**
+ * Blocks until a buffer is available from this channel's own buffer pool. Implementations must
+ * first await upstream readiness and must be invoked outside the drainer lock.
+ */
+ Buffer requestRecoveryBufferBlocking() throws InterruptedException, IOException;
+
+ /**
+ * Invoked by the consume path the moment it polls the {@code EndOfFetchedChannelStateEvent}
+ * sentinel, i.e. once all recovered buffers have been consumed. Implementations flip out of
+ * recovery, release any upstream events held back during recovery, and reopen the upstream so
+ * live data may flow again.
+ */
+ void onRecoveredStateConsumed() throws IOException;
+}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RecoveredInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RecoveredInputChannel.java
index d9b7885815bd12..ec878b69c62ab5 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RecoveredInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RecoveredInputChannel.java
@@ -59,7 +59,7 @@ public abstract class RecoveredInputChannel extends InputChannel implements Chan
private static final Logger LOG = LoggerFactory.getLogger(RecoveredInputChannel.class);
private final ArrayDeque receivedBuffers = new ArrayDeque<>();
- private final CompletableFuture stateConsumedFuture = new CompletableFuture<>();
+ private final CompletableFuture stateConsumedFuture = new CompletableFuture<>();
protected final BufferManager bufferManager;
/**
@@ -105,7 +105,7 @@ public abstract class RecoveredInputChannel extends InputChannel implements Chan
numBytesIn,
numBuffersIn);
- bufferManager = new BufferManager(inputGate.getMemorySegmentProvider(), this, 0);
+ bufferManager = new BufferManager(inputGate.getMemorySegmentProvider(), this, 0, true);
this.networkBuffersPerChannel = networkBuffersPerChannel;
}
@@ -115,23 +115,53 @@ public void setChannelStateWriter(ChannelStateWriter channelStateWriter) {
this.channelStateWriter = checkNotNull(channelStateWriter);
}
- public final InputChannel toInputChannel() throws IOException {
- Preconditions.checkState(
- bufferFilteringCompleteFuture.isDone(), "buffer filtering is not complete");
- if (!inputGate.isCheckpointingDuringRecoveryEnabled()) {
- Preconditions.checkState(
- stateConsumedFuture.isDone(), "recovered state is not fully consumed");
+ public final InputChannel toInputChannel(boolean needsRecovery) throws IOException {
+ if (needsRecovery) {
+ return toInputChannelInRecovery();
+ }
+ synchronized (receivedBuffers) {
+ Preconditions.checkState(receivedBuffers.isEmpty(), "Received buffer should be empty.");
}
- // Extract remaining buffers before conversion.
- // These buffers have been filtered but not yet consumed by the Task.
- final ArrayDeque remainingBuffers;
+ final InputChannel inputChannel = toInputChannelInternal(needsRecovery);
+ inputChannel.setup();
+ inputChannel.checkpointStopped(lastStoppedCheckpointId);
+ return inputChannel;
+ }
+
+ /**
+ * FLINK-38544 transitional: removed when the spilling backend lands. Creates the physical
+ * channel in recovery state and synchronously hands every queued recovered buffer over through
+ * the push interface. The legacy {@link EndOfInputChannelStateEvent} in the queue is dropped in
+ * translation; the {@link EndOfFetchedChannelStateEvent} sentinel takes its place. The sentinel
+ * is appended directly instead of via {@link
+ * RecoverableInputChannel#finishRecoveredBufferDelivery()} because that method waits for
+ * upstream readiness, which cannot happen while the mailbox thread is still converting channels
+ * (partitions are requested only after conversion).
+ */
+ private InputChannel toInputChannelInRecovery() throws IOException {
+ final Buffer[] remainingBuffers;
synchronized (receivedBuffers) {
- remainingBuffers = new ArrayDeque<>(receivedBuffers);
+ remainingBuffers = receivedBuffers.toArray(new Buffer[0]);
receivedBuffers.clear();
}
- final InputChannel inputChannel = toInputChannelInternal(remainingBuffers);
+ final InputChannel inputChannel = toInputChannelInternal(true);
+ inputChannel.setup();
+ final RecoverableInputChannel recoverableChannel = (RecoverableInputChannel) inputChannel;
+ for (int i = 0; i < remainingBuffers.length; i++) {
+ final Buffer buffer = remainingBuffers[i];
+ if (isEndOfInputChannelStateEvent(buffer)) {
+ Preconditions.checkState(
+ i == remainingBuffers.length - 1,
+ "EndOfInputChannelStateEvent must be the last recovered buffer.");
+ buffer.recycleBuffer();
+ } else {
+ recoverableChannel.onRecoveredStateBuffer(buffer);
+ }
+ }
+ recoverableChannel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(EndOfFetchedChannelStateEvent.INSTANCE, false));
inputChannel.checkpointStopped(lastStoppedCheckpointId);
return inputChannel;
}
@@ -142,13 +172,10 @@ public void checkpointStopped(long checkpointId) {
}
/**
- * Creates the physical InputChannel from this recovered channel.
- *
- * @param remainingBuffers buffers that have been filtered but not yet consumed by the Task.
- * These buffers will be migrated to the new physical channel.
- * @return the physical InputChannel (LocalInputChannel or RemoteInputChannel)
+ * Creates the physical {@link InputChannel}; {@code needsRecovery} controls whether it starts
+ * in recovery.
*/
- protected abstract InputChannel toInputChannelInternal(ArrayDeque remainingBuffers)
+ protected abstract InputChannel toInputChannelInternal(boolean needsRecovery)
throws IOException;
/**
@@ -159,17 +186,15 @@ CompletableFuture getBufferFilteringCompleteFuture() {
return bufferFilteringCompleteFuture;
}
- CompletableFuture getStateConsumedFuture() {
+ @Override
+ public CompletableFuture getStateConsumedFuture() {
return stateConsumedFuture;
}
public void onRecoveredStateBuffer(Buffer buffer) {
boolean recycleBuffer = true;
NetworkActionsLogger.traceRecover(
- "InputChannelRecoveredStateHandler#recover",
- buffer,
- inputGate.getOwningTaskName(),
- channelInfo);
+ "NoSpillingHandler#recover", buffer, inputGate.getOwningTaskName(), channelInfo);
try {
final boolean wasEmpty;
synchronized (receivedBuffers) {
@@ -307,7 +332,7 @@ boolean isReleased() {
}
}
- void releaseAllResources() throws IOException {
+ public void releaseAllResources() throws IOException {
ArrayDeque releasedBuffers = new ArrayDeque<>();
boolean shouldRelease = false;
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
index 645446120d09e8..8b9241e19da968 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannel.java
@@ -25,6 +25,7 @@
import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.checkpoint.CheckpointFailureReason;
import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter;
+import org.apache.flink.runtime.checkpoint.channel.RecoveryCheckpointBarrier;
import org.apache.flink.runtime.event.AbstractEvent;
import org.apache.flink.runtime.event.TaskEvent;
import org.apache.flink.runtime.execution.CancelTaskException;
@@ -62,9 +63,12 @@
import java.util.List;
import java.util.Optional;
import java.util.OptionalLong;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.apache.flink.runtime.io.network.buffer.Buffer.DataType.RECOVERY_METADATA;
import static org.apache.flink.util.Preconditions.checkArgument;
@@ -72,7 +76,7 @@
import static org.apache.flink.util.Preconditions.checkState;
/** An input channel, which requests a remote partition queue. */
-public class RemoteInputChannel extends InputChannel {
+public class RemoteInputChannel extends InputChannel implements RecoverableInputChannel {
private static final Logger LOG = LoggerFactory.getLogger(RemoteInputChannel.class);
private static final int NONE = -1;
@@ -107,6 +111,8 @@ public class RemoteInputChannel extends InputChannel {
/** The initial number of exclusive buffers assigned to this channel. */
private final int initialCredit;
+ private final boolean needsRecovery;
+
/** The milliseconds timeout for partition request listener in result partition manager. */
private final int partitionRequestListenerTimeout;
@@ -123,6 +129,45 @@ public class RemoteInputChannel extends InputChannel {
private final ChannelStatePersister channelStatePersister;
+ /**
+ * Whether the channel is still replaying recovered state. Recovered buffers delivered by the
+ * spill drain are appended directly to {@link #receivedBuffers}, so the consume path needs no
+ * recovery-specific branch. Starts {@code false} for channels that do not need recovery and is
+ * flipped to {@code false} the moment the consume path polls the {@code
+ * EndOfFetchedChannelStateEvent} sentinel that the drain appended after the last recovered
+ * buffer (see {@link #onRecoveredStateConsumed()}).
+ */
+ @GuardedBy("receivedBuffers")
+ private boolean inRecovery;
+
+ private final CompletableFuture stateConsumedFuture = new CompletableFuture<>();
+
+ /**
+ * Sequence number assigned to recovered buffers, starting at {@link Integer#MIN_VALUE},
+ * consistent with {@link RecoveredInputChannel}.
+ */
+ @GuardedBy("receivedBuffers")
+ private int recoverySequenceNumber = Integer.MIN_VALUE;
+
+ /**
+ * Ordinary (non-priority) upstream events received while recovery is still in progress. They
+ * cannot enter {@link #receivedBuffers} ahead of the recovered buffers, so they are stashed
+ * here and appended once recovery delivery finishes. Credit is suppressed during recovery, so
+ * the upstream can only send events (never data buffers) before {@link
+ * #finishRecoveredBufferDelivery()}.
+ */
+ @GuardedBy("receivedBuffers")
+ private final ArrayDeque recoveryEventStash = new ArrayDeque<>();
+
+ /**
+ * One-shot latch that opens once the upstream reader is registered and the connection is live
+ * (signalled by the first {@link #onBuffer} or by {@link #releaseAllResources()}).
+ * Recovery-side awaiters block on it before handing off; once open, {@link
+ * CountDownLatch#countDown()} on the hot path is a cheap idempotent no-op, unlike completing a
+ * {@code CompletableFuture}.
+ */
+ private final CountDownLatch upstreamReady;
+
private long totalQueueSizeInBytes;
public RemoteInputChannel(
@@ -139,7 +184,42 @@ public RemoteInputChannel(
Counter numBytesIn,
Counter numBuffersIn,
ChannelStateWriter stateWriter,
- ArrayDeque initialRecoveredBuffers) {
+ boolean needsRecovery) {
+ this(
+ inputGate,
+ channelIndex,
+ partitionId,
+ consumedSubpartitionIndexSet,
+ connectionId,
+ connectionManager,
+ initialBackOff,
+ maxBackoff,
+ partitionRequestListenerTimeout,
+ networkBuffersPerChannel,
+ numBytesIn,
+ numBuffersIn,
+ stateWriter,
+ needsRecovery,
+ new CountDownLatch(1));
+ }
+
+ @VisibleForTesting
+ RemoteInputChannel(
+ SingleInputGate inputGate,
+ int channelIndex,
+ ResultPartitionID partitionId,
+ ResultSubpartitionIndexSet consumedSubpartitionIndexSet,
+ ConnectionID connectionId,
+ ConnectionManager connectionManager,
+ int initialBackOff,
+ int maxBackoff,
+ int partitionRequestListenerTimeout,
+ int networkBuffersPerChannel,
+ Counter numBytesIn,
+ Counter numBuffersIn,
+ ChannelStateWriter stateWriter,
+ boolean needsRecovery,
+ CountDownLatch upstreamReady) {
super(
inputGate,
@@ -156,30 +236,15 @@ public RemoteInputChannel(
this.initialCredit = networkBuffersPerChannel;
this.connectionId = checkNotNull(connectionId);
this.connectionManager = checkNotNull(connectionManager);
- this.bufferManager = new BufferManager(inputGate.getMemorySegmentProvider(), this, 0);
- this.channelStatePersister = new ChannelStatePersister(stateWriter, getChannelInfo());
-
- // Migrate recovered buffers from RecoveredInputChannel if provided.
- // These buffers have been filtered but not yet consumed by the Task.
- if (!initialRecoveredBuffers.isEmpty()) {
- final int expectedCount = initialRecoveredBuffers.size();
- // Sequence number starts at Integer.MIN_VALUE, consistent with RecoveredInputChannel.
- int seqNum = Integer.MIN_VALUE;
- for (Buffer buffer : initialRecoveredBuffers) {
- // subpartitionId is set to 0 for recovered buffers. This is correct because:
- // 1) For single-subpartition channels, the only valid subpartition is 0.
- // 2) For multi-subpartition channels (consumedSubpartitionIndexSet.size() > 1),
- // RecoveryMetadata events embedded in the recovered buffer sequence track
- // the actual subpartition context for proper routing.
- SequenceBuffer sequenceBuffer = new SequenceBuffer(buffer, seqNum++, 0);
- receivedBuffers.add(sequenceBuffer);
- totalQueueSizeInBytes += buffer.getSize();
- }
- checkState(
- receivedBuffers.size() == expectedCount,
- "Buffer migration failed: expected %s buffers but got %s",
- expectedCount,
- receivedBuffers.size());
+ this.needsRecovery = needsRecovery;
+ this.bufferManager =
+ new BufferManager(inputGate.getMemorySegmentProvider(), this, 0, !needsRecovery);
+ this.channelStatePersister =
+ new ChannelStatePersister(checkNotNull(stateWriter), getChannelInfo());
+ this.inRecovery = needsRecovery;
+ this.upstreamReady = checkNotNull(upstreamReady);
+ if (!needsRecovery) {
+ stateConsumedFuture.complete(null);
}
}
@@ -201,6 +266,117 @@ void setup() throws IOException {
bufferManager.requestExclusiveBuffers(initialCredit);
}
+ // ------------------------------------------------------------------------
+ // RecoverableInputChannel implementation
+ // ------------------------------------------------------------------------
+
+ @Override
+ public void onRecoveredStateBuffer(Buffer buffer) {
+ boolean wasEmpty;
+ synchronized (receivedBuffers) {
+ if (isReleased.get()) {
+ buffer.recycleBuffer();
+ return;
+ }
+ // Migrate recovered buffers from RecoveredInputChannel. These buffers have been
+ // filtered but not yet consumed by the Task. They are appended to receivedBuffers so
+ // the consume path stays identical to the non-recovery case.
+ wasEmpty = appendRecoveredBuffer(buffer);
+ }
+ if (wasEmpty) {
+ notifyChannelNonEmpty();
+ }
+ }
+
+ @Override
+ public void finishRecoveredBufferDelivery() throws IOException, InterruptedException {
+ upstreamReady.await();
+ boolean wasEmpty;
+ synchronized (receivedBuffers) {
+ // A release may have opened the latch instead of the first buffer; bail out so we never
+ // append to a queue that releaseAllResources() already cleared.
+ if (isReleased.get()) {
+ return;
+ }
+ checkState(inRecovery, "Recovery delivery already finished.");
+ // Append the sentinel after the last recovered buffer. The consume path flips out of
+ // recovery (unstash + reopen credit) only once it polls this sentinel, guaranteeing all
+ // recovered buffers are consumed first.
+ wasEmpty =
+ appendRecoveredBuffer(
+ EventSerializer.toBuffer(
+ EndOfFetchedChannelStateEvent.INSTANCE, false));
+ }
+ if (wasEmpty) {
+ notifyChannelNonEmpty();
+ }
+ }
+
+ /**
+ * Flips out of recovery once the consume path polls the {@code EndOfFetchedChannelStateEvent}
+ * sentinel: releases the upstream events stashed during recovery so they are consumed after the
+ * recovered buffers, then reopens the suppressed credit notifications.
+ */
+ @Override
+ public void onRecoveredStateConsumed() throws IOException {
+ synchronized (receivedBuffers) {
+ checkState(inRecovery, "Recovery already finished.");
+ inRecovery = false;
+ recoveryEventStash.forEach(receivedBuffers::add);
+ recoveryEventStash.clear();
+ }
+ notifyChannelNonEmpty();
+ // Credit notifications are suppressed while recovery borrows the exclusive buffers.
+ bufferManager.enableNotify();
+ stateConsumedFuture.complete(null);
+ }
+
+ @Override
+ public CompletableFuture getStateConsumedFuture() {
+ return stateConsumedFuture;
+ }
+
+ @Override
+ public Buffer requestRecoveryBufferBlocking() throws InterruptedException, IOException {
+ upstreamReady.await();
+ // If a release opened the latch instead of the first buffer, requestBufferBlocking()
+ // detects the released channel and throws CancelTaskException.
+ return bufferManager.requestBufferBlocking();
+ }
+
+ @Override
+ public void insertRecoveryCheckpointBarrierIfInRecovery(long checkpointId) throws IOException {
+ boolean wasEmpty = false;
+ synchronized (receivedBuffers) {
+ if (!isReleased.get() && inRecovery) {
+ wasEmpty =
+ appendRecoveredBuffer(
+ EventSerializer.toBuffer(
+ new RecoveryCheckpointBarrier(checkpointId), false));
+ }
+ }
+ if (wasEmpty) {
+ notifyChannelNonEmpty();
+ }
+ }
+
+ /**
+ * Appends a recovered buffer (or {@code RecoveryCheckpointBarrier} sentinel) to {@link
+ * #receivedBuffers} with a recovery sequence number.
+ *
+ * @return {@code true} iff {@code receivedBuffers} transitioned from empty to non-empty.
+ */
+ @GuardedBy("receivedBuffers")
+ private boolean appendRecoveredBuffer(Buffer buffer) {
+ boolean wasEmpty = receivedBuffers.isEmpty();
+ // Recovered buffers carry no per-buffer subpartition id (NONE): they are snapshotted via
+ // the recovery path, never via getInflightBuffersUnsafe which is the only consumer of that
+ // field.
+ receivedBuffers.add(new SequenceBuffer(buffer, recoverySequenceNumber++, NONE));
+ totalQueueSizeInBytes += buffer.getSize();
+ return wasEmpty;
+ }
+
// ------------------------------------------------------------------------
// Consume
// ------------------------------------------------------------------------
@@ -343,14 +519,21 @@ public boolean isReleased() {
@Override
void releaseAllResources() throws IOException {
if (isReleased.compareAndSet(false, true)) {
+ // Unblock any thread awaiting upstreamReady (drain still in flight) so it falls
+ // through and observes the released state instead of deadlocking.
+ upstreamReady.countDown();
+
+ // Recovery will never be consumed on a released channel; unblock anyone gating on it.
+ stateConsumedFuture.completeExceptionally(new CancelTaskException("Channel released."));
final ArrayDeque releasedBuffers;
synchronized (receivedBuffers) {
releasedBuffers =
- receivedBuffers.stream()
+ Stream.concat(receivedBuffers.stream(), recoveryEventStash.stream())
.map(sb -> sb.buffer)
.collect(Collectors.toCollection(ArrayDeque::new));
receivedBuffers.clear();
+ recoveryEventStash.clear();
}
bufferManager.releaseAllBuffers(releasedBuffers);
@@ -558,6 +741,10 @@ public int getInitialCredit() {
return initialCredit;
}
+ public boolean needsRecovery() {
+ return needsRecovery;
+ }
+
public BufferProvider getBufferProvider() throws IOException {
if (isReleased.get()) {
return null;
@@ -596,6 +783,11 @@ public void onBuffer(Buffer buffer, int sequenceNumber, int backlog, int subpart
throws IOException {
boolean recycleBuffer = true;
+ // The first buffer from the producer proves the upstream reader is registered and the
+ // connection is live; release any recovery-side awaiter. On later buffers this is a cheap
+ // idempotent no-op (the latch count is already zero).
+ upstreamReady.countDown();
+
try {
if (expectedSequenceNumber != sequenceNumber) {
onError(new BufferReorderingException(expectedSequenceNumber, sequenceNumber));
@@ -633,7 +825,20 @@ public void onBuffer(Buffer buffer, int sequenceNumber, int backlog, int subpart
firstPriorityEvent = addPriorityBuffer(sequenceBuffer);
recycleBuffer = false;
} else {
- receivedBuffers.add(sequenceBuffer);
+ if (inRecovery) {
+ // The upstream has no credit until recovery delivery finishes, so it can
+ // only
+ // send events here, never data buffers. Stash ordinary events so they are
+ // consumed after the recovered buffers; data buffers are a protocol
+ // violation.
+ checkState(
+ !buffer.isBuffer(),
+ "Received live data buffer during recovery on channel %s",
+ getChannelInfo());
+ recoveryEventStash.add(sequenceBuffer);
+ } else {
+ receivedBuffers.add(sequenceBuffer);
+ }
recycleBuffer = false;
if (dataType.requiresAnnouncement()) {
firstPriorityEvent = addPriorityBuffer(announce(sequenceBuffer));
@@ -713,30 +918,153 @@ private void checkAnnouncedOnlyOnce(SequenceBuffer sequenceBuffer) {
}
/**
- * Spills all queued buffers on checkpoint start. If barrier has already been received (and
- * reordered), spill only the overtaken buffers.
+ * Persists inflight data on checkpoint start. During recovery, persists recovered buffers
+ * before the matching RecoveryCheckpointBarrier sentinel; after recovery, uses the normal
+ * remote-channel barrier sequence tracking and persists overtaken live buffers.
*/
public void checkpointStarted(CheckpointBarrier barrier) throws CheckpointException {
- synchronized (receivedBuffers) {
- if (barrier.getId() < lastBarrierId) {
- throw new CheckpointException(
- String.format(
- "Sequence number for checkpoint %d is not known (it was likely been overwritten by a newer checkpoint %d)",
- barrier.getId(), lastBarrierId),
- CheckpointFailureReason
- .CHECKPOINT_SUBSUMED); // currently, at most one active unaligned
- // checkpoint is possible
- } else if (barrier.getId() > lastBarrierId) {
- // This channel has received some obsolete barrier, older compared to the
- // checkpointId
- // which we are processing right now, and we should ignore that obsoleted checkpoint
- // barrier sequence number.
- resetLastBarrier();
+ try {
+ List toPersist;
+ synchronized (receivedBuffers) {
+ if (inRecovery) {
+ toPersist = collectPreRecoveryBarrier(barrier.getId());
+ } else {
+ if (barrier.getId() < lastBarrierId) {
+ // Currently, at most one active unaligned checkpoint is possible.
+ throw new CheckpointException(
+ String.format(
+ "Sequence number for checkpoint %d is not known (it was likely been overwritten by a newer checkpoint %d)",
+ barrier.getId(), lastBarrierId),
+ CheckpointFailureReason.CHECKPOINT_SUBSUMED);
+ } else if (barrier.getId() > lastBarrierId) {
+ // This channel has received some obsolete barrier, older compared to the
+ // checkpointId which we are processing right now, and we should ignore that
+ // obsoleted checkpoint barrier sequence number.
+ resetLastBarrier();
+ }
+ toPersist = getInflightBuffersUnsafe(barrier.getId());
+ }
+ channelStatePersister.startPersisting(barrier.getId(), toPersist);
}
+ } catch (IOException e) {
+ throw new CheckpointException(
+ "Failed to extract recovered buffers for checkpoint " + barrier.getId(),
+ CheckpointFailureReason.CHECKPOINT_DECLINED,
+ e);
+ }
+ }
- channelStatePersister.startPersisting(
- barrier.getId(), getInflightBuffersUnsafe(barrier.getId()));
+ /**
+ * Walks {@link #receivedBuffers} (skipping priority events) up to the {@link
+ * RecoveryCheckpointBarrier} sentinel matching {@code checkpointId}, retaining each pre-barrier
+ * recovered data buffer and removing the sentinel. During recovery the upstream has no credit,
+ * so {@code receivedBuffers} holds only recovered buffers, sentinels, and priority events — no
+ * live data buffers.
+ *
+ *
The scan runs on any checkpoint that starts while the channel is still {@code inRecovery},
+ * whether or not the drain has finished delivering. It must skip the {@code
+ * EndOfFetchedChannelStateEvent} because that sentinel can sit ahead of the {@code
+ * RecoveryCheckpointBarrier}: once the drain finishes it appends {@code
+ * EndOfFetchedChannelStateEvent}, and if a checkpoint is then triggered before the consume path
+ * polls it, {@link #insertRecoveryCheckpointBarrierIfInRecovery} appends the {@code
+ * RecoveryCheckpointBarrier} behind it. The incoming {@link CheckpointBarrier} is a priority
+ * event kept at the head, skipped by advancing past the priority region.
+ *
+ * @throws IOException if a barrier for a later checkpoint is encountered before {@code
+ * checkpointId}, or if no sentinel matching {@code checkpointId} is found (the snapshot
+ * protocol guarantees one must be present while the channel is in recovery).
+ */
+ @GuardedBy("receivedBuffers")
+ private List collectPreRecoveryBarrier(long checkpointId) throws IOException {
+ assert Thread.holdsLock(receivedBuffers);
+ List retained = new ArrayList<>();
+ List subsumed = new ArrayList<>();
+ SequenceBuffer sentinel = null;
+ try {
+ Iterator it = receivedBuffers.iterator();
+ // Priority events are stored separately at the head and never carry recovered data.
+ Iterators.advance(it, receivedBuffers.getNumPriorityElements());
+ while (it.hasNext()) {
+ SequenceBuffer sb = it.next();
+ RecoveryCheckpointBarrier barrier = asRecoveryCheckpointBarrier(sb.buffer);
+ if (barrier != null) {
+ long barrierId = barrier.getCheckpointId();
+ if (barrierId == checkpointId) {
+ sentinel = sb;
+ break;
+ }
+ if (barrierId > checkpointId) {
+ throw new IOException(
+ "Found RecoveryCheckpointBarrier for a later checkpoint "
+ + barrierId
+ + " before the target checkpoint "
+ + checkpointId
+ + " in receivedBuffers for channel "
+ + getChannelInfo());
+ }
+ // barrierId < checkpointId: the checkpoint was subsumed; drop its stale
+ // barrier and keep scanning for the target.
+ LOG.warn(
+ "Discarding subsumed RecoveryCheckpointBarrier for checkpoint {} while "
+ + "collecting checkpoint {} on channel {}.",
+ barrierId,
+ checkpointId,
+ getChannelInfo());
+ subsumed.add(sb);
+ continue;
+ }
+ // Skip non-data events (e.g. the EndOfFetchedChannelStateEvent sentinel appended
+ // after the recovered buffers): only recovered data buffers are snapshotted.
+ if (sb.buffer.isBuffer()) {
+ retained.add(sb.buffer.retainBuffer());
+ }
+ }
+ } catch (IOException e) {
+ releaseRetainedBuffers(retained);
+ throw e;
+ }
+ if (sentinel == null) {
+ releaseRetainedBuffers(retained);
+ throw new IOException(
+ "Missing RecoveryCheckpointBarrier for checkpoint "
+ + checkpointId
+ + " in receivedBuffers for channel "
+ + getChannelInfo());
+ }
+ // receivedBuffers is a PrioritizedDeque whose iterator() is read-only; remove matched and
+ // subsumed sentinels by identity through its mutable removal API.
+ for (SequenceBuffer s : subsumed) {
+ removeRecoverySentinel(s);
+ }
+ removeRecoverySentinel(sentinel);
+ return retained;
+ }
+
+ @GuardedBy("receivedBuffers")
+ private void removeRecoverySentinel(SequenceBuffer sentinel) {
+ receivedBuffers.getAndRemove(sb -> sb == sentinel);
+ totalQueueSizeInBytes -= sentinel.buffer.getSize();
+ sentinel.buffer.recycleBuffer();
+ }
+
+ private static void releaseRetainedBuffers(List retained) {
+ for (Buffer buffer : retained) {
+ buffer.recycleBuffer();
+ }
+ }
+
+ @Nullable
+ private static RecoveryCheckpointBarrier asRecoveryCheckpointBarrier(Buffer b)
+ throws IOException {
+ if (b.isBuffer()) {
+ return null;
}
+ AbstractEvent event =
+ EventSerializer.fromBuffer(b, RecoveryCheckpointBarrier.class.getClassLoader());
+ b.setReaderIndex(0);
+ return event instanceof RecoveryCheckpointBarrier
+ ? (RecoveryCheckpointBarrier) event
+ : null;
}
public void checkpointStopped(long checkpointId) {
@@ -909,13 +1237,13 @@ public void onError(Throwable cause) {
}
/**
- * When receivedBuffers contains migrated buffers from RecoveredInputChannel, they can be read
- * before requestSubpartitions(). In that case only check for errors. Once migrated buffers are
- * drained, require full client initialization check.
+ * Allows reads while recovery data or already queued network data is available before the
+ * remote partition request is fully initialized. If neither recovery nor queued data can
+ * satisfy the read, require the partition request client to be initialized.
*/
private void checkReadability() throws IOException {
assert Thread.holdsLock(receivedBuffers);
- if (receivedBuffers.isEmpty()) {
+ if (!inRecovery && receivedBuffers.isEmpty()) {
checkPartitionRequestQueueInitialized();
} else {
checkError();
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteRecoveredInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteRecoveredInputChannel.java
index 2cfff6f5e7972a..b76aa347fe6445 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteRecoveredInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteRecoveredInputChannel.java
@@ -20,13 +20,11 @@
import org.apache.flink.runtime.io.network.ConnectionID;
import org.apache.flink.runtime.io.network.ConnectionManager;
-import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.io.network.metrics.InputChannelMetrics;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.io.network.partition.ResultSubpartitionIndexSet;
import java.io.IOException;
-import java.util.ArrayDeque;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -68,8 +66,7 @@ public class RemoteRecoveredInputChannel extends RecoveredInputChannel {
}
@Override
- protected InputChannel toInputChannelInternal(ArrayDeque remainingBuffers)
- throws IOException {
+ protected InputChannel toInputChannelInternal(boolean needsRecovery) throws IOException {
RemoteInputChannel remoteInputChannel =
new RemoteInputChannel(
inputGate,
@@ -85,8 +82,7 @@ protected InputChannel toInputChannelInternal(ArrayDeque remainingBuffer
numBytesIn,
numBuffersIn,
channelStateWriter,
- remainingBuffers);
- remoteInputChannel.setup();
+ needsRecovery);
return remoteInputChannel;
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
index 438efa2f58bd5b..4a7c9c6617e3de 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGate.java
@@ -323,9 +323,7 @@ public CompletableFuture getStateConsumedFuture() {
synchronized (requestLock) {
List> futures = new ArrayList<>(numberOfInputChannels);
for (InputChannel inputChannel : inputChannels()) {
- if (inputChannel instanceof RecoveredInputChannel) {
- futures.add(((RecoveredInputChannel) inputChannel).getStateConsumedFuture());
- }
+ futures.add(inputChannel.getStateConsumedFuture());
}
return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0]));
}
@@ -358,6 +356,11 @@ public CompletableFuture getBufferFilteringCompleteFuture() {
@Override
public void requestPartitions() {
+ requestPartitions(false);
+ }
+
+ @Override
+ public void requestPartitions(boolean needsRecovery) {
synchronized (requestLock) {
if (!requestedPartitionsFlag) {
if (closeFuture.isDone()) {
@@ -376,7 +379,7 @@ public void requestPartitions() {
numInputChannels, numberOfInputChannels));
}
- convertRecoveredInputChannels();
+ convertRecoveredInputChannels(needsRecovery);
internalRequestPartitions();
}
@@ -390,12 +393,18 @@ public void requestPartitions() {
}
}
+ @VisibleForTesting
+ public void convertRecoveredInputChannels() {
+ convertRecoveredInputChannels(false);
+ }
+
/**
* Converts all {@link RecoveredInputChannel}s to their real channel types ({@link
- * LocalInputChannel} or {@link RemoteInputChannel}).
+ * LocalInputChannel} or {@link RemoteInputChannel}). {@code needsRecovery} controls whether the
+ * converted physical channels start in recovery.
*/
@VisibleForTesting
- public void convertRecoveredInputChannels() {
+ public void convertRecoveredInputChannels(boolean needsRecovery) {
LOG.debug("Converting recovered input channels ({} channels)", getNumberOfInputChannels());
for (Map inputChannelsForCurrentPartition :
inputChannels.values()) {
@@ -413,7 +422,7 @@ public void convertRecoveredInputChannels() {
// order with onRecoveredStateBuffer() which acquires receivedBuffers
// first and then inputChannelsWithData.
InputChannel realInputChannel =
- ((RecoveredInputChannel) inputChannel).toInputChannel();
+ ((RecoveredInputChannel) inputChannel).toInputChannel(needsRecovery);
inputChannel.releaseAllResources();
int buffersInUseCount = realInputChannel.getBuffersInUseCount();
@@ -595,6 +604,11 @@ public InputChannel getChannel(int channelIndex) {
return channels[channelIndex];
}
+ @Override
+ public InputChannel getChannel(InputChannelInfo channelInfo) {
+ return channels[channelInfo.getInputChannelIdx()];
+ }
+
// ------------------------------------------------------------------------
// Setup/Life-cycle
// ------------------------------------------------------------------------
@@ -957,7 +971,7 @@ private Optional readRecoveredOrNormalBuffer(InputChannel inputChannel)
// Firstly, read the buffers from the recovered channel
if (inputChannel instanceof RecoveredInputChannel && !inputChannel.isReleased()) {
Optional buffer = readBufferFromInputChannel(inputChannel);
- if (!((RecoveredInputChannel) inputChannel).getStateConsumedFuture().isDone()) {
+ if (!inputChannel.getStateConsumedFuture().isDone()) {
return buffer;
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java
index dda71c63be38f4..37aee532ae55ee 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGate.java
@@ -176,6 +176,13 @@ public InputChannel getChannel(int channelIndex) {
.getChannel(channelIndex - inputGateChannelIndexOffsets[gateIndex]);
}
+ @Override
+ public InputChannel getChannel(InputChannelInfo channelInfo) {
+ // The member gate's local channel index is carried directly in channelInfo, so resolve
+ // through the owning gate instead of the gate-global getChannel(int) addressing.
+ return inputGatesByGateIndex.get(channelInfo.getGateIdx()).getChannel(channelInfo);
+ }
+
@Override
public boolean isFinished() {
return inputGatesWithRemainingData.isEmpty();
@@ -361,8 +368,13 @@ public CompletableFuture getBufferFilteringCompleteFuture() {
@Override
public void requestPartitions() throws IOException {
+ requestPartitions(false);
+ }
+
+ @Override
+ public void requestPartitions(boolean needsRecovery) throws IOException {
for (InputGate inputGate : inputGatesByGateIndex.values()) {
- inputGate.requestPartitions();
+ inputGate.requestPartitions(needsRecovery);
}
}
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnknownInputChannel.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnknownInputChannel.java
index 15182cedadb9fc..9db9df9c613c14 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnknownInputChannel.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/UnknownInputChannel.java
@@ -31,12 +31,13 @@
import org.apache.flink.runtime.io.network.partition.ResultPartitionManager;
import org.apache.flink.runtime.io.network.partition.ResultSubpartitionIndexSet;
import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.concurrent.FutureUtils;
import javax.annotation.Nullable;
import java.io.IOException;
-import java.util.ArrayDeque;
import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
import static org.apache.flink.runtime.checkpoint.CheckpointFailureReason.CHECKPOINT_DECLINED_TASK_NOT_READY;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -100,6 +101,11 @@ public UnknownInputChannel(
this.networkBuffersPerChannel = networkBuffersPerChannel;
}
+ @Override
+ public CompletableFuture getStateConsumedFuture() {
+ return FutureUtils.completedVoidFuture();
+ }
+
@Override
public void resumeConsumption() {
throw new UnsupportedOperationException("UnknownInputChannel should never be blocked.");
@@ -171,21 +177,23 @@ public String toString() {
public RemoteInputChannel toRemoteInputChannel(
ConnectionID producerAddress, ResultPartitionID resultPartitionID) {
- return new RemoteInputChannel(
- inputGate,
- getChannelIndex(),
- resultPartitionID,
- consumedSubpartitionIndexSet,
- checkNotNull(producerAddress),
- connectionManager,
- initialBackoff,
- maxBackoff,
- partitionRequestListenerTimeout,
- networkBuffersPerChannel,
- metrics.getNumBytesInRemoteCounter(),
- metrics.getNumBuffersInRemoteCounter(),
- channelStateWriter == null ? ChannelStateWriter.NO_OP : channelStateWriter,
- new ArrayDeque<>());
+ RemoteInputChannel channel =
+ new RemoteInputChannel(
+ inputGate,
+ getChannelIndex(),
+ resultPartitionID,
+ consumedSubpartitionIndexSet,
+ checkNotNull(producerAddress),
+ connectionManager,
+ initialBackoff,
+ maxBackoff,
+ partitionRequestListenerTimeout,
+ networkBuffersPerChannel,
+ metrics.getNumBytesInRemoteCounter(),
+ metrics.getNumBuffersInRemoteCounter(),
+ channelStateWriter == null ? ChannelStateWriter.NO_OP : channelStateWriter,
+ false);
+ return channel;
}
public LocalInputChannel toLocalInputChannel(ResultPartitionID resultPartitionID) {
@@ -201,7 +209,8 @@ public LocalInputChannel toLocalInputChannel(ResultPartitionID resultPartitionID
metrics.getNumBytesInLocalCounter(),
metrics.getNumBuffersInLocalCounter(),
channelStateWriter == null ? ChannelStateWriter.NO_OP : channelStateWriter,
- new ArrayDeque<>());
+ networkBuffersPerChannel,
+ false);
}
@Override
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java
index bff412f53b3303..695d897c92ffcd 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java
@@ -80,6 +80,11 @@ public InputChannel getChannel(int channelIndex) {
return inputGate.getChannel(channelIndex);
}
+ @Override
+ public InputChannel getChannel(InputChannelInfo channelInfo) {
+ return inputGate.getChannel(channelInfo);
+ }
+
@Override
public int getGateIndex() {
return inputGate.getGateIndex();
@@ -130,6 +135,11 @@ public void requestPartitions() throws IOException {
inputGate.requestPartitions();
}
+ @Override
+ public void requestPartitions(boolean needsRecovery) throws IOException {
+ inputGate.requestPartitions(needsRecovery);
+ }
+
@Override
public void setChannelStateWriter(ChannelStateWriter channelStateWriter) {
inputGate.setChannelStateWriter(channelStateWriter);
diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/AbstractStreamTaskNetworkInput.java b/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/AbstractStreamTaskNetworkInput.java
index a3743edec7f940..26f0b5f8dfff08 100644
--- a/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/AbstractStreamTaskNetworkInput.java
+++ b/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/AbstractStreamTaskNetworkInput.java
@@ -326,8 +326,7 @@ public void close() throws IOException {
// terminate the TM
Exception err = null;
for (InputChannelInfo channelInfo : new ArrayList<>(recordDeserializers.keySet())) {
- final boolean hadError =
- checkpointedInputGate.getChannel(channelInfo.getInputChannelIdx()).hasError();
+ final boolean hadError = checkpointedInputGate.getChannel(channelInfo).hasError();
try {
releaseDeserializer(channelInfo);
} catch (Exception e) {
diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/StreamMultipleInputProcessor.java b/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/StreamMultipleInputProcessor.java
index 565447b25fdbd0..a5ef79bc3be3f4 100644
--- a/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/StreamMultipleInputProcessor.java
+++ b/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/StreamMultipleInputProcessor.java
@@ -79,7 +79,17 @@ public DataInputStatus processInput() throws Exception {
readingInputIndex = selectFirstReadingInputIndex();
}
if (readingInputIndex == InputSelection.NONE_AVAILABLE) {
- return DataInputStatus.NOTHING_AVAILABLE;
+ // If all inputs are already finished, there is nothing left to read: report
+ // END_OF_INPUT rather than NOTHING_AVAILABLE. Otherwise, because getAvailableFuture()
+ // returns AVAILABLE once all inputs are finished, the mailbox loop would never block
+ // and
+ // would spin on processInput. This matters when processInput is invoked again after the
+ // operator already finished -- e.g. a task that reached END_OF_INPUT during recovery
+ // and
+ // is resumed once recovery completes (see StreamTask#processInput).
+ return inputSelectionHandler.areAllInputsFinished()
+ ? DataInputStatus.END_OF_INPUT
+ : DataInputStatus.NOTHING_AVAILABLE;
}
lastReadInputIndex = readingInputIndex;
diff --git a/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/checkpointing/CheckpointedInputGate.java b/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/checkpointing/CheckpointedInputGate.java
index ad0dcb5b0bb74b..26f335386a198d 100644
--- a/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/checkpointing/CheckpointedInputGate.java
+++ b/flink-runtime/src/main/java/org/apache/flink/streaming/runtime/io/checkpointing/CheckpointedInputGate.java
@@ -29,9 +29,11 @@
import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
import org.apache.flink.runtime.io.network.api.EventAnnouncement;
import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent;
+import org.apache.flink.runtime.io.network.partition.consumer.EndOfFetchedChannelStateEvent;
import org.apache.flink.runtime.io.network.partition.consumer.EndOfOutputChannelStateEvent;
import org.apache.flink.runtime.io.network.partition.consumer.InputChannel;
import org.apache.flink.runtime.io.network.partition.consumer.InputGate;
+import org.apache.flink.runtime.io.network.partition.consumer.RecoverableInputChannel;
import org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput;
import org.slf4j.Logger;
@@ -202,6 +204,15 @@ private Optional handleEvent(BufferOrEvent bufferOrEvent) throws
bufferOrEvent.getChannelInfo());
} else if (bufferOrEvent.getEvent().getClass() == EndOfOutputChannelStateEvent.class) {
upstreamRecoveryTracker.handleEndOfRecovery(bufferOrEvent.getChannelInfo());
+ } else if (eventClass == EndOfFetchedChannelStateEvent.class) {
+ // Tail of the recovered buffers: only a RecoverableInputChannel can produce this
+ // sentinel, so anything else here is a bug rather than something to tolerate.
+ InputChannel channel = inputGate.getChannel(bufferOrEvent.getChannelInfo());
+ checkState(
+ channel instanceof RecoverableInputChannel,
+ "EndOfFetchedChannelStateEvent received on a non-recoverable channel %s",
+ bufferOrEvent.getChannelInfo());
+ ((RecoverableInputChannel) channel).onRecoveredStateConsumed();
}
return Optional.of(bufferOrEvent);
}
@@ -296,6 +307,10 @@ public InputChannel getChannel(int channelIndex) {
return inputGate.getChannel(channelIndex);
}
+ public InputChannel getChannel(InputChannelInfo channelInfo) {
+ return inputGate.getChannel(channelInfo);
+ }
+
public List getChannelInfos() {
return inputGate.getChannelInfos();
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java
index b073f6cbed1210..1601b06bda9986 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/asyncprocessing/operators/AbstractAsyncRunnableStreamOperatorTest.java
@@ -253,8 +253,11 @@ void testCheckpointDrain() throws Exception {
});
((AbstractAsyncRunnableStreamOperator) testHarness.getOperator())
.postProcessElement();
- assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(1);
- unblockAsyncRequest.complete(null);
+ try {
+ assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(1);
+ } finally {
+ unblockAsyncRequest.complete(null);
+ }
testHarness.drainAsyncRequests();
assertThat(asyncExecutionController.getInFlightRecordNum()).isEqualTo(0);
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationTest.java
index 711c3ef5cf385e..1f2c9c7a3031c1 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StateAssignmentOperationTest.java
@@ -66,6 +66,7 @@
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
@@ -96,6 +97,7 @@
import static org.apache.flink.runtime.util.JobVertexConnectionUtils.connectNewDataSetAsInput;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests to verify state assignment operation. */
class StateAssignmentOperationTest {
@@ -1315,6 +1317,76 @@ private List buildOperatorIds(int numOperators) {
.collect(Collectors.toList());
}
+ /**
+ * A keyed vertex whose chained operators recorded different maximum parallelism cannot be
+ * restored. The rejection must not depend on the order in which the operator states are
+ * reconciled onto the shared vertex, so both orders are exercised.
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void restoreRejectsKeyedVertexWithConflictingMaxParallelism(boolean keyedStateFirst)
+ throws Exception {
+ final OperatorID keyedOperator = new OperatorID();
+ final OperatorID chainedOperator = new OperatorID();
+ final int keyedMaxParallelism = 128;
+ final int chainedMaxParallelism = 64;
+
+ OperatorState keyedState =
+ new OperatorState(null, null, keyedOperator, 1, keyedMaxParallelism);
+ keyedState.putState(
+ 0,
+ OperatorSubtaskState.builder()
+ .setManagedKeyedState(
+ StateObjectCollection.singleton(
+ createNewKeyedStateHandle(
+ KeyGroupRange.of(0, keyedMaxParallelism - 1))))
+ .build());
+ OperatorState chainedState =
+ new OperatorState(null, null, chainedOperator, 1, chainedMaxParallelism);
+ chainedState.putState(
+ 0,
+ OperatorSubtaskState.builder()
+ .setManagedOperatorState(
+ StateObjectCollection.singleton(
+ createNewOperatorStateHandle(2, new Random())))
+ .build());
+
+ JobVertex jobVertex =
+ new JobVertex(
+ "keyed-chain",
+ new JobVertexID(),
+ asList(
+ OperatorIDPair.generatedIDOnly(keyedOperator),
+ OperatorIDPair.generatedIDOnly(chainedOperator)));
+ jobVertex.setInvokableClass(NoOpInvokable.class);
+ jobVertex.setParallelism(1);
+ ExecutionJobVertex executionJobVertex =
+ ExecutionGraphTestUtils.getExecutionJobVertex(jobVertex);
+
+ Map oldState = new LinkedHashMap<>();
+ if (keyedStateFirst) {
+ oldState.put(keyedOperator, keyedState);
+ oldState.put(chainedOperator, chainedState);
+ } else {
+ oldState.put(chainedOperator, chainedState);
+ oldState.put(keyedOperator, keyedState);
+ }
+
+ TaskStateAssignment taskStateAssignment =
+ new TaskStateAssignment(
+ executionJobVertex, oldState, new HashMap<>(), new HashMap<>(), false);
+ StateAssignmentOperation stateAssignmentOperation =
+ new StateAssignmentOperation(
+ 1L, Collections.singleton(executionJobVertex), oldState, false, false);
+
+ assertThatThrownBy(
+ () ->
+ stateAssignmentOperation.checkParallelismPreconditions(
+ taskStateAssignment))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("recorded different maximum parallelism");
+ }
+
/**
* Asserts the upstream output buffer state for a specific subtask by verifying the expected
* upstream subtask and subpartition mappings.
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/InputChannelRecoveredStateHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/InputChannelRecoveredStateHandlerTest.java
index 9c4aab0bc7a5da..18f8a863352b65 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/InputChannelRecoveredStateHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/InputChannelRecoveredStateHandlerTest.java
@@ -40,12 +40,13 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-/** Test of different implementation of {@link InputChannelRecoveredStateHandler}. */
+/** Test of different implementation of {@link AbstractInputChannelRecoveredStateHandler}. */
class InputChannelRecoveredStateHandlerTest extends RecoveredChannelStateHandlerTest {
private static final int preAllocatedSegments = 3;
private NetworkBufferPool networkBufferPool;
private SingleInputGate inputGate;
- private InputChannelRecoveredStateHandler icsHandler;
+ // NoSpillingHandler: checkpointingDuringRecoveryEnabled=false, filteringHandler=null
+ private AbstractInputChannelRecoveredStateHandler icsHandler;
private InputChannelInfo channelInfo;
@BeforeEach
@@ -61,14 +62,14 @@ void setUp() {
.setSegmentProvider(networkBufferPool)
.build();
- icsHandler = buildInputChannelStateHandler(inputGate);
+ icsHandler = buildNoSpillingHandler(inputGate);
channelInfo = new InputChannelInfo(0, 0);
}
- private InputChannelRecoveredStateHandler buildInputChannelStateHandler(
+ private AbstractInputChannelRecoveredStateHandler buildNoSpillingHandler(
SingleInputGate inputGate) {
- return new InputChannelRecoveredStateHandler(
+ return AbstractInputChannelRecoveredStateHandler.create(
new InputGate[] {inputGate},
new InflightDataRescalingDescriptor(
new InflightDataRescalingDescriptor
@@ -82,11 +83,12 @@ private InputChannelRecoveredStateHandler buildInputChannelStateHandler(
.InflightDataGateOrPartitionRescalingDescriptor
.MappingType.IDENTITY)
}),
+ false,
null,
MemoryManager.DEFAULT_PAGE_SIZE);
}
- private InputChannelRecoveredStateHandler buildMultiChannelHandler() {
+ private AbstractInputChannelRecoveredStateHandler buildMultiChannelHandler() {
// Setup multi-channel scenario to trigger distribution constraint validation
SingleInputGate multiChannelGate =
new SingleInputGateBuilder()
@@ -95,7 +97,7 @@ private InputChannelRecoveredStateHandler buildMultiChannelHandler() {
.setSegmentProvider(networkBufferPool)
.build();
- return new InputChannelRecoveredStateHandler(
+ return AbstractInputChannelRecoveredStateHandler.create(
new InputGate[] {multiChannelGate},
new InflightDataRescalingDescriptor(
new InflightDataRescalingDescriptor
@@ -110,39 +112,42 @@ private InputChannelRecoveredStateHandler buildMultiChannelHandler() {
.InflightDataGateOrPartitionRescalingDescriptor
.MappingType.RESCALING)
}),
+ false,
null,
MemoryManager.DEFAULT_PAGE_SIZE);
}
/** Builds a handler in filtering mode (non-null filtering handler, no-op stub). */
- private InputChannelRecoveredStateHandler buildFilteringInputChannelStateHandler() {
+ private FilteringHandler buildFilteringInputChannelStateHandler() {
// Empty GateFilterHandler array: filtering is "enabled" structurally, but no gate-level
// filter logic runs. Suitable for exercising getBuffer() routing only.
ChannelStateFilteringHandler stubFilteringHandler =
new ChannelStateFilteringHandler(
new ChannelStateFilteringHandler.GateFilterHandler[0]);
- return new InputChannelRecoveredStateHandler(
- new InputGate[] {inputGate},
- new InflightDataRescalingDescriptor(
- new InflightDataRescalingDescriptor
- .InflightDataGateOrPartitionRescalingDescriptor[] {
- new InflightDataRescalingDescriptor
- .InflightDataGateOrPartitionRescalingDescriptor(
- new int[] {1},
- RescaleMappings.identity(1, 1),
- new HashSet<>(),
- InflightDataRescalingDescriptor
- .InflightDataGateOrPartitionRescalingDescriptor
- .MappingType.IDENTITY)
- }),
- stubFilteringHandler,
- MemoryManager.DEFAULT_PAGE_SIZE);
+ return (FilteringHandler)
+ AbstractInputChannelRecoveredStateHandler.create(
+ new InputGate[] {inputGate},
+ new InflightDataRescalingDescriptor(
+ new InflightDataRescalingDescriptor
+ .InflightDataGateOrPartitionRescalingDescriptor[] {
+ new InflightDataRescalingDescriptor
+ .InflightDataGateOrPartitionRescalingDescriptor(
+ new int[] {1},
+ RescaleMappings.identity(1, 1),
+ new HashSet<>(),
+ InflightDataRescalingDescriptor
+ .InflightDataGateOrPartitionRescalingDescriptor
+ .MappingType.IDENTITY)
+ }),
+ true,
+ stubFilteringHandler,
+ MemoryManager.DEFAULT_PAGE_SIZE);
}
@Test
void testBufferDistributedToMultipleInputChannelsThrowsException() throws Exception {
// Test constraint that prevents buffer distribution to multiple channels
- try (InputChannelRecoveredStateHandler handler = buildMultiChannelHandler()) {
+ try (AbstractInputChannelRecoveredStateHandler handler = buildMultiChannelHandler()) {
assertThatThrownBy(() -> handler.getBuffer(channelInfo))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining(
@@ -187,8 +192,7 @@ void testRecycleBufferAfterRecoverWasCalled() throws Exception {
@Test
void testPreFilterBufferIsolationFromNetworkBufferPool() throws Exception {
- try (InputChannelRecoveredStateHandler filteringHandler =
- buildFilteringInputChannelStateHandler()) {
+ try (FilteringHandler filteringHandler = buildFilteringInputChannelStateHandler()) {
int availableBefore = networkBufferPool.getNumberOfAvailableMemorySegments();
RecoveredChannelStateHandler.BufferWithContext bufferWithContext =
@@ -229,8 +233,7 @@ void testNonFilteringModeUsesNetworkBufferPool() throws Exception {
@Test
void testPreFilterSegmentReusedAcrossCalls() throws Exception {
- try (InputChannelRecoveredStateHandler filteringHandler =
- buildFilteringInputChannelStateHandler()) {
+ try (FilteringHandler filteringHandler = buildFilteringInputChannelStateHandler()) {
// First getBuffer() lazily allocates the segment.
RecoveredChannelStateHandler.BufferWithContext first =
filteringHandler.getBuffer(channelInfo);
@@ -259,8 +262,7 @@ void testPreFilterSegmentReusedAcrossCalls() throws Exception {
@Test
void testGetBufferThrowsWhenPriorBufferNotRecycled() throws Exception {
- try (InputChannelRecoveredStateHandler filteringHandler =
- buildFilteringInputChannelStateHandler()) {
+ try (FilteringHandler filteringHandler = buildFilteringInputChannelStateHandler()) {
RecoveredChannelStateHandler.BufferWithContext first =
filteringHandler.getBuffer(channelInfo);
try {
@@ -283,8 +285,7 @@ void testGetBufferThrowsWhenPriorBufferNotRecycled() throws Exception {
@Test
void testPreFilterSegmentFreedOnClose() throws Exception {
- InputChannelRecoveredStateHandler filteringHandler =
- buildFilteringInputChannelStateHandler();
+ FilteringHandler filteringHandler = buildFilteringInputChannelStateHandler();
RecoveredChannelStateHandler.BufferWithContext bufferWithContext =
filteringHandler.getBuffer(channelInfo);
bufferWithContext.context.recycleBuffer();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandlerTest.java
index 01f7c43920ae94..db302d23f73376 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecoveredChannelStateHandlerTest.java
@@ -22,7 +22,7 @@
/**
* Base class which contains all tests which should be implemented for every implementation of
- * {@link InputChannelRecoveredStateHandler}.
+ * {@link AbstractInputChannelRecoveredStateHandler}.
*/
abstract class RecoveredChannelStateHandlerTest {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecoveryCheckpointBarrierTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecoveryCheckpointBarrierTest.java
new file mode 100644
index 00000000000000..0dd2c18286a7ac
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/channel/RecoveryCheckpointBarrierTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.checkpoint.channel;
+
+import org.apache.flink.core.memory.DataOutputSerializer;
+import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
+import org.apache.flink.runtime.io.network.buffer.Buffer;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link RecoveryCheckpointBarrier}. */
+class RecoveryCheckpointBarrierTest {
+
+ @Test
+ void testReflectiveWriteIsUnsupported() {
+ RecoveryCheckpointBarrier barrier = new RecoveryCheckpointBarrier(1L);
+
+ assertThatThrownBy(() -> barrier.write(new DataOutputSerializer(Long.BYTES)))
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining("dedicated type-tag path");
+ }
+
+ @Test
+ void testEventSerializerHandlesRecoveryCheckpointBarrier() throws Exception {
+ long checkpointId = 123L;
+
+ Buffer buffer =
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(checkpointId), false);
+ Object deserialized =
+ EventSerializer.fromBuffer(
+ buffer, RecoveryCheckpointBarrier.class.getClassLoader());
+
+ assertThat(deserialized).isEqualTo(new RecoveryCheckpointBarrier(checkpointId));
+ }
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java
index 2adbded7d9e90e..d165f6a50cb4d8 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CancelPartitionRequestTest.java
@@ -106,7 +106,8 @@ void testCancelPartitionRequest() throws Exception {
pid,
new ResultSubpartitionIndexSet(0),
new InputChannelID(),
- Integer.MAX_VALUE))
+ Integer.MAX_VALUE,
+ false))
.await();
// Wait for the notification
@@ -170,7 +171,8 @@ void testDuplicateCancel() throws Exception {
pid,
new ResultSubpartitionIndexSet(0),
inputChannelId,
- Integer.MAX_VALUE))
+ Integer.MAX_VALUE,
+ false))
.await();
// Wait for the notification
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedPartitionRequestClientHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedPartitionRequestClientHandlerTest.java
index d4b162304b8e6b..d58018036961a3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedPartitionRequestClientHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedPartitionRequestClientHandlerTest.java
@@ -68,7 +68,6 @@
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.util.ArrayDeque;
import java.util.stream.Stream;
import static org.apache.flink.runtime.io.network.netty.PartitionRequestQueueTest.blockChannel;
@@ -956,7 +955,7 @@ private static class TestRemoteInputChannelForError extends RemoteInputChannel {
new SimpleCounter(),
new SimpleCounter(),
ChannelStateWriter.NO_OP,
- new ArrayDeque<>());
+ false);
this.expectedMessage = expectedMessage;
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReaderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReaderTest.java
index cd4a3103240ee7..ba686adce04959 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReaderTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/CreditBasedSequenceNumberingViewReaderTest.java
@@ -88,7 +88,7 @@ private static CreditBasedSequenceNumberingViewReader createNetworkSequenceViewR
channel.close();
CreditBasedSequenceNumberingViewReader reader =
new CreditBasedSequenceNumberingViewReader(
- new InputChannelID(), initialCredit, queue);
+ new InputChannelID(), initialCredit, false, queue);
reader.notifySubpartitionsCreated(
TestingResultPartition.newBuilder()
.setCreateSubpartitionViewFunction(
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageServerSideSerializationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageServerSideSerializationTest.java
index 22b5420b616d2b..f2e0e8146c8da6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageServerSideSerializationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/NettyMessageServerSideSerializationTest.java
@@ -67,7 +67,8 @@ void testPartitionRequest() {
new ResultPartitionID(),
new ResultSubpartitionIndexSet(queueIndex),
new InputChannelID(),
- random.nextInt());
+ random.nextInt(),
+ random.nextBoolean());
NettyMessage.PartitionRequest actual = encodeAndDecode(expected, channel);
@@ -75,6 +76,7 @@ void testPartitionRequest() {
assertThat(actual.queueIndexSet).isEqualTo(expected.queueIndexSet);
assertThat(actual.receiverId).isEqualTo(expected.receiverId);
assertThat(actual.credit).isEqualTo(expected.credit);
+ assertThat(actual.needsRecovery).isEqualTo(expected.needsRecovery);
}
@Test
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java
index 7046a8518093be..a4e15e4cf90d6c 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestQueueTest.java
@@ -90,7 +90,8 @@ public void testNotifyReaderPartitionTimeout() throws Exception {
ResultPartitionManager resultPartitionManager = new ResultPartitionManager();
ResultPartitionID resultPartitionId = new ResultPartitionID();
CreditBasedSequenceNumberingViewReader reader =
- new CreditBasedSequenceNumberingViewReader(new InputChannelID(0, 0), 10, queue);
+ new CreditBasedSequenceNumberingViewReader(
+ new InputChannelID(0, 0), 10, false, queue);
reader.requestSubpartitionViewOrRegisterListener(
resultPartitionManager, resultPartitionId, new ResultSubpartitionIndexSet(0));
@@ -132,9 +133,11 @@ void testNotifyReaderNonEmptyOnEmptyReaders() throws Exception {
EmbeddedChannel channel = new EmbeddedChannel(queue);
CreditBasedSequenceNumberingViewReader reader1 =
- new CreditBasedSequenceNumberingViewReader(new InputChannelID(0, 0), 10, queue);
+ new CreditBasedSequenceNumberingViewReader(
+ new InputChannelID(0, 0), 10, false, queue);
CreditBasedSequenceNumberingViewReader reader2 =
- new CreditBasedSequenceNumberingViewReader(new InputChannelID(1, 1), 10, queue);
+ new CreditBasedSequenceNumberingViewReader(
+ new InputChannelID(1, 1), 10, false, queue);
ResultSubpartitionView view1 = new EmptyAlwaysAvailableResultSubpartitionView();
reader1.notifySubpartitionsCreated(
@@ -192,7 +195,8 @@ private void testBufferWriting(ResultSubpartitionView view) throws IOException {
final InputChannelID receiverId = new InputChannelID();
final PartitionRequestQueue queue = new PartitionRequestQueue();
final CreditBasedSequenceNumberingViewReader reader =
- new CreditBasedSequenceNumberingViewReader(receiverId, Integer.MAX_VALUE, queue);
+ new CreditBasedSequenceNumberingViewReader(
+ receiverId, Integer.MAX_VALUE, false, queue);
final EmbeddedChannel channel = new EmbeddedChannel(queue);
reader.notifySubpartitionsCreated(partition, new ResultSubpartitionIndexSet(0));
@@ -287,7 +291,7 @@ void testEnqueueReaderByNotifyingEventBuffer() throws Exception {
final InputChannelID receiverId = new InputChannelID();
final PartitionRequestQueue queue = new PartitionRequestQueue();
final CreditBasedSequenceNumberingViewReader reader =
- new CreditBasedSequenceNumberingViewReader(receiverId, 0, queue);
+ new CreditBasedSequenceNumberingViewReader(receiverId, 0, false, queue);
final EmbeddedChannel channel = new EmbeddedChannel(queue);
reader.notifySubpartitionsCreated(partition, new ResultSubpartitionIndexSet(0));
@@ -340,7 +344,7 @@ void testEnqueueReaderByNotifyingBufferAndCredit() throws Exception {
final InputChannelID receiverId = new InputChannelID();
final PartitionRequestQueue queue = new PartitionRequestQueue();
final CreditBasedSequenceNumberingViewReader reader =
- new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
+ new CreditBasedSequenceNumberingViewReader(receiverId, 2, false, queue);
final EmbeddedChannel channel = new EmbeddedChannel(queue);
reader.addCredit(-2);
@@ -421,7 +425,7 @@ void testEnqueueReaderByResumingConsumption() throws Exception {
InputChannelID receiverId = new InputChannelID();
PartitionRequestQueue queue = new PartitionRequestQueue();
CreditBasedSequenceNumberingViewReader reader =
- new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
+ new CreditBasedSequenceNumberingViewReader(receiverId, 2, false, queue);
EmbeddedChannel channel = new EmbeddedChannel(queue);
reader.notifySubpartitionsCreated(partition, new ResultSubpartitionIndexSet(0));
@@ -461,7 +465,7 @@ void testAnnounceBacklog() throws Exception {
PartitionRequestQueue queue = new PartitionRequestQueue();
InputChannelID receiverId = new InputChannelID();
CreditBasedSequenceNumberingViewReader reader =
- new CreditBasedSequenceNumberingViewReader(receiverId, 0, queue);
+ new CreditBasedSequenceNumberingViewReader(receiverId, 0, false, queue);
EmbeddedChannel channel = new EmbeddedChannel(queue);
reader.notifySubpartitionsCreated(partition, new ResultSubpartitionIndexSet(0));
@@ -498,7 +502,7 @@ private void testCancelPartitionRequest(boolean isAvailableView) throws Exceptio
final InputChannelID receiverId = new InputChannelID();
final PartitionRequestQueue queue = new PartitionRequestQueue();
final CreditBasedSequenceNumberingViewReader reader =
- new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
+ new CreditBasedSequenceNumberingViewReader(receiverId, 2, false, queue);
final EmbeddedChannel channel = new EmbeddedChannel(queue);
reader.notifySubpartitionsCreated(partition, new ResultSubpartitionIndexSet(0));
@@ -546,7 +550,7 @@ void testNotifyNewBufferSize() throws Exception {
InputChannelID receiverId = new InputChannelID();
PartitionRequestQueue queue = new PartitionRequestQueue();
CreditBasedSequenceNumberingViewReader reader =
- new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
+ new CreditBasedSequenceNumberingViewReader(receiverId, 2, false, queue);
EmbeddedChannel channel = new EmbeddedChannel(queue);
reader.notifySubpartitionsCreated(partition, new ResultSubpartitionIndexSet(0));
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestRegistrationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestRegistrationTest.java
index e3cfb55e3400ff..86d9588094f9a3 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestRegistrationTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestRegistrationTest.java
@@ -44,7 +44,6 @@
import org.junit.jupiter.api.Test;
-import java.util.ArrayDeque;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -96,7 +95,8 @@ void testRegisterResultPartitionBeforeRequest() throws Exception {
resultPartition.getPartitionId(),
new ResultSubpartitionIndexSet(0),
new InputChannelID(),
- Integer.MAX_VALUE))
+ Integer.MAX_VALUE,
+ false))
.await();
// Wait for the notification
@@ -144,7 +144,8 @@ void testRegisterResultPartitionAfterRequest() throws Exception {
resultPartition.getPartitionId(),
new ResultSubpartitionIndexSet(0),
new InputChannelID(),
- Integer.MAX_VALUE))
+ Integer.MAX_VALUE,
+ false))
.await();
// Register result partition after partition request
@@ -213,7 +214,8 @@ public void releasePartitionRequestListener(
pid,
new ResultSubpartitionIndexSet(0),
remoteInputChannel.getInputChannelId(),
- Integer.MAX_VALUE))
+ Integer.MAX_VALUE,
+ false))
.await();
// Wait for the notification
@@ -250,7 +252,7 @@ private static class TestRemoteInputChannelForPartitionNotFound extends RemoteIn
new SimpleCounter(),
new SimpleCounter(),
ChannelStateWriter.NO_OP,
- new ArrayDeque<>());
+ false);
this.latch = latch;
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandlerTest.java
index 7f19b199582e7c..1fe0c00e79a188 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandlerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/PartitionRequestServerHandlerTest.java
@@ -81,7 +81,7 @@ void testAcknowledgeAllRecordsProcessed() throws IOException {
// Creates and registers the view to netty.
NetworkSequenceViewReader viewReader =
new CreditBasedSequenceNumberingViewReader(
- inputChannelID, 2, partitionRequestQueue);
+ inputChannelID, 2, false, partitionRequestQueue);
viewReader.notifySubpartitionsCreated(resultPartition, new ResultSubpartitionIndexSet(0));
partitionRequestQueue.notifyReaderCreated(viewReader);
@@ -149,7 +149,7 @@ private static class TestViewReader extends CreditBasedSequenceNumberingViewRead
TestViewReader(
InputChannelID receiverId, int initialCredit, PartitionRequestQueue requestQueue) {
- super(receiverId, initialCredit, requestQueue);
+ super(receiverId, initialCredit, false, requestQueue);
}
@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ServerTransportErrorHandlingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ServerTransportErrorHandlingTest.java
index a4f753fe1e7d26..bfc4e01a153cc6 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ServerTransportErrorHandlingTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/netty/ServerTransportErrorHandlingTest.java
@@ -103,7 +103,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) {
new ResultPartitionID(),
new ResultSubpartitionIndexSet(0),
new InputChannelID(),
- Integer.MAX_VALUE));
+ Integer.MAX_VALUE,
+ false));
// Wait for the notification
assertThat(sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS))
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannelBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannelBuilder.java
index 08f65d9fe72658..e69f8414b43277 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannelBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannelBuilder.java
@@ -34,7 +34,7 @@
import org.apache.flink.runtime.io.network.partition.ResultSubpartitionIndexSet;
import java.net.InetSocketAddress;
-import java.util.ArrayDeque;
+import java.util.concurrent.CountDownLatch;
import static org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateTest.TestingResultPartitionManager;
@@ -56,6 +56,8 @@ public class InputChannelBuilder {
private int maxBackoff = 0;
private int partitionRequestListenerTimeout = 0;
private int networkBuffersPerChannel = 2;
+ private boolean needsRecovery = false;
+ private CountDownLatch upstreamReady = new CountDownLatch(1);
private InputChannelMetrics metrics =
InputChannelTestUtils.newUnregisteredInputChannelMetrics();
@@ -115,6 +117,16 @@ public InputChannelBuilder setNetworkBuffersPerChannel(int networkBuffersPerChan
return this;
}
+ public InputChannelBuilder setNeedsRecovery(boolean needsRecovery) {
+ this.needsRecovery = needsRecovery;
+ return this;
+ }
+
+ public InputChannelBuilder setUpstreamReady(CountDownLatch upstreamReady) {
+ this.upstreamReady = upstreamReady;
+ return this;
+ }
+
public InputChannelBuilder setMetrics(InputChannelMetrics metrics) {
this.metrics = metrics;
return this;
@@ -166,7 +178,9 @@ public LocalInputChannel buildLocalChannel(SingleInputGate inputGate) {
metrics.getNumBytesInLocalCounter(),
metrics.getNumBuffersInLocalCounter(),
stateWriter,
- new ArrayDeque<>());
+ networkBuffersPerChannel,
+ needsRecovery,
+ upstreamReady);
}
public RemoteInputChannel buildRemoteChannel(SingleInputGate inputGate) {
@@ -184,7 +198,8 @@ public RemoteInputChannel buildRemoteChannel(SingleInputGate inputGate) {
metrics.getNumBytesInRemoteCounter(),
metrics.getNumBuffersInRemoteCounter(),
stateWriter,
- new ArrayDeque<>());
+ needsRecovery,
+ upstreamReady);
}
public LocalRecoveredInputChannel buildLocalRecoveredChannel(SingleInputGate inputGate) {
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannelTest.java
index 866636a30614e8..a6ade5890d1b40 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputChannelTest.java
@@ -27,6 +27,7 @@
import java.io.IOException;
import java.util.Optional;
+import java.util.concurrent.CompletableFuture;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
@@ -145,6 +146,11 @@ private MockInputChannel(
@Override
public void resumeConsumption() {}
+ @Override
+ public CompletableFuture getStateConsumedFuture() {
+ return CompletableFuture.completedFuture(null);
+ }
+
@Override
public void acknowledgeAllRecordsProcessed() throws IOException {}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
index 44c4b48cb7a630..d7dfa095818ba0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java
@@ -19,10 +19,13 @@
package org.apache.flink.runtime.io.network.partition.consumer;
import org.apache.flink.metrics.SimpleCounter;
+import org.apache.flink.runtime.checkpoint.CheckpointException;
+import org.apache.flink.runtime.checkpoint.CheckpointFailureReason;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter;
import org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter;
+import org.apache.flink.runtime.checkpoint.channel.RecoveryCheckpointBarrier;
import org.apache.flink.runtime.execution.CancelTaskException;
import org.apache.flink.runtime.io.disk.NoOpFileChannelManager;
import org.apache.flink.runtime.io.network.TaskEventDispatcher;
@@ -61,7 +64,6 @@
import org.mockito.stubbing.Answer;
import java.io.IOException;
-import java.util.ArrayDeque;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -69,6 +71,7 @@
import java.util.TimerTask;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicInteger;
@@ -669,7 +672,7 @@ void testReceivingBuffersInUseBeforeSubpartitionViewInitialization() throws Exce
@Test
void testGetBuffersInUseCountIncludesToBeConsumedBuffers() throws Exception {
- // given: Local input channel with recovered buffers in toBeConsumedBuffers
+ // given: Local input channel with recovered buffers in the recovery queue
ResultSubpartitionView subpartitionView =
InputChannelTestUtils.createResultSubpartitionView(
createFilledFinishedBufferConsumer(4096),
@@ -678,12 +681,6 @@ void testGetBuffersInUseCountIncludesToBeConsumedBuffers() throws Exception {
new TestingResultPartitionManager(subpartitionView);
final SingleInputGate inputGate = createSingleInputGate(1);
- // Create 3 recovered buffers
- ArrayDeque recoveredBuffers = new ArrayDeque<>();
- recoveredBuffers.add(TestBufferFactory.createBuffer(32));
- recoveredBuffers.add(TestBufferFactory.createBuffer(32));
- recoveredBuffers.add(TestBufferFactory.createBuffer(32));
-
final LocalInputChannel localChannel =
new LocalInputChannel(
inputGate,
@@ -697,10 +694,16 @@ void testGetBuffersInUseCountIncludesToBeConsumedBuffers() throws Exception {
new SimpleCounter(),
new SimpleCounter(),
ChannelStateWriter.NO_OP,
- recoveredBuffers);
+ 2,
+ true);
inputGate.setInputChannels(localChannel);
+ // Create 3 recovered buffers
+ localChannel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(32));
+ localChannel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(32));
+ localChannel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(32));
+
// then: Before requesting subpartitions, buffers in use should include recovered buffers
assertThat(localChannel.getBuffersInUseCount()).isEqualTo(3);
assertThat(localChannel.unsynchronizedGetNumberOfQueuedBuffers()).isEqualTo(3);
@@ -718,10 +721,7 @@ void testGetNextBufferWithMigratedRecoveredBuffers() throws Exception {
// given: LocalInputChannel with recovered buffers migrated from RecoveredInputChannel
SingleInputGate inputGate = createSingleInputGate(1);
- ArrayDeque recoveredBuffers = new ArrayDeque<>();
- recoveredBuffers.add(TestBufferFactory.createBuffer(10));
- recoveredBuffers.add(TestBufferFactory.createBuffer(20));
-
+ CountDownLatch upstreamReady = new CountDownLatch(1);
LocalInputChannel channel =
new LocalInputChannel(
inputGate,
@@ -735,10 +735,17 @@ void testGetNextBufferWithMigratedRecoveredBuffers() throws Exception {
new SimpleCounter(),
new SimpleCounter(),
ChannelStateWriter.NO_OP,
- recoveredBuffers);
+ 2,
+ true,
+ upstreamReady);
inputGate.setInputChannels(channel);
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(10));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(20));
+ upstreamReady.countDown();
+ channel.finishRecoveredBufferDelivery();
+
// then: Can read recovered buffers even before requestSubpartitions()
Optional first = channel.getNextBuffer();
assertThat(first).isPresent();
@@ -755,11 +762,6 @@ void testCheckpointStartedPersistsRecoveredBuffers() throws Exception {
// given: Local input channel with recovered buffers
SingleInputGate inputGate = new SingleInputGateBuilder().build();
- ArrayDeque recoveredBuffers = new ArrayDeque<>();
- recoveredBuffers.add(TestBufferFactory.createBuffer(10));
- recoveredBuffers.add(TestBufferFactory.createBuffer(20));
- recoveredBuffers.add(TestBufferFactory.createBuffer(30));
-
RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
LocalInputChannel channel =
@@ -775,53 +777,51 @@ void testCheckpointStartedPersistsRecoveredBuffers() throws Exception {
new SimpleCounter(),
new SimpleCounter(),
stateWriter,
- recoveredBuffers);
+ 2,
+ true);
inputGate.setInputChannels(channel);
- // when: Checkpoint is started
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(10));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(20));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(30));
+ // Mirror snapshotAndInsertBarriers: push the sentinel before
+ // checkpointStarted scans recoveredQueue.
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+
CheckpointOptions options =
CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
stateWriter.start(1L, options);
CheckpointBarrier barrier = new CheckpointBarrier(1L, 0L, options);
+ // when: Checkpoint is started
channel.checkpointStarted(barrier);
- // then: All 3 recovered buffers should be persisted as inflight data
List persistedBuffers = stateWriter.getAddedInput().get(channel.getChannelInfo());
+ // then: All 3 recovered buffers should be persisted as inflight data
assertThat(persistedBuffers).isNotNull().hasSize(3);
assertThat(persistedBuffers.stream().mapToInt(Buffer::getSize).toArray())
.containsExactly(10, 20, 30);
}
+ /** Port of the FLINK-40016 double-persist regression test to the push-based recovery API. */
@Test
void testRecoveredBuffersNotPersistedAgainWhenConsumedDuringCheckpoint() throws Exception {
- // given: Channel with 2 recovered buffers
+ // given: Channel with 2 recovered buffers, followed by the RecoveryCheckpointBarrier
+ // sentinel that snapshotAndInsertBarriers pushes before checkpointStarted scans
+ // recoveredBuffers
SingleInputGate inputGate = new SingleInputGateBuilder().build();
-
- ArrayDeque recoveredBuffers = new ArrayDeque<>();
- recoveredBuffers.add(TestBufferFactory.createBuffer(10));
- recoveredBuffers.add(TestBufferFactory.createBuffer(20));
-
RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
-
- LocalInputChannel channel =
- new LocalInputChannel(
- inputGate,
- 0,
- new ResultPartitionID(),
- new ResultSubpartitionIndexSet(0),
- new ResultPartitionManager(),
- new TaskEventDispatcher(),
- 0,
- 0,
- new SimpleCounter(),
- new SimpleCounter(),
- stateWriter,
- recoveredBuffers);
-
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, stateWriter);
inputGate.setInputChannels(channel);
- // when: Checkpoint starts — checkpointStarted spills recovered buffers as inflight data
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(10));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(20));
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+
+ // when: Checkpoint starts — checkpointStarted persists the pre-barrier recovered buffers
+ // as inflight data
CheckpointOptions options =
CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
stateWriter.start(1L, options);
@@ -830,15 +830,22 @@ void testRecoveredBuffersNotPersistedAgainWhenConsumedDuringCheckpoint() throws
// then: exactly 2 buffers persisted so far
assertThat(stateWriter.getAddedInput().get(channel.getChannelInfo())).hasSize(2);
- // when: Consume the recovered buffers while checkpoint is still in BARRIER_PENDING state
- channel.getNextBuffer();
- channel.getNextBuffer();
+ // when: Consume the recovered buffers while the checkpoint is still in BARRIER_PENDING
+ // state
+ Optional first = channel.getNextBuffer();
+ assertThat(first).isPresent();
+ assertThat(first.get().buffer().getSize()).isEqualTo(10);
+ Optional second = channel.getNextBuffer();
+ assertThat(second).isPresent();
+ assertThat(second.get().buffer().getSize()).isEqualTo(20);
// then: total persisted count must still be 2 — recovered buffers must not be
// re-persisted via maybePersist() when consumed (that would make it 4 without the fix)
- assertThat(stateWriter.getAddedInput().get(channel.getChannelInfo()))
+ List persisted = stateWriter.getAddedInput().get(channel.getChannelInfo());
+ assertThat(persisted)
.as("recovered buffers must not be persisted again when consumed")
.hasSize(2);
+ assertThat(persisted.stream().mapToInt(Buffer::getSize).toArray()).containsExactly(10, 20);
}
@Test
@@ -872,9 +879,6 @@ void testPriorityEventFailsFastWhenSubpartitionViewIsNull() throws Exception {
// given: Local input channel with recovered buffers but NO subpartition view initialized
SingleInputGate inputGate = new SingleInputGateBuilder().build();
- ArrayDeque recoveredBuffers = new ArrayDeque<>();
- recoveredBuffers.add(TestBufferFactory.createBuffer(10));
-
LocalInputChannel channel =
new LocalInputChannel(
inputGate,
@@ -888,9 +892,11 @@ void testPriorityEventFailsFastWhenSubpartitionViewIsNull() throws Exception {
new SimpleCounter(),
new SimpleCounter(),
ChannelStateWriter.NO_OP,
- recoveredBuffers);
+ 2,
+ true);
inputGate.setInputChannels(channel);
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(10));
// Do NOT call channel.requestSubpartitions() — subpartitionView stays null
channel.notifyPriorityEvent(0);
@@ -1010,11 +1016,6 @@ private static ChannelAndSubpartition createChannelWithRecoveredBuffers(
TestingResultPartitionManager partitionManager =
new TestingResultPartitionManager(subpartitionView);
- ArrayDeque recoveredBuffers = new ArrayDeque<>();
- for (int size : recoveredBufferSizes) {
- recoveredBuffers.add(TestBufferFactory.createBuffer(size));
- }
-
LocalInputChannel channel =
new LocalInputChannel(
inputGate,
@@ -1028,9 +1029,15 @@ private static ChannelAndSubpartition createChannelWithRecoveredBuffers(
new SimpleCounter(),
new SimpleCounter(),
stateWriter,
- recoveredBuffers);
+ 2,
+ true);
inputGate.setInputChannels(channel);
+
+ for (int size : recoveredBufferSizes) {
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(size));
+ }
+
channel.requestSubpartitions();
return new ChannelAndSubpartition(channel, subpartition);
@@ -1046,6 +1053,351 @@ private static class ChannelAndSubpartition {
}
}
+ // ---------------------------------------------------------------------------------------------
+ // RecoverableInputChannel push-based recovery tests
+ // ---------------------------------------------------------------------------------------------
+
+ private static LocalInputChannel newPushOnlyLocalChannel(
+ SingleInputGate inputGate, ChannelStateWriter stateWriter) {
+ return newPushOnlyLocalChannel(inputGate, stateWriter, new CountDownLatch(1));
+ }
+
+ private static LocalInputChannel newPushOnlyLocalChannel(
+ SingleInputGate inputGate,
+ ChannelStateWriter stateWriter,
+ CountDownLatch upstreamReady) {
+ return new LocalInputChannel(
+ inputGate,
+ 0,
+ new ResultPartitionID(),
+ new ResultSubpartitionIndexSet(0),
+ new ResultPartitionManager(),
+ new TaskEventDispatcher(),
+ 0,
+ 0,
+ new SimpleCounter(),
+ new SimpleCounter(),
+ stateWriter,
+ 2,
+ true,
+ upstreamReady);
+ }
+
+ @Test
+ void testOnRecoveredStateBufferEnqueues() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, ChannelStateWriter.NO_OP);
+ inputGate.setInputChannels(channel);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(11));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(22));
+
+ Optional first = channel.getNextBuffer();
+ assertThat(first).isPresent();
+ assertThat(first.get().buffer().getSize()).isEqualTo(11);
+ Optional second = channel.getNextBuffer();
+ assertThat(second).isPresent();
+ assertThat(second.get().buffer().getSize()).isEqualTo(22);
+ }
+
+ @Test
+ void testOnRecoveredStateBufferOnReleasedChannelIsSilentlyRecycled() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, ChannelStateWriter.NO_OP);
+ inputGate.setInputChannels(channel);
+ channel.releaseAllResources();
+
+ Buffer b = TestBufferFactory.createBuffer(33);
+ channel.onRecoveredStateBuffer(b);
+ assertThat(b.isRecycled()).isTrue();
+ }
+
+ @Test
+ void testOnRecoveredStateBufferNotifiesChannelNonEmptyOnEmptyToNonEmptyTransition()
+ throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, ChannelStateWriter.NO_OP);
+ inputGate.setInputChannels(channel);
+
+ CompletableFuture availability = inputGate.getAvailableFuture();
+ assertThat(availability).isNotDone();
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ assertThat(availability).isDone();
+ }
+
+ @Test
+ void testInRecoveryBoundaryFlagFalseQueueEmptyReturnsEmpty() throws Exception {
+ // Drive into the (flag=false, queue=empty) boundary by pushing one buffer, polling it,
+ // and verifying the channel does not yet expose a master-path result.
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, ChannelStateWriter.NO_OP);
+ inputGate.setInputChannels(channel);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ channel.getNextBuffer();
+ // Queue is empty; no finishRecoveredBufferDelivery was called. Without the subpartitionView
+ // active, the channel returns empty.
+ Optional result = channel.getNextBuffer();
+ assertThat(result).isNotPresent();
+ }
+
+ @Test
+ void testInRecoveryBoundaryFlagFalseQueueNonEmptyPolls() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, ChannelStateWriter.NO_OP);
+ inputGate.setInputChannels(channel);
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(7));
+ Optional r = channel.getNextBuffer();
+ assertThat(r).isPresent();
+ assertThat(r.get().buffer().getSize()).isEqualTo(7);
+ }
+
+ @Test
+ void testInRecoveryBoundaryFlagTrueQueueNonEmptyPolls() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ CountDownLatch upstreamReady = new CountDownLatch(1);
+ LocalInputChannel channel =
+ newPushOnlyLocalChannel(inputGate, ChannelStateWriter.NO_OP, upstreamReady);
+ inputGate.setInputChannels(channel);
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(8));
+ upstreamReady.countDown();
+ channel.finishRecoveredBufferDelivery();
+ Optional r = channel.getNextBuffer();
+ assertThat(r).isPresent();
+ assertThat(r.get().buffer().getSize()).isEqualTo(8);
+ }
+
+ @Test
+ void testFinishWithNoRecoveredBuffersEmitsSentinelThenFallsToMasterPath() throws Exception {
+ // With no recovered buffers, finish still appends the EndOfFetchedChannelStateEvent
+ // sentinel; getNextBuffer returns it. Consuming the sentinel flips the channel out of
+ // recovery, after which the master path takes over: without a subpartition view it raises
+ // IllegalStateException via checkAndWaitForSubpartitionView, proving the recovery branch is
+ // no longer swallowing the call.
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ CountDownLatch upstreamReady = new CountDownLatch(1);
+ LocalInputChannel channel =
+ newPushOnlyLocalChannel(inputGate, ChannelStateWriter.NO_OP, upstreamReady);
+ inputGate.setInputChannels(channel);
+ upstreamReady.countDown();
+ channel.finishRecoveredBufferDelivery();
+
+ Optional sentinel = channel.getNextBuffer();
+ assertThat(sentinel).isPresent();
+ assertThat(EventSerializer.fromBuffer(sentinel.get().buffer(), getClass().getClassLoader()))
+ .isInstanceOf(EndOfFetchedChannelStateEvent.class);
+
+ channel.onRecoveredStateConsumed();
+ assertThatThrownBy(channel::getNextBuffer)
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Queried for a buffer before requesting the subpartition");
+ }
+
+ @Test
+ void testPriorityEventDuringRecoveryFetchedFromSubpartitionView() throws Exception {
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ ChannelAndSubpartition ctx = createChannelWithRecoveredBuffers(stateWriter, 10, 20);
+
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ ctx.subpartition.add(
+ EventSerializer.toBufferConsumer(new CheckpointBarrier(1L, 0L, options), true));
+ ctx.channel.notifyPriorityEvent(0);
+
+ Optional first = ctx.channel.getNextBuffer();
+ assertThat(first).isPresent();
+ assertThat(first.get().buffer().getDataType().hasPriority()).isTrue();
+ }
+
+ @Test
+ void testPriorityEventDuringRecoveryResetAfterNonPriority() throws Exception {
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ ChannelAndSubpartition ctx = createChannelWithRecoveredBuffers(stateWriter, 10);
+
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ ctx.subpartition.add(
+ EventSerializer.toBufferConsumer(new CheckpointBarrier(1L, 0L, options), true));
+ ctx.subpartition.add(createFilledFinishedBufferConsumer(32));
+ ctx.channel.notifyPriorityEvent(0);
+
+ Optional priority = ctx.channel.getNextBuffer();
+ assertThat(priority).isPresent();
+ Optional recovered = ctx.channel.getNextBuffer();
+ assertThat(recovered).isPresent();
+ assertThat(recovered.get().buffer().isBuffer()).isTrue();
+ assertThat(recovered.get().buffer().getSize()).isEqualTo(10);
+ }
+
+ @Test
+ void testCheckpointStartedScansRecoveredBuffersUpToBarrier() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, stateWriter);
+ inputGate.setInputChannels(channel);
+
+ Buffer b1 = TestBufferFactory.createBuffer(1);
+ Buffer b2 = TestBufferFactory.createBuffer(2);
+ Buffer b3 = TestBufferFactory.createBuffer(3);
+ channel.onRecoveredStateBuffer(b1);
+ channel.onRecoveredStateBuffer(b2);
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+ channel.onRecoveredStateBuffer(b3);
+
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ stateWriter.start(1L, options);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, options));
+
+ List persisted = stateWriter.getAddedInput().get(channel.getChannelInfo());
+ assertThat(persisted).hasSize(2);
+ assertThat(persisted.stream().mapToInt(Buffer::getSize).toArray()).containsExactly(1, 2);
+ }
+
+ @Test
+ void testCheckpointStartedDeclinesWhenRecoveryBarrierIsMissing() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, stateWriter);
+ inputGate.setInputChannels(channel);
+
+ Buffer b1 = TestBufferFactory.createBuffer(1);
+ channel.onRecoveredStateBuffer(b1);
+ int refCntBefore = b1.refCnt();
+
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ stateWriter.start(1L, options);
+
+ // The snapshot protocol guarantees a RecoveryCheckpointBarrier sentinel is present while
+ // the channel is in recovery, so a missing sentinel is a protocol violation: the checkpoint
+ // is declined and the recovered buffer is neither dropped nor persisted.
+ assertThatThrownBy(() -> channel.checkpointStarted(new CheckpointBarrier(1L, 0L, options)))
+ .isInstanceOfSatisfying(
+ CheckpointException.class,
+ e ->
+ assertThat(e.getCheckpointFailureReason())
+ .isEqualTo(CheckpointFailureReason.CHECKPOINT_DECLINED))
+ .cause()
+ .hasMessageContaining("Missing RecoveryCheckpointBarrier");
+ assertThat(b1.refCnt()).isEqualTo(refCntBefore);
+ assertThat(stateWriter.getAddedInput().get(channel.getChannelInfo())).isEmpty();
+ }
+
+ @Test
+ void testCheckpointStartedRetainsPreBarrierBuffers() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, stateWriter);
+ inputGate.setInputChannels(channel);
+
+ Buffer b1 = TestBufferFactory.createBuffer(1);
+ channel.onRecoveredStateBuffer(b1);
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+
+ int before = b1.refCnt();
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ stateWriter.start(1L, options);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, options));
+ // Pre-barrier buffers are retained for the writer; their ref-count stays at or above the
+ // pre-checkpoint value.
+ assertThat(b1.refCnt()).isGreaterThanOrEqualTo(before);
+ }
+
+ @Test
+ void testCheckpointStartedRemovesSentinel() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, stateWriter);
+ inputGate.setInputChannels(channel);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(2));
+
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ stateWriter.start(1L, options);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, options));
+
+ Optional h1 = channel.getNextBuffer();
+ assertThat(h1).isPresent();
+ Optional h2 = channel.getNextBuffer();
+ assertThat(h2).isPresent();
+ assertThat(h2.get().buffer().getSize()).isEqualTo(2);
+ }
+
+ @Test
+ void testCheckpointStartedNestedCpIds() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, stateWriter);
+ inputGate.setInputChannels(channel);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(2));
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(2L), false));
+
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ stateWriter.start(1L, options);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, options));
+
+ List persisted = stateWriter.getAddedInput().get(channel.getChannelInfo());
+ assertThat(persisted).hasSize(1);
+ assertThat(persisted.get(0).getSize()).isEqualTo(1);
+ }
+
+ @Test
+ void testCheckpointStartedNotInRecoveryUsesMasterPath() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ CountDownLatch upstreamReady = new CountDownLatch(1);
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, stateWriter, upstreamReady);
+ inputGate.setInputChannels(channel);
+ // Channel starts in-recovery; finish appends the sentinel and consuming it flips the
+ // channel to not-in-recovery so checkpointStarted exercises the master path instead of the
+ // recovery branch.
+ upstreamReady.countDown();
+ channel.finishRecoveredBufferDelivery();
+ channel.getNextBuffer();
+ channel.onRecoveredStateConsumed();
+
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ stateWriter.start(1L, options);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, options));
+
+ assertThat(stateWriter.getAddedInput().get(channel.getChannelInfo())).isNullOrEmpty();
+ }
+
+ @Test
+ void testReceivedBuffersHasNoLiveDataBufferIsTrueOnLocal() throws Exception {
+ // Local has no receivedBuffers; the helper is trivially true. We exercise the
+ // in-recovery checkpointStarted branch without any "live data" infrastructure to confirm
+ // the channel does not crash.
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ LocalInputChannel channel = newPushOnlyLocalChannel(inputGate, stateWriter);
+ inputGate.setInputChannels(channel);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+
+ CheckpointOptions options =
+ CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, getDefault());
+ stateWriter.start(1L, options);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, options));
+ }
+
// ---------------------------------------------------------------------------------------------
/** Returns the configured number of buffers for each channel in a random order. */
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalRecoveredInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalRecoveredInputChannelTest.java
new file mode 100644
index 00000000000000..fdacf6ec1ee897
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalRecoveredInputChannelTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.io.network.partition.consumer;
+
+import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter;
+import org.apache.flink.runtime.io.network.buffer.Buffer;
+import org.apache.flink.runtime.io.network.util.TestBufferFactory;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class LocalRecoveredInputChannelTest {
+
+ @Test
+ void testToInputChannelRequiresEmptyRecoveredBuffers() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ LocalRecoveredInputChannel recoveredChannel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .buildLocalRecoveredChannel(inputGate);
+
+ Buffer buffer = TestBufferFactory.createBuffer(11);
+ recoveredChannel.onRecoveredStateBuffer(buffer);
+
+ try {
+ recoveredChannel.finishReadRecoveredState();
+ assertThatThrownBy(() -> recoveredChannel.toInputChannel(false))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Received buffer should be empty");
+ } finally {
+ recoveredChannel.releaseAllResources();
+ }
+ }
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RecoveredInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RecoveredInputChannelTest.java
index f40fd09702ede8..87a6c0466deed0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RecoveredInputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RecoveredInputChannelTest.java
@@ -22,14 +22,15 @@
import org.apache.flink.runtime.checkpoint.CheckpointException;
import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.io.network.api.CheckpointBarrier;
+import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
import org.apache.flink.runtime.io.network.buffer.Buffer;
+import org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils;
import org.apache.flink.runtime.io.network.partition.ResultPartitionID;
import org.apache.flink.runtime.io.network.partition.ResultSubpartitionIndexSet;
import org.junit.jupiter.api.Test;
import java.io.IOException;
-import java.util.ArrayDeque;
import static org.apache.flink.runtime.checkpoint.CheckpointOptions.unaligned;
import static org.apache.flink.runtime.state.CheckpointStorageLocationReference.getDefault;
@@ -39,16 +40,6 @@
/** Tests for {@link RecoveredInputChannel}. */
class RecoveredInputChannelTest {
- @Test
- void testConversionOnlyPossibleAfterBufferFilteringComplete() {
- // toInputChannel() always checks bufferFilteringCompleteFuture regardless of config
- for (boolean configEnabled : new boolean[] {true, false}) {
- assertThatThrownBy(() -> buildChannel(configEnabled).toInputChannel())
- .isInstanceOf(IllegalStateException.class)
- .hasMessageContaining("buffer filtering is not complete");
- }
- }
-
@Test
void testRequestPartitionsImpossible() {
assertThatThrownBy(() -> buildChannel(false).requestSubpartitions())
@@ -71,93 +62,83 @@ void testCheckpointStartImpossible() {
}
@Test
- void testToInputChannelAllowedWhenBufferFilteringCompleteAndConfigEnabled() throws IOException {
- // When config is enabled, conversion is allowed when bufferFilteringCompleteFuture is done
- TestableRecoveredInputChannel channel = buildTestableChannel(true);
-
- // Initially, conversion should fail
- assertThatThrownBy(() -> channel.toInputChannel())
- .isInstanceOf(IllegalStateException.class)
- .hasMessageContaining("buffer filtering is not complete");
-
- // After finishReadRecoveredState(), bufferFilteringCompleteFuture should be done
- channel.finishReadRecoveredState();
- assertThat(channel.getBufferFilteringCompleteFuture()).isDone();
- assertThat(channel.getStateConsumedFuture()).isNotDone();
-
- // Conversion should now succeed (no exception)
- InputChannel converted = channel.toInputChannel();
- assertThat(converted).isNotNull();
- }
-
- @Test
- void testToInputChannelAllowedWhenStateConsumedAndConfigDisabled() throws IOException {
- // When config is disabled, conversion requires both bufferFilteringCompleteFuture
- // and stateConsumedFuture to be done
+ void testToInputChannelRejectedWhileRecoveredStateUnconsumed() throws IOException {
+ // Conversion is rejected while recovered state is still queued: finishReadRecoveredState()
+ // enqueues the EndOfInputChannelStateEvent sentinel, so receivedBuffers is non-empty until
+ // it is consumed. The empty-queue check thus also guarantees stateConsumedFuture is done.
TestableRecoveredInputChannel channel = buildTestableChannel(false);
- // Initially, conversion should fail (buffer filtering not complete)
- assertThatThrownBy(() -> channel.toInputChannel())
- .isInstanceOf(IllegalStateException.class)
- .hasMessageContaining("buffer filtering is not complete");
-
- // After finishReadRecoveredState(), bufferFilteringCompleteFuture is done
- // but stateConsumedFuture is not
channel.finishReadRecoveredState();
- assertThat(channel.getBufferFilteringCompleteFuture()).isDone();
assertThat(channel.getStateConsumedFuture()).isNotDone();
- // Conversion should still fail because stateConsumedFuture is not done
- assertThatThrownBy(() -> channel.toInputChannel())
+ // Conversion fails because the sentinel is still queued.
+ assertThatThrownBy(() -> channel.toInputChannel(false))
.isInstanceOf(IllegalStateException.class)
- .hasMessageContaining("recovered state is not fully consumed");
+ .hasMessageContaining("Received buffer should be empty");
- // Consume the EndOfInputChannelStateEvent to complete stateConsumedFuture
+ // Consuming the EndOfInputChannelStateEvent should complete the future.
+ // getNextBuffer() returns empty when it encounters the event internally.
assertThat(channel.getNextBuffer()).isNotPresent();
assertThat(channel.getStateConsumedFuture()).isDone();
// Now conversion should succeed
- InputChannel converted = channel.toInputChannel();
+ InputChannel converted = channel.toInputChannel(true);
assertThat(converted).isNotNull();
}
@Test
- void testBufferFilteringCompleteFutureAlwaysCompletes() throws IOException {
- // finishReadRecoveredState() unconditionally completes bufferFilteringCompleteFuture
- for (boolean configEnabled : new boolean[] {true, false}) {
- RecoveredInputChannel channel = buildChannel(configEnabled);
- assertThat(channel.getBufferFilteringCompleteFuture()).isNotDone();
- channel.finishReadRecoveredState();
- assertThat(channel.getBufferFilteringCompleteFuture()).isDone();
- }
+ void testToInputChannelRequiresEmptyRecoveredBuffers() throws IOException {
+ TestableRecoveredInputChannel channel = buildTestableChannel(true);
+
+ channel.onRecoveredStateBuffer(BufferBuilderTestUtils.buildSomeBuffer());
+ channel.finishReadRecoveredState();
+
+ assertThatThrownBy(() -> channel.toInputChannel(false))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Received buffer should be empty");
}
@Test
- void testStateConsumedFutureCompletesAfterConsumingAllBuffers() throws IOException {
- // This test verifies that stateConsumedFuture completes after consuming
- // EndOfInputChannelStateEvent regardless of the config setting
- for (boolean configEnabled : new boolean[] {true, false}) {
- RecoveredInputChannel channel = buildChannel(configEnabled);
+ void testToInputChannelPushesQueuedBuffersWhenNeedsRecovery() throws IOException {
+ // FLINK-38544 transitional: removed when the spilling backend lands (recovered state then
+ // goes to disk, the queue is always empty at conversion, and toInputChannel(true) asserts
+ // emptiness instead of pushing).
+ TestableRecoveredInputChannel channel = buildTestableChannel(true);
- assertThat(channel.getStateConsumedFuture()).isNotDone();
+ channel.onRecoveredStateBuffer(BufferBuilderTestUtils.buildSomeBuffer(42));
+ channel.finishReadRecoveredState();
- channel.finishReadRecoveredState();
- assertThat(channel.getStateConsumedFuture()).isNotDone();
+ TestInputChannel converted = (TestInputChannel) channel.toInputChannel(true);
+
+ // The queued data buffer is handed over through the push interface, the legacy
+ // EndOfInputChannelStateEvent is dropped in translation, and the
+ // EndOfFetchedChannelStateEvent sentinel is appended after the last recovered buffer.
+ assertThat(converted.getRecoveredBuffersSpy()).hasSize(2);
+ Buffer data = converted.getRecoveredBuffersSpy().pollFirst();
+ assertThat(data.isBuffer()).isTrue();
+ assertThat(data.getSize()).isEqualTo(42);
+ Buffer sentinel = converted.getRecoveredBuffersSpy().pollFirst();
+ assertThat(sentinel.isBuffer()).isFalse();
+ assertThat(EventSerializer.fromBuffer(sentinel, getClass().getClassLoader()))
+ .isInstanceOf(EndOfFetchedChannelStateEvent.class);
+ }
- // Consuming the EndOfInputChannelStateEvent should complete the future.
- // getNextBuffer() returns empty when it encounters the event internally.
- assertThat(channel.getNextBuffer()).isNotPresent();
- assertThat(channel.getStateConsumedFuture()).isDone();
- }
+ @Test
+ void testStateConsumedFutureCompletesAfterLegacySentinelIsConsumed() throws IOException {
+ RecoveredInputChannel channel = buildChannel(false);
+
+ assertThat(channel.getStateConsumedFuture()).isNotDone();
+
+ channel.finishReadRecoveredState();
+ assertThat(channel.getStateConsumedFuture()).isNotDone();
+
+ assertThat(channel.getNextBuffer()).isNotPresent();
+ assertThat(channel.getStateConsumedFuture()).isDone();
}
private RecoveredInputChannel buildChannel(boolean checkpointingDuringRecoveryEnabled) {
try {
- SingleInputGate inputGate =
- new SingleInputGateBuilder()
- .setCheckpointingDuringRecoveryEnabled(
- checkpointingDuringRecoveryEnabled)
- .build();
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
return new RecoveredInputChannel(
inputGate,
0,
@@ -169,7 +150,7 @@ private RecoveredInputChannel buildChannel(boolean checkpointingDuringRecoveryEn
new SimpleCounter(),
10) {
@Override
- protected InputChannel toInputChannelInternal(ArrayDeque remainingBuffers) {
+ protected InputChannel toInputChannelInternal(boolean needsRecovery) {
throw new AssertionError("channel conversion succeeded");
}
};
@@ -181,11 +162,7 @@ protected InputChannel toInputChannelInternal(ArrayDeque remainingBuffer
private TestableRecoveredInputChannel buildTestableChannel(
boolean checkpointingDuringRecoveryEnabled) {
try {
- SingleInputGate inputGate =
- new SingleInputGateBuilder()
- .setCheckpointingDuringRecoveryEnabled(
- checkpointingDuringRecoveryEnabled)
- .build();
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
return new TestableRecoveredInputChannel(inputGate);
} catch (Exception e) {
throw new AssertionError("channel creation failed", e);
@@ -210,7 +187,7 @@ private static class TestableRecoveredInputChannel extends RecoveredInputChannel
}
@Override
- protected InputChannel toInputChannelInternal(ArrayDeque remainingBuffers) {
+ protected InputChannel toInputChannelInternal(boolean needsRecovery) {
return new TestInputChannel(inputGate, 0);
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
index e47de93c9e8bdf..9b1ccaabdb8922 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteInputChannelTest.java
@@ -23,10 +23,14 @@
import org.apache.flink.core.testutils.OneShotLatch;
import org.apache.flink.metrics.SimpleCounter;
import org.apache.flink.runtime.checkpoint.CheckpointException;
+import org.apache.flink.runtime.checkpoint.CheckpointFailureReason;
import org.apache.flink.runtime.checkpoint.CheckpointOptions;
import org.apache.flink.runtime.checkpoint.CheckpointType;
import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter;
import org.apache.flink.runtime.checkpoint.channel.InputChannelInfo;
+import org.apache.flink.runtime.checkpoint.channel.RecordingChannelStateWriter;
+import org.apache.flink.runtime.checkpoint.channel.RecoveryCheckpointBarrier;
+import org.apache.flink.runtime.clusterframework.types.ResourceID;
import org.apache.flink.runtime.execution.CancelTaskException;
import org.apache.flink.runtime.execution.ExecutionState;
import org.apache.flink.runtime.io.network.ConnectionID;
@@ -37,6 +41,7 @@
import org.apache.flink.runtime.io.network.TestingConnectionManager;
import org.apache.flink.runtime.io.network.TestingPartitionRequestClient;
import org.apache.flink.runtime.io.network.api.CheckpointBarrier;
+import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
import org.apache.flink.runtime.io.network.api.serialization.EventSerializer;
import org.apache.flink.runtime.io.network.buffer.Buffer;
import org.apache.flink.runtime.io.network.buffer.Buffer.DataType;
@@ -72,6 +77,7 @@
import javax.annotation.Nullable;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;
@@ -80,6 +86,7 @@
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
@@ -2079,15 +2086,9 @@ void testGetNextBufferWithMigratedRecoveredBuffers() throws Exception {
// given: RemoteInputChannel with recovered buffers migrated from RecoveredInputChannel
SingleInputGate inputGate = createSingleInputGate(1);
- ArrayDeque recoveredBuffers = new ArrayDeque<>();
- recoveredBuffers.add(TestBufferFactory.createBuffer(10));
- recoveredBuffers.add(TestBufferFactory.createBuffer(20));
-
ConnectionID connectionId =
- new ConnectionID(
- org.apache.flink.runtime.clusterframework.types.ResourceID.generate(),
- new java.net.InetSocketAddress("localhost", 0),
- 0);
+ new ConnectionID(ResourceID.generate(), new InetSocketAddress("localhost", 0), 0);
+ CountDownLatch upstreamReady = new CountDownLatch(1);
RemoteInputChannel channel =
new RemoteInputChannel(
inputGate,
@@ -2104,10 +2105,16 @@ void testGetNextBufferWithMigratedRecoveredBuffers() throws Exception {
new SimpleCounter(),
new SimpleCounter(),
ChannelStateWriter.NO_OP,
- recoveredBuffers);
+ true,
+ upstreamReady);
inputGate.setInputChannels(channel);
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(10));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(20));
+ upstreamReady.countDown();
+ channel.finishRecoveredBufferDelivery();
+
// then: Can read recovered buffers even before requestSubpartitions()
Optional first = channel.getNextBuffer();
assertThat(first).isPresent();
@@ -2119,6 +2126,499 @@ void testGetNextBufferWithMigratedRecoveredBuffers() throws Exception {
assertThat(second.get().buffer().getSize()).isEqualTo(20);
}
+ // ---------------------------------------------------------------------------------------------
+ // RecoverableInputChannel push-based recovery tests
+ // ---------------------------------------------------------------------------------------------
+
+ @Test
+ void testOnRecoveredStateBufferEnqueues() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ Buffer b1 = TestBufferFactory.createBuffer(11);
+ Buffer b2 = TestBufferFactory.createBuffer(22);
+ channel.onRecoveredStateBuffer(b1);
+ channel.onRecoveredStateBuffer(b2);
+
+ Optional first = channel.getNextBuffer();
+ assertThat(first).isPresent();
+ assertThat(first.get().buffer().getSize()).isEqualTo(11);
+ Optional second = channel.getNextBuffer();
+ assertThat(second).isPresent();
+ assertThat(second.get().buffer().getSize()).isEqualTo(22);
+ }
+
+ @Test
+ void testRecoveredBuffersConsumedBeforeStashedEventsThenSentinelFlipsRecovery()
+ throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ CountDownLatch upstreamReady = new CountDownLatch(1);
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .setNeedsRecovery(true)
+ .setUpstreamReady(upstreamReady)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ // Recovered buffer arrives via the drain; an ordinary upstream event arrives via onBuffer
+ // while still in recovery and must be stashed (events carry no credit).
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(11));
+ // backlog=-1: events carry no backlog, and this channel has no floating-buffer pool wired.
+ channel.onBuffer(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), 0, -1, 0);
+ upstreamReady.countDown();
+ channel.finishRecoveredBufferDelivery();
+
+ // Recovered buffer is consumed first.
+ Optional recovered = channel.getNextBuffer();
+ assertThat(recovered).isPresent();
+ assertThat(recovered.get().buffer().getSize()).isEqualTo(11);
+
+ // Then the sentinel; the stashed event is not yet visible (still in recovery).
+ Optional sentinel = channel.getNextBuffer();
+ assertThat(sentinel).isPresent();
+ assertThat(EventSerializer.fromBuffer(sentinel.get().buffer(), getClass().getClassLoader()))
+ .isInstanceOf(EndOfFetchedChannelStateEvent.class);
+
+ // The gate consumes the sentinel externally: flips out of recovery and unstashes the event.
+ channel.onRecoveredStateConsumed();
+ Optional stashed = channel.getNextBuffer();
+ assertThat(stashed).isPresent();
+ assertThat(EventSerializer.fromBuffer(stashed.get().buffer(), getClass().getClassLoader()))
+ .isInstanceOf(EndOfPartitionEvent.class);
+ }
+
+ @Test
+ void testOnBufferRejectsLiveDataBufferDuringRecovery() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ assertThatThrownBy(() -> channel.onBuffer(TestBufferFactory.createBuffer(1), 0, 0, 0))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Received live data buffer during recovery");
+ }
+
+ @Test
+ void testOnRecoveredStateBufferOnReleasedChannelIsSilentlyRecycled() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+ channel.releaseAllResources();
+
+ Buffer b = TestBufferFactory.createBuffer(33);
+ channel.onRecoveredStateBuffer(b);
+
+ assertThat(b.isRecycled()).isTrue();
+ }
+
+ @Test
+ void testOnRecoveredStateBufferNotifiesChannelNonEmptyOnEmptyToNonEmptyTransition()
+ throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ CompletableFuture availability = inputGate.getAvailableFuture();
+ assertThat(availability).isNotDone();
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ assertThat(availability).isDone();
+ }
+
+ @Test
+ void testInRecoveryBoundaryFlagFalseQueueEmptyReturnsEmpty() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ // Force in-recovery + empty queue: push then poll a buffer, then push another while
+ // delaying finish. After the consumer drains the staged buffer, queue=empty and
+ // flag=false (no finishRecoveredBufferDelivery called yet). getNextBuffer must return
+ // empty.
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ channel.getNextBuffer();
+ // Simulate an explicit recovery context where the producer signals "not done yet".
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(2));
+ channel.getNextBuffer();
+ // The boundary case "flag=false (drain still running) + queue empty" should return empty.
+ // To set this state explicitly, we deliberately do not call finishReadRecoveredState.
+ Optional result = channel.getNextBuffer();
+ assertThat(result).isNotPresent();
+ }
+
+ @Test
+ void testInRecoveryBoundaryFlagFalseQueueNonEmptyPolls() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(7));
+
+ Optional r = channel.getNextBuffer();
+ assertThat(r).isPresent();
+ assertThat(r.get().buffer().getSize()).isEqualTo(7);
+ }
+
+ @Test
+ void testInRecoveryBoundaryFlagTrueQueueNonEmptyPolls() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ CountDownLatch upstreamReady = new CountDownLatch(1);
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .setNeedsRecovery(true)
+ .setUpstreamReady(upstreamReady)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(8));
+ upstreamReady.countDown();
+ channel.finishRecoveredBufferDelivery();
+
+ Optional r = channel.getNextBuffer();
+ assertThat(r).isPresent();
+ assertThat(r.get().buffer().getSize()).isEqualTo(8);
+ }
+
+ @Test
+ void testFinishWithNoRecoveredBuffersEmitsSentinelThenFallsToMasterPath() throws Exception {
+ // Wire a real network pool so requestSubpartitions() can succeed and the master path can
+ // poll receivedBuffers.
+ final NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 4096);
+ final CountDownLatch upstreamReady = new CountDownLatch(1);
+ try {
+ SingleInputGate inputGate =
+ new SingleInputGateBuilder()
+ .setBufferPoolFactory(networkBufferPool.createBufferPool(1, 4))
+ .setSegmentProvider(networkBufferPool)
+ .setChannelFactory(
+ (builder, gate) ->
+ builder.setNeedsRecovery(true)
+ .setUpstreamReady(upstreamReady)
+ .buildRemoteChannel(gate))
+ .build();
+ inputGate.setup();
+ RemoteInputChannel channel = (RemoteInputChannel) inputGate.getChannel(0);
+ upstreamReady.countDown();
+ // Even with no recovered buffers, finish appends the EndOfFetchedChannelStateEvent
+ // sentinel so the consume path can flip out of recovery in order.
+ channel.finishRecoveredBufferDelivery();
+ inputGate.requestPartitions();
+
+ Optional sentinel = channel.getNextBuffer();
+ assertThat(sentinel).isPresent();
+ assertThat(
+ EventSerializer.fromBuffer(
+ sentinel.get().buffer(), getClass().getClassLoader()))
+ .isInstanceOf(EndOfFetchedChannelStateEvent.class);
+
+ // Consuming the sentinel (done externally by the gate) flips the channel out of
+ // recovery; afterwards the master path is taken and there is no more queued data.
+ channel.onRecoveredStateConsumed();
+ assertThat(channel.getNextBuffer()).isNotPresent();
+ } finally {
+ networkBufferPool.destroy();
+ }
+ }
+
+ @Test
+ void testMoreAvailableNoneWhenLastRecoveredBufferAndDrainNotFinished() throws Exception {
+ // While the channel is in recovery and the drain has not finished, the last currently
+ // queued recovered buffer must report NONE as its next data type: no live data can enter
+ // receivedBuffers (the upstream has no credit), so there is nothing else to expose yet.
+ final NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 4096);
+ try {
+ SingleInputGate inputGate =
+ new SingleInputGateBuilder()
+ .setBufferPoolFactory(networkBufferPool.createBufferPool(1, 4))
+ .setSegmentProvider(networkBufferPool)
+ .setChannelFactory(
+ (builder, gate) ->
+ builder.setNeedsRecovery(true).buildRemoteChannel(gate))
+ .build();
+ inputGate.setup();
+ RemoteInputChannel channel = (RemoteInputChannel) inputGate.getChannel(0);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(11));
+
+ Optional recoveredBuf = channel.getNextBuffer();
+ assertThat(recoveredBuf).isPresent();
+ assertThat(recoveredBuf.get().buffer().getSize()).isEqualTo(11);
+ // Drain not finished and queue now empty: nothing more is available yet.
+ assertThat(recoveredBuf.get().moreAvailable()).isFalse();
+ } finally {
+ networkBufferPool.destroy();
+ }
+ }
+
+ @Test
+ void testPriorityEventDuringRecoveryViaAddPriorityBuffer() throws Exception {
+ final NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 4096);
+ try {
+ SingleInputGate inputGate =
+ new SingleInputGateBuilder()
+ .setBufferPoolFactory(networkBufferPool.createBufferPool(1, 4))
+ .setSegmentProvider(networkBufferPool)
+ .setChannelFactory(
+ (builder, gate) ->
+ builder.setNeedsRecovery(true).buildRemoteChannel(gate))
+ .build();
+ inputGate.setup();
+ RemoteInputChannel channel = (RemoteInputChannel) inputGate.getChannel(0);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(11));
+
+ CheckpointBarrier barrier = new CheckpointBarrier(1L, 0L, UNALIGNED);
+ channel.onBuffer(toBuffer(barrier, true), 0, 0, 0);
+
+ Optional first = channel.getNextBuffer();
+ assertThat(first).isPresent();
+ assertThat(first.get().buffer().getDataType().hasPriority()).isTrue();
+
+ Optional second = channel.getNextBuffer();
+ assertThat(second).isPresent();
+ assertThat(second.get().buffer().getSize()).isEqualTo(11);
+ } finally {
+ networkBufferPool.destroy();
+ }
+ }
+
+ @Test
+ void testCheckpointStartedScansRecoveredBuffersUpToBarrier() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(stateWriter)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ Buffer b1 = TestBufferFactory.createBuffer(1);
+ Buffer b2 = TestBufferFactory.createBuffer(2);
+ Buffer b3 = TestBufferFactory.createBuffer(3);
+ channel.onRecoveredStateBuffer(b1);
+ channel.onRecoveredStateBuffer(b2);
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+ channel.onRecoveredStateBuffer(b3);
+
+ stateWriter.start(1L, UNALIGNED);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, UNALIGNED));
+
+ List persisted = stateWriter.getAddedInput().get(channel.getChannelInfo());
+ assertThat(persisted).hasSize(2);
+ assertThat(persisted.stream().mapToInt(Buffer::getSize).toArray()).containsExactly(1, 2);
+ }
+
+ @Test
+ void testCheckpointStartedDeclinesWhenRecoveryBarrierIsMissing() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(stateWriter)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ Buffer b1 = TestBufferFactory.createBuffer(1);
+ channel.onRecoveredStateBuffer(b1);
+ int refCntBefore = b1.refCnt();
+
+ stateWriter.start(1L, UNALIGNED);
+
+ // The snapshot protocol guarantees a RecoveryCheckpointBarrier sentinel is present while
+ // the channel is in recovery, so a missing sentinel is a protocol violation: the checkpoint
+ // is declined and the recovered buffer is neither dropped nor persisted.
+ assertThatThrownBy(
+ () -> channel.checkpointStarted(new CheckpointBarrier(1L, 0L, UNALIGNED)))
+ .isInstanceOfSatisfying(
+ CheckpointException.class,
+ e ->
+ assertThat(e.getCheckpointFailureReason())
+ .isEqualTo(CheckpointFailureReason.CHECKPOINT_DECLINED))
+ .cause()
+ .hasMessageContaining("Missing RecoveryCheckpointBarrier");
+ assertThat(b1.refCnt()).isEqualTo(refCntBefore);
+ assertThat(stateWriter.getAddedInput().get(channel.getChannelInfo())).isEmpty();
+ }
+
+ @Test
+ void testCheckpointStartedRetainsPreBarrierBuffers() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(stateWriter)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ Buffer b1 = TestBufferFactory.createBuffer(1);
+ channel.onRecoveredStateBuffer(b1);
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+
+ int before = b1.refCnt();
+ stateWriter.start(1L, UNALIGNED);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, UNALIGNED));
+ // After retainBuffer the buffer remains live for both the queue read and the writer copy.
+ assertThat(b1.refCnt()).isGreaterThanOrEqualTo(before);
+ }
+
+ @Test
+ void testCheckpointStartedRemovesSentinel() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(stateWriter)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(2));
+
+ stateWriter.start(1L, UNALIGNED);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, UNALIGNED));
+
+ Optional head = channel.getNextBuffer();
+ assertThat(head).isPresent();
+ Optional nextHead = channel.getNextBuffer();
+ assertThat(nextHead).isPresent();
+ assertThat(nextHead.get().buffer().getSize()).isEqualTo(2);
+ }
+
+ @Test
+ void testCheckpointStartedNestedCpIds() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(stateWriter)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(2));
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(2L), false));
+
+ stateWriter.start(1L, UNALIGNED);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, UNALIGNED));
+
+ List persisted1 = stateWriter.getAddedInput().get(channel.getChannelInfo());
+ assertThat(persisted1).hasSize(1);
+ assertThat(persisted1.get(0).getSize()).isEqualTo(1);
+ }
+
+ @Test
+ void testCheckpointStartedNotInRecoveryUsesMasterPath() throws Exception {
+ SingleInputGate inputGate = createSingleInputGate(1);
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ RemoteInputChannel channel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(stateWriter)
+ .buildRemoteChannel(inputGate);
+ inputGate.setInputChannels(channel);
+ channel.requestSubpartitions();
+ stateWriter.start(7L, UNALIGNED);
+ channel.checkpointStarted(new CheckpointBarrier(7L, 0L, UNALIGNED));
+
+ assertThat(stateWriter.getAddedInput().get(channel.getChannelInfo())).isNullOrEmpty();
+ }
+
+ @Test
+ void testReceivedBuffersHasNoLiveDataBufferDetectsLiveData() throws Exception {
+ final NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 4096);
+ try {
+ SingleInputGate inputGate =
+ new SingleInputGateBuilder()
+ .setBufferPoolFactory(networkBufferPool.createBufferPool(1, 4))
+ .setSegmentProvider(networkBufferPool)
+ .setChannelFactory(
+ (builder, gate) ->
+ builder.setNeedsRecovery(true).buildRemoteChannel(gate))
+ .build();
+ inputGate.setup();
+ RemoteInputChannel channel = (RemoteInputChannel) inputGate.getChannel(0);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ // During recovery the upstream has no credit and can only send events. A live data
+ // buffer is a protocol violation that onBuffer must reject at the entry point.
+ assertThatThrownBy(() -> channel.onBuffer(TestBufferFactory.createBuffer(1), 0, 0, 0))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Received live data buffer during recovery");
+ } finally {
+ networkBufferPool.destroy();
+ }
+ }
+
+ @Test
+ void testReceivedBuffersHasNoLiveDataBufferAcceptsPriorityOnly() throws Exception {
+ final NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 4096);
+ try {
+ RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
+ SingleInputGate inputGate =
+ new SingleInputGateBuilder()
+ .setBufferPoolFactory(networkBufferPool.createBufferPool(1, 4))
+ .setSegmentProvider(networkBufferPool)
+ .setChannelFactory(
+ (builder, gate) ->
+ builder.setStateWriter(stateWriter)
+ .setNeedsRecovery(true)
+ .buildRemoteChannel(gate))
+ .build();
+ inputGate.setup();
+ RemoteInputChannel channel = (RemoteInputChannel) inputGate.getChannel(0);
+
+ channel.onRecoveredStateBuffer(TestBufferFactory.createBuffer(1));
+ // Mirror what snapshotAndInsertBarriers does: push the
+ // RecoveryCheckpointBarrier sentinel so collectPreRecoveryBarrier finds it.
+ channel.onRecoveredStateBuffer(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(1L), false));
+ // Priority event in receivedBuffers is OK (!isBuffer()).
+ CheckpointBarrier priorityBarrier = new CheckpointBarrier(1L, 0L, UNALIGNED);
+ channel.onBuffer(toBuffer(priorityBarrier, true), 0, 0, 0);
+
+ stateWriter.start(1L, UNALIGNED);
+ channel.checkpointStarted(new CheckpointBarrier(1L, 0L, UNALIGNED));
+ } finally {
+ networkBufferPool.destroy();
+ }
+ }
+
private static final class TestBufferPool extends NoOpBufferPool {
@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteRecoveredInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteRecoveredInputChannelTest.java
new file mode 100644
index 00000000000000..7aa0461e0d9fce
--- /dev/null
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/RemoteRecoveredInputChannelTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.runtime.io.network.partition.consumer;
+
+import org.apache.flink.runtime.checkpoint.channel.ChannelStateWriter;
+import org.apache.flink.runtime.io.network.buffer.Buffer;
+import org.apache.flink.runtime.io.network.util.TestBufferFactory;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+class RemoteRecoveredInputChannelTest {
+
+ @Test
+ void testToInputChannelRequiresEmptyRecoveredBuffers() throws Exception {
+ SingleInputGate inputGate = new SingleInputGateBuilder().build();
+ RemoteRecoveredInputChannel recoveredChannel =
+ InputChannelBuilder.newBuilder()
+ .setStateWriter(ChannelStateWriter.NO_OP)
+ .buildRemoteRecoveredChannel(inputGate);
+
+ Buffer buffer = TestBufferFactory.createBuffer(13);
+ recoveredChannel.onRecoveredStateBuffer(buffer);
+
+ try {
+ recoveredChannel.finishReadRecoveredState();
+ assertThatThrownBy(() -> recoveredChannel.toInputChannel(false))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("Received buffer should be empty");
+ } finally {
+ recoveredChannel.releaseAllResources();
+ }
+ }
+}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestInputChannel.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestInputChannel.java
index 14a3654a666399..8fd0a57f93c4b0 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestInputChannel.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/TestInputChannel.java
@@ -19,6 +19,7 @@
package org.apache.flink.runtime.io.network.partition.consumer;
import org.apache.flink.metrics.SimpleCounter;
+import org.apache.flink.runtime.checkpoint.channel.RecoveryCheckpointBarrier;
import org.apache.flink.runtime.event.TaskEvent;
import org.apache.flink.runtime.io.network.api.EndOfData;
import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent;
@@ -31,8 +32,10 @@
import javax.annotation.Nullable;
import java.io.IOException;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Deque;
import java.util.Optional;
import java.util.Queue;
import java.util.concurrent.CompletableFuture;
@@ -45,7 +48,7 @@
import static org.assertj.core.api.Assertions.assertThat;
/** A mocked input channel. */
-public class TestInputChannel extends InputChannel {
+public class TestInputChannel extends InputChannel implements RecoverableInputChannel {
private final Queue buffers = new ConcurrentLinkedQueue<>();
@@ -259,6 +262,50 @@ public void notifyRequiredSegmentId(int subpartitionId, int segmentId) {
requiredSegmentIdFuture.complete(segmentId);
}
+ private final Deque recoveredBuffersSpy = new ArrayDeque<>();
+ private boolean finishRecoveredBufferDeliveryCalled = false;
+
+ @Override
+ public void onRecoveredStateBuffer(Buffer buffer) {
+ recoveredBuffersSpy.add(buffer);
+ }
+
+ @Override
+ public void finishRecoveredBufferDelivery() {
+ finishRecoveredBufferDeliveryCalled = true;
+ }
+
+ @Override
+ public void insertRecoveryCheckpointBarrierIfInRecovery(long checkpointId) throws IOException {
+ if (!finishRecoveredBufferDeliveryCalled || !recoveredBuffersSpy.isEmpty()) {
+ recoveredBuffersSpy.add(
+ EventSerializer.toBuffer(new RecoveryCheckpointBarrier(checkpointId), false));
+ }
+ }
+
+ @Override
+ public Buffer requestRecoveryBufferBlocking() {
+ throw new UnsupportedOperationException("TestInputChannel does not back recovery drain");
+ }
+
+ @Override
+ public void onRecoveredStateConsumed() {
+ // No-op in this test stub.
+ }
+
+ @Override
+ public CompletableFuture getStateConsumedFuture() {
+ return CompletableFuture.completedFuture(null);
+ }
+
+ public Deque getRecoveredBuffersSpy() {
+ return recoveredBuffersSpy;
+ }
+
+ public boolean isFinishRecoveredBufferDeliveryCalled() {
+ return finishRecoveredBufferDeliveryCalled;
+ }
+
public void assertReturnedEventsAreRecycled() {
assertReturnedBuffersAreRecycled(false, true);
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java
index 1ed1a42a66ea01..a7325d153757ca 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/UnionInputGateTest.java
@@ -325,9 +325,7 @@ private static RecoveredInputChannel buildRecoveredChannel(SingleInputGate input
new SimpleCounter(),
10) {
@Override
- protected InputChannel toInputChannelInternal(
- java.util.ArrayDeque
- remainingBuffers) {
+ protected InputChannel toInputChannelInternal(boolean needsRecovery) {
throw new UnsupportedOperationException();
}
};
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
index c50eface312b06..101ff4386cb29f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironment.java
@@ -179,13 +179,14 @@ protected MockEnvironment(
TaskManagerRuntimeInfo taskManagerRuntimeInfo,
MemoryManager memManager,
ExternalResourceInfoProvider externalResourceInfoProvider,
- ChannelStateWriteRequestExecutorFactory channelStateExecutorFactory) {
+ ChannelStateWriteRequestExecutorFactory channelStateExecutorFactory,
+ Configuration jobConfiguration) {
this.jobInfo = new JobInfoImpl(jobID, jobName);
this.jobVertexID = jobVertexID;
this.jobType = jobType;
this.taskInfo = new TaskInfoImpl(taskName, maxParallelism, subtaskIndex, parallelism, 0);
- this.jobConfiguration = new Configuration();
+ this.jobConfiguration = jobConfiguration;
this.taskConfiguration = taskConfiguration;
this.inputs = new LinkedList<>();
this.outputs = new LinkedList();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironmentBuilder.java b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironmentBuilder.java
index 2a8c7c273a867a..c69a1e275e26ce 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironmentBuilder.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/operators/testutils/MockEnvironmentBuilder.java
@@ -67,6 +67,7 @@ public class MockEnvironmentBuilder {
ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES;
private ChannelStateWriteRequestExecutorFactory channelStateExecutorFactory =
new ChannelStateWriteRequestExecutorFactory(jobID);
+ private Configuration jobConfiguration = new Configuration();
private MemoryManager buildMemoryManager(long memorySize) {
return MemoryManagerBuilder.newBuilder().setMemorySize(memorySize).build();
@@ -181,6 +182,11 @@ public MockEnvironmentBuilder setJobType(JobType jobType) {
return this;
}
+ public MockEnvironmentBuilder setJobConfiguration(Configuration jobConfiguration) {
+ this.jobConfiguration = jobConfiguration;
+ return this;
+ }
+
public MockEnvironment build() {
if (ioManager == null) {
ioManager = new IOManagerAsync();
@@ -206,6 +212,7 @@ public MockEnvironment build() {
taskManagerRuntimeInfo,
memoryManager,
externalResourceInfoProvider,
- channelStateExecutorFactory);
+ channelStateExecutorFactory,
+ jobConfiguration);
}
}
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/timeline/RescaleTimelineITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/timeline/RescaleTimelineITCase.java
index a25d0c9722368f..e5920f282cbcff 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/timeline/RescaleTimelineITCase.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adaptive/timeline/RescaleTimelineITCase.java
@@ -269,6 +269,17 @@ void testRescaleTerminatedBySucceeded() {}
@TestTemplate
void testRescaleTerminatedByJobFinished() throws Exception {
+ // This case only asserts on the recorded rescale history; skip the disabled-history
+ // parameter before the cluster rebuild below so it does not pay for an unused cluster.
+ assumeThat(enabledRescaleHistory(configuration)).isTrue();
+
+ // With the short shared cooldown the DefaultStateTransitionManager re-enters Idling on a
+ // wall-clock timer and terminates the in-progress rescale with
+ // NO_RESOURCES_OR_PARALLELISMS_CHANGE before the job finishes; goToFinished then finds it
+ // already terminated and its JOB_FINISHED stamp is ignored, so waiting cannot win.
+ // Widen the cooldown to keep the rescale in-progress until the job finishes.
+ rebuildClusterWithExecutingTimeouts(Duration.ofSeconds(60), Duration.ofSeconds(60));
+
final MiniCluster miniCluster = miniClusterResource.getMiniCluster();
final JobGraph jobGraph = createBlockingJobGraph(PARALLELISM);
@@ -276,8 +287,6 @@ void testRescaleTerminatedByJobFinished() throws Exception {
waitForVertexParallelismReachedAndJobRunning(jobGraph, JOB_VERTEX_ID, PARALLELISM);
- assumeThat(enabledRescaleHistory(configuration)).isTrue();
-
updateJobResourceRequirements(miniCluster, jobGraph, 1, PARALLELISM * 2);
// The upper bound (PARALLELISM * 2) exceeds the available slots, so this rescale is only
@@ -289,13 +298,14 @@ void testRescaleTerminatedByJobFinished() throws Exception {
OnceBlockingNoOpInvokable.unblock();
+ // Generous budget: on a loaded CI leg the unblock-to-finish window can itself exceed 10s.
waitUntilConditionWithTimeout(
() -> {
List rescaleHistory = getRescaleHistory(miniCluster, jobGraph);
return hasRescaleHistoryMetCondition(
rescaleHistory, 2, TerminatedReason.JOB_FINISHED);
},
- 10000);
+ 60000);
}
@TestTemplate
@@ -567,7 +577,11 @@ void testRecordNonTerminatedRescaleMergingWithNewRecoverableFailureTriggerCause(
miniCluster.terminateTaskManager(0);
- waitForVertexParallelismReachedAndJobRunning(jobGraph, JOB_VERTEX_ID, PARALLELISM);
+ // Wait for the failover to complete before snapshotting: the merge that re-stamps the
+ // trigger cause to RECOVERABLE_FAILOVER runs before the job is RUNNING again at the
+ // reduced parallelism (one TaskManager left).
+ waitForVertexParallelismReachedAndJobRunning(
+ jobGraph, JOB_VERTEX_ID, NUMBER_SLOTS_PER_TASK_MANAGER);
final ExecutionGraphInfo executionGraphInfo =
miniCluster.getExecutionGraphInfo(jobGraph.getJobID()).join();
diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/MockIndexedInputGate.java b/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/MockIndexedInputGate.java
index 584aeb7eb9089f..5cc443777078f7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/MockIndexedInputGate.java
+++ b/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/MockIndexedInputGate.java
@@ -86,6 +86,11 @@ public InputChannel getChannel(int channelIndex) {
throw new UnsupportedOperationException();
}
+ @Override
+ public InputChannel getChannel(InputChannelInfo channelInfo) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public void setChannelStateWriter(ChannelStateWriter channelStateWriter) {}
diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/MockInputGate.java b/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/MockInputGate.java
index 71b2c43f3306aa..e63a49be53b8a7 100644
--- a/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/MockInputGate.java
+++ b/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/MockInputGate.java
@@ -101,6 +101,11 @@ public InputChannel getChannel(int channelIndex) {
throw new UnsupportedOperationException();
}
+ @Override
+ public InputChannel getChannel(InputChannelInfo channelInfo) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public List getChannelInfos() {
return IntStream.range(0, numberOfChannels)
diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/SingleInputGateBenchmarkFactory.java b/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/SingleInputGateBenchmarkFactory.java
index b850a7cc553702..b3f0aec9dc5e53 100644
--- a/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/SingleInputGateBenchmarkFactory.java
+++ b/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/SingleInputGateBenchmarkFactory.java
@@ -37,7 +37,6 @@
import org.apache.flink.runtime.taskmanager.NettyShuffleEnvironmentConfiguration;
import java.io.IOException;
-import java.util.ArrayDeque;
/**
* A benchmark-specific input gate factory which overrides the respective methods of creating {@link
@@ -130,7 +129,8 @@ public TestLocalInputChannel(
metrics.getNumBytesInLocalCounter(),
metrics.getNumBuffersInLocalCounter(),
ChannelStateWriter.NO_OP,
- new ArrayDeque<>());
+ 0,
+ false);
}
@Override
@@ -186,7 +186,7 @@ public TestRemoteInputChannel(
metrics.getNumBytesInRemoteCounter(),
metrics.getNumBuffersInRemoteCounter(),
ChannelStateWriter.NO_OP,
- new ArrayDeque<>());
+ false);
}
@Override
diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/checkpointing/AlignedCheckpointsMassiveRandomTest.java b/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/checkpointing/AlignedCheckpointsMassiveRandomTest.java
index 619873c387d08f..e27f6b28937d88 100644
--- a/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/checkpointing/AlignedCheckpointsMassiveRandomTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/streaming/runtime/io/checkpointing/AlignedCheckpointsMassiveRandomTest.java
@@ -178,6 +178,11 @@ public InputChannel getChannel(int channelIndex) {
throw new UnsupportedOperationException();
}
+ @Override
+ public InputChannel getChannel(InputChannelInfo channelInfo) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public List getChannelInfos() {
return IntStream.range(0, numberOfChannels)
diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/util/KeyedMultiInputStreamOperatorTestHarness.java b/flink-runtime/src/test/java/org/apache/flink/streaming/util/KeyedMultiInputStreamOperatorTestHarness.java
index 9d3f5fa431babb..15ee270fe85cdd 100644
--- a/flink-runtime/src/test/java/org/apache/flink/streaming/util/KeyedMultiInputStreamOperatorTestHarness.java
+++ b/flink-runtime/src/test/java/org/apache/flink/streaming/util/KeyedMultiInputStreamOperatorTestHarness.java
@@ -22,6 +22,7 @@
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.ClosureCleaner;
import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
import org.apache.flink.streaming.api.operators.MultipleInputStreamOperator;
import org.apache.flink.streaming.api.operators.StreamOperatorFactory;
@@ -43,6 +44,17 @@ public KeyedMultiInputStreamOperatorTestHarness(
config.serializeAllConfigs();
}
+ public KeyedMultiInputStreamOperatorTestHarness(
+ StreamOperatorFactory operator,
+ TypeInformation keyType,
+ MockEnvironment environment)
+ throws Exception {
+ super(operator, environment);
+ config.setStateKeySerializer(
+ keyType.createSerializer(executionConfig.getSerializerConfig()));
+ config.serializeAllConfigs();
+ }
+
public KeyedMultiInputStreamOperatorTestHarness(
StreamOperatorFactory operatorFactory,
int maxParallelism,
diff --git a/flink-runtime/src/test/java/org/apache/flink/streaming/util/MultiInputStreamOperatorTestHarness.java b/flink-runtime/src/test/java/org/apache/flink/streaming/util/MultiInputStreamOperatorTestHarness.java
index 058fc57ee8bbc2..95f3c0bf57aa3f 100644
--- a/flink-runtime/src/test/java/org/apache/flink/streaming/util/MultiInputStreamOperatorTestHarness.java
+++ b/flink-runtime/src/test/java/org/apache/flink/streaming/util/MultiInputStreamOperatorTestHarness.java
@@ -18,6 +18,7 @@
package org.apache.flink.streaming.util;
+import org.apache.flink.runtime.operators.testutils.MockEnvironment;
import org.apache.flink.streaming.api.operators.Input;
import org.apache.flink.streaming.api.operators.MultipleInputStreamOperator;
import org.apache.flink.streaming.api.operators.StreamOperatorFactory;
@@ -50,6 +51,12 @@ public MultiInputStreamOperatorTestHarness(
super(operatorFactory, maxParallelism, numSubtasks, subtaskIndex);
}
+ public MultiInputStreamOperatorTestHarness(
+ StreamOperatorFactory operatorFactory, MockEnvironment environment)
+ throws Exception {
+ super(operatorFactory, environment);
+ }
+
public void processElement(int idx, StreamRecord element) throws Exception {
Input input = getCastedOperator().getInputs().get(idx);
input.setKeyContextElement(element);
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/RocksDBNativeMetricMonitor.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/RocksDBNativeMetricMonitor.java
index e51ec32c7daec9..83a885a4f477f5 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/RocksDBNativeMetricMonitor.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/RocksDBNativeMetricMonitor.java
@@ -22,6 +22,7 @@
import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.View;
+import org.apache.flink.util.IOUtils;
import org.rocksdb.ColumnFamilyHandle;
import org.rocksdb.RocksDB;
@@ -107,7 +108,7 @@ void registerColumnFamily(String columnFamilyName, ColumnFamilyHandle handle) {
/** Updates the value of metricView if the reference is still valid. */
private void setProperty(RocksDBNativePropertyMetricView metricView) {
- if (metricView.isClosed()) {
+ if (metricView.isClosed() || rocksDB == null) {
return;
}
try {
@@ -126,11 +127,11 @@ private void setProperty(RocksDBNativePropertyMetricView metricView) {
}
private void setStatistics(RocksDBNativeStatisticsMetricView metricView) {
- if (metricView.isClosed()) {
+ if (metricView.isClosed() || statistics == null) {
return;
}
- if (statistics != null) {
- synchronized (lock) {
+ synchronized (lock) {
+ if (statistics != null) {
metricView.setValue(statistics.getTickerCount(metricView.tickerType));
}
}
@@ -140,6 +141,8 @@ private void setStatistics(RocksDBNativeStatisticsMetricView metricView) {
public void close() {
synchronized (lock) {
rocksDB = null;
+ // Wrapper holds a JNI shared_ptr that leaks without explicit close. See FLINK-39923.
+ IOUtils.closeQuietly(statistics);
statistics = null;
}
}
diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/restore/RocksDBHandle.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/restore/RocksDBHandle.java
index c233922d9d2b6b..5cbd60b3a154c8 100644
--- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/restore/RocksDBHandle.java
+++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/restore/RocksDBHandle.java
@@ -37,6 +37,7 @@
import org.rocksdb.DBOptions;
import org.rocksdb.ExportImportFilesMetaData;
import org.rocksdb.RocksDB;
+import org.rocksdb.Statistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -90,6 +91,8 @@ class RocksDBHandle implements AutoCloseable {
private RocksDB db;
private ColumnFamilyHandle defaultColumnFamilyHandle;
private RocksDBNativeMetricMonitor nativeMetricMonitor;
+ // Released in close() for the partial-init case; on success the monitor closes it.
+ private Statistics statistics;
private final Long writeBufferManagerCapacity;
protected RocksDBHandle(
@@ -147,12 +150,16 @@ private void loadDb() throws IOException {
dbOptions);
// remove the default column family which is located at the first index
defaultColumnFamilyHandle = columnFamilyHandles.remove(0);
- // init native metrics monitor if configured
+
+ if (!nativeMetricOptions.isEnabled()) {
+ return;
+ }
+ // dbOptions.statistics() returns a new Java wrapper around a fresh shared_ptr
+ // aliasing the existing native StatisticsImpl. The original wrapper is closed by
+ // RocksDBResourceContainer (via dbOptions); this one must also be closed. See FLINK-39923.
+ statistics = dbOptions.statistics();
nativeMetricMonitor =
- nativeMetricOptions.isEnabled()
- ? new RocksDBNativeMetricMonitor(
- nativeMetricOptions, metricGroup, db, dbOptions.statistics())
- : null;
+ new RocksDBNativeMetricMonitor(nativeMetricOptions, metricGroup, db, statistics);
}
RocksDbKvStateInfo getOrRegisterStateColumnFamilyHandle(
@@ -306,7 +313,13 @@ public DBOptions getDbOptions() {
@Override
public void close() throws Exception {
IOUtils.closeQuietly(defaultColumnFamilyHandle);
- IOUtils.closeQuietly(nativeMetricMonitor);
+ if (nativeMetricMonitor != null) {
+ // Monitor owns the statistics wrapper.
+ IOUtils.closeQuietly(nativeMetricMonitor);
+ } else {
+ // Monitor construction never completed; release the wrapper directly.
+ IOUtils.closeQuietly(statistics);
+ }
IOUtils.closeQuietly(db);
// Making sure the already created column family options will be closed
columnFamilyDescriptors.forEach((cfd) -> IOUtils.closeQuietly(cfd.getOptions()));
diff --git a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
index 56cf110880ac2a..cc01f96e7ece27 100644
--- a/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
+++ b/flink-table/flink-sql-gateway/src/main/java/org/apache/flink/table/gateway/service/materializedtable/MaterializedTableManager.java
@@ -955,7 +955,7 @@ private ResultFetcher callAlterMaterializedTableChangeOperation(
op.getTableIdentifier(),
oldTable -> op.getTableChanges(),
suspendMaterializedTable,
- op.getSinkModifyQuery());
+ op.getAsQueryOperation());
operationExecutor.callExecutableOperation(
handle, alterMaterializedTableChangeOperation);
@@ -1014,7 +1014,7 @@ private ResultFetcher callAlterMaterializedTableChangeOperation(
tableIdentifier,
oldTable -> tableChanges,
oldMaterializedTable,
- op.getSinkModifyQuery());
+ op.getAsQueryOperation());
operationExecutor.callExecutableOperation(
handle, alterMaterializedTableChangeOperation);
@@ -1036,7 +1036,7 @@ private AlterMaterializedTableChangeOperation generateRollbackAlterMaterializedT
op.getTableIdentifier(),
oldTable -> List.of(),
oldMaterializedTable,
- op.getSinkModifyQuery());
+ op.getAsQueryOperation());
}
private TableChange.ModifyRefreshHandler generateResetSavepointTableChange(
diff --git a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
index b7e6d7ab9ea3f4..db08bbfbf2c329 100644
--- a/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
+++ b/flink-table/flink-sql-parser/src/main/codegen/data/Parser.tdd
@@ -173,6 +173,7 @@
"org.apache.flink.sql.parser.type.ExtendedSqlCollectionTypeNameSpec"
"org.apache.flink.sql.parser.type.ExtendedSqlRowTypeNameSpec"
"org.apache.flink.sql.parser.type.SqlBitmapTypeNameSpec"
+ "org.apache.flink.sql.parser.type.SqlGeographyTypeNameSpec"
"org.apache.flink.sql.parser.type.SqlRawTypeNameSpec"
"org.apache.flink.sql.parser.type.SqlStructuredTypeNameSpec"
"org.apache.flink.sql.parser.type.SqlTimestampLtzTypeNameSpec"
@@ -225,6 +226,7 @@
"FROM_TIMESTAMP"
"FUNCTIONS"
"FRESHNESS"
+ "GEOGRAPHY"
"HASH"
"IF"
"JSON_EXECUTION_PLAN"
@@ -719,6 +721,7 @@
"ExtendedSqlRowTypeName()"
"SqlStructuredTypeName()"
"SqlBitmapTypeName()"
+ "SqlGeographyTypeName()"
]
# List of methods for parsing builtin function calls.
diff --git a/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl b/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl
index 2b8250ea98ed9e..ede54c27ca86e0 100644
--- a/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl
+++ b/flink-table/flink-sql-parser/src/main/codegen/includes/parserImpls.ftl
@@ -2666,6 +2666,17 @@ SqlTypeNameSpec SqlBitmapTypeName() :
}
}
+/** Parses GEOGRAPHY type. */
+SqlTypeNameSpec SqlGeographyTypeName() :
+{
+}
+{
+
+ {
+ return new SqlGeographyTypeNameSpec(getPos());
+ }
+}
+
/**
* Parse a "name1 type1 [ NULL | NOT NULL] [ comment ]
* [, name2 type2 [ NULL | NOT NULL] [ comment ] ]* ..." list.
diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java
new file mode 100644
index 00000000000000..55a3d7ff4fd7d6
--- /dev/null
+++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlGeographyTypeNameSpec.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.sql.parser.type;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.calcite.ExtendedRelTypeFactory;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlTypeNameSpec;
+import org.apache.calcite.sql.SqlWriter;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.util.Litmus;
+
+/** Represents the GEOGRAPHY data type. */
+@Internal
+public final class SqlGeographyTypeNameSpec extends SqlTypeNameSpec {
+
+ private static final String GEOGRAPHY_TYPE_NAME = "GEOGRAPHY";
+
+ public SqlGeographyTypeNameSpec(SqlParserPos pos) {
+ super(new SqlIdentifier(GEOGRAPHY_TYPE_NAME, pos), pos);
+ }
+
+ @Override
+ public RelDataType deriveType(SqlValidator validator) {
+ return ((ExtendedRelTypeFactory) validator.getTypeFactory()).createGeographyType();
+ }
+
+ @Override
+ public void unparse(SqlWriter writer, int leftPrec, int rightPrec) {
+ writer.keyword(GEOGRAPHY_TYPE_NAME);
+ }
+
+ @Override
+ public boolean equalsDeep(SqlTypeNameSpec spec, Litmus litmus) {
+ if (!(spec instanceof SqlGeographyTypeNameSpec)) {
+ return litmus.fail("{} != {}", this, spec);
+ }
+ return litmus.succeed();
+ }
+}
diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java
index 57b9bc615ec930..57be54e4cb6bd4 100644
--- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java
+++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/table/calcite/ExtendedRelTypeFactory.java
@@ -44,4 +44,7 @@ RelDataType createStructuredType(
/** Creates a BITMAP type. */
RelDataType createBitmapType();
+
+ /** Creates a GEOGRAPHY type. */
+ RelDataType createGeographyType();
}
diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java
index 92f8e34c770a00..ee33ea2e37daf2 100644
--- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java
+++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java
@@ -4074,4 +4074,23 @@ void testBitmapType() {
sql("CREATE TABLE t (\n" + "^bitmap^ INT" + "\n)")
.fails("(?s).*Encountered \"bitmap\" at line 2, column 1.\n.*");
}
+
+ @Test
+ void testGeographyType() {
+ sql("CREATE TABLE t (\n" + "g geography" + "\n)")
+ .ok("CREATE TABLE `T` (\n" + " `G` GEOGRAPHY\n" + ")");
+
+ sql("CREATE TABLE t (\n" + "g geography NOT NULL" + "\n)")
+ .ok("CREATE TABLE `T` (\n" + " `G` GEOGRAPHY NOT NULL\n" + ")");
+
+ // GEOGRAPHY takes no parameters
+ sql("CREATE TABLE t (\n" + "g geography^(^1)" + "\n)")
+ .fails("(?s).*Encountered \"\\(\" at line 2, column 12.\n.*");
+ sql("CREATE TABLE t (\n" + "g geography^(^4326)" + "\n)")
+ .fails("(?s).*Encountered \"\\(\" at line 2, column 12.\n.*");
+
+ // GEOGRAPHY is a reserved keyword and cannot be used as an identifier without escaping
+ sql("CREATE TABLE t (\n" + "^geography^ INT" + "\n)")
+ .fails("(?s).*Encountered \"geography\" at line 2, column 1.\n.*");
+ }
}
diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java
index 34fe9926079b3c..070c77905a335f 100644
--- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java
+++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/TestRelDataTypeFactory.java
@@ -53,6 +53,11 @@ public RelDataType createBitmapType() {
return canonize(new DummyBitmapType());
}
+ @Override
+ public RelDataType createGeographyType() {
+ return canonize(new DummyGeographyType());
+ }
+
private static class DummyRawType extends RelDataTypeImpl {
private final String className;
@@ -117,4 +122,16 @@ protected void generateTypeString(StringBuilder sb, boolean withDetail) {
sb.append("BITMAP");
}
}
+
+ private static class DummyGeographyType extends RelDataTypeImpl {
+
+ DummyGeographyType() {
+ computeDigest();
+ }
+
+ @Override
+ protected void generateTypeString(StringBuilder sb, boolean withDetail) {
+ sb.append("GEOGRAPHY");
+ }
+ }
}
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java
index 074224ae696fda..72a63e8bd082f6 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/BaseExpressions.java
@@ -2179,7 +2179,7 @@ public OutType sha2(InType hashLength) {
*
* lit("\"abc\"").isJson() // true
* lit("abc").isJson() // false
- * nullOf(DataTypes.STRING()).isJson() // false
+ * nullOf(DataTypes.STRING()).isJson() // null
*
* lit("1").isJson(JsonType.SCALAR) // true
* lit("1").isJson(JsonType.ARRAY) // false
@@ -2192,7 +2192,7 @@ public OutType sha2(InType hashLength) {
*
* @param type The type of JSON object to validate against.
* @return {@code true} if the string is a valid JSON of the given {@param type}, {@code false}
- * otherwise.
+ * otherwise, or {@code NULL} if the input is {@code NULL}.
*/
public OutType isJson(JsonType type) {
return toApiSpecificExpression(unresolvedCall(IS_JSON, toExpr(), valueLiteral(type)));
@@ -2203,7 +2203,8 @@ public OutType isJson(JsonType type) {
*
*
This is a shortcut for {@code isJson(JsonType.VALUE)}. See {@link #isJson(JsonType)}.
*
- * @return {@code true} if the string is a valid JSON value, {@code false} otherwise.
+ * @return {@code true} if the string is a valid JSON value, {@code false} otherwise, or {@code
+ * NULL} if the input is {@code NULL}.
*/
public OutType isJson() {
return toApiSpecificExpression(unresolvedCall(IS_JSON, toExpr()));
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableAsQueryOperation.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableAsQueryOperation.java
index 9da8302ca60420..0dcbfd72926658 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableAsQueryOperation.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableAsQueryOperation.java
@@ -42,8 +42,8 @@ public AlterMaterializedTableAsQueryOperation(
ObjectIdentifier tableIdentifier,
Function> tableChangesForTable,
ResolvedCatalogMaterializedTable oldTable,
- QueryOperation sinkModifyQuery) {
- super(tableIdentifier, tableChangesForTable, oldTable, sinkModifyQuery);
+ QueryOperation asQueryOperation) {
+ super(tableIdentifier, tableChangesForTable, oldTable, asQueryOperation);
}
@Override
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableChangeOperation.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableChangeOperation.java
index 87d588a920da50..299c72d6cfea3b 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableChangeOperation.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/AlterMaterializedTableChangeOperation.java
@@ -57,7 +57,7 @@ public class AlterMaterializedTableChangeOperation extends AlterMaterializedTabl
implements ModifyOperation {
private final Function> tableChangeForTable;
- private final QueryOperation sinkModifyQuery;
+ private final QueryOperation asQueryOperation;
private ResolvedCatalogMaterializedTable oldTable;
private MaterializedTableChangeHandler handler;
private CatalogMaterializedTable newTable;
@@ -75,15 +75,15 @@ public AlterMaterializedTableChangeOperation(
ObjectIdentifier tableIdentifier,
Function> tableChangeForTable,
ResolvedCatalogMaterializedTable oldTable,
- QueryOperation sinkModifyQuery) {
+ QueryOperation asQueryOperation) {
super(tableIdentifier);
this.tableChangeForTable = tableChangeForTable;
this.oldTable = oldTable;
- this.sinkModifyQuery = sinkModifyQuery;
+ this.asQueryOperation = asQueryOperation;
}
- public QueryOperation getSinkModifyQuery() {
- return sinkModifyQuery;
+ public QueryOperation getAsQueryOperation() {
+ return asQueryOperation;
}
public List getTableChanges() {
@@ -95,7 +95,7 @@ public List getTableChanges() {
public AlterMaterializedTableChangeOperation copyAsTableChangeOperation() {
return new AlterMaterializedTableChangeOperation(
- tableIdentifier, tableChangeForTable, oldTable, sinkModifyQuery);
+ tableIdentifier, tableChangeForTable, oldTable, asQueryOperation);
}
public CatalogMaterializedTable getNewTable() {
@@ -165,7 +165,7 @@ public String asSummaryString() {
@Override
public QueryOperation getChild() {
- return this.sinkModifyQuery;
+ return this.asQueryOperation;
}
@Override
diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/CreateMaterializedTableOperation.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/CreateMaterializedTableOperation.java
index ff425fe53aa6c3..070e0f4948e791 100644
--- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/CreateMaterializedTableOperation.java
+++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/operations/materializedtable/CreateMaterializedTableOperation.java
@@ -41,15 +41,15 @@ public class CreateMaterializedTableOperation
private final ObjectIdentifier tableIdentifier;
private final ResolvedCatalogMaterializedTable materializedTable;
- private final QueryOperation sinkModifyingQuery;
+ private final QueryOperation asQueryOperation;
public CreateMaterializedTableOperation(
ObjectIdentifier tableIdentifier,
ResolvedCatalogMaterializedTable materializedTable,
- QueryOperation sinkModifyQuery) {
+ QueryOperation asQueryOperation) {
this.tableIdentifier = tableIdentifier;
this.materializedTable = materializedTable;
- this.sinkModifyingQuery = sinkModifyQuery;
+ this.asQueryOperation = asQueryOperation;
}
@Override
@@ -67,8 +67,8 @@ public ResolvedCatalogMaterializedTable getCatalogMaterializedTable() {
return materializedTable;
}
- public QueryOperation getSinkModifyingQuery() {
- return sinkModifyingQuery;
+ public QueryOperation getAsQueryOperation() {
+ return asQueryOperation;
}
@Override
@@ -83,7 +83,7 @@ public String asSummaryString() {
@Override
public QueryOperation getChild() {
- return this.sinkModifyingQuery;
+ return this.asQueryOperation;
}
@Override
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.0/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.0/serializer-snapshot
new file mode 100644
index 00000000000000..976c3c220a4043
Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.0/serializer-snapshot differ
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.0/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.0/test-data
similarity index 100%
rename from flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.0/test-data
rename to flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.0/test-data
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.1/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.1/serializer-snapshot
new file mode 100644
index 00000000000000..976c3c220a4043
Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.1/serializer-snapshot differ
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.1/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.1/test-data
similarity index 100%
rename from flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.1/test-data
rename to flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.1/test-data
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.2/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.2/serializer-snapshot
new file mode 100644
index 00000000000000..976c3c220a4043
Binary files /dev/null and b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.2/serializer-snapshot differ
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.2/test-data b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.2/test-data
similarity index 100%
rename from flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.2/test-data
rename to flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-list-buffer-2.2/test-data
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.0/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.0/serializer-snapshot
deleted file mode 100644
index 5b74ce2bb36b29..00000000000000
Binary files a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.0/serializer-snapshot and /dev/null differ
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.1/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.1/serializer-snapshot
deleted file mode 100644
index 5b74ce2bb36b29..00000000000000
Binary files a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.1/serializer-snapshot and /dev/null differ
diff --git a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.2/serializer-snapshot b/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.2/serializer-snapshot
deleted file mode 100644
index 5b74ce2bb36b29..00000000000000
Binary files a/flink-table/flink-table-api-scala/src/test/resources/traversable-serializer-mutable-list-2.2/serializer-snapshot and /dev/null differ
diff --git a/flink-table/flink-table-api-scala/src/test/scala/org/apache/flink/table/api/typeutils/TraversableSerializerUpgradeTest.scala b/flink-table/flink-table-api-scala/src/test/scala/org/apache/flink/table/api/typeutils/TraversableSerializerUpgradeTest.scala
index 2a89f5f78234e1..d9d531ae879918 100644
--- a/flink-table/flink-table-api-scala/src/test/scala/org/apache/flink/table/api/typeutils/TraversableSerializerUpgradeTest.scala
+++ b/flink-table/flink-table-api-scala/src/test/scala/org/apache/flink/table/api/typeutils/TraversableSerializerUpgradeTest.scala
@@ -28,7 +28,6 @@ import org.apache.flink.table.api.typeutils.TraversableSerializerUpgradeTest.Typ
import org.apache.flink.test.util.MigrationTest
import org.assertj.core.api.Condition
-import org.junit.jupiter.api.Disabled
import java.util
import java.util.Objects
@@ -37,7 +36,6 @@ import java.util.function.Supplier
import scala.collection.{mutable, BitSet, LinearSeq}
/** A [[TypeSerializerUpgradeTestBase]] for [[TraversableSerializer]]. */
-@Disabled("FLINK-36334")
class TraversableSerializerUpgradeTest
extends TypeSerializerUpgradeTestBase[TraversableOnce[_], TraversableOnce[_]] {
@@ -71,10 +69,10 @@ class TraversableSerializerUpgradeTest
classOf[MapSerializerVerifier]))
testSpecifications.add(
new TestSpecification[mutable.ListBuffer[Int], mutable.ListBuffer[Int]](
- "traversable-serializer-mutable-list",
+ "traversable-serializer-list-buffer",
migrationVersion,
- classOf[MutableListSerializerSetup],
- classOf[MutableListSerializerVerifier]))
+ classOf[ListBufferSerializerSetup],
+ classOf[ListBufferSerializerVerifier]))
testSpecifications.add(
new TestSpecification[Seq[Int], Seq[Int]](
"traversable-serializer-seq",
@@ -130,7 +128,7 @@ object TraversableSerializerUpgradeTest {
val mapTypeInfo = implicitly[TypeInformation[Map[String, Int]]]
val setTypeInfo = implicitly[TypeInformation[Set[Int]]]
val bitsetTypeInfo = implicitly[TypeInformation[BitSet]]
- val mutableListTypeInfo =
+ val listBufferTypeInfo =
implicitly[TypeInformation[mutable.ListBuffer[Int]]]
val seqTupleTypeInfo = implicitly[TypeInformation[Seq[(Int, String)]]]
val seqPojoTypeInfo = implicitly[TypeInformation[Seq[Pojo]]]
@@ -224,18 +222,18 @@ object TraversableSerializerUpgradeTest {
TypeSerializerConditions.isCompatibleAsIs[Map[String, Int]]()
}
- final class MutableListSerializerSetup
+ final class ListBufferSerializerSetup
extends TypeSerializerUpgradeTestBase.PreUpgradeSetup[mutable.ListBuffer[Int]] {
override def createPriorSerializer: TypeSerializer[mutable.ListBuffer[Int]] =
- new TypeSerializerSupplier(mutableListTypeInfo).get()
+ new TypeSerializerSupplier(listBufferTypeInfo).get()
override def createTestData: mutable.ListBuffer[Int] = mutable.ListBuffer(1, 2, 3)
}
- final class MutableListSerializerVerifier
+ final class ListBufferSerializerVerifier
extends TypeSerializerUpgradeTestBase.UpgradeVerifier[mutable.ListBuffer[Int]] {
override def createUpgradedSerializer: TypeSerializer[mutable.ListBuffer[Int]] =
- new TypeSerializerSupplier(mutableListTypeInfo).get()
+ new TypeSerializerSupplier(listBufferTypeInfo).get()
override def testDataCondition: Condition[mutable.ListBuffer[Int]] =
new Condition[mutable.ListBuffer[Int]](
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java
index af1e718d90e04c..3912c178328526 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/api/DataTypes.java
@@ -44,6 +44,7 @@
import org.apache.flink.table.types.logical.DescriptorType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
import org.apache.flink.table.types.logical.LogicalType;
@@ -1075,6 +1076,15 @@ public static DataType BITMAP() {
return new AtomicDataType(new BitmapType());
}
+ /**
+ * Data type of geography data.
+ *
+ * @see GeographyType
+ */
+ public static DataType GEOGRAPHY() {
+ return new AtomicDataType(new GeographyType());
+ }
+
// --------------------------------------------------------------------------------------------
// Helper functions
// --------------------------------------------------------------------------------------------
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java
index 811c9be70e6b87..728d3c228f05da 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/ArrayData.java
@@ -123,6 +123,12 @@ default Bitmap getBitmap(int pos) {
"This ArrayData implementation does not support Bitmap type.");
}
+ /** Returns the geography value at the given position. */
+ default GeographyData getGeography(int pos) {
+ throw new UnsupportedOperationException(
+ "This ArrayData implementation does not support Geography type.");
+ }
+
// ------------------------------------------------------------------------------------------
// Conversion Utilities
// ------------------------------------------------------------------------------------------
@@ -225,6 +231,9 @@ static ElementGetter createElementGetter(LogicalType elementType) {
case BITMAP:
elementGetter = ArrayData::getBitmap;
break;
+ case GEOGRAPHY:
+ elementGetter = ArrayData::getGeography;
+ break;
case NULL:
case SYMBOL:
case UNRESOLVED:
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java
index f609c2f22f1311..08856cdb0ceb1e 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericArrayData.java
@@ -265,6 +265,11 @@ public Bitmap getBitmap(int pos) {
return (Bitmap) getObject(pos);
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ return (GeographyData) getObject(pos);
+ }
+
private Object getObject(int pos) {
return ((Object[]) array)[pos];
}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java
index b234edef62d05b..98c77bca28f06d 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GenericRowData.java
@@ -192,6 +192,11 @@ public byte[] getBinary(int pos) {
return (byte[]) this.fields[pos];
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ return (GeographyData) this.fields[pos];
+ }
+
@Override
public ArrayData getArray(int pos) {
return (ArrayData) this.fields[pos];
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java
new file mode 100644
index 00000000000000..f65075be637c80
--- /dev/null
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/GeographyData.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.data;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.table.data.binary.BinaryGeographyData;
+import org.apache.flink.table.types.logical.GeographyType;
+
+/** An internal data structure representing data of {@link GeographyType}. */
+@PublicEvolving
+public interface GeographyData {
+
+ /** ISO WKB subtype ID for Point geometries. */
+ int POINT = 1;
+
+ /** ISO WKB subtype ID for LineString geometries. */
+ int LINE_STRING = 2;
+
+ /** ISO WKB subtype ID for Polygon geometries. */
+ int POLYGON = 3;
+
+ /** ISO WKB subtype ID for MultiPoint geometries. */
+ int MULTI_POINT = 4;
+
+ /** ISO WKB subtype ID for MultiLineString geometries. */
+ int MULTI_LINE_STRING = 5;
+
+ /** ISO WKB subtype ID for MultiPolygon geometries. */
+ int MULTI_POLYGON = 6;
+
+ /** ISO WKB subtype ID for GeometryCollection geometries. */
+ int GEOMETRY_COLLECTION = 7;
+
+ /**
+ * Converts this {@link GeographyData} object to an ISO WKB byte array.
+ *
+ *
Note: The returned byte array may be reused.
+ */
+ byte[] toBytes();
+
+ /** Returns the ISO WKB subtype ID. */
+ int subtypeId();
+
+ /** Returns the size in bytes of the ISO WKB payload. */
+ int sizeInBytes();
+
+ // ------------------------------------------------------------------------------------------
+ // Construction Utilities
+ // ------------------------------------------------------------------------------------------
+
+ /**
+ * Creates an instance of {@link GeographyData} from the given ISO WKB byte array. Returns
+ * {@code null} if the input is {@code null}.
+ */
+ static GeographyData fromBytes(byte[] bytes) {
+ return BinaryGeographyData.fromBytes(bytes);
+ }
+
+ /**
+ * Creates an instance of {@link GeographyData} from the given ISO WKB byte range. Returns
+ * {@code null} if the input is {@code null}.
+ */
+ static GeographyData fromBytes(byte[] bytes, int offset, int numBytes) {
+ return BinaryGeographyData.fromBytes(bytes, offset, numBytes);
+ }
+}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java
index ca43f1608f99ef..a783ecb572a698 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/RowData.java
@@ -111,6 +111,8 @@
* +--------------------------------+-----------------------------------------+
* | BITMAP | {@link Bitmap} |
* +--------------------------------+-----------------------------------------+
+ * | GEOGRAPHY | {@link GeographyData} |
+ * +--------------------------------+-----------------------------------------+
*
*
*
Nullability is always handled by the container data structure.
@@ -214,6 +216,12 @@ default Bitmap getBitmap(int pos) {
"This RowData implementation does not support Bitmap type.");
}
+ /** Returns the geography value at the given position. */
+ default GeographyData getGeography(int pos) {
+ throw new UnsupportedOperationException(
+ "This RowData implementation does not support Geography type.");
+ }
+
// ------------------------------------------------------------------------------------------
// Access Utilities
// ------------------------------------------------------------------------------------------
@@ -299,6 +307,9 @@ static FieldGetter createFieldGetter(LogicalType fieldType, int fieldPos) {
case BITMAP:
fieldGetter = row -> row.getBitmap(fieldPos);
break;
+ case GEOGRAPHY:
+ fieldGetter = row -> row.getGeography(fieldPos);
+ break;
case NULL:
case SYMBOL:
case UNRESOLVED:
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java
index 10a8b3e6ef71c7..f09f07d25996c1 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryArrayData.java
@@ -23,6 +23,7 @@
import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RawValueData;
import org.apache.flink.table.data.RowData;
@@ -95,6 +96,7 @@ public static int calculateFixLengthPartSize(LogicalType type) {
case RAW:
case VARIANT:
case BITMAP:
+ case GEOGRAPHY:
// long and double are 8 bytes;
// otherwise it stores the length and offset of the variable-length part for types
// such as is string, map, etc.
@@ -269,6 +271,14 @@ public byte[] getBinary(int pos) {
return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndSize);
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getElementOffset(pos, 8);
+ final long offsetAndSize = BinarySegmentUtils.getLong(segments, fieldOffset);
+ return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndSize);
+ }
+
@Override
public ArrayData getArray(int pos) {
assertIndexIsValid(pos);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java
new file mode 100644
index 00000000000000..4256f786a86816
--- /dev/null
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryGeographyData.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.data.binary;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.core.memory.MemorySegment;
+import org.apache.flink.core.memory.MemorySegmentFactory;
+import org.apache.flink.table.api.TableRuntimeException;
+import org.apache.flink.table.data.GeographyData;
+
+import java.util.Arrays;
+
+/**
+ * A binary implementation of {@link GeographyData} backed by raw ISO WKB bytes.
+ *
+ *
GEOGRAPHY uses OGC:CRS84 by contract, but ISO WKB does not encode CRS or SRID metadata. This
+ * container stores the raw ISO WKB payload only; CRS validation, CRS transformation, and EWKB/SRID
+ * handling belong to constructors, functions, and connector schema mapping.
+ */
+@Internal
+public final class BinaryGeographyData extends BinarySection implements GeographyData {
+
+ private static final int MIN_WKB_HEADER_SIZE = 5;
+ private static final int WKB_COUNT_SIZE = 4;
+ private static final int WKB_POINT_COORDINATE_SIZE = 16;
+ private static final int BIG_ENDIAN = 0;
+ private static final int LITTLE_ENDIAN = 1;
+
+ private final int subtypeId;
+
+ private BinaryGeographyData(MemorySegment[] segments, int offset, int sizeInBytes) {
+ super(segments, offset, sizeInBytes);
+ this.subtypeId = readSubtypeId(segments, offset, sizeInBytes);
+ }
+
+ /** Creates a {@link BinaryGeographyData} instance from the given address and length. */
+ public static BinaryGeographyData fromAddress(
+ MemorySegment[] segments, int offset, int numBytes) {
+ return new BinaryGeographyData(segments, offset, numBytes);
+ }
+
+ /** Creates a {@link BinaryGeographyData} instance from the given ISO WKB bytes. */
+ public static BinaryGeographyData fromBytes(byte[] bytes) {
+ return bytes == null ? null : fromBytes(bytes, 0, bytes.length);
+ }
+
+ /** Creates a {@link BinaryGeographyData} instance from the given ISO WKB byte range. */
+ public static BinaryGeographyData fromBytes(byte[] bytes, int offset, int numBytes) {
+ if (bytes == null) {
+ return null;
+ }
+ checkRange(bytes, offset, numBytes);
+ byte[] copy = Arrays.copyOfRange(bytes, offset, offset + numBytes);
+ return fromAddress(new MemorySegment[] {MemorySegmentFactory.wrap(copy)}, 0, copy.length);
+ }
+
+ @Override
+ public byte[] toBytes() {
+ return BinarySegmentUtils.copyToBytes(segments, offset, sizeInBytes);
+ }
+
+ @Override
+ public int subtypeId() {
+ return subtypeId;
+ }
+
+ @Override
+ public int sizeInBytes() {
+ return sizeInBytes;
+ }
+
+ private static void checkRange(byte[] bytes, int offset, int numBytes) {
+ if (offset < 0 || numBytes < 0 || offset > bytes.length - numBytes) {
+ throw new TableRuntimeException(
+ String.format(
+ "Invalid ISO WKB byte range: offset %d, length %d, array length %d.",
+ offset, numBytes, bytes.length));
+ }
+ }
+
+ private static int readSubtypeId(MemorySegment[] segments, int offset, int sizeInBytes) {
+ final long endOffset = (long) offset + sizeInBytes;
+ final GeometryHeader header = readHeader(segments, offset, endOffset);
+ final long consumedOffset = validateGeometry(segments, offset, endOffset);
+ if (consumedOffset != endOffset) {
+ throw new TableRuntimeException(
+ String.format(
+ "Malformed ISO WKB payload. Found %d trailing byte(s).",
+ endOffset - consumedOffset));
+ }
+ return header.subtypeId;
+ }
+
+ private static long validateGeometry(
+ MemorySegment[] segments, long geometryOffset, long endOffset) {
+ final GeometryHeader header = readHeader(segments, geometryOffset, endOffset);
+ long cursor = geometryOffset + MIN_WKB_HEADER_SIZE;
+
+ switch (header.subtypeId) {
+ case GeographyData.POINT:
+ return requireBytes(
+ cursor, WKB_POINT_COORDINATE_SIZE, endOffset, "POINT coordinates")
+ + WKB_POINT_COORDINATE_SIZE;
+ case GeographyData.LINE_STRING:
+ return skipCoordinateSequence(segments, cursor, endOffset, header.byteOrder);
+ case GeographyData.POLYGON:
+ return skipPolygon(segments, cursor, endOffset, header.byteOrder);
+ case GeographyData.MULTI_POINT:
+ return skipTypedGeometryCollection(
+ segments, cursor, endOffset, header.byteOrder, GeographyData.POINT);
+ case GeographyData.MULTI_LINE_STRING:
+ return skipTypedGeometryCollection(
+ segments, cursor, endOffset, header.byteOrder, GeographyData.LINE_STRING);
+ case GeographyData.MULTI_POLYGON:
+ return skipTypedGeometryCollection(
+ segments, cursor, endOffset, header.byteOrder, GeographyData.POLYGON);
+ case GeographyData.GEOMETRY_COLLECTION:
+ return skipGeometryCollection(segments, cursor, endOffset, header.byteOrder);
+ default:
+ throw new TableRuntimeException(
+ String.format(
+ "Malformed ISO WKB payload. Unsupported geography subtype ID %d.",
+ header.subtypeId));
+ }
+ }
+
+ private static long skipCoordinateSequence(
+ MemorySegment[] segments, long offset, long endOffset, int byteOrder) {
+ final long numPoints =
+ readUnsignedInt(segments, offset, endOffset, byteOrder, "point count");
+ long cursor = offset + WKB_COUNT_SIZE;
+ final long coordinateSequenceSize =
+ multiplyExact(numPoints, WKB_POINT_COORDINATE_SIZE, "coordinate sequence");
+ return requireBytes(cursor, coordinateSequenceSize, endOffset, "coordinate sequence")
+ + coordinateSequenceSize;
+ }
+
+ private static long skipPolygon(
+ MemorySegment[] segments, long offset, long endOffset, int byteOrder) {
+ final long numRings = readUnsignedInt(segments, offset, endOffset, byteOrder, "ring count");
+ long cursor = offset + WKB_COUNT_SIZE;
+ for (long i = 0; i < numRings; i++) {
+ cursor = skipCoordinateSequence(segments, cursor, endOffset, byteOrder);
+ }
+ return cursor;
+ }
+
+ private static long skipTypedGeometryCollection(
+ MemorySegment[] segments,
+ long offset,
+ long endOffset,
+ int byteOrder,
+ int expectedSubtypeId) {
+ final long numGeometries =
+ readUnsignedInt(segments, offset, endOffset, byteOrder, "geometry count");
+ long cursor = offset + WKB_COUNT_SIZE;
+ for (long i = 0; i < numGeometries; i++) {
+ final GeometryHeader nestedHeader = readHeader(segments, cursor, endOffset);
+ if (nestedHeader.subtypeId != expectedSubtypeId) {
+ throw new TableRuntimeException(
+ String.format(
+ "Malformed ISO WKB payload. Expected nested subtype ID %d but found %d.",
+ expectedSubtypeId, nestedHeader.subtypeId));
+ }
+ cursor = validateGeometry(segments, cursor, endOffset);
+ }
+ return cursor;
+ }
+
+ private static long skipGeometryCollection(
+ MemorySegment[] segments, long offset, long endOffset, int byteOrder) {
+ final long numGeometries =
+ readUnsignedInt(segments, offset, endOffset, byteOrder, "geometry count");
+ long cursor = offset + WKB_COUNT_SIZE;
+ for (long i = 0; i < numGeometries; i++) {
+ cursor = validateGeometry(segments, cursor, endOffset);
+ }
+ return cursor;
+ }
+
+ private static GeometryHeader readHeader(
+ MemorySegment[] segments, long offset, long endOffset) {
+ requireBytes(offset, MIN_WKB_HEADER_SIZE, endOffset, "WKB header");
+
+ final int byteOrder = BinarySegmentUtils.getByte(segments, (int) offset) & 0xFF;
+ if (byteOrder != BIG_ENDIAN && byteOrder != LITTLE_ENDIAN) {
+ throw new TableRuntimeException(
+ String.format(
+ "Malformed ISO WKB payload. Unsupported byte order %d.", byteOrder));
+ }
+
+ final long subtypeId =
+ readUnsignedInt(segments, offset + 1, endOffset, byteOrder, "subtype ID");
+
+ if (subtypeId < GeographyData.POINT || subtypeId > GeographyData.GEOMETRY_COLLECTION) {
+ throw new TableRuntimeException(
+ String.format(
+ "Malformed ISO WKB payload. Unsupported geography subtype ID %d.",
+ subtypeId));
+ }
+ return new GeometryHeader(byteOrder, (int) subtypeId);
+ }
+
+ private static long readUnsignedInt(
+ MemorySegment[] segments,
+ long offset,
+ long endOffset,
+ int byteOrder,
+ String fieldName) {
+ requireBytes(offset, WKB_COUNT_SIZE, endOffset, fieldName);
+
+ if (byteOrder == LITTLE_ENDIAN) {
+ return (BinarySegmentUtils.getByte(segments, (int) offset) & 0xFFL)
+ | ((BinarySegmentUtils.getByte(segments, (int) offset + 1) & 0xFFL) << 8)
+ | ((BinarySegmentUtils.getByte(segments, (int) offset + 2) & 0xFFL) << 16)
+ | ((BinarySegmentUtils.getByte(segments, (int) offset + 3) & 0xFFL) << 24);
+ }
+ return ((BinarySegmentUtils.getByte(segments, (int) offset) & 0xFFL) << 24)
+ | ((BinarySegmentUtils.getByte(segments, (int) offset + 1) & 0xFFL) << 16)
+ | ((BinarySegmentUtils.getByte(segments, (int) offset + 2) & 0xFFL) << 8)
+ | (BinarySegmentUtils.getByte(segments, (int) offset + 3) & 0xFFL);
+ }
+
+ private static long requireBytes(
+ long offset, long numBytes, long endOffset, String componentName) {
+ if (offset > endOffset || endOffset - offset < numBytes) {
+ throw new TableRuntimeException(
+ String.format(
+ "Malformed ISO WKB payload. Incomplete %s: expected %d byte(s) but found %d.",
+ componentName, numBytes, Math.max(0, endOffset - offset)));
+ }
+ return offset;
+ }
+
+ private static long multiplyExact(long value, int factor, String componentName) {
+ final long result = value * factor;
+ if (value != 0 && result / value != factor) {
+ throw new TableRuntimeException(
+ String.format("Malformed ISO WKB payload. %s size overflows.", componentName));
+ }
+ return result;
+ }
+
+ private static final class GeometryHeader {
+ private final int byteOrder;
+ private final int subtypeId;
+
+ private GeometryHeader(int byteOrder, int subtypeId) {
+ this.byteOrder = byteOrder;
+ this.subtypeId = subtypeId;
+ }
+ }
+}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java
index 45bcf7cd3f1128..04c9a0e50ed43d 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinaryRowData.java
@@ -22,6 +22,7 @@
import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RawValueData;
import org.apache.flink.table.data.RowData;
@@ -372,6 +373,14 @@ public byte[] getBinary(int pos) {
return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen);
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndLen = segments[0].getLong(fieldOffset);
+ return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndLen);
+ }
+
@Override
public ArrayData getArray(int pos) {
assertIndexIsValid(pos);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java
index 653fc559df9bce..2dd12978d6bf5f 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/BinarySegmentUtils.java
@@ -1056,6 +1056,34 @@ public static byte[] readBinary(
}
}
+ /**
+ * Get geography data, if len less than 8, it will be included in variablePartOffsetAndLen.
+ *
+ * @param baseOffset base offset of composite binary format.
+ * @param fieldOffset absolute start offset of 'variablePartOffsetAndLen'.
+ * @param variablePartOffsetAndLen a long value, real data or offset and len.
+ */
+ public static BinaryGeographyData readGeographyData(
+ MemorySegment[] segments,
+ int baseOffset,
+ int fieldOffset,
+ long variablePartOffsetAndLen) {
+ long mark = variablePartOffsetAndLen & HIGHEST_FIRST_BIT;
+ if (mark == 0) {
+ final int subOffset = (int) (variablePartOffsetAndLen >> 32);
+ final int len = (int) variablePartOffsetAndLen;
+ return BinaryGeographyData.fromAddress(segments, baseOffset + subOffset, len);
+ } else {
+ int len = (int) ((variablePartOffsetAndLen & HIGHEST_SECOND_TO_EIGHTH_BIT) >>> 56);
+ if (BinarySegmentUtils.LITTLE_ENDIAN) {
+ return BinaryGeographyData.fromAddress(segments, fieldOffset, len);
+ } else {
+ // fieldOffset + 1 to skip header.
+ return BinaryGeographyData.fromAddress(segments, fieldOffset + 1, len);
+ }
+ }
+ }
+
/**
* Get binary string, if len less than 8, will be include in variablePartOffsetAndLen.
*
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java
index 27085b487ae50d..225d3e103fe103 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/binary/NestedRowData.java
@@ -21,6 +21,7 @@
import org.apache.flink.core.memory.MemorySegmentFactory;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RawValueData;
import org.apache.flink.table.data.RowData;
@@ -294,6 +295,14 @@ public byte[] getBinary(int pos) {
return BinarySegmentUtils.readBinary(segments, offset, fieldOffset, offsetAndLen);
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ assertIndexIsValid(pos);
+ int fieldOffset = getFieldOffset(pos);
+ final long offsetAndLen = BinarySegmentUtils.getLong(segments, fieldOffset);
+ return BinarySegmentUtils.readGeographyData(segments, offset, fieldOffset, offsetAndLen);
+ }
+
@Override
public RowData getRow(int pos, int numFields) {
assertIndexIsValid(pos);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java
index a3b8e7dd56daf9..9c6e6ad8c317d0 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarArrayData.java
@@ -21,6 +21,7 @@
import org.apache.flink.annotation.Internal;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RawValueData;
import org.apache.flink.table.data.RowData;
@@ -147,6 +148,12 @@ public byte[] getBinary(int pos) {
}
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ BytesColumnVector.Bytes byteArray = getByteArray(pos);
+ return GeographyData.fromBytes(byteArray.data, byteArray.offset, byteArray.len);
+ }
+
@Override
public ArrayData getArray(int pos) {
return ((ArrayColumnVector) data).getArray(offset + pos);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java
index bd3dce1edf6bae..0413ac980f737a 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/columnar/ColumnarRowData.java
@@ -21,6 +21,7 @@
import org.apache.flink.annotation.Internal;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RawValueData;
import org.apache.flink.table.data.RowData;
@@ -152,6 +153,12 @@ public byte[] getBinary(int pos) {
}
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ Bytes byteArray = vectorizedColumnBatch.getByteArray(rowId, pos);
+ return GeographyData.fromBytes(byteArray.data, byteArray.offset, byteArray.len);
+ }
+
@Override
public RowData getRow(int pos, int numFields) {
return vectorizedColumnBatch.getRow(rowId, pos);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java
index b9eff05a6c388f..d1420debfdb07a 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/JoinedRowData.java
@@ -21,6 +21,7 @@
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RawValueData;
import org.apache.flink.table.data.RowData;
@@ -269,6 +270,15 @@ public Bitmap getBitmap(int pos) {
}
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ if (pos < row1.getArity()) {
+ return row1.getGeography(pos);
+ } else {
+ return row2.getGeography(pos - row1.getArity());
+ }
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java
index 7f29ca5e613236..e6a173bb4cd991 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/data/utils/ProjectedRowData.java
@@ -22,6 +22,7 @@
import org.apache.flink.table.connector.Projection;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RawValueData;
import org.apache.flink.table.data.RowData;
@@ -178,6 +179,11 @@ public Bitmap getBitmap(int pos) {
return row.getBitmap(indexMapping[pos]);
}
+ @Override
+ public GeographyData getGeography(int pos) {
+ return row.getGeography(indexMapping[pos]);
+ }
+
@Override
public boolean equals(Object o) {
throw new UnsupportedOperationException("Projected row data cannot be compared");
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
index 0828f2bb8b4e92..b95f6f6910688d 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
@@ -946,7 +946,8 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL)
.inputTypeStrategy(LATERAL_SNAPSHOT_INPUT_TYPE_STRATEGY)
.outputTypeStrategy(LATERAL_SNAPSHOT_OUTPUT_TYPE_STRATEGY)
.runtimeProvided()
- // TODO: disableSystemArguments(true), once we have a dedicated translation rule
+ // SNAPSHOT does not support the implicit PTF system arguments (on_time, uid)
+ .disableSystemArguments(true)
.notDeterministic()
.build();
@@ -2929,7 +2930,7 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL)
sequence(
logical(LogicalTypeFamily.CHARACTER_STRING),
symbol(JsonType.class))))
- .outputTypeStrategy(explicit(BOOLEAN().notNull()))
+ .outputTypeStrategy(nullableIfArgs(explicit(BOOLEAN())))
.runtimeDeferred()
.build();
@@ -3115,6 +3116,63 @@ ANY, and(logical(LogicalTypeRoot.BOOLEAN), LITERAL)
"org.apache.flink.table.runtime.functions.scalar.TryParseJsonFunction")
.build();
+ // --------------------------------------------------------------------------------------------
+ // Geography functions
+ // --------------------------------------------------------------------------------------------
+
+ public static final BuiltInFunctionDefinition ST_GEOGFROMTEXT =
+ BuiltInFunctionDefinition.newBuilder()
+ .name("ST_GEOGFROMTEXT")
+ .kind(SCALAR)
+ .inputTypeStrategy(
+ sequence(
+ Collections.singletonList("text"),
+ Collections.singletonList(
+ logical(LogicalTypeFamily.CHARACTER_STRING))))
+ .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.GEOGRAPHY())))
+ .runtimeClass(
+ "org.apache.flink.table.runtime.functions.scalar.StGeogFromTextFunction")
+ .build();
+
+ public static final BuiltInFunctionDefinition ST_GEOGFROMWKB =
+ BuiltInFunctionDefinition.newBuilder()
+ .name("ST_GEOGFROMWKB")
+ .kind(SCALAR)
+ .inputTypeStrategy(
+ sequence(
+ Collections.singletonList("bytes"),
+ Collections.singletonList(
+ logical(LogicalTypeFamily.BINARY_STRING))))
+ .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.GEOGRAPHY())))
+ .runtimeClass(
+ "org.apache.flink.table.runtime.functions.scalar.StGeogFromWkbFunction")
+ .build();
+
+ public static final BuiltInFunctionDefinition ST_ASTEXT =
+ BuiltInFunctionDefinition.newBuilder()
+ .name("ST_ASTEXT")
+ .kind(SCALAR)
+ .inputTypeStrategy(
+ sequence(
+ Collections.singletonList("geography"),
+ Collections.singletonList(logical(LogicalTypeRoot.GEOGRAPHY))))
+ .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.STRING())))
+ .runtimeClass(
+ "org.apache.flink.table.runtime.functions.scalar.StAsTextFunction")
+ .build();
+
+ public static final BuiltInFunctionDefinition ST_ASWKB =
+ BuiltInFunctionDefinition.newBuilder()
+ .name("ST_ASWKB")
+ .kind(SCALAR)
+ .inputTypeStrategy(
+ sequence(
+ Collections.singletonList("geography"),
+ Collections.singletonList(logical(LogicalTypeRoot.GEOGRAPHY))))
+ .outputTypeStrategy(nullableIfArgs(explicit(DataTypes.BYTES())))
+ .runtimeClass("org.apache.flink.table.runtime.functions.scalar.StAsWkbFunction")
+ .build();
+
// --------------------------------------------------------------------------------------------
// Bitmap functions
// --------------------------------------------------------------------------------------------
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/CastInputTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/CastInputTypeStrategy.java
index fbd4bf188d7519..3bd66e01cd5ecc 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/CastInputTypeStrategy.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/CastInputTypeStrategy.java
@@ -35,6 +35,7 @@
import java.util.List;
import java.util.Optional;
+import static org.apache.flink.table.types.logical.utils.LogicalTypeCasts.getUnsupportedCastHint;
import static org.apache.flink.table.types.logical.utils.LogicalTypeCasts.supportsExplicitCast;
/**
@@ -69,6 +70,15 @@ public Optional> inferInputTypes(
return Optional.of(argumentDataTypes);
}
if (!supportsExplicitCast(fromType, toType)) {
+ final Optional hint = getUnsupportedCastHint(fromType, toType);
+ if (hint.isPresent()) {
+ return callContext.fail(
+ throwOnFailure,
+ "Unsupported cast from '%s' to '%s'. %s",
+ fromType,
+ toType,
+ hint.get());
+ }
return callContext.fail(
throwOnFailure, "Unsupported cast from '%s' to '%s'.", fromType, toType);
}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/LateralSnapshotTypeStrategy.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/LateralSnapshotTypeStrategy.java
index 8ec2d4899c975d..49bead4730d099 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/LateralSnapshotTypeStrategy.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/inference/strategies/LateralSnapshotTypeStrategy.java
@@ -69,21 +69,31 @@
@Internal
public final class LateralSnapshotTypeStrategy {
- /** Argument index of the {@code input} TABLE. */
+ /** The {@code input} TABLE argument. */
public static final int INPUT_ARG_INDEX = 0;
- /** Argument index of the {@code load_completed_condition} STRING. */
+ public static final String INPUT_ARG_NAME = "input";
+
+ /** The {@code load_completed_condition} STRING argument. */
public static final int LOAD_COMPLETED_CONDITION_ARG_INDEX = 1;
- /** Argument index of the {@code load_completed_time} TIMESTAMP_LTZ. */
+ public static final String LOAD_COMPLETED_CONDITION_ARG_NAME = "load_completed_condition";
+
+ /** The {@code load_completed_time} TIMESTAMP_LTZ argument. */
public static final int LOAD_COMPLETED_TIME_ARG_INDEX = 2;
- /** Argument index of the {@code load_completed_idle_timeout} INTERVAL. */
+ public static final String LOAD_COMPLETED_TIME_ARG_NAME = "load_completed_time";
+
+ /** The {@code load_completed_idle_timeout} INTERVAL argument. */
public static final int LOAD_COMPLETED_IDLE_TIMEOUT_ARG_INDEX = 3;
- /** Argument index of the {@code state_ttl} INTERVAL. */
+ public static final String LOAD_COMPLETED_IDLE_TIMEOUT_ARG_NAME = "load_completed_idle_timeout";
+
+ /** The {@code state_ttl} INTERVAL argument. */
public static final int STATE_TTL_ARG_INDEX = 4;
+ public static final String STATE_TTL_ARG_NAME = "state_ttl";
+
/** Default value for {@code load_completed_condition}. */
public static final String LOAD_COMPLETED_CONDITION_COMPILE_TIME = "compile_time";
@@ -122,11 +132,13 @@ public Optional> inferInputTypes(
public List getExpectedSignatures(final FunctionDefinition definition) {
return List.of(
Signature.of(
- Argument.of("input", "TABLE"),
- Argument.of("load_completed_condition", "STRING"),
- Argument.of("load_completed_time", "TIMESTAMP_LTZ(3)"),
- Argument.of("load_completed_idle_timeout", "INTERVAL SECOND"),
- Argument.of("state_ttl", "INTERVAL SECOND")));
+ Argument.of(INPUT_ARG_NAME, "TABLE"),
+ Argument.of(LOAD_COMPLETED_CONDITION_ARG_NAME, "STRING"),
+ Argument.of(LOAD_COMPLETED_TIME_ARG_NAME, "TIMESTAMP_LTZ(3)"),
+ Argument.of(
+ LOAD_COMPLETED_IDLE_TIMEOUT_ARG_NAME,
+ "INTERVAL SECOND"),
+ Argument.of(STATE_TTL_ARG_NAME, "INTERVAL SECOND")));
}
};
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java
new file mode 100644
index 00000000000000..403ee88e0cfccd
--- /dev/null
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/GeographyType.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.types.logical;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.table.data.GeographyData;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Data type of geography data.
+ *
+ *
The serializable string representation of this type is {@code GEOGRAPHY}.
+ */
+@PublicEvolving
+public final class GeographyType extends LogicalType {
+
+ private static final long serialVersionUID = 1L;
+
+ private static final String FORMAT = "GEOGRAPHY";
+
+ private static final Class INPUT_OUTPUT_CONVERSION = GeographyData.class;
+
+ public GeographyType(boolean isNullable) {
+ super(isNullable, LogicalTypeRoot.GEOGRAPHY);
+ }
+
+ public GeographyType() {
+ this(true);
+ }
+
+ @Override
+ public LogicalType copy(boolean isNullable) {
+ return new GeographyType(isNullable);
+ }
+
+ @Override
+ public String asSerializableString() {
+ return withNullability(FORMAT);
+ }
+
+ @Override
+ public boolean supportsInputConversion(Class clazz) {
+ return INPUT_OUTPUT_CONVERSION.isAssignableFrom(clazz);
+ }
+
+ @Override
+ public boolean supportsOutputConversion(Class clazz) {
+ return INPUT_OUTPUT_CONVERSION.isAssignableFrom(clazz);
+ }
+
+ @Override
+ public Class getDefaultConversion() {
+ return INPUT_OUTPUT_CONVERSION;
+ }
+
+ @Override
+ public List getChildren() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public R accept(LogicalTypeVisitor visitor) {
+ return visitor.visit(this);
+ }
+}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java
index 6c823add433bfb..c21abce4af7461 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeRoot.java
@@ -145,7 +145,9 @@ public enum LogicalTypeRoot {
VARIANT(LogicalTypeFamily.EXTENSION),
- BITMAP(LogicalTypeFamily.EXTENSION);
+ BITMAP(LogicalTypeFamily.EXTENSION),
+
+ GEOGRAPHY(LogicalTypeFamily.EXTENSION);
private final Set families;
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java
index 6a0e5614466d11..b35ffe05541f44 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/LogicalTypeVisitor.java
@@ -101,5 +101,9 @@ default R visit(BitmapType bitmapType) {
return visit((LogicalType) bitmapType);
}
+ default R visit(GeographyType geographyType) {
+ return visit((LogicalType) geographyType);
+ }
+
R visit(LogicalType other);
}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java
index a0afca2caa09c3..58f474ec0eb70b 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeCasts.java
@@ -37,6 +37,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.BiPredicate;
@@ -79,6 +80,7 @@
import static org.apache.flink.table.types.logical.LogicalTypeRoot.TINYINT;
import static org.apache.flink.table.types.logical.LogicalTypeRoot.VARBINARY;
import static org.apache.flink.table.types.logical.LogicalTypeRoot.VARCHAR;
+import static org.apache.flink.table.types.logical.LogicalTypeRoot.VARIANT;
import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getDayPrecision;
import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getFractionalPrecision;
import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.getLength;
@@ -646,6 +648,9 @@ private static boolean supportsCasting(
// BITMAP can only be cast to BYTES (unbounded VARBINARY), because trimming or padding
// would corrupt the serialized bitmap data.
return allowExplicit && getLength(targetType) == VarBinaryType.MAX_LENGTH;
+ } else if (sourceRoot == VARIANT) {
+ // a VARIANT can only be explicitly cast to a supported scalar type
+ return allowExplicit && supportsVariantToScalarCast(targetType);
}
if (implicitCastingRules.get(targetRoot).contains(sourceRoot)) {
@@ -726,6 +731,45 @@ private static boolean supportsConstructedCasting(
return false;
}
+ private static boolean supportsVariantToScalarCast(LogicalType targetType) {
+ switch (targetType.getTypeRoot()) {
+ case BOOLEAN:
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+ case FLOAT:
+ case DOUBLE:
+ case DECIMAL:
+ case BINARY:
+ case VARBINARY:
+ case DATE:
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return true;
+ default:
+ // TIME has no counterpart in the Variant type model. CHARACTER_STRING is handled by
+ // the display-oriented VariantToStringCastRule and is intentionally not offered as
+ // a
+ // user-facing cast here.
+ return false;
+ }
+ }
+
+ /**
+ * Returns a hint pointing to the function that performs a conceptually related operation when
+ * an explicit cast is unsupported, or empty when no specific hint applies.
+ */
+ public static Optional getUnsupportedCastHint(
+ LogicalType sourceType, LogicalType targetType) {
+ if (sourceType.is(VARIANT) && targetType.is(CHARACTER_STRING)) {
+ return Optional.of(
+ "Use the JSON_STRING function to convert a VARIANT to its JSON string "
+ + "representation.");
+ }
+ return Optional.empty();
+ }
+
private static CastingRuleBuilder castTo(LogicalTypeRoot targetType) {
return new CastingRuleBuilder(targetType);
}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java
index 056ec5953ccdc3..43237c443f56e9 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeDefaultVisitor.java
@@ -31,6 +31,7 @@
import org.apache.flink.table.types.logical.DistinctType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
import org.apache.flink.table.types.logical.LogicalType;
@@ -203,6 +204,11 @@ public R visit(BitmapType bitmapType) {
return defaultMethod(bitmapType);
}
+ @Override
+ public R visit(GeographyType geographyType) {
+ return defaultMethod(geographyType);
+ }
+
@Override
public R visit(LogicalType other) {
return defaultMethod(other);
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java
index 3711ac76f4fd2a..dfa2e7cf358147 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeParser.java
@@ -36,6 +36,7 @@
import org.apache.flink.table.types.logical.DescriptorType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LegacyTypeInformationType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
@@ -336,7 +337,8 @@ private enum Keyword {
DESCRIPTOR,
STRUCTURED,
VARIANT,
- BITMAP
+ BITMAP,
+ GEOGRAPHY
}
private static final Set KEYWORDS =
@@ -586,6 +588,8 @@ private LogicalType parseTypeByKeyword() {
return new VariantType();
case BITMAP:
return new BitmapType();
+ case GEOGRAPHY:
+ return new GeographyType();
default:
throw parsingError("Unsupported type: " + token().value);
}
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java
index 98629b15fc1770..fa994ed160f3c7 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/utils/LogicalTypeUtils.java
@@ -21,6 +21,7 @@
import org.apache.flink.annotation.Internal;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.data.MapData;
import org.apache.flink.table.data.RawValueData;
import org.apache.flink.table.data.RowData;
@@ -115,6 +116,8 @@ public static Class toInternalConversionClass(LogicalType type) {
return Variant.class;
case BITMAP:
return Bitmap.class;
+ case GEOGRAPHY:
+ return GeographyData.class;
case SYMBOL:
case UNRESOLVED:
default:
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java
index 9aa83f3cbee1d3..929aaec786f490 100644
--- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/utils/LogicalTypeDataTypeConverter.java
@@ -38,6 +38,7 @@
import org.apache.flink.table.types.logical.DistinctType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
import org.apache.flink.table.types.logical.LogicalType;
@@ -265,6 +266,11 @@ public DataType visit(BitmapType bitmapType) {
return new AtomicDataType(bitmapType);
}
+ @Override
+ public DataType visit(GeographyType geographyType) {
+ return new AtomicDataType(geographyType);
+ }
+
@Override
public DataType visit(LogicalType other) {
if (other.is(LogicalTypeRoot.UNRESOLVED)) {
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java
index 7b54e40f387935..7b4aef14cc3a80 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/api/SchemaTest.java
@@ -51,5 +51,21 @@ void testFromResolvedSchema() {
assertThat(newSchema.resolve(new TestSchemaResolver())).isEqualTo(originalSchema);
}
+
+ @Test
+ void testGeographyColumn() {
+ final Schema schema =
+ Schema.newBuilder()
+ .column("location", DataTypes.GEOGRAPHY())
+ .column("required_location", DataTypes.GEOGRAPHY().notNull())
+ .build();
+
+ assertThat(schema.resolve(new TestSchemaResolver()))
+ .isEqualTo(
+ ResolvedSchema.of(
+ Column.physical("location", DataTypes.GEOGRAPHY()),
+ Column.physical(
+ "required_location", DataTypes.GEOGRAPHY().notNull())));
+ }
}
}
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java
new file mode 100644
index 00000000000000..4e45d2d28549aa
--- /dev/null
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/data/binary/BinaryGeographyDataTest.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.data.binary;
+
+import org.apache.flink.core.memory.MemorySegment;
+import org.apache.flink.core.memory.MemorySegmentFactory;
+import org.apache.flink.table.api.TableRuntimeException;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.GeographyData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.GeographyType;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.Arrays;
+import java.util.stream.Stream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Tests for {@link BinaryGeographyData}. */
+class BinaryGeographyDataTest {
+
+ private static final int BIG_ENDIAN = 0;
+ private static final int LITTLE_ENDIAN = 1;
+ private static final byte[] POINT_WKB = pointWkb(LITTLE_ENDIAN);
+
+ @Test
+ void testValidWkbRoundTrip() {
+ final GeographyData geography = GeographyData.fromBytes(POINT_WKB);
+
+ assertThat(geography).isInstanceOf(BinaryGeographyData.class);
+ assertThat(geography.toBytes()).isEqualTo(POINT_WKB);
+ assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT);
+ assertThat(geography.sizeInBytes()).isEqualTo(POINT_WKB.length);
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("supportedSubtypePayloads")
+ void testSupported2dSubtypePayloads(String name, int subtypeId, byte[] wkb) {
+ final BinaryGeographyData geography = BinaryGeographyData.fromBytes(wkb);
+
+ assertThat(geography.subtypeId()).isEqualTo(subtypeId);
+ assertThat(geography.toBytes()).isEqualTo(wkb);
+ assertThat(geography.sizeInBytes()).isEqualTo(wkb.length);
+ }
+
+ static Stream supportedSubtypePayloads() {
+ return Stream.of(
+ Arguments.of("Point", GeographyData.POINT, pointWkb(LITTLE_ENDIAN)),
+ Arguments.of("LineString", GeographyData.LINE_STRING, lineStringWkb(LITTLE_ENDIAN)),
+ Arguments.of("Polygon", GeographyData.POLYGON, polygonWkb(LITTLE_ENDIAN)),
+ Arguments.of("MultiPoint", GeographyData.MULTI_POINT, multiPointWkb(LITTLE_ENDIAN)),
+ Arguments.of(
+ "MultiLineString",
+ GeographyData.MULTI_LINE_STRING,
+ multiLineStringWkb(LITTLE_ENDIAN)),
+ Arguments.of(
+ "MultiPolygon",
+ GeographyData.MULTI_POLYGON,
+ multiPolygonWkb(LITTLE_ENDIAN)),
+ Arguments.of(
+ "GeometryCollection",
+ GeographyData.GEOMETRY_COLLECTION,
+ geometryCollectionWkb(LITTLE_ENDIAN)));
+ }
+
+ @Test
+ void testBigEndianSubtypeExtraction() {
+ final BinaryGeographyData point = BinaryGeographyData.fromBytes(pointWkb(BIG_ENDIAN));
+ final BinaryGeographyData lineString =
+ BinaryGeographyData.fromBytes(lineStringWkb(BIG_ENDIAN));
+
+ assertThat(point.subtypeId()).isEqualTo(GeographyData.POINT);
+ assertThat(lineString.subtypeId()).isEqualTo(GeographyData.LINE_STRING);
+ }
+
+ @Test
+ void testNullHandling() {
+ assertThat(GeographyData.fromBytes((byte[]) null)).isNull();
+ assertThat(GeographyData.fromBytes(null, 0, 0)).isNull();
+ assertThat(BinaryGeographyData.fromBytes((byte[]) null)).isNull();
+ assertThat(BinaryGeographyData.fromBytes(null, 0, 0)).isNull();
+ }
+
+ @Test
+ void testGenericRowDataNullPath() {
+ final RowData.FieldGetter getter = RowData.createFieldGetter(new GeographyType(), 0);
+ final GenericRowData nullRow = GenericRowData.of((Object) null);
+ final GeographyData geography = GeographyData.fromBytes(POINT_WKB);
+ final GenericRowData valueRow = GenericRowData.of(geography);
+
+ assertThat(getter.getFieldOrNull(nullRow)).isNull();
+ assertThat(getter.getFieldOrNull(valueRow)).isSameAs(geography);
+ }
+
+ @Test
+ void testBinaryRowDataNullPath() {
+ final RowData.FieldGetter getter = RowData.createFieldGetter(new GeographyType(), 0);
+ final int fixedLength = BinaryRowData.calculateFixPartSizeInBytes(1);
+ final BinaryRowData nullRow = new BinaryRowData(1);
+ nullRow.pointTo(MemorySegmentFactory.wrap(new byte[fixedLength]), 0, fixedLength);
+ nullRow.setNullAt(0);
+
+ assertThat(getter.getFieldOrNull(nullRow)).isNull();
+ }
+
+ @Test
+ void testBinaryRowDataValuePath() {
+ final BinaryRowData row = binaryRowWithGeography(POINT_WKB);
+
+ assertThat(row.getGeography(0).toBytes()).isEqualTo(POINT_WKB);
+ assertThat(row.getGeography(0).subtypeId()).isEqualTo(GeographyData.POINT);
+ }
+
+ @Test
+ void testByteRangeCopiesOnlySelectedPayload() {
+ final byte[] bytes = concat(bytes(42), POINT_WKB, bytes(99));
+ final BinaryGeographyData geography =
+ BinaryGeographyData.fromBytes(bytes, 1, POINT_WKB.length);
+
+ bytes[1] = 0;
+
+ assertThat(geography.toBytes()).isEqualTo(POINT_WKB);
+ assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT);
+ }
+
+ @Test
+ void testFromAddressSupportsSegmentBoundary() {
+ final byte[] first = concat(new byte[14], Arrays.copyOfRange(POINT_WKB, 0, 2));
+ final byte[] second = Arrays.copyOfRange(POINT_WKB, 2, 18);
+ final byte[] third =
+ concat(Arrays.copyOfRange(POINT_WKB, 18, POINT_WKB.length), new byte[13]);
+ final MemorySegment[] segments =
+ new MemorySegment[] {
+ MemorySegmentFactory.wrap(first),
+ MemorySegmentFactory.wrap(second),
+ MemorySegmentFactory.wrap(third)
+ };
+
+ final BinaryGeographyData geography =
+ BinaryGeographyData.fromAddress(segments, 14, POINT_WKB.length);
+
+ assertThat(geography.toBytes()).isEqualTo(POINT_WKB);
+ assertThat(geography.subtypeId()).isEqualTo(GeographyData.POINT);
+ }
+
+ @Test
+ void testMalformedPayloadBoundaries() {
+ assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes()))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Incomplete WKB header");
+
+ assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes(1, 1, 0, 0)))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Incomplete WKB header");
+
+ assertThatThrownBy(() -> BinaryGeographyData.fromBytes(bytes(2, 1, 0, 0, 0)))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Unsupported byte order 2");
+
+ assertThatThrownBy(() -> BinaryGeographyData.fromBytes(header(LITTLE_ENDIAN, 0)))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Unsupported geography subtype ID 0");
+
+ assertThatThrownBy(() -> BinaryGeographyData.fromBytes(header(LITTLE_ENDIAN, 8)))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Unsupported geography subtype ID 8");
+ }
+
+ @Test
+ void testStructurallyIncompletePayloads() {
+ assertThatThrownBy(
+ () ->
+ BinaryGeographyData.fromBytes(
+ header(LITTLE_ENDIAN, GeographyData.POINT)))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Incomplete POINT coordinates");
+
+ assertThatThrownBy(
+ () ->
+ BinaryGeographyData.fromBytes(
+ concat(
+ header(LITTLE_ENDIAN, GeographyData.LINE_STRING),
+ unsignedInt(LITTLE_ENDIAN, 1),
+ new byte[15])))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Incomplete coordinate sequence");
+
+ assertThatThrownBy(
+ () ->
+ BinaryGeographyData.fromBytes(
+ concat(
+ header(LITTLE_ENDIAN, GeographyData.MULTI_POINT),
+ unsignedInt(LITTLE_ENDIAN, 1),
+ lineStringWkb(LITTLE_ENDIAN))))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Expected nested subtype ID 1 but found 2");
+
+ assertThatThrownBy(
+ () ->
+ BinaryGeographyData.fromBytes(
+ concat(pointWkb(LITTLE_ENDIAN), bytes(42))))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("trailing byte");
+ }
+
+ @Test
+ void testInvalidByteRanges() {
+ assertThatThrownBy(() -> BinaryGeographyData.fromBytes(POINT_WKB, -1, POINT_WKB.length))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Invalid ISO WKB byte range");
+
+ assertThatThrownBy(() -> BinaryGeographyData.fromBytes(POINT_WKB, 0, POINT_WKB.length + 1))
+ .isInstanceOf(TableRuntimeException.class)
+ .hasMessageContaining("Invalid ISO WKB byte range");
+ }
+
+ private static BinaryRowData binaryRowWithGeography(byte[] wkb) {
+ final int fixedLength = BinaryRowData.calculateFixPartSizeInBytes(1);
+ final byte[] rowBytes = new byte[fixedLength + wkb.length];
+ final MemorySegment segment = MemorySegmentFactory.wrap(rowBytes);
+ segment.putLong(8, ((long) fixedLength << 32) | wkb.length);
+ segment.put(fixedLength, wkb, 0, wkb.length);
+
+ final BinaryRowData row = new BinaryRowData(1);
+ row.pointTo(segment, 0, rowBytes.length);
+ return row;
+ }
+
+ private static byte[] pointWkb(int byteOrder) {
+ return concat(header(byteOrder, GeographyData.POINT), new byte[16]);
+ }
+
+ private static byte[] lineStringWkb(int byteOrder) {
+ return concat(
+ header(byteOrder, GeographyData.LINE_STRING),
+ unsignedInt(byteOrder, 2),
+ new byte[32]);
+ }
+
+ private static byte[] polygonWkb(int byteOrder) {
+ return concat(
+ header(byteOrder, GeographyData.POLYGON),
+ unsignedInt(byteOrder, 1),
+ unsignedInt(byteOrder, 4),
+ new byte[64]);
+ }
+
+ private static byte[] multiPointWkb(int byteOrder) {
+ return concat(
+ header(byteOrder, GeographyData.MULTI_POINT),
+ unsignedInt(byteOrder, 1),
+ pointWkb(byteOrder));
+ }
+
+ private static byte[] multiLineStringWkb(int byteOrder) {
+ return concat(
+ header(byteOrder, GeographyData.MULTI_LINE_STRING),
+ unsignedInt(byteOrder, 1),
+ lineStringWkb(byteOrder));
+ }
+
+ private static byte[] multiPolygonWkb(int byteOrder) {
+ return concat(
+ header(byteOrder, GeographyData.MULTI_POLYGON),
+ unsignedInt(byteOrder, 1),
+ polygonWkb(byteOrder));
+ }
+
+ private static byte[] geometryCollectionWkb(int byteOrder) {
+ return concat(
+ header(byteOrder, GeographyData.GEOMETRY_COLLECTION),
+ unsignedInt(byteOrder, 2),
+ pointWkb(byteOrder),
+ lineStringWkb(byteOrder));
+ }
+
+ private static byte[] header(int byteOrder, int subtypeId) {
+ return concat(bytes(byteOrder), unsignedInt(byteOrder, subtypeId));
+ }
+
+ private static byte[] unsignedInt(int byteOrder, int value) {
+ if (byteOrder == LITTLE_ENDIAN) {
+ return bytes(value, value >>> 8, value >>> 16, value >>> 24);
+ }
+ return bytes(value >>> 24, value >>> 16, value >>> 8, value);
+ }
+
+ private static byte[] concat(byte[]... values) {
+ int length = 0;
+ for (byte[] value : values) {
+ length += value.length;
+ }
+
+ final byte[] result = new byte[length];
+ int offset = 0;
+ for (byte[] value : values) {
+ System.arraycopy(value, 0, result, offset, value.length);
+ offset += value.length;
+ }
+ return result;
+ }
+
+ private static byte[] bytes(int... values) {
+ final byte[] result = new byte[values.length];
+ for (int i = 0; i < values.length; i++) {
+ result[i] = (byte) values[i];
+ }
+ return result;
+ }
+}
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java
index 8c274ed719137e..3c0bf063631459 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/DataTypesTest.java
@@ -21,6 +21,7 @@
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.common.typeutils.base.VoidSerializer;
import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.types.logical.ArrayType;
import org.apache.flink.table.types.logical.BigIntType;
import org.apache.flink.table.types.logical.BinaryType;
@@ -32,6 +33,7 @@
import org.apache.flink.table.types.logical.DecimalType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
import org.apache.flink.table.types.logical.LogicalType;
@@ -82,6 +84,7 @@
import static org.apache.flink.table.api.DataTypes.DOUBLE;
import static org.apache.flink.table.api.DataTypes.FIELD;
import static org.apache.flink.table.api.DataTypes.FLOAT;
+import static org.apache.flink.table.api.DataTypes.GEOGRAPHY;
import static org.apache.flink.table.api.DataTypes.INT;
import static org.apache.flink.table.api.DataTypes.INTERVAL;
import static org.apache.flink.table.api.DataTypes.MAP;
@@ -231,6 +234,9 @@ private static Stream testData() {
TestSpec.forDataType(BITMAP())
.expectLogicalType(new BitmapType())
.expectConversionClass(Bitmap.class),
+ TestSpec.forDataType(GEOGRAPHY())
+ .expectLogicalType(new GeographyType())
+ .expectConversionClass(GeographyData.class),
TestSpec.forUnresolvedDataType(RAW(Types.VOID))
.expectUnresolvedString("[RAW('java.lang.Void', '?')]")
.lookupReturns(dummyRaw(Void.class))
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java
index 3d0dec239974f8..89cc1341dfe2a8 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeCastsTest.java
@@ -31,9 +31,11 @@
import org.apache.flink.table.types.logical.DecimalType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.MapType;
import org.apache.flink.table.types.logical.NullType;
import org.apache.flink.table.types.logical.RawType;
import org.apache.flink.table.types.logical.RowType;
@@ -46,6 +48,7 @@
import org.apache.flink.table.types.logical.TinyIntType;
import org.apache.flink.table.types.logical.VarBinaryType;
import org.apache.flink.table.types.logical.VarCharType;
+import org.apache.flink.table.types.logical.VariantType;
import org.apache.flink.table.types.logical.YearMonthIntervalType;
import org.apache.flink.table.types.logical.ZonedTimestampType;
import org.apache.flink.table.types.logical.utils.LogicalTypeCasts;
@@ -57,6 +60,7 @@
import org.junit.jupiter.params.provider.MethodSource;
import java.util.Arrays;
+import java.util.List;
import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
@@ -262,7 +266,43 @@ private static Stream testData() {
new RawType<>(Integer.class, IntSerializer.INSTANCE),
VarCharType.STRING_TYPE,
false,
- true));
+ true),
+
+ // variant to scalar is explicit only
+ Arguments.of(new VariantType(), new BooleanType(), false, true),
+ Arguments.of(new VariantType(), new TinyIntType(), false, true),
+ Arguments.of(new VariantType(), new IntType(), false, true),
+ Arguments.of(new VariantType(), new BigIntType(), false, true),
+ Arguments.of(new VariantType(), new DoubleType(), false, true),
+ Arguments.of(new VariantType(), new DecimalType(10, 2), false, true),
+ Arguments.of(new VariantType(), new DateType(), false, true),
+ Arguments.of(new VariantType(), new TimestampType(), false, true),
+ Arguments.of(new VariantType(), new TimestampType(3), false, true),
+ Arguments.of(new VariantType(), new LocalZonedTimestampType(), false, true),
+ Arguments.of(new VariantType(), new LocalZonedTimestampType(9), false, true),
+ Arguments.of(
+ new VariantType(),
+ new VarBinaryType(VarBinaryType.MAX_LENGTH),
+ false,
+ true),
+ // variant identity cast is implicit
+ Arguments.of(new VariantType(), new VariantType(), true, true),
+ // TIME, character strings and constructed targets are not castable from variant
+ Arguments.of(new VariantType(), new TimeType(), false, false),
+ Arguments.of(new VariantType(), VarCharType.STRING_TYPE, false, false),
+ Arguments.of(new VariantType(), new ArrayType(new IntType()), false, false),
+ Arguments.of(new VariantType(), new RowType(List.of()), false, false),
+ Arguments.of(
+ new VariantType(),
+ new MapType(new IntType(), new CharType()),
+ false,
+ false),
+
+ // GEOGRAPHY construction and serialization require explicit functions.
+ Arguments.of(new GeographyType(), VarCharType.STRING_TYPE, false, false),
+ Arguments.of(VarCharType.STRING_TYPE, new GeographyType(), false, false),
+ Arguments.of(new GeographyType(), new VarBinaryType(), false, false),
+ Arguments.of(new VarBinaryType(), new GeographyType(), false, false));
}
@ParameterizedTest(name = "{index}: [From: {0}, To: {1}, Implicit: {2}, Explicit: {3}]")
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java
index 258b397c871878..e1c96b1fbc9e7c 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypeParserTest.java
@@ -39,6 +39,7 @@
import org.apache.flink.table.types.logical.DescriptorType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LegacyTypeInformationType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
@@ -309,6 +310,8 @@ private static Stream testData() {
TestSpec.forString("VARIANT NOT NULL").expectType(new VariantType(false)),
TestSpec.forString("BITMAP").expectType(new BitmapType()),
TestSpec.forString("BITMAP NOT NULL").expectType(new BitmapType(false)),
+ TestSpec.forString("GEOGRAPHY").expectType(new GeographyType()),
+ TestSpec.forString("GEOGRAPHY NOT NULL").expectType(new GeographyType(false)),
// error message testing
diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java
index ee40463cf0b92d..b608d923cc3b6a 100644
--- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java
+++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java
@@ -26,6 +26,8 @@
import org.apache.flink.table.api.ValidationException;
import org.apache.flink.table.catalog.ObjectIdentifier;
import org.apache.flink.table.catalog.UnresolvedIdentifier;
+import org.apache.flink.table.data.GeographyData;
+import org.apache.flink.table.data.binary.BinaryGeographyData;
import org.apache.flink.table.expressions.TimeIntervalUnit;
import org.apache.flink.table.legacy.types.logical.TypeInformationRawType;
import org.apache.flink.table.types.logical.ArrayType;
@@ -40,6 +42,7 @@
import org.apache.flink.table.types.logical.DistinctType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
import org.apache.flink.table.types.logical.LogicalType;
@@ -629,6 +632,20 @@ void testBitmapType() {
new BitmapType(false)));
}
+ @Test
+ void testGeographyType() {
+ assertThat(new GeographyType())
+ .isJavaSerializable()
+ .satisfies(
+ baseAssertions(
+ "GEOGRAPHY",
+ "GEOGRAPHY",
+ new Class[] {GeographyData.class, BinaryGeographyData.class},
+ new Class[] {GeographyData.class, BinaryGeographyData.class},
+ new LogicalType[] {},
+ new GeographyType(false)));
+ }
+
@Test
void testTypeInformationRawType() {
final TypeInformationRawType rawType =
diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml
index b8f76f7f969929..b1686bf1f0d28a 100644
--- a/flink-table/flink-table-planner/pom.xml
+++ b/flink-table/flink-table-planner/pom.xml
@@ -158,6 +158,11 @@ under the License.
flink-table-runtime${project.version}
+
+ org.apache.flink
+ flink-table-type-utils
+ ${project.version}
+
@@ -183,6 +188,13 @@ under the License.
test
+
+ org.apache.flink
+ flink-table-code-splitter
+ ${project.version}
+ test
+
+
org.apache.flink
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java
index 7b6112f9672dc8..d819a145e94cdd 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlCastFunction.java
@@ -269,16 +269,22 @@ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean throwOnFail
private boolean canCastFrom(RelDataType toType, RelDataType fromType) {
SqlTypeName fromTypeName = fromType.getSqlTypeName();
+ SqlTypeName toTypeName = toType.getSqlTypeName();
// Cast to Variant is not support at the moment.
// TODO: Support cast to variant (FLINK-37925，FLINK-37926)
- if (toType.getSqlTypeName() == SqlTypeName.VARIANT) {
+ if (toTypeName == SqlTypeName.VARIANT) {
return false;
}
// Cast to BITMAP is not supported at the moment.
if (toType instanceof BitmapRelDataType) {
return false;
}
+ if (toTypeName == SqlTypeName.OTHER) {
+ return LogicalTypeCasts.supportsExplicitCast(
+ FlinkTypeFactory.toLogicalType(fromType),
+ FlinkTypeFactory.toLogicalType(toType));
+ }
switch (fromTypeName) {
case ARRAY:
case MAP:
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
deleted file mode 100644
index 0f655a998b4ada..00000000000000
--- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java
+++ /dev/null
@@ -1,2711 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to you under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.calcite.sql.fun;
-
-import com.google.common.base.Suppliers;
-import com.google.common.collect.ImmutableList;
-import org.apache.calcite.avatica.util.TimeUnit;
-import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.SqlAsOperator;
-import org.apache.calcite.sql.SqlBasicCall;
-import org.apache.calcite.sql.SqlBasicFunction;
-import org.apache.calcite.sql.SqlBinaryOperator;
-import org.apache.calcite.sql.SqlCall;
-import org.apache.calcite.sql.SqlDescriptorOperator;
-import org.apache.calcite.sql.SqlFilterOperator;
-import org.apache.calcite.sql.SqlFunction;
-import org.apache.calcite.sql.SqlFunctionCategory;
-import org.apache.calcite.sql.SqlGroupedWindowFunction;
-import org.apache.calcite.sql.SqlHopTableFunction;
-import org.apache.calcite.sql.SqlInternalOperator;
-import org.apache.calcite.sql.SqlJsonConstructorNullClause;
-import org.apache.calcite.sql.SqlKind;
-import org.apache.calcite.sql.SqlLateralOperator;
-import org.apache.calcite.sql.SqlLiteral;
-import org.apache.calcite.sql.SqlMatchFunction;
-import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.SqlNullTreatmentOperator;
-import org.apache.calcite.sql.SqlNumericLiteral;
-import org.apache.calcite.sql.SqlOperandCountRange;
-import org.apache.calcite.sql.SqlOperator;
-import org.apache.calcite.sql.SqlOverOperator;
-import org.apache.calcite.sql.SqlPostfixOperator;
-import org.apache.calcite.sql.SqlPrefixOperator;
-import org.apache.calcite.sql.SqlProcedureCallOperator;
-import org.apache.calcite.sql.SqlRankFunction;
-import org.apache.calcite.sql.SqlSampleSpec;
-import org.apache.calcite.sql.SqlSessionTableFunction;
-import org.apache.calcite.sql.SqlSetOperator;
-import org.apache.calcite.sql.SqlSetSemanticsTableOperator;
-import org.apache.calcite.sql.SqlSpecialOperator;
-import org.apache.calcite.sql.SqlSyntax;
-import org.apache.calcite.sql.SqlTumbleTableFunction;
-import org.apache.calcite.sql.SqlUnnestOperator;
-import org.apache.calcite.sql.SqlUtil;
-import org.apache.calcite.sql.SqlValuesOperator;
-import org.apache.calcite.sql.SqlWindow;
-import org.apache.calcite.sql.SqlWithinDistinctOperator;
-import org.apache.calcite.sql.SqlWithinGroupOperator;
-import org.apache.calcite.sql.SqlWriter;
-import org.apache.calcite.sql.type.InferTypes;
-import org.apache.calcite.sql.type.OperandTypes;
-import org.apache.calcite.sql.type.ReturnTypes;
-import org.apache.calcite.sql.type.SqlOperandCountRanges;
-import org.apache.calcite.sql.type.SqlReturnTypeInference;
-import org.apache.calcite.sql.type.SqlTypeFamily;
-import org.apache.calcite.sql.type.SqlTypeName;
-import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable;
-import org.apache.calcite.sql.validate.SqlConformance;
-import org.apache.calcite.sql.validate.SqlConformanceEnum;
-import org.apache.calcite.sql.validate.SqlModality;
-import org.apache.calcite.sql2rel.AuxiliaryConverter;
-import org.apache.calcite.util.Litmus;
-import org.apache.calcite.util.Optionality;
-import org.apache.calcite.util.Pair;
-import org.checkerframework.checker.nullness.qual.Nullable;
-
-import java.util.List;
-import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-import java.util.function.Supplier;
-
-import static java.util.Objects.requireNonNull;
-import static org.apache.calcite.linq4j.Nullness.castNonNull;
-import static org.apache.calcite.util.Static.RESOURCE;
-
-/**
- * Implementation of {@link org.apache.calcite.sql.SqlOperatorTable} containing the standard
- * operators and functions.
- *
- *
Lines 828 ~ 830, 848 ~ 850, 859 ~ 861, 870 ~ 872, 881 ~ 883, 892 ~ 894, 903 ~ 905, Flink
- * changes the return type of the {@code IS [NOT] JSON ...} predicates from {@link
- * ReturnTypes#BOOLEAN_NULLABLE} to {@link ReturnTypes#BOOLEAN} so that they always return a
- * non-nullable {@code BOOLEAN}, see also FLINK-39943.
- */
-public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable {
-
- // ~ Static fields/initializers ---------------------------------------------
-
- /** The standard operator table. */
- private static final Supplier INSTANCE =
- Suppliers.memoize(() -> (SqlStdOperatorTable) new SqlStdOperatorTable().init());
-
- // -------------------------------------------------------------
- // SET OPERATORS
- // -------------------------------------------------------------
- // The set operators can be compared to the arithmetic operators
- // UNION -> +
- // EXCEPT -> -
- // INTERSECT -> *
- // which explains the different precedence values
- public static final SqlSetOperator UNION =
- new SqlSetOperator("UNION", SqlKind.UNION, 12, false);
-
- public static final SqlSetOperator UNION_ALL =
- new SqlSetOperator("UNION ALL", SqlKind.UNION, 12, true);
-
- public static final SqlSetOperator EXCEPT =
- new SqlSetOperator("EXCEPT", SqlKind.EXCEPT, 12, false);
-
- public static final SqlSetOperator EXCEPT_ALL =
- new SqlSetOperator("EXCEPT ALL", SqlKind.EXCEPT, 12, true);
-
- public static final SqlSetOperator INTERSECT =
- new SqlSetOperator("INTERSECT", SqlKind.INTERSECT, 14, false);
-
- public static final SqlSetOperator INTERSECT_ALL =
- new SqlSetOperator("INTERSECT ALL", SqlKind.INTERSECT, 14, true);
-
- /** The {@code MULTISET UNION DISTINCT} operator. */
- public static final SqlMultisetSetOperator MULTISET_UNION_DISTINCT =
- new SqlMultisetSetOperator("MULTISET UNION DISTINCT", 12, false);
-
- /** The {@code MULTISET UNION [ALL]} operator. */
- public static final SqlMultisetSetOperator MULTISET_UNION =
- new SqlMultisetSetOperator("MULTISET UNION ALL", 12, true);
-
- /** The {@code MULTISET EXCEPT DISTINCT} operator. */
- public static final SqlMultisetSetOperator MULTISET_EXCEPT_DISTINCT =
- new SqlMultisetSetOperator("MULTISET EXCEPT DISTINCT", 12, false);
-
- /** The {@code MULTISET EXCEPT [ALL]} operator. */
- public static final SqlMultisetSetOperator MULTISET_EXCEPT =
- new SqlMultisetSetOperator("MULTISET EXCEPT ALL", 12, true);
-
- /** The {@code MULTISET INTERSECT DISTINCT} operator. */
- public static final SqlMultisetSetOperator MULTISET_INTERSECT_DISTINCT =
- new SqlMultisetSetOperator("MULTISET INTERSECT DISTINCT", 14, false);
-
- /** The {@code MULTISET INTERSECT [ALL]} operator. */
- public static final SqlMultisetSetOperator MULTISET_INTERSECT =
- new SqlMultisetSetOperator("MULTISET INTERSECT ALL", 14, true);
-
- // -------------------------------------------------------------
- // BINARY OPERATORS
- // -------------------------------------------------------------
-
- /** Logical AND operator. */
- public static final SqlBinaryOperator AND =
- new SqlBinaryOperator(
- "AND",
- SqlKind.AND,
- 24,
- true,
- ReturnTypes.BOOLEAN_NULLABLE_OPTIMIZED,
- InferTypes.BOOLEAN,
- OperandTypes.BOOLEAN_BOOLEAN);
-
- /** AS operator associates an expression in the SELECT clause with an alias. */
- public static final SqlAsOperator AS = new SqlAsOperator();
-
- /**
- * ARGUMENT_ASSIGNMENT operator (=<) assigns an argument to a
- * function call to a particular named parameter.
- */
- public static final SqlSpecialOperator ARGUMENT_ASSIGNMENT =
- new SqlArgumentAssignmentOperator();
-
- /**
- * DEFAULT operator indicates that an argument to a function call is to take its
- * default value.
- */
- public static final SqlSpecialOperator DEFAULT = new SqlDefaultOperator();
-
- /** FILTER operator filters which rows are included in an aggregate function. */
- public static final SqlFilterOperator FILTER = new SqlFilterOperator();
-
- /** WITHIN_GROUP operator performs aggregations on ordered data input. */
- public static final SqlWithinGroupOperator WITHIN_GROUP = new SqlWithinGroupOperator();
-
- /** WITHIN_DISTINCT operator performs aggregations on distinct data input. */
- public static final SqlWithinDistinctOperator WITHIN_DISTINCT = new SqlWithinDistinctOperator();
-
- /**
- * {@code CUBE} operator, occurs within {@code GROUP BY} clause or nested within a {@code
- * GROUPING SETS}.
- */
- public static final SqlInternalOperator CUBE = new SqlRollupOperator("CUBE", SqlKind.CUBE);
-
- /**
- * {@code ROLLUP} operator, occurs within {@code GROUP BY} clause or nested within a {@code
- * GROUPING SETS}.
- */
- public static final SqlInternalOperator ROLLUP =
- new SqlRollupOperator("ROLLUP", SqlKind.ROLLUP);
-
- /**
- * {@code GROUPING SETS} operator, occurs within {@code GROUP BY} clause or nested within a
- * {@code GROUPING SETS}.
- */
- public static final SqlInternalOperator GROUPING_SETS =
- new SqlRollupOperator("GROUPING SETS", SqlKind.GROUPING_SETS);
-
- /**
- * {@code GROUPING(c1 [, c2, ...])} function.
- *
- *
Occurs in similar places to an aggregate function ({@code SELECT}, {@code HAVING} clause,
- * etc. of an aggregate query), but not technically an aggregate function.
- */
- public static final SqlAggFunction GROUPING = new SqlGroupingFunction("GROUPING");
-
- /** {@code GROUP_ID()} function. (Oracle-specific.) */
- public static final SqlAggFunction GROUP_ID = new SqlGroupIdFunction();
-
- /**
- * {@code GROUPING_ID} function is a synonym for {@code GROUPING}.
- *
- *
Some history. The {@code GROUPING} function is in the SQL standard, and originally
- * supported only one argument. {@code GROUPING_ID} is not standard (though supported in Oracle
- * and SQL Server) and supports one or more arguments.
- *
- *
The SQL standard has changed to allow {@code GROUPING} to have multiple arguments. It is
- * now equivalent to {@code GROUPING_ID}, so we made {@code GROUPING_ID} a synonym for {@code
- * GROUPING}.
- */
- public static final SqlAggFunction GROUPING_ID = new SqlGroupingFunction("GROUPING_ID");
-
- /** {@code EXTEND} operator. */
- public static final SqlInternalOperator EXTEND = new SqlExtendOperator();
-
- /**
- * String and array-to-array concatenation operator, '||'.
- *
- * @see SqlLibraryOperators#CONCAT_FUNCTION
- */
- public static final SqlBinaryOperator CONCAT =
- new SqlBinaryOperator(
- "||",
- SqlKind.OTHER,
- 60,
- true,
- ReturnTypes.ARG0.andThen(
- (opBinding, typeToTransform) -> {
- SqlReturnTypeInference returnType =
- typeToTransform.getSqlTypeName().getFamily()
- == SqlTypeFamily.ARRAY
- ? ReturnTypes.LEAST_RESTRICTIVE
- : ReturnTypes.DYADIC_STRING_SUM_PRECISION_NULLABLE;
- RelDataType type = returnType.inferReturnType(opBinding);
- if (type == null) {
- throw opBinding.newError(
- RESOURCE.cannotInferReturnType(
- opBinding.getOperator().toString(),
- opBinding.collectOperandTypes().toString()));
- }
- return type;
- }),
- null,
- OperandTypes.STRING_SAME_SAME_OR_ARRAY_SAME_SAME);
-
- /** Arithmetic division operator, '/'. */
- public static final SqlBinaryOperator DIVIDE =
- new SqlBinaryOperator(
- "/",
- SqlKind.DIVIDE,
- 60,
- true,
- ReturnTypes.QUOTIENT_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.DIVISION_OPERATOR);
-
- /** Checked version of arithmetic division operator, '/'. */
- public static final SqlBinaryOperator CHECKED_DIVIDE =
- new SqlBinaryOperator(
- "/",
- SqlKind.CHECKED_DIVIDE,
- 60,
- true,
- ReturnTypes.QUOTIENT_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.DIVISION_OPERATOR);
-
- /**
- * Arithmetic remainder operator, '%', an alternative to {@link #MOD} allowed if
- * under certain conformance levels.
- *
- * @see SqlConformance#isPercentRemainderAllowed
- */
- public static final SqlBinaryOperator PERCENT_REMAINDER =
- new SqlBinaryOperator(
- "%",
- SqlKind.MOD,
- 60,
- true,
- ReturnTypes.NULLABLE_MOD,
- null,
- OperandTypes.EXACT_NUMERIC_EXACT_NUMERIC);
-
- /**
- * The {@code RAND_INTEGER([seed, ] bound)} function, which yields a random integer, optionally
- * with seed.
- */
- public static final SqlRandIntegerFunction RAND_INTEGER = new SqlRandIntegerFunction();
-
- /** The {@code RAND([seed])} function, which yields a random double, optionally with seed. */
- public static final SqlBasicFunction RAND =
- SqlBasicFunction.create(
- "RAND",
- ReturnTypes.DOUBLE,
- OperandTypes.NILADIC.or(OperandTypes.NUMERIC),
- SqlFunctionCategory.NUMERIC)
- .withDeterministic(false)
- .withDynamic(true);
-
- /**
- * Internal integer arithmetic division operator, '/INT'. This is only used to
- * adjust scale for numerics. We distinguish it from user-requested division since some
- * personalities want a floating-point computation, whereas for the internal scaling use of
- * division, we always want integer division.
- */
- public static final SqlBinaryOperator DIVIDE_INTEGER =
- new SqlBinaryOperator(
- "/INT",
- SqlKind.DIVIDE,
- 60,
- true,
- ReturnTypes.INTEGER_QUOTIENT_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.DIVISION_OPERATOR);
-
- /** Checked version of integer division. @see DIVIDE_INTEGER. */
- public static final SqlBinaryOperator CHECKED_DIVIDE_INTEGER =
- new SqlBinaryOperator(
- "/INT",
- SqlKind.CHECKED_DIVIDE,
- 60,
- true,
- ReturnTypes.INTEGER_QUOTIENT_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.DIVISION_OPERATOR);
-
- /** Dot operator, '.', used for referencing fields of records. */
- public static final SqlOperator DOT = new SqlDotOperator();
-
- /** Logical equals operator, '='. */
- public static final SqlBinaryOperator EQUALS =
- new SqlBinaryOperator(
- "=",
- SqlKind.EQUALS,
- 30,
- true,
- ReturnTypes.BOOLEAN_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
-
- /** Logical greater-than operator, '>'. */
- public static final SqlBinaryOperator GREATER_THAN =
- new SqlBinaryOperator(
- ">",
- SqlKind.GREATER_THAN,
- 30,
- true,
- ReturnTypes.BOOLEAN_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_ORDERED_COMPARABLE_ORDERED);
-
- /** IS DISTINCT FROM operator. */
- public static final SqlBinaryOperator IS_DISTINCT_FROM =
- new SqlBinaryOperator(
- "IS DISTINCT FROM",
- SqlKind.IS_DISTINCT_FROM,
- 30,
- true,
- ReturnTypes.BOOLEAN,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
-
- /**
- * IS NOT DISTINCT FROM operator. Is equivalent to NOT(x
- * IS DISTINCT FROM y)
- */
- public static final SqlBinaryOperator IS_NOT_DISTINCT_FROM =
- new SqlBinaryOperator(
- "IS NOT DISTINCT FROM",
- SqlKind.IS_NOT_DISTINCT_FROM,
- 30,
- true,
- ReturnTypes.BOOLEAN,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
-
- /**
- * The internal $IS_DIFFERENT_FROM operator is the same as the user-level {@link
- * #IS_DISTINCT_FROM} in all respects except that the test for equality on character datatypes
- * treats trailing spaces as significant.
- */
- public static final SqlBinaryOperator IS_DIFFERENT_FROM =
- new SqlBinaryOperator(
- "$IS_DIFFERENT_FROM",
- SqlKind.OTHER,
- 30,
- true,
- ReturnTypes.BOOLEAN,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
-
- /** Logical greater-than-or-equal operator, '>='. */
- public static final SqlBinaryOperator GREATER_THAN_OR_EQUAL =
- new SqlBinaryOperator(
- ">=",
- SqlKind.GREATER_THAN_OR_EQUAL,
- 30,
- true,
- ReturnTypes.BOOLEAN_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_ORDERED_COMPARABLE_ORDERED);
-
- /**
- * IN operator tests for a value's membership in a sub-query or a list of values.
- */
- public static final SqlBinaryOperator IN = new SqlInOperator(SqlKind.IN);
-
- /**
- * NOT IN operator tests for a value's membership in a sub-query or a list of
- * values.
- */
- public static final SqlBinaryOperator NOT_IN = new SqlInOperator(SqlKind.NOT_IN);
-
- /**
- * Operator that tests whether its left operand is included in the range of values covered by
- * search arguments.
- */
- public static final SqlInternalOperator SEARCH = new SqlSearchOperator();
-
- /** The < SOME operator (synonymous with < ANY). */
- public static final SqlQuantifyOperator SOME_LT =
- new SqlQuantifyOperator(SqlKind.SOME, SqlKind.LESS_THAN);
-
- public static final SqlQuantifyOperator SOME_LE =
- new SqlQuantifyOperator(SqlKind.SOME, SqlKind.LESS_THAN_OR_EQUAL);
-
- public static final SqlQuantifyOperator SOME_GT =
- new SqlQuantifyOperator(SqlKind.SOME, SqlKind.GREATER_THAN);
-
- public static final SqlQuantifyOperator SOME_GE =
- new SqlQuantifyOperator(SqlKind.SOME, SqlKind.GREATER_THAN_OR_EQUAL);
-
- public static final SqlQuantifyOperator SOME_EQ =
- new SqlQuantifyOperator(SqlKind.SOME, SqlKind.EQUALS);
-
- public static final SqlQuantifyOperator SOME_NE =
- new SqlQuantifyOperator(SqlKind.SOME, SqlKind.NOT_EQUALS);
-
- /** The < ALL operator. */
- public static final SqlQuantifyOperator ALL_LT =
- new SqlQuantifyOperator(SqlKind.ALL, SqlKind.LESS_THAN);
-
- public static final SqlQuantifyOperator ALL_LE =
- new SqlQuantifyOperator(SqlKind.ALL, SqlKind.LESS_THAN_OR_EQUAL);
-
- public static final SqlQuantifyOperator ALL_GT =
- new SqlQuantifyOperator(SqlKind.ALL, SqlKind.GREATER_THAN);
-
- public static final SqlQuantifyOperator ALL_GE =
- new SqlQuantifyOperator(SqlKind.ALL, SqlKind.GREATER_THAN_OR_EQUAL);
-
- public static final SqlQuantifyOperator ALL_EQ =
- new SqlQuantifyOperator(SqlKind.ALL, SqlKind.EQUALS);
-
- public static final SqlQuantifyOperator ALL_NE =
- new SqlQuantifyOperator(SqlKind.ALL, SqlKind.NOT_EQUALS);
-
- public static final List QUANTIFY_OPERATORS =
- ImmutableList.of(
- SqlStdOperatorTable.SOME_EQ,
- SqlStdOperatorTable.SOME_GT,
- SqlStdOperatorTable.SOME_GE,
- SqlStdOperatorTable.SOME_LE,
- SqlStdOperatorTable.SOME_LT,
- SqlStdOperatorTable.SOME_NE,
- SqlStdOperatorTable.ALL_EQ,
- SqlStdOperatorTable.ALL_GT,
- SqlStdOperatorTable.ALL_GE,
- SqlStdOperatorTable.ALL_LE,
- SqlStdOperatorTable.ALL_LT,
- SqlStdOperatorTable.ALL_NE);
-
- /** Logical less-than operator, '<'. */
- public static final SqlBinaryOperator LESS_THAN =
- new SqlBinaryOperator(
- "<",
- SqlKind.LESS_THAN,
- 30,
- true,
- ReturnTypes.BOOLEAN_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_ORDERED_COMPARABLE_ORDERED);
-
- /** Logical less-than-or-equal operator, '<='. */
- public static final SqlBinaryOperator LESS_THAN_OR_EQUAL =
- new SqlBinaryOperator(
- "<=",
- SqlKind.LESS_THAN_OR_EQUAL,
- 30,
- true,
- ReturnTypes.BOOLEAN_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_ORDERED_COMPARABLE_ORDERED);
-
- /**
- * Infix arithmetic minus operator, '-'.
- *
- *
Its precedence is less than the prefix {@link #UNARY_PLUS +} and {@link #UNARY_MINUS -}
- * operators.
- */
- public static final SqlBinaryOperator MINUS =
- new SqlMonotonicBinaryOperator(
- "-",
- SqlKind.MINUS,
- 40,
- true,
-
- // Same type inference strategy as sum
- ReturnTypes.NULLABLE_SUM,
- InferTypes.FIRST_KNOWN,
- OperandTypes.MINUS_OPERATOR);
-
- /** Checked version of infix arithmetic minus operator, '-'. */
- public static final SqlBinaryOperator CHECKED_MINUS =
- new SqlMonotonicBinaryOperator(
- "-",
- SqlKind.CHECKED_MINUS,
- 40,
- true,
-
- // Same type inference strategy as sum
- ReturnTypes.NULLABLE_SUM,
- InferTypes.FIRST_KNOWN,
- OperandTypes.MINUS_OPERATOR);
-
- /** Arithmetic multiplication operator, '*'. */
- public static final SqlBinaryOperator MULTIPLY =
- new SqlMonotonicBinaryOperator(
- "*",
- SqlKind.TIMES,
- 60,
- true,
- ReturnTypes.PRODUCT_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.MULTIPLY_OPERATOR);
-
- /** Checked version of arithmetic multiplication operator, '*'. */
- public static final SqlBinaryOperator CHECKED_MULTIPLY =
- new SqlMonotonicBinaryOperator(
- "*",
- SqlKind.CHECKED_TIMES,
- 60,
- true,
- ReturnTypes.PRODUCT_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.MULTIPLY_OPERATOR);
-
- /** Logical not-equals operator, '<>'. */
- public static final SqlBinaryOperator NOT_EQUALS =
- new SqlBinaryOperator(
- "<>",
- SqlKind.NOT_EQUALS,
- 30,
- true,
- ReturnTypes.BOOLEAN_NULLABLE,
- InferTypes.FIRST_KNOWN,
- OperandTypes.COMPARABLE_UNORDERED_COMPARABLE_UNORDERED);
-
- /** Logical OR operator. */
- public static final SqlBinaryOperator OR =
- new SqlBinaryOperator(
- "OR",
- SqlKind.OR,
- 22,
- true,
- ReturnTypes.BOOLEAN_NULLABLE_OPTIMIZED,
- InferTypes.BOOLEAN,
- OperandTypes.BOOLEAN_BOOLEAN);
-
- /** Infix arithmetic plus operator, '+'. */
- public static final SqlBinaryOperator PLUS =
- new SqlMonotonicBinaryOperator(
- "+",
- SqlKind.PLUS,
- 40,
- true,
- ReturnTypes.NULLABLE_SUM,
- InferTypes.FIRST_KNOWN,
- OperandTypes.PLUS_OPERATOR);
-
- /** Checked version of infix arithmetic plus operator, '+'. */
- public static final SqlBinaryOperator CHECKED_PLUS =
- new SqlMonotonicBinaryOperator(
- "+",
- SqlKind.CHECKED_PLUS,
- 40,
- true,
- ReturnTypes.NULLABLE_SUM,
- InferTypes.FIRST_KNOWN,
- OperandTypes.PLUS_OPERATOR);
-
- /** Infix datetime plus operator, 'DATETIME + INTERVAL'. */
- public static final SqlSpecialOperator DATETIME_PLUS = new SqlDatetimePlusOperator();
-
- /** Interval expression, 'INTERVAL n timeUnit'. */
- public static final SqlSpecialOperator INTERVAL = new SqlIntervalOperator();
-
- /**
- * Multiset {@code MEMBER OF}, which returns whether a element belongs to a multiset.
- *
- *
For example, the following returns false:
- *
- *
- *
- * 'green' MEMBER OF MULTISET ['red','almost green','blue']
- *
- *
- */
- public static final SqlBinaryOperator MEMBER_OF = new SqlMultisetMemberOfOperator();
-
- /**
- * Submultiset. Checks to see if an multiset is a sub-set of another multiset.
- *
- *
This operator is special since it needs to hold the additional interval qualifier
- * specification.
- */
- public static final SqlDatetimeSubtractionOperator MINUS_DATE =
- new SqlDatetimeSubtractionOperator("-", ReturnTypes.ARG2_NULLABLE);
-
- /** The MULTISET Value Constructor. e.g. "MULTISET[1,2,3]". */
- public static final SqlMultisetValueConstructor MULTISET_VALUE =
- new SqlMultisetValueConstructor();
-
- /**
- * The MULTISET Query Constructor. e.g. "SELECT dname, MULTISET(SELECT
- * FROM emp WHERE deptno = dept.deptno) FROM dept".
- */
- public static final SqlMultisetQueryConstructor MULTISET_QUERY =
- new SqlMultisetQueryConstructor();
-
- /**
- * The ARRAY Query Constructor. e.g. "SELECT dname, ARRAY(SELECT
- * FROM emp WHERE deptno = dept.deptno) FROM dept".
- */
- public static final SqlMultisetQueryConstructor ARRAY_QUERY = new SqlArrayQueryConstructor();
-
- /**
- * The MAP Query Constructor. e.g. "MAP(SELECT empno, deptno
- * FROM emp)".
- */
- public static final SqlMultisetQueryConstructor MAP_QUERY = new SqlMapQueryConstructor();
-
- /**
- * The CURSOR constructor. e.g. "SELECT * FROM
- * TABLE(DEDUP(CURSOR(SELECT * FROM EMPS), 'name'))".
- */
- public static final SqlCursorConstructor CURSOR = new SqlCursorConstructor();
-
- /**
- * The COLUMN_LIST constructor. e.g. the ROW() call in "SELECT * FROM
- * TABLE(DEDUP(CURSOR(SELECT * FROM EMPS), ROW(name, empno)))".
- */
- public static final SqlColumnListConstructor COLUMN_LIST = new SqlColumnListConstructor();
-
- /** The UNNEST operator. */
- public static final SqlUnnestOperator UNNEST = new SqlUnnestOperator(false);
-
- /** The UNNEST WITH ORDINALITY operator. */
- @LibraryOperator(libraries = {}) // do not include in index
- public static final SqlUnnestOperator UNNEST_WITH_ORDINALITY = new SqlUnnestOperator(true);
-
- /** The LATERAL operator. */
- public static final SqlSpecialOperator LATERAL = new SqlLateralOperator(SqlKind.LATERAL);
-
- /**
- * The "table function derived table" operator, which a table-valued function into a relation,
- * e.g. "SELECT * FROM
- * TABLE(ramp(5))".
- *
- *
This operator has function syntax (with one argument), whereas {@link #EXPLICIT_TABLE} is
- * a prefix operator.
- */
- public static final SqlSpecialOperator COLLECTION_TABLE =
- new SqlCollectionTableOperator("TABLE", SqlModality.RELATION);
-
- public static final SqlOverlapsOperator OVERLAPS = new SqlOverlapsOperator(SqlKind.OVERLAPS);
-
- public static final SqlOverlapsOperator CONTAINS = new SqlOverlapsOperator(SqlKind.CONTAINS);
-
- public static final SqlOverlapsOperator PRECEDES = new SqlOverlapsOperator(SqlKind.PRECEDES);
-
- public static final SqlOverlapsOperator IMMEDIATELY_PRECEDES =
- new SqlOverlapsOperator(SqlKind.IMMEDIATELY_PRECEDES);
-
- public static final SqlOverlapsOperator SUCCEEDS = new SqlOverlapsOperator(SqlKind.SUCCEEDS);
-
- public static final SqlOverlapsOperator IMMEDIATELY_SUCCEEDS =
- new SqlOverlapsOperator(SqlKind.IMMEDIATELY_SUCCEEDS);
-
- public static final SqlOverlapsOperator PERIOD_EQUALS =
- new SqlOverlapsOperator(SqlKind.PERIOD_EQUALS);
-
- public static final SqlSpecialOperator VALUES = new SqlValuesOperator();
-
- public static final SqlLiteralChainOperator LITERAL_CHAIN = new SqlLiteralChainOperator();
-
- public static final SqlThrowOperator THROW = new SqlThrowOperator();
-
- public static final SqlFunction JSON_EXISTS = new SqlJsonExistsFunction();
-
- public static final SqlFunction JSON_VALUE = new SqlJsonValueFunction("JSON_VALUE");
-
- public static final SqlFunction JSON_QUERY = new SqlJsonQueryFunction();
-
- public static final SqlFunction JSON_OBJECT = new SqlJsonObjectFunction();
-
- public static final SqlJsonObjectAggAggFunction JSON_OBJECTAGG =
- new SqlJsonObjectAggAggFunction(
- SqlKind.JSON_OBJECTAGG, SqlJsonConstructorNullClause.NULL_ON_NULL);
-
- public static final SqlFunction JSON_ARRAY = new SqlJsonArrayFunction();
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_TYPE = new SqlJsonTypeFunction();
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_DEPTH = new SqlJsonDepthFunction();
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_LENGTH = new SqlJsonLengthFunction();
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_KEYS = new SqlJsonKeysFunction();
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_PRETTY = new SqlJsonPrettyFunction();
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_REMOVE = new SqlJsonRemoveFunction();
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_STORAGE_SIZE = new SqlJsonStorageSizeFunction();
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_INSERT = new SqlJsonModifyFunction("JSON_INSERT");
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_REPLACE = new SqlJsonModifyFunction("JSON_REPLACE");
-
- @Deprecated // to be removed before 2.0
- public static final SqlFunction JSON_SET = new SqlJsonModifyFunction("JSON_SET");
-
- public static final SqlJsonArrayAggAggFunction JSON_ARRAYAGG =
- new SqlJsonArrayAggAggFunction(
- SqlKind.JSON_ARRAYAGG, SqlJsonConstructorNullClause.ABSENT_ON_NULL);
-
- public static final SqlBetweenOperator BETWEEN =
- new SqlBetweenOperator(SqlBetweenOperator.Flag.ASYMMETRIC, false);
-
- public static final SqlBetweenOperator SYMMETRIC_BETWEEN =
- new SqlBetweenOperator(SqlBetweenOperator.Flag.SYMMETRIC, false);
-
- public static final SqlBetweenOperator NOT_BETWEEN =
- new SqlBetweenOperator(SqlBetweenOperator.Flag.ASYMMETRIC, true);
-
- public static final SqlBetweenOperator SYMMETRIC_NOT_BETWEEN =
- new SqlBetweenOperator(SqlBetweenOperator.Flag.SYMMETRIC, true);
-
- public static final SqlSpecialOperator NOT_LIKE =
- new SqlLikeOperator("NOT LIKE", SqlKind.LIKE, true, true);
-
- public static final SqlSpecialOperator LIKE =
- new SqlLikeOperator("LIKE", SqlKind.LIKE, false, true);
-
- public static final SqlSpecialOperator NOT_SIMILAR_TO =
- new SqlLikeOperator("NOT SIMILAR TO", SqlKind.SIMILAR, true, true);
-
- public static final SqlSpecialOperator SIMILAR_TO =
- new SqlLikeOperator("SIMILAR TO", SqlKind.SIMILAR, false, true);
-
- public static final SqlBinaryOperator POSIX_REGEX_CASE_SENSITIVE =
- new SqlPosixRegexOperator(
- "POSIX REGEX CASE SENSITIVE", SqlKind.POSIX_REGEX_CASE_SENSITIVE, true, false);
-
- public static final SqlBinaryOperator POSIX_REGEX_CASE_INSENSITIVE =
- new SqlPosixRegexOperator(
- "POSIX REGEX CASE INSENSITIVE",
- SqlKind.POSIX_REGEX_CASE_INSENSITIVE,
- false,
- false);
-
- public static final SqlBinaryOperator NEGATED_POSIX_REGEX_CASE_SENSITIVE =
- new SqlPosixRegexOperator(
- "NEGATED POSIX REGEX CASE SENSITIVE",
- SqlKind.POSIX_REGEX_CASE_SENSITIVE,
- true,
- true);
-
- public static final SqlBinaryOperator NEGATED_POSIX_REGEX_CASE_INSENSITIVE =
- new SqlPosixRegexOperator(
- "NEGATED POSIX REGEX CASE INSENSITIVE",
- SqlKind.POSIX_REGEX_CASE_INSENSITIVE,
- false,
- true);
-
- /** Internal operator used to represent the ESCAPE clause of a LIKE or SIMILAR TO expression. */
- public static final SqlSpecialOperator ESCAPE =
- new SqlSpecialOperator("ESCAPE", SqlKind.ESCAPE, 0);
-
- public static final SqlCaseOperator CASE = SqlCaseOperator.INSTANCE;
-
- public static final SqlOperator PROCEDURE_CALL = new SqlProcedureCallOperator();
-
- public static final SqlOperator NEW = new SqlNewOperator();
-
- /**
- * The OVER operator, which applies an aggregate functions to a {@link SqlWindow
- * window}.
- *
- *
Operands are as follows:
- *
- *
- *
name of window function ({@link org.apache.calcite.sql.SqlCall})
- *
window name ({@link org.apache.calcite.sql.SqlLiteral}) or window in-line specification
- * ({@code org.apache.calcite.sql.SqlWindow.SqlWindowOperator})
- *
- */
- public static final SqlBinaryOperator OVER = new SqlOverOperator();
-
- /**
- * An REINTERPRET operator is internal to the planner. When the physical storage of
- * two types is the same, this operator may be used to reinterpret values of one type as the
- * other. This operator is similar to a cast, except that it does not alter the data value. Like
- * a regular cast it accepts one operand and stores the target type as the return type. It
- * performs an overflow check if it has any second operand, whether true or not.
- */
- public static final SqlSpecialOperator REINTERPRET =
- new SqlSpecialOperator("Reinterpret", SqlKind.REINTERPRET) {
- @Override
- public SqlOperandCountRange getOperandCountRange() {
- return SqlOperandCountRanges.between(1, 2);
- }
- };
-
- // -------------------------------------------------------------
- // FUNCTIONS
- // -------------------------------------------------------------
-
- /**
- * The character substring function: SUBSTRING(string FROM start [FOR
- * length]).
- *
- *
If the length parameter is a constant, the length of the result is the minimum of the
- * length of the input and that length. Otherwise it is the length of the input.
- */
- public static final SqlFunction SUBSTRING = new SqlSubstringFunction();
-
- /**
- * The {@code REPLACE(string, search, replace)} function. Not standard SQL, but in Oracle,
- * PostgreSQL and Microsoft SQL Server.
- *
- *
REPLACE behaves a little different in Microsoft SQL Server, whose search pattern is
- * case-insensitive during matching.
- *
- *
For example, {@code REPLACE(('ciAao', 'a', 'ciao'))} returns "ciAciaoo" in both Oracle and
- * PostgreSQL, but returns "ciciaociaoo" in Microsoft SQL Server.
- */
- public static final SqlFunction REPLACE =
- SqlBasicFunction.create(
- "REPLACE",
- ReturnTypes.VARCHAR_NULLABLE,
- OperandTypes.STRING_STRING_STRING,
- SqlFunctionCategory.STRING);
-
- /**
- * The {@code CONVERT(charValue, srcCharsetName, destCharsetName)} function converts {@code
- * charValue} with {@code destCharsetName}, whose original encoding is specified by {@code
- * srcCharsetName}.
- *
- *
The SQL standard defines {@code CONVERT(charValue USING transcodingName)}, and MySQL
- * implements it; Calcite supports this in the following TRANSLATE function.
- *
- *
MySQL and Microsoft SQL Server have a {@code CONVERT(type, value)} function; Calcite does
- * not currently support this, either.
- */
- public static final SqlFunction CONVERT = new SqlConvertFunction("CONVERT");
-
- /**
- * The TRANSLATE/CONVERT(char_value USING transcodingName) function
- * alters the character set of a string value from one base character set to transcodingName.
- *
- *
It is defined in the SQL standard. See also the non-standard {@link
- * SqlLibraryOperators#TRANSLATE3}, which has a different purpose.
- */
- public static final SqlFunction TRANSLATE = new SqlTranslateFunction("TRANSLATE");
-
- public static final SqlFunction OVERLAY = new SqlOverlayFunction();
-
- /** The "TRIM" function. */
- public static final SqlFunction TRIM = SqlTrimFunction.INSTANCE;
-
- public static final SqlFunction POSITION = new SqlPositionFunction("POSITION");
-
- public static final SqlBasicFunction CHAR_LENGTH =
- SqlBasicFunction.create(
- SqlKind.CHAR_LENGTH, ReturnTypes.INTEGER_NULLABLE, OperandTypes.CHARACTER);
-
- /** Alias for {@link #CHAR_LENGTH}. */
- public static final SqlFunction CHARACTER_LENGTH = CHAR_LENGTH.withName("CHARACTER_LENGTH");
-
- public static final SqlFunction OCTET_LENGTH =
- SqlBasicFunction.create(
- "OCTET_LENGTH",
- ReturnTypes.INTEGER_NULLABLE,
- OperandTypes.BINARY,
- SqlFunctionCategory.NUMERIC);
-
- public static final SqlFunction UPPER =
- SqlBasicFunction.create(
- "UPPER",
- ReturnTypes.ARG0_NULLABLE,
- OperandTypes.CHARACTER,
- SqlFunctionCategory.STRING);
-
- public static final SqlFunction LOWER =
- SqlBasicFunction.create(
- "LOWER",
- ReturnTypes.ARG0_NULLABLE,
- OperandTypes.CHARACTER,
- SqlFunctionCategory.STRING);
-
- public static final SqlFunction INITCAP =
- SqlBasicFunction.create(
- "INITCAP",
- ReturnTypes.ARG0_NULLABLE,
- OperandTypes.CHARACTER,
- SqlFunctionCategory.STRING);
-
- public static final SqlFunction ASCII =
- SqlBasicFunction.create(
- "ASCII",
- ReturnTypes.INTEGER_NULLABLE,
- OperandTypes.CHARACTER,
- SqlFunctionCategory.STRING);
-
- /**
- * The {@code POWER(numeric, numeric)} function.
- *
- *
The return type is always {@code DOUBLE} since we don't know what the result type will be
- * by just looking at the operand types. For example {@code POWER(INTEGER, INTEGER)} can return
- * a non-INTEGER if the second operand is negative.
- */
- public static final SqlBasicFunction POWER =
- SqlBasicFunction.create(
- "POWER",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC_NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code SQRT(numeric)} function. */
- public static final SqlFunction SQRT =
- SqlBasicFunction.create("SQRT", ReturnTypes.DOUBLE_NULLABLE, OperandTypes.NUMERIC);
-
- /**
- * Arithmetic remainder function {@code MOD}.
- *
- * @see #PERCENT_REMAINDER
- */
- public static final SqlFunction MOD =
- // Return type is same as divisor (2nd operand)
- // SQL2003 Part2 Section 6.27, Syntax Rules 9
- SqlBasicFunction.create(
- SqlKind.MOD,
- ReturnTypes.NULLABLE_MOD,
- OperandTypes.EXACT_NUMERIC_EXACT_NUMERIC)
- .withFunctionType(SqlFunctionCategory.NUMERIC);
-
- /** The {@code LN(numeric)} function. */
- public static final SqlFunction LN =
- SqlBasicFunction.create(
- "LN",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code LOG10(numeric)} function. */
- public static final SqlFunction LOG10 =
- SqlBasicFunction.create(
- "LOG10",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code ABS(numeric)} function. */
- public static final SqlFunction ABS =
- SqlBasicFunction.create(
- "ABS",
- ReturnTypes.ARG0,
- OperandTypes.NUMERIC_OR_INTERVAL,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code ACOS(numeric)} function. */
- public static final SqlFunction ACOS =
- SqlBasicFunction.create(
- "ACOS",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code ASIN(numeric)} function. */
- public static final SqlFunction ASIN =
- SqlBasicFunction.create(
- "ASIN",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code ATAN(numeric)} function. */
- public static final SqlFunction ATAN =
- SqlBasicFunction.create(
- "ATAN",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code ATAN2(numeric, numeric)} function. */
- public static final SqlFunction ATAN2 =
- SqlBasicFunction.create(
- "ATAN2",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC_NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code CBRT(numeric)} function. */
- public static final SqlFunction CBRT =
- SqlBasicFunction.create(
- "CBRT",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code COS(numeric)} function. */
- public static final SqlFunction COS =
- SqlBasicFunction.create(
- "COS",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code COT(numeric)} function. */
- public static final SqlFunction COT =
- SqlBasicFunction.create(
- "COT",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code DEGREES(numeric)} function. */
- public static final SqlFunction DEGREES =
- SqlBasicFunction.create(
- "DEGREES",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code EXP(numeric)} function. */
- public static final SqlFunction EXP =
- SqlBasicFunction.create(
- "EXP",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code RADIANS(numeric)} function. */
- public static final SqlFunction RADIANS =
- SqlBasicFunction.create(
- "RADIANS",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code ROUND(numeric [, integer])} function. */
- public static final SqlFunction ROUND =
- SqlBasicFunction.create(
- "ROUND",
- ReturnTypes.ARG0_NULLABLE,
- OperandTypes.NUMERIC.or(OperandTypes.NUMERIC_INT32),
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code SIGN(numeric)} function. */
- public static final SqlFunction SIGN =
- SqlBasicFunction.create(
- "SIGN", ReturnTypes.ARG0, OperandTypes.NUMERIC, SqlFunctionCategory.NUMERIC);
-
- /** The {@code SIN(numeric)} function. */
- public static final SqlFunction SIN =
- SqlBasicFunction.create(
- "SIN",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code TAN(numeric)} function. */
- public static final SqlFunction TAN =
- SqlBasicFunction.create(
- "TAN",
- ReturnTypes.DOUBLE_NULLABLE,
- OperandTypes.NUMERIC,
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code TRUNCATE(numeric [, integer])} function. */
- public static final SqlBasicFunction TRUNCATE =
- SqlBasicFunction.create(
- "TRUNCATE",
- ReturnTypes.ARG0_NULLABLE,
- OperandTypes.NUMERIC.or(OperandTypes.NUMERIC_INT32),
- SqlFunctionCategory.NUMERIC);
-
- /** The {@code PI} function. */
- public static final SqlFunction PI =
- SqlBasicFunction.create(
- "PI",
- ReturnTypes.DOUBLE,
- OperandTypes.NILADIC,
- SqlFunctionCategory.NUMERIC)
- .withSyntax(SqlSyntax.FUNCTION_ID_CONSTANT);
-
- /** The {@code TYPEOF} function. */
- public static final SqlFunction TYPEOF =
- SqlBasicFunction.create(
- "TYPEOF",
- ReturnTypes.VARCHAR,
- OperandTypes.VARIANT,
- SqlFunctionCategory.STRING);
-
- /** The {@code VARIANTNULL} function. */
- public static final SqlFunction VARIANTNULL =
- SqlBasicFunction.create(
- "VARIANTNULL",
- ReturnTypes.VARIANT,
- OperandTypes.NILADIC,
- SqlFunctionCategory.SYSTEM);
-
- /** {@code FIRST} function to be used within {@code MATCH_RECOGNIZE}. */
- public static final SqlFunction FIRST =
- SqlBasicFunction.create(
- SqlKind.FIRST, ReturnTypes.ARG0_NULLABLE, OperandTypes.ANY_NUMERIC)
- .withFunctionType(SqlFunctionCategory.MATCH_RECOGNIZE);
-
- /** {@code LAST} function to be used within {@code MATCH_RECOGNIZE}. */
- public static final SqlMatchFunction LAST =
- new SqlMatchFunction(
- "LAST",
- SqlKind.LAST,
- ReturnTypes.ARG0_NULLABLE,
- null,
- OperandTypes.ANY_NUMERIC,
- SqlFunctionCategory.MATCH_RECOGNIZE);
-
- /** {@code PREV} function to be used within {@code MATCH_RECOGNIZE}. */
- public static final SqlMatchFunction PREV =
- new SqlMatchFunction(
- "PREV",
- SqlKind.PREV,
- ReturnTypes.ARG0_NULLABLE,
- null,
- OperandTypes.ANY_NUMERIC,
- SqlFunctionCategory.MATCH_RECOGNIZE);
-
- /** {@code NEXT} function to be used within {@code MATCH_RECOGNIZE}. */
- public static final SqlFunction NEXT =
- SqlBasicFunction.create(
- SqlKind.NEXT, ReturnTypes.ARG0_NULLABLE, OperandTypes.ANY_NUMERIC)
- .withFunctionType(SqlFunctionCategory.MATCH_RECOGNIZE);
-
- /** {@code CLASSIFIER} function to be used within {@code MATCH_RECOGNIZE}. */
- public static final SqlMatchFunction CLASSIFIER =
- new SqlMatchFunction(
- "CLASSIFIER",
- SqlKind.CLASSIFIER,
- ReturnTypes.VARCHAR_2000,
- null,
- OperandTypes.NILADIC,
- SqlFunctionCategory.MATCH_RECOGNIZE);
-
- /** {@code MATCH_NUMBER} function to be used within {@code MATCH_RECOGNIZE}. */
- public static final SqlFunction MATCH_NUMBER =
- SqlBasicFunction.create(
- SqlKind.MATCH_NUMBER, ReturnTypes.BIGINT_NULLABLE, OperandTypes.NILADIC)
- .withFunctionType(SqlFunctionCategory.MATCH_RECOGNIZE);
-
- public static final SqlFunction NULLIF = new SqlNullifFunction();
-
- /** The COALESCE builtin function. */
- public static final SqlFunction COALESCE = new SqlCoalesceFunction();
-
- /** The FLOOR function. */
- public static final SqlFunction FLOOR = new SqlFloorFunction(SqlKind.FLOOR);
-
- /** The CEIL function. */
- public static final SqlFunction CEIL = new SqlFloorFunction(SqlKind.CEIL);
-
- /** The USER function. */
- public static final SqlFunction USER = new SqlStringContextVariable("USER");
-
- /** The CURRENT_USER function. */
- public static final SqlFunction CURRENT_USER = new SqlStringContextVariable("CURRENT_USER");
-
- /** The SESSION_USER function. */
- public static final SqlFunction SESSION_USER = new SqlStringContextVariable("SESSION_USER");
-
- /** The SYSTEM_USER function. */
- public static final SqlFunction SYSTEM_USER = new SqlStringContextVariable("SYSTEM_USER");
-
- /** The CURRENT_PATH function. */
- public static final SqlFunction CURRENT_PATH = new SqlStringContextVariable("CURRENT_PATH");
-
- /** The CURRENT_ROLE function. */
- public static final SqlFunction CURRENT_ROLE = new SqlStringContextVariable("CURRENT_ROLE");
-
- /** The CURRENT_CATALOG function. */
- public static final SqlFunction CURRENT_CATALOG =
- new SqlStringContextVariable("CURRENT_CATALOG");
-
- /** The CURRENT_SCHEMA function. */
- public static final SqlFunction CURRENT_SCHEMA = new SqlStringContextVariable("CURRENT_SCHEMA");
-
- /** The LOCALTIME [(precision)] function. */
- public static final SqlFunction LOCALTIME =
- new SqlAbstractTimeFunction("LOCALTIME", SqlTypeName.TIME);
-
- /** The LOCALTIMESTAMP [(precision)] function. */
- public static final SqlFunction LOCALTIMESTAMP =
- new SqlAbstractTimeFunction("LOCALTIMESTAMP", SqlTypeName.TIMESTAMP);
-
- /** The CURRENT_TIME [(precision)] function. */
- public static final SqlFunction CURRENT_TIME =
- new SqlAbstractTimeFunction("CURRENT_TIME", SqlTypeName.TIME);
-
- /** The CURRENT_TIMESTAMP [(precision)] function. */
- public static final SqlFunction CURRENT_TIMESTAMP =
- new SqlAbstractTimeFunction("CURRENT_TIMESTAMP", SqlTypeName.TIMESTAMP);
-
- /** The CURRENT_DATE function. */
- public static final SqlFunction CURRENT_DATE = new SqlCurrentDateFunction();
-
- /** The TIMESTAMPADD function. */
- public static final SqlFunction TIMESTAMP_ADD = new SqlTimestampAddFunction("TIMESTAMPADD");
-
- /** The TIMESTAMPDIFF function. */
- public static final SqlFunction TIMESTAMP_DIFF =
- new SqlTimestampDiffFunction(
- "TIMESTAMPDIFF",
- OperandTypes.family(
- SqlTypeFamily.ANY, SqlTypeFamily.DATETIME, SqlTypeFamily.DATETIME));
-
- /**
- * The SQL CAST operator.
- *
- *
The SQL syntax is
- *
- *
- *
- * CAST(expression AS type)
- *
- *
- *
- *
When the CAST operator is applies as a {@link SqlCall}, it has two arguments: the
- * expression and the type. The type must not include a constraint, so
- * CAST(x AS INTEGER NOT NULL), for instance, is invalid.
- *
- *
When the CAST operator is applied as a RexCall, the target type is simply
- * stored as the return type, not an explicit operand. For example, the expression
- * CAST(1 + 2 AS DOUBLE) will become a call to CAST with the expression
- * 1 + 2 as its only operand.
- *
- *
The RexCall form can also have a type which contains a NOT NULL
- * constraint. When this expression is implemented, if the value is NULL, an exception will be
- * thrown.
- */
- public static final SqlFunction CAST = new SqlCastFunction();
-
- /**
- * The SQL EXTRACT operator. Extracts a specified field value from a DATETIME or an
- * INTERVAL. E.g.
- * EXTRACT(HOUR FROM INTERVAL '364 23:59:59') returns
- * 23
- */
- public static final SqlFunction EXTRACT = new SqlExtractFunction("EXTRACT", false);
-
- /**
- * The SQL YEAR operator. Returns the Year from a DATETIME E.g.
- * YEAR(date '2008-9-23') returns
- * 2008
- */
- public static final SqlDatePartFunction YEAR = new SqlDatePartFunction("YEAR", TimeUnit.YEAR);
-
- /**
- * The SQL QUARTER operator. Returns the Quarter from a DATETIME E.g.
- * QUARTER(date '2008-9-23') returns
- * 3
- */
- public static final SqlDatePartFunction QUARTER =
- new SqlDatePartFunction("QUARTER", TimeUnit.QUARTER);
-
- /**
- * The SQL MONTH operator. Returns the Month from a DATETIME E.g.
- * MONTH(date '2008-9-23') returns
- * 9
- */
- public static final SqlDatePartFunction MONTH =
- new SqlDatePartFunction("MONTH", TimeUnit.MONTH);
-
- /**
- * The SQL WEEK operator. Returns the Week from a DATETIME E.g.
- * WEEK(date '2008-9-23') returns
- * 39
- */
- public static final SqlDatePartFunction WEEK = new SqlDatePartFunction("WEEK", TimeUnit.WEEK);
-
- /**
- * The SQL DAYOFYEAR operator. Returns the DOY from a DATETIME E.g.
- * DAYOFYEAR(date '2008-9-23') returns
- * 267
- */
- public static final SqlDatePartFunction DAYOFYEAR =
- new SqlDatePartFunction("DAYOFYEAR", TimeUnit.DOY);
-
- /**
- * The SQL DAYOFMONTH operator. Returns the Day from a DATETIME E.g.
- * DAYOFMONTH(date '2008-9-23') returns
- * 23
- */
- public static final SqlDatePartFunction DAYOFMONTH =
- new SqlDatePartFunction("DAYOFMONTH", TimeUnit.DAY);
-
- /**
- * The SQL DAYOFWEEK operator. Returns the DOW from a DATETIME E.g.
- * DAYOFWEEK(date '2008-9-23') returns
- * 2
- */
- public static final SqlDatePartFunction DAYOFWEEK =
- new SqlDatePartFunction("DAYOFWEEK", TimeUnit.DOW);
-
- /**
- * The SQL HOUR operator. Returns the Hour from a DATETIME E.g.
- * HOUR(timestamp '2008-9-23 01:23:45') returns
- * 1
- */
- public static final SqlDatePartFunction HOUR = new SqlDatePartFunction("HOUR", TimeUnit.HOUR);
-
- /**
- * The SQL MINUTE operator. Returns the Minute from a DATETIME E.g.
- * MINUTE(timestamp '2008-9-23 01:23:45') returns
- * 23
- */
- public static final SqlDatePartFunction MINUTE =
- new SqlDatePartFunction("MINUTE", TimeUnit.MINUTE);
-
- /**
- * The SQL SECOND operator. Returns the Second from a DATETIME E.g.
- * SECOND(timestamp '2008-9-23 01:23:45') returns
- * 45
- */
- public static final SqlDatePartFunction SECOND =
- new SqlDatePartFunction("SECOND", TimeUnit.SECOND);
-
- /** The {@code LAST_DAY(date)} function. */
- public static final SqlFunction LAST_DAY =
- SqlBasicFunction.create(
- "LAST_DAY",
- ReturnTypes.DATE_NULLABLE,
- OperandTypes.DATETIME,
- SqlFunctionCategory.TIMEDATE);
-
- /**
- * The ELEMENT operator, used to convert a multiset with only one item to a "regular" type.
- * Example ... log(ELEMENT(MULTISET[1])) ...
- */
- public static final SqlFunction ELEMENT =
- SqlBasicFunction.create(
- "ELEMENT",
- ReturnTypes.MULTISET_ELEMENT_FORCE_NULLABLE,
- OperandTypes.COLLECTION);
-
- /**
- * The item operator {@code [ ... ]}, used to access a given element of an array, map or struct.
- * For example, {@code myArray[3]}, {@code "myMap['foo']"}, {@code myStruct[2]} or {@code
- * myStruct['fieldName']}.
- *
- *
The SQL standard calls the ARRAY variant a <array element reference>. Index is
- * 1-based. The standard says to raise "data exception - array element error" but we currently
- * return null.
- *
- *
MAP is not standard SQL.
- */
- public static final SqlOperator ITEM =
- new SqlItemOperator("ITEM", OperandTypes.ARRAY_OR_MAP_OR_VARIANT, 1, true);
-
- /** The ARRAY Value Constructor. e.g. "ARRAY[1, 2, 3]". */
- public static final SqlArrayValueConstructor ARRAY_VALUE_CONSTRUCTOR =
- new SqlArrayValueConstructor();
-
- /** The MAP Value Constructor, e.g. "MAP['washington', 1, 'obama', 44]". */
- public static final SqlMapValueConstructor MAP_VALUE_CONSTRUCTOR = new SqlMapValueConstructor();
-
- /**
- * The internal "$SLICE" operator takes a multiset of records and returns a multiset of the
- * first column of those records.
- *
- *
It is introduced when multisets of scalar types are created, in order to keep types
- * consistent. For example, MULTISET [5] has type INTEGER MULTISET but
- * is translated to an expression of type RECORD(INTEGER EXPR$0) MULTISET because
- * in our internal representation of multisets, every element must be a record. Applying the
- * "$SLICE" operator to this result converts the type back to an
- * INTEGER MULTISET multiset value.
- *
- *
$SLICE is often translated away when the multiset type is converted back to
- * scalar values.
- */
- public static final SqlInternalOperator SLICE =
- new SqlInternalOperator(
- "$SLICE",
- SqlKind.OTHER,
- 0,
- false,
- ReturnTypes.MULTISET_PROJECT0,
- null,
- OperandTypes.RECORD_COLLECTION) {};
-
- /**
- * The internal "$ELEMENT_SLICE" operator returns the first field of the only element of a
- * multiset.
- *
- *
It is introduced when multisets of scalar types are created, in order to keep types
- * consistent. For example, ELEMENT(MULTISET [5]) is translated to
- * $ELEMENT_SLICE(MULTISET (VALUES ROW (5
- * EXPR$0)) It is translated away when the multiset type is converted back to scalar
- * values.
- *
- *
NOTE: jhyde, 2006/1/9: Usages of this operator are commented out, but I'm not deleting the
- * operator, because some multiset tests are disabled, and we may need this operator to get them
- * working!
- */
- public static final SqlInternalOperator ELEMENT_SLICE =
- new SqlInternalOperator(
- "$ELEMENT_SLICE",
- SqlKind.OTHER,
- 0,
- false,
- ReturnTypes.MULTISET_RECORD,
- null,
- OperandTypes.MULTISET) {
- @Override
- public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
- SqlUtil.unparseFunctionSyntax(this, writer, call, false);
- }
- };
-
- /**
- * The internal "$SCALAR_QUERY" operator returns a scalar value from a record type. It assumes
- * the record type only has one field, and returns that field as the output.
- */
- public static final SqlInternalOperator SCALAR_QUERY =
- new SqlInternalOperator(
- "$SCALAR_QUERY",
- SqlKind.SCALAR_QUERY,
- 0,
- false,
- ReturnTypes.RECORD_TO_SCALAR,
- null,
- OperandTypes.RECORD_TO_SCALAR) {
- @Override
- public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
- final SqlWriter.Frame frame = writer.startList("(", ")");
- call.operand(0).unparse(writer, 0, 0);
- writer.endList(frame);
- }
-
- @Override
- public boolean argumentMustBeScalar(int ordinal) {
- // Obvious, really.
- return false;
- }
- };
-
- /**
- * The internal {@code $STRUCT_ACCESS} operator is used to access a field of a record.
- *
- *
In contrast with {@link #DOT} operator, it never appears in an {@link SqlNode} tree and
- * allows to access fields by position and not by name.
- */
- public static final SqlInternalOperator STRUCT_ACCESS =
- new SqlInternalOperator("$STRUCT_ACCESS", SqlKind.OTHER);
-
- /**
- * The CARDINALITY operator, used to retrieve the number of elements in a MULTISET, ARRAY or
- * MAP.
- */
- public static final SqlFunction CARDINALITY =
- SqlBasicFunction.create(
- "CARDINALITY", ReturnTypes.INTEGER_NULLABLE, OperandTypes.COLLECTION_OR_MAP);
-
- /** The COLLECT operator. Multiset aggregator function. */
- public static final SqlAggFunction COLLECT =
- SqlBasicAggFunction.create(SqlKind.COLLECT, ReturnTypes.TO_MULTISET, OperandTypes.ANY)
- .withFunctionType(SqlFunctionCategory.SYSTEM)
- .withGroupOrder(Optionality.OPTIONAL);
-
- /**
- * {@code PERCENTILE_CONT} inverse distribution aggregate function.
- *
- *
The argument must be a numeric literal in the range 0 to 1 inclusive (representing a
- * percentage), and the return type is the type of the {@code ORDER BY} expression.
- */
- public static final SqlAggFunction PERCENTILE_CONT =
- SqlBasicAggFunction.create(
- SqlKind.PERCENTILE_CONT,
- ReturnTypes.PERCENTILE_DISC_CONT,
- OperandTypes.UNIT_INTERVAL_NUMERIC_LITERAL)
- .withFunctionType(SqlFunctionCategory.SYSTEM)
- .withGroupOrder(Optionality.MANDATORY)
- .withPercentile(true)
- .withAllowsFraming(false);
-
- /**
- * {@code PERCENTILE_DISC} inverse distribution aggregate function.
- *
- *
The argument must be a numeric literal in the range 0 to 1 inclusive (representing a
- * percentage), and the return type is the type of the {@code ORDER BY} expression.
- */
- public static final SqlAggFunction PERCENTILE_DISC =
- SqlBasicAggFunction.create(
- SqlKind.PERCENTILE_DISC,
- ReturnTypes.PERCENTILE_DISC_CONT,
- OperandTypes.UNIT_INTERVAL_NUMERIC_LITERAL)
- .withFunctionType(SqlFunctionCategory.SYSTEM)
- .withGroupOrder(Optionality.MANDATORY)
- .withPercentile(true)
- .withAllowsFraming(false);
-
- /** The LISTAGG operator. String aggregator function. */
- public static final SqlAggFunction LISTAGG =
- new SqlListaggAggFunction(SqlKind.LISTAGG, ReturnTypes.ARG0_NULLABLE);
-
- /** The FUSION operator. Multiset aggregator function. */
- public static final SqlAggFunction FUSION =
- SqlBasicAggFunction.create(SqlKind.FUSION, ReturnTypes.ARG0, OperandTypes.MULTISET)
- .withFunctionType(SqlFunctionCategory.SYSTEM);
-
- /** The INTERSECTION operator. Multiset aggregator function. */
- public static final SqlAggFunction INTERSECTION =
- SqlBasicAggFunction.create(
- SqlKind.INTERSECTION, ReturnTypes.ARG0, OperandTypes.MULTISET)
- .withFunctionType(SqlFunctionCategory.SYSTEM);
-
- /** The sequence next value function: NEXT VALUE FOR sequence. */
- public static final SqlOperator NEXT_VALUE = new SqlSequenceValueOperator(SqlKind.NEXT_VALUE);
-
- /**
- * The sequence current value function: CURRENT VALUE FOR
- * sequence.
- */
- public static final SqlOperator CURRENT_VALUE =
- new SqlSequenceValueOperator(SqlKind.CURRENT_VALUE);
-
- /**
- * The TABLESAMPLE operator.
- *
- *
<query> TABLESAMPLE BERNOULLI(<percent>)
- * [REPEATABLE(<seed>)] (standard, but not implemented for FTRS yet)
- *
<query> TABLESAMPLE SYSTEM(<percent>)
- * [REPEATABLE(<seed>)] (standard, but not implemented for FTRS yet)
- *
- *
- *
Operand #0 is a query or table; Operand #1 is a {@link SqlSampleSpec} wrapped in a {@link
- * SqlLiteral}.
- */
- public static final SqlSpecialOperator TABLESAMPLE =
- new SqlSpecialOperator(
- "TABLESAMPLE",
- SqlKind.TABLESAMPLE,
- 20,
- true,
- ReturnTypes.ARG0,
- null,
- OperandTypes.VARIADIC) {
- @Override
- public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
- call.operand(0).unparse(writer, leftPrec, 0);
- writer.keyword("TABLESAMPLE");
- call.operand(1).unparse(writer, 0, rightPrec);
- }
- };
-
- /** DESCRIPTOR(column_name, ...). */
- public static final SqlOperator DESCRIPTOR = new SqlDescriptorOperator();
-
- /** TUMBLE as a table function. */
- public static final SqlFunction TUMBLE = new SqlTumbleTableFunction();
-
- /** HOP as a table function. */
- public static final SqlFunction HOP = new SqlHopTableFunction();
-
- /** SESSION as a table function. */
- public static final SqlFunction SESSION = new SqlSessionTableFunction();
-
- /**
- * The {@code TUMBLE} group function.
- *
- *
This operator is named "$TUMBLE" (not "TUMBLE") because it is created directly by the
- * parser, not by looking up an operator by name.
- *
- *
Why did we add TUMBLE to the parser? Because we plan to support TUMBLE as a table function
- * (see [CALCITE-3272]); "TUMBLE" as a name will only be used by the TUMBLE table function.
- *
- *
After the TUMBLE table function is introduced, we plan to deprecate this TUMBLE group
- * function, and in fact all group functions. See [CALCITE-3340] for details.
- */
- public static final SqlGroupedWindowFunction TUMBLE_OLD =
- new SqlGroupedWindowFunction(
- "$TUMBLE",
- SqlKind.TUMBLE,
- null,
- ReturnTypes.ARG0,
- null,
- OperandTypes.DATETIME_INTERVAL.or(OperandTypes.DATETIME_INTERVAL_TIME),
- SqlFunctionCategory.SYSTEM) {
- @Override
- public List getAuxiliaryFunctions() {
- return ImmutableList.of(TUMBLE_START, TUMBLE_END);
- }
- };
-
- /** The {@code TUMBLE_START} auxiliary function of the {@code TUMBLE} group function. */
- public static final SqlGroupedWindowFunction TUMBLE_START =
- TUMBLE_OLD.auxiliary(SqlKind.TUMBLE_START);
-
- /** The {@code TUMBLE_END} auxiliary function of the {@code TUMBLE} group function. */
- public static final SqlGroupedWindowFunction TUMBLE_END =
- TUMBLE_OLD.auxiliary(SqlKind.TUMBLE_END);
-
- /** The {@code HOP} group function. */
- public static final SqlGroupedWindowFunction HOP_OLD =
- new SqlGroupedWindowFunction(
- "$HOP",
- SqlKind.HOP,
- null,
- ReturnTypes.ARG0,
- null,
- OperandTypes.DATETIME_INTERVAL_INTERVAL.or(
- OperandTypes.DATETIME_INTERVAL_INTERVAL_TIME),
- SqlFunctionCategory.SYSTEM) {
- @Override
- public List getAuxiliaryFunctions() {
- return ImmutableList.of(HOP_START, HOP_END);
- }
- };
-
- /** The {@code HOP_START} auxiliary function of the {@code HOP} group function. */
- public static final SqlGroupedWindowFunction HOP_START = HOP_OLD.auxiliary(SqlKind.HOP_START);
-
- /** The {@code HOP_END} auxiliary function of the {@code HOP} group function. */
- public static final SqlGroupedWindowFunction HOP_END = HOP_OLD.auxiliary(SqlKind.HOP_END);
-
- /** The {@code SESSION} group function. */
- public static final SqlGroupedWindowFunction SESSION_OLD =
- new SqlGroupedWindowFunction(
- "$SESSION",
- SqlKind.SESSION,
- null,
- ReturnTypes.ARG0,
- null,
- OperandTypes.DATETIME_INTERVAL.or(OperandTypes.DATETIME_INTERVAL_TIME),
- SqlFunctionCategory.SYSTEM) {
- @Override
- public List getAuxiliaryFunctions() {
- return ImmutableList.of(SESSION_START, SESSION_END);
- }
- };
-
- /** The {@code SESSION_START} auxiliary function of the {@code SESSION} group function. */
- public static final SqlGroupedWindowFunction SESSION_START =
- SESSION_OLD.auxiliary(SqlKind.SESSION_START);
-
- /** The {@code SESSION_END} auxiliary function of the {@code SESSION} group function. */
- public static final SqlGroupedWindowFunction SESSION_END =
- SESSION_OLD.auxiliary(SqlKind.SESSION_END);
-
- /**
- * {@code |} operator to create alternate patterns within {@code MATCH_RECOGNIZE}.
- *
- *
If {@code p1} and {@code p2} are patterns then {@code p1 | p2} is a pattern that matches
- * {@code p1} or {@code p2}.
- */
- public static final SqlBinaryOperator PATTERN_ALTER =
- new SqlBinaryOperator("|", SqlKind.PATTERN_ALTER, 70, true, null, null, null);
-
- /**
- * Operator to concatenate patterns within {@code MATCH_RECOGNIZE}.
- *
- *
If {@code p1} and {@code p2} are patterns then {@code p1 p2} is a pattern that matches
- * {@code p1} followed by {@code p2}.
- */
- public static final SqlBinaryOperator PATTERN_CONCAT =
- new SqlBinaryOperator("", SqlKind.PATTERN_CONCAT, 80, true, null, null, null);
-
- /**
- * Operator to quantify patterns within {@code MATCH_RECOGNIZE}.
- *
- *
If {@code p} is a pattern then {@code p{3, 5}} is a pattern that matches between 3 and 5
- * occurrences of {@code p}.
- */
- public static final SqlSpecialOperator PATTERN_QUANTIFIER =
- new SqlSpecialOperator("PATTERN_QUANTIFIER", SqlKind.PATTERN_QUANTIFIER, 90) {
- @Override
- public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
- call.operand(0).unparse(writer, this.getLeftPrec(), this.getRightPrec());
- int startNum = ((SqlNumericLiteral) call.operand(1)).intValue(true);
- SqlNumericLiteral endRepNum = call.operand(2);
- boolean isReluctant = ((SqlLiteral) call.operand(3)).booleanValue();
- int endNum = endRepNum.intValue(true);
- if (startNum == endNum) {
- writer.keyword("{ " + startNum + " }");
- } else {
- if (endNum == -1) {
- if (startNum == 0) {
- writer.keyword("*");
- } else if (startNum == 1) {
- writer.keyword("+");
- } else {
- writer.keyword("{ " + startNum + ", }");
- }
- } else {
- if (startNum == 0 && endNum == 1) {
- writer.keyword("?");
- } else if (startNum == -1) {
- writer.keyword("{ , " + endNum + " }");
- } else {
- writer.keyword("{ " + startNum + ", " + endNum + " }");
- }
- }
- if (isReluctant) {
- writer.keyword("?");
- }
- }
- }
- };
-
- /**
- * {@code PERMUTE} operator to combine patterns within {@code MATCH_RECOGNIZE}.
- *
- *
If {@code p1} and {@code p2} are patterns then {@code PERMUTE (p1, p2)} is a pattern that
- * matches all permutations of {@code p1} and {@code p2}.
- */
- public static final SqlSpecialOperator PATTERN_PERMUTE =
- new SqlSpecialOperator("PATTERN_PERMUTE", SqlKind.PATTERN_PERMUTE, 100) {
- @Override
- public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
- writer.keyword("PERMUTE");
- SqlWriter.Frame frame = writer.startList("(", ")");
- for (int i = 0; i < call.getOperandList().size(); i++) {
- SqlNode pattern = call.getOperandList().get(i);
- pattern.unparse(writer, 0, 0);
- if (i != call.getOperandList().size() - 1) {
- writer.print(",");
- }
- }
- writer.endList(frame);
- }
- };
-
- /**
- * {@code EXCLUDE} operator within {@code MATCH_RECOGNIZE}.
- *
- *
If {@code p} is a pattern then {@code {- p -} }} is a pattern that excludes {@code p} from
- * the output.
- */
- public static final SqlSpecialOperator PATTERN_EXCLUDE =
- new SqlSpecialOperator("PATTERN_EXCLUDE", SqlKind.PATTERN_EXCLUDED, 100) {
- @Override
- public void unparse(SqlWriter writer, SqlCall call, int leftPrec, int rightPrec) {
- SqlWriter.Frame frame = writer.startList("{-", "-}");
- SqlNode node = call.getOperandList().get(0);
- node.unparse(writer, 0, 0);
- writer.endList(frame);
- }
- };
-
- /** SetSemanticsTable represents as an input table with set semantics. */
- public static final SqlInternalOperator SET_SEMANTICS_TABLE =
- new SqlSetSemanticsTableOperator();
-
- // ~ Methods ----------------------------------------------------------------
-
- /** Returns the standard operator table, creating it if necessary. */
- public static SqlStdOperatorTable instance() {
- return INSTANCE.get();
- }
-
- @Override
- protected void lookUpOperators(
- String name, boolean caseSensitive, Consumer consumer) {
- // Only UDFs are looked up using case-sensitive search.
- // Always look up built-in operators case-insensitively. Even in sessions
- // with unquotedCasing=UNCHANGED and caseSensitive=true.
- super.lookUpOperators(name, false, consumer);
- }
-
- /**
- * Returns the group function for which a given kind is an auxiliary function, or null if it is
- * not an auxiliary function.
- */
- public static @Nullable SqlGroupedWindowFunction auxiliaryToGroup(SqlKind kind) {
- switch (kind) {
- case TUMBLE_START:
- case TUMBLE_END:
- return TUMBLE_OLD;
- case HOP_START:
- case HOP_END:
- return HOP_OLD;
- case SESSION_START:
- case SESSION_END:
- return SESSION_OLD;
- default:
- return null;
- }
- }
-
- /**
- * Converts a call to a grouped auxiliary function to a call to the grouped window function. For
- * other calls returns null.
- *
- *
For example, converts {@code TUMBLE_START(rowtime, INTERVAL '1' HOUR))} to {@code
- * TUMBLE(rowtime, INTERVAL '1' HOUR))}.
- */
- public static @Nullable SqlCall convertAuxiliaryToGroupCall(SqlCall call) {
- final SqlOperator op = call.getOperator();
- if (op instanceof SqlGroupedWindowFunction && op.isGroupAuxiliary()) {
- final SqlGroupedWindowFunction fun = (SqlGroupedWindowFunction) op;
- return copy(call, requireNonNull(fun.groupFunction, "groupFunction"));
- }
- return null;
- }
-
- /**
- * Converts a call to a grouped window function to a call to its auxiliary window function(s).
- */
- @Deprecated // to be removed before 2.0
- public static List> convertGroupToAuxiliaryCalls(
- SqlCall call) {
- ImmutableList.Builder> builder = ImmutableList.builder();
- convertGroupToAuxiliaryCalls(call, (k, v) -> builder.add(Pair.of(k, v)));
- return builder.build();
- }
-
- /**
- * Converts a call to a grouped window function to a call to its auxiliary window function(s).
- *
- *
For example, converts {@code TUMBLE_START(rowtime, INTERVAL '1' HOUR))} to {@code
- * TUMBLE(rowtime, INTERVAL '1' HOUR))}.
- */
- public static void convertGroupToAuxiliaryCalls(
- SqlCall call, BiConsumer consumer) {
- final SqlOperator op = call.getOperator();
- if (op instanceof SqlGroupedWindowFunction && op.isGroup()) {
- final SqlGroupedWindowFunction fun = (SqlGroupedWindowFunction) op;
- fun.getAuxiliaryFunctions()
- .forEach(f -> consumer.accept(copy(call, f), new AuxiliaryConverter.Impl(f)));
- }
- }
-
- /** Creates a copy of a call with a new operator. */
- private static SqlCall copy(SqlCall call, SqlOperator operator) {
- return new SqlBasicCall(operator, call.getOperandList(), call.getParserPosition());
- }
-
- /** Returns the operator for {@code SOME comparisonKind}. */
- public static SqlQuantifyOperator some(SqlKind comparisonKind) {
- switch (comparisonKind) {
- case EQUALS:
- return SOME_EQ;
- case NOT_EQUALS:
- return SOME_NE;
- case LESS_THAN:
- return SOME_LT;
- case LESS_THAN_OR_EQUAL:
- return SOME_LE;
- case GREATER_THAN:
- return SOME_GT;
- case GREATER_THAN_OR_EQUAL:
- return SOME_GE;
- default:
- throw new AssertionError(comparisonKind);
- }
- }
-
- /** Returns the operator for {@code ALL comparisonKind}. */
- public static SqlQuantifyOperator all(SqlKind comparisonKind) {
- switch (comparisonKind) {
- case EQUALS:
- return ALL_EQ;
- case NOT_EQUALS:
- return ALL_NE;
- case LESS_THAN:
- return ALL_LT;
- case LESS_THAN_OR_EQUAL:
- return ALL_LE;
- case GREATER_THAN:
- return ALL_GT;
- case GREATER_THAN_OR_EQUAL:
- return ALL_GE;
- default:
- throw new AssertionError(comparisonKind);
- }
- }
-
- /**
- * Returns the binary operator that corresponds to this operator but in the opposite direction.
- * Or returns this, if its kind is not reversible.
- *
- *
For example, {@code reverse(GREATER_THAN)} returns {@link #LESS_THAN}.
- *
- * @deprecated Use {@link SqlOperator#reverse()}, but beware that it has slightly different
- * semantics
- */
- @Deprecated // to be removed before 2.0
- public static SqlOperator reverse(SqlOperator operator) {
- switch (operator.getKind()) {
- case GREATER_THAN:
- return LESS_THAN;
- case GREATER_THAN_OR_EQUAL:
- return LESS_THAN_OR_EQUAL;
- case LESS_THAN:
- return GREATER_THAN;
- case LESS_THAN_OR_EQUAL:
- return GREATER_THAN_OR_EQUAL;
- default:
- return operator;
- }
- }
-
- /** Returns the operator for {@code LIKE} with given case-sensitivity, optionally negated. */
- public static SqlOperator like(boolean negated, boolean caseSensitive) {
- if (negated) {
- if (caseSensitive) {
- return NOT_LIKE;
- } else {
- return SqlLibraryOperators.NOT_ILIKE;
- }
- } else {
- if (caseSensitive) {
- return LIKE;
- } else {
- return SqlLibraryOperators.ILIKE;
- }
- }
- }
-
- /**
- * Returns the operator for {@code FLOOR} and {@code CEIL} with given floor flag and library.
- */
- public static SqlOperator floorCeil(boolean floor, SqlConformance conformance) {
- if (SqlConformanceEnum.BIG_QUERY == conformance) {
- return floor ? SqlLibraryOperators.FLOOR_BIG_QUERY : SqlLibraryOperators.CEIL_BIG_QUERY;
- } else {
- return floor ? SqlStdOperatorTable.FLOOR : SqlStdOperatorTable.CEIL;
- }
- }
-
- /**
- * Returns the operator for standard {@code CONVERT} and Oracle's {@code CONVERT} with the given
- * library.
- */
- public static SqlOperator getConvertFuncByConformance(SqlConformance conformance) {
- if (SqlConformanceEnum.ORACLE_10 == conformance
- || SqlConformanceEnum.ORACLE_12 == conformance) {
- return SqlLibraryOperators.CONVERT_ORACLE;
- } else {
- return SqlStdOperatorTable.CONVERT;
- }
- }
-}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkRelBuilder.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkRelBuilder.java
index 9045b7d9e38503..c68cdfed8f8ac8 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkRelBuilder.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkRelBuilder.java
@@ -242,8 +242,16 @@ public RelBuilder watermark(int rowtimeFieldIndex, RexNode watermarkExpr) {
}
public RelBuilder queryOperation(QueryOperation queryOperation) {
- final RelNode relNode = queryOperation.accept(toRelNodeConverter);
- return push(relNode);
+ // Only the outermost call has a root (its top-level ORDER BY is kept); subquery
+ // re-entries have none, so their fetch-less sorts are dropped. Restore afterwards.
+ final QueryOperation previousRoot = toRelNodeConverter.getRootOperation();
+ toRelNodeConverter.setRootOperation(previousRoot == null ? queryOperation : null);
+ try {
+ final RelNode relNode = queryOperation.accept(toRelNodeConverter);
+ return push(relNode);
+ } finally {
+ toRelNodeConverter.setRootOperation(previousRoot);
+ }
}
@Override
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java
index bb99230775c855..703c5707583975 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/FlinkTypeFactory.java
@@ -29,6 +29,7 @@
import org.apache.flink.table.legacy.types.logical.TypeInformationRawType;
import org.apache.flink.table.planner.plan.schema.BitmapRelDataType;
import org.apache.flink.table.planner.plan.schema.GenericRelDataType;
+import org.apache.flink.table.planner.plan.schema.GeographyRelDataType;
import org.apache.flink.table.planner.plan.schema.RawRelDataType;
import org.apache.flink.table.planner.plan.schema.StructuredRelDataType;
import org.apache.flink.table.planner.plan.schema.TimeIndicatorRelDataType;
@@ -46,6 +47,7 @@
import org.apache.flink.table.types.logical.DescriptorType;
import org.apache.flink.table.types.logical.DoubleType;
import org.apache.flink.table.types.logical.FloatType;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.flink.table.types.logical.IntType;
import org.apache.flink.table.types.logical.LegacyTypeInformationType;
import org.apache.flink.table.types.logical.LocalZonedTimestampType;
@@ -65,6 +67,7 @@
import org.apache.flink.table.types.logical.VarBinaryType;
import org.apache.flink.table.types.logical.VarCharType;
import org.apache.flink.table.types.logical.VariantType;
+import org.apache.flink.table.types.logical.utils.LogicalTypeMerging;
import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo;
import org.apache.flink.table.utils.TableSchemaUtils;
import org.apache.flink.util.Preconditions;
@@ -160,6 +163,11 @@ public RelDataType createBitmapType() {
return canonize(new BitmapRelDataType(new BitmapType()));
}
+ @Override
+ public RelDataType createGeographyType() {
+ return canonize(new GeographyRelDataType(new GeographyType()));
+ }
+
@Override
public RelDataType createArrayType(RelDataType elementType, long maxCardinality) {
// Just validate type, make sure there is a failure in validate phase.
@@ -230,6 +238,8 @@ public RelDataType createTypeWithNullability(RelDataType relDataType, boolean is
newType = ((StructuredRelDataType) relDataType).createWithNullability(isNullable);
} else if (relDataType instanceof BitmapRelDataType) {
newType = ((BitmapRelDataType) relDataType).createWithNullability(isNullable);
+ } else if (relDataType instanceof GeographyRelDataType) {
+ newType = ((GeographyRelDataType) relDataType).createWithNullability(isNullable);
} else if (relDataType instanceof GenericRelDataType) {
final GenericRelDataType generic = (GenericRelDataType) relDataType;
newType = new GenericRelDataType(generic.genericType(), isNullable, getTypeSystem());
@@ -254,8 +264,20 @@ public RelDataType createTypeWithNullability(RelDataType relDataType, boolean is
@Override
public RelDataType leastRestrictive(List types) {
final Optional resolved = resolveAllIdenticalTypes(types);
- final RelDataType leastRestrictive =
- resolved.orElseGet(() -> super.leastRestrictive(types));
+ if (resolved.isPresent()) {
+ return normalizeLeastRestrictive(resolved.get());
+ }
+
+ if (containsFlinkExtensionType(types)) {
+ return normalizeLeastRestrictive(
+ resolveCommonTypeForFlinkExtensions(types).orElse(null));
+ }
+
+ final RelDataType leastRestrictive = super.leastRestrictive(types);
+ return normalizeLeastRestrictive(leastRestrictive);
+ }
+
+ private RelDataType normalizeLeastRestrictive(RelDataType leastRestrictive) {
// NULL is reserved for untyped literals only
if (leastRestrictive == null || leastRestrictive.getSqlTypeName() == SqlTypeName.NULL) {
return null;
@@ -263,6 +285,24 @@ public RelDataType leastRestrictive(List types) {
return leastRestrictive;
}
+ private Optional resolveCommonTypeForFlinkExtensions(List types) {
+ return LogicalTypeMerging.findCommonType(
+ types.stream()
+ .map(FlinkTypeFactory::toLogicalType)
+ .collect(Collectors.toList()))
+ .map(this::createFieldTypeFromLogicalType);
+ }
+
+ private boolean containsFlinkExtensionType(List types) {
+ return types.stream().anyMatch(FlinkTypeFactory::isFlinkExtensionType);
+ }
+
+ private static boolean isFlinkExtensionType(RelDataType type) {
+ return type instanceof RawRelDataType
+ || type instanceof BitmapRelDataType
+ || type instanceof GeographyRelDataType;
+ }
+
private Optional resolveAllIdenticalTypes(List types) {
final RelDataType head = types.get(0);
// check if all types are the same
@@ -492,6 +532,9 @@ private RelDataType newRelDataType(LogicalType logicalType) {
case BITMAP:
return new BitmapRelDataType((BitmapType) logicalType);
+ case GEOGRAPHY:
+ return new GeographyRelDataType((GeographyType) logicalType);
+
default:
throw new TableException("Type is not supported: " + logicalType);
}
@@ -865,6 +908,8 @@ private static LogicalType toLogicalTypeWithoutNullability(RelDataType relDataTy
return ((RawRelDataType) relDataType).getRawType();
} else if (relDataType instanceof BitmapRelDataType) {
return ((BitmapRelDataType) relDataType).getBitmapType();
+ } else if (relDataType instanceof GeographyRelDataType) {
+ return ((GeographyRelDataType) relDataType).getGeographyType();
} else {
throw new TableException("Type is not supported: " + relDataType);
}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.java
index 262665155789d5..86954858c16085 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.java
@@ -30,6 +30,7 @@
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalExpand;
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalIntersect;
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalJoin;
+import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLateralSnapshotJoin;
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalLegacySink;
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalMatch;
import org.apache.flink.table.planner.plan.nodes.logical.FlinkLogicalMinus;
@@ -164,6 +165,8 @@ public RelNode visit(RelNode node) {
return visitCalc((FlinkLogicalCalc) node);
} else if (node instanceof FlinkLogicalCorrelate) {
return visitCorrelate((FlinkLogicalCorrelate) node);
+ } else if (node instanceof FlinkLogicalLateralSnapshotJoin) {
+ return visitLateralSnapshotJoin((FlinkLogicalLateralSnapshotJoin) node);
} else if (node instanceof FlinkLogicalJoin) {
return visitJoin((FlinkLogicalJoin) node);
} else if (node instanceof FlinkLogicalMultiJoin) {
@@ -367,6 +370,43 @@ public RexNode visitInputRef(RexInputRef inputRef) {
}
}
+ private RelNode visitLateralSnapshotJoin(FlinkLogicalLateralSnapshotJoin join) {
+ RelNode newLeft = join.getLeft().accept(this);
+ RelNode newRight = join.getRight().accept(this);
+
+ // Materialize the build-side (right) proc-time attributes
+ newRight = materializeProcTime(newRight);
+
+ List leftRightFields = new ArrayList<>();
+ leftRightFields.addAll(newLeft.getRowType().getFieldList());
+ leftRightFields.addAll(newRight.getRowType().getFieldList());
+
+ RexNode newCondition =
+ join.getCondition()
+ .accept(
+ new RexShuttle() {
+ @Override
+ public RexNode visitInputRef(RexInputRef inputRef) {
+ if (isTimeIndicatorType(inputRef.getType())) {
+ return RexInputRef.of(
+ inputRef.getIndex(), leftRightFields);
+ } else {
+ return super.visitInputRef(inputRef);
+ }
+ }
+ });
+
+ return FlinkLogicalLateralSnapshotJoin.create(
+ newLeft,
+ newRight,
+ newCondition,
+ join.getJoinType(),
+ join.getLoadCompletedCondition(),
+ join.getLoadCompletedTime(),
+ join.getLoadCompletedIdleTimeoutMs(),
+ join.getStateTtlMs());
+ }
+
private RelNode visitCorrelate(FlinkLogicalCorrelate correlate) {
// visit children and update inputs
RelNode newLeft = correlate.getLeft().accept(this);
@@ -544,6 +584,7 @@ private FlinkLogicalWindowAggregate visitWindowAggregate(FlinkLogicalWindowAggre
return new FlinkLogicalWindowAggregate(
agg.getCluster(),
agg.getTraitSet(),
+ agg.getHints(),
newInput,
agg.getGroupSet(),
updatedAggCalls,
@@ -556,6 +597,7 @@ private RelNode visitWindowTableAggregate(FlinkLogicalWindowTableAggregate table
new FlinkLogicalWindowAggregate(
tableAgg.getCluster(),
tableAgg.getTraitSet(),
+ List.of(),
tableAgg.getInput(),
tableAgg.getGroupSet(),
tableAgg.getAggCallList(),
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java
index 3a532d6f15f5df..9e9b0da7a20e16 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/expressions/converter/ExpressionConverter.java
@@ -18,11 +18,13 @@
package org.apache.flink.table.planner.expressions.converter;
+import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.TableException;
import org.apache.flink.table.catalog.Catalog;
import org.apache.flink.table.catalog.ContextResolvedModel;
import org.apache.flink.table.catalog.DataTypeFactory;
import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GeographyData;
import org.apache.flink.table.expressions.CallExpression;
import org.apache.flink.table.expressions.Expression;
import org.apache.flink.table.expressions.ExpressionVisitor;
@@ -36,6 +38,7 @@
import org.apache.flink.table.expressions.ValueLiteralExpression;
import org.apache.flink.table.factories.FactoryUtil;
import org.apache.flink.table.factories.ModelProviderFactory;
+import org.apache.flink.table.functions.BuiltInFunctionDefinitions;
import org.apache.flink.table.ml.ModelProvider;
import org.apache.flink.table.module.Module;
import org.apache.flink.table.planner.calcite.FlinkContext;
@@ -141,6 +144,10 @@ public RexNode visit(ValueLiteralExpression valueLiteral) {
.collect(Collectors.toList()));
}
+ if (type.getTypeRoot() == LogicalTypeRoot.GEOGRAPHY) {
+ return convertGeographyLiteral(valueLiteral);
+ }
+
Object value;
switch (type.getTypeRoot()) {
case DECIMAL:
@@ -220,6 +227,28 @@ public RexNode visit(ValueLiteralExpression valueLiteral) {
true);
}
+ private RexNode convertGeographyLiteral(ValueLiteralExpression valueLiteral) {
+ final GeographyData geography =
+ valueLiteral
+ .getValueAs(GeographyData.class)
+ .orElseThrow(
+ () ->
+ new TableException(
+ String.format(
+ "GEOGRAPHY literals require values of class '%s' but found '%s'.",
+ GeographyData.class.getName(),
+ extractValue(valueLiteral, Object.class)
+ .getClass()
+ .getName())));
+ return visit(
+ CallExpression.permanent(
+ BuiltInFunctionDefinitions.ST_GEOGFROMWKB,
+ List.of(
+ new ValueLiteralExpression(
+ geography.toBytes(), DataTypes.BYTES().notNull())),
+ valueLiteral.getOutputDataType()));
+ }
+
@Override
public RexNode visit(FieldReferenceExpression fieldReference) {
// We can not use inputCount+inputIndex+FieldIndex to construct field of calcite.
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CastRuleProvider.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CastRuleProvider.java
index b18b44e126e171..133542ec6f58e6 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CastRuleProvider.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/CastRuleProvider.java
@@ -97,6 +97,7 @@ public class CastRuleProvider {
.addRule(RowToRowCastRule.INSTANCE)
// Variant rules
.addRule(VariantToStringCastRule.INSTANCE)
+ .addRule(VariantToPrimitiveCastRule.INSTANCE)
// Bitmap rules
.addRule(BitmapToStringCastRule.INSTANCE)
.addRule(BitmapToBinaryCastRule.INSTANCE)
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java
new file mode 100644
index 00000000000000..8a278f41498b57
--- /dev/null
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/casting/VariantToPrimitiveCastRule.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.planner.functions.casting;
+
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.TimestampData;
+import org.apache.flink.table.planner.functions.casting.CastRuleUtils.CodeWriter;
+import org.apache.flink.table.types.logical.DecimalType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.flink.table.types.logical.utils.LogicalTypeChecks;
+import org.apache.flink.types.variant.Variant;
+
+import java.math.BigDecimal;
+import java.util.Arrays;
+
+import static org.apache.flink.table.planner.codegen.CodeGenUtils.className;
+import static org.apache.flink.table.planner.codegen.CodeGenUtils.newName;
+import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.arrayLength;
+import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.cast;
+import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.constructorCall;
+import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.methodCall;
+import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.staticCall;
+import static org.apache.flink.table.planner.functions.casting.CastRuleUtils.ternaryOperator;
+
+/**
+ * {@link LogicalTypeRoot#VARIANT} to primitive type cast rule.
+ *
+ *
Numeric targets are lenient and follow regular numeric cast semantics; other targets require
+ * the stored value to match the target kind. On a mismatch {@code CAST} fails and {@code TRY_CAST}
+ * returns {@code null}.
+ *
+ *
{@code CHARACTER_STRING} is handled by {@link VariantToStringCastRule}; {@code TIME} has no
+ * variant counterpart and is unsupported.
+ */
+class VariantToPrimitiveCastRule extends AbstractNullAwareCodeGeneratorCastRule {
+
+ static final VariantToPrimitiveCastRule INSTANCE = new VariantToPrimitiveCastRule();
+
+ private VariantToPrimitiveCastRule() {
+ super(
+ CastRulePredicate.builder()
+ .predicate(
+ (input, target) ->
+ input.is(LogicalTypeRoot.VARIANT)
+ && isSupportedTarget(target))
+ .build());
+ }
+
+ private static boolean isSupportedTarget(LogicalType targetType) {
+ switch (targetType.getTypeRoot()) {
+ case BOOLEAN:
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+ case FLOAT:
+ case DOUBLE:
+ case DECIMAL:
+ case BINARY:
+ case VARBINARY:
+ case DATE:
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ @Override
+ public boolean canFail(LogicalType inputLogicalType, LogicalType targetLogicalType) {
+ return true;
+ }
+
+ /**
+ * Treats a variant that stores a JSON {@code null} as a {@code NULL} input, so it casts to SQL
+ * {@code NULL} instead of failing in the type-specific accessor. Only applied for a nullable
+ * target: a {@code NOT NULL} result cannot carry {@code NULL}, so a null-valued variant then
+ * fails as a regular type mismatch.
+ */
+ @Override
+ public CastCodeBlock generateCodeBlock(
+ CodeGeneratorCastRule.Context context,
+ String inputTerm,
+ String inputIsNullTerm,
+ LogicalType inputLogicalType,
+ LogicalType targetLogicalType) {
+ if (!targetLogicalType.isNullable()) {
+ return super.generateCodeBlock(
+ context, inputTerm, inputIsNullTerm, inputLogicalType, targetLogicalType);
+ }
+ final String isNullTerm =
+ "(" + inputIsNullTerm + " || " + methodCall(inputTerm, "isNull") + ")";
+ return super.generateCodeBlock(
+ context, inputTerm, isNullTerm, inputLogicalType, targetLogicalType);
+ }
+
+ @Override
+ protected String generateCodeBlockInternal(
+ CodeGeneratorCastRule.Context context,
+ String inputTerm,
+ String returnVariable,
+ LogicalType inputLogicalType,
+ LogicalType targetLogicalType) {
+ final CodeWriter writer = new CastRuleUtils.CodeWriter();
+ switch (targetLogicalType.getTypeRoot()) {
+ case BOOLEAN:
+ writer.assignStmt(returnVariable, methodCall(inputTerm, "getBoolean"));
+ break;
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+ case FLOAT:
+ case DOUBLE:
+ case DECIMAL:
+ writer.assignStmt(returnVariable, numericExpression(inputTerm, targetLogicalType));
+ break;
+ case BINARY:
+ case VARBINARY:
+ generateToBytes(context, inputTerm, returnVariable, targetLogicalType, writer);
+ break;
+ case DATE:
+ writer.assignStmt(
+ returnVariable,
+ cast("int", methodCall(methodCall(inputTerm, "getDate"), "toEpochDay")));
+ break;
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ writer.assignStmt(
+ returnVariable,
+ staticCall(
+ TimestampData.class,
+ "fromLocalDateTime",
+ methodCall(inputTerm, "getDateTime")));
+ break;
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
+ writer.assignStmt(
+ returnVariable,
+ staticCall(
+ TimestampData.class,
+ "fromInstant",
+ methodCall(inputTerm, "getInstant")));
+ break;
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported target type for casting from VARIANT: " + targetLogicalType);
+ }
+ return writer.toString();
+ }
+
+ /**
+ * Converts a numeric variant to the numeric {@code target} via the matching {@link Number}
+ * accessor, mirroring regular numeric cast semantics. A non-numeric variant raises {@link
+ * ClassCastException}, failing {@code CAST} and yielding {@code null} for {@code TRY_CAST}.
+ */
+ private static String numericExpression(String inputTerm, LogicalType target) {
+ final String number = cast(className(Number.class), methodCall(inputTerm, "get"));
+ if (!target.is(LogicalTypeRoot.DECIMAL)) {
+ return methodCall(number, numberAccessor(target));
+ }
+ final DecimalType decimalType = (DecimalType) target;
+ return staticCall(
+ DecimalData.class,
+ "fromBigDecimal",
+ constructorCall(BigDecimal.class, methodCall(number, "toString")),
+ decimalType.getPrecision(),
+ decimalType.getScale());
+ }
+
+ private static String numberAccessor(LogicalType target) {
+ switch (target.getTypeRoot()) {
+ case TINYINT:
+ return "byteValue";
+ case SMALLINT:
+ return "shortValue";
+ case INTEGER:
+ return "intValue";
+ case BIGINT:
+ return "longValue";
+ case FLOAT:
+ return "floatValue";
+ case DOUBLE:
+ return "doubleValue";
+ default:
+ throw new IllegalArgumentException(
+ "Unsupported numeric target for casting from VARIANT: " + target);
+ }
+ }
+
+ private static void generateToBytes(
+ CodeGeneratorCastRule.Context context,
+ String inputTerm,
+ String returnVariable,
+ LogicalType targetLogicalType,
+ CodeWriter writer) {
+ final int targetLength = LogicalTypeChecks.getLength(targetLogicalType);
+ // Read the bytes once to avoid decoding the variant twice.
+ final String bytesTerm = newName(context.getCodeGeneratorContext(), "variantBytes");
+ writer.declStmt("byte[]", bytesTerm, methodCall(inputTerm, "getBytes"));
+ if (BinaryToBinaryCastRule.couldPad(targetLogicalType, targetLength)) {
+ // BINARY(n): pad or trim to the exact target length.
+ writer.assignStmt(
+ returnVariable,
+ ternaryOperator(
+ arrayLength(bytesTerm) + " == " + targetLength,
+ bytesTerm,
+ staticCall(Arrays.class, "copyOf", bytesTerm, targetLength)));
+ } else if (BinaryToBinaryCastRule.couldTrim(targetLength)) {
+ // VARBINARY(n): trim only when longer than the target length.
+ writer.assignStmt(
+ returnVariable,
+ ternaryOperator(
+ arrayLength(bytesTerm) + " <= " + targetLength,
+ bytesTerm,
+ staticCall(Arrays.class, "copyOf", bytesTerm, targetLength)));
+ } else {
+ writer.assignStmt(returnVariable, bytesTerm);
+ }
+ }
+}
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
index 32a2a20d679660..33da605f632aab 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/functions/sql/FlinkSqlOperatorTable.java
@@ -25,6 +25,7 @@
import org.apache.flink.table.planner.functions.sql.ml.SqlVectorSearchTableFunction;
import org.apache.flink.table.planner.plan.type.FlinkReturnTypes;
import org.apache.flink.table.planner.plan.type.NumericExceptFirstOperandChecker;
+import org.apache.flink.table.types.logical.GeographyType;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlFunction;
@@ -72,6 +73,16 @@ public class FlinkSqlOperatorTable extends ReflectiveSqlOperatorTable {
/** The table of contains Flink-specific operators. */
private static final Map cachedInstances = new HashMap<>();
+ private static final SqlReturnTypeInference GEOGRAPHY_NULLABLE_IF_ARGS =
+ opBinding -> {
+ boolean nullable = false;
+ for (int i = 0; i < opBinding.getOperandCount(); i++) {
+ nullable |= opBinding.getOperandType(i).isNullable();
+ }
+ return ((FlinkTypeFactory) opBinding.getTypeFactory())
+ .createFieldTypeFromLogicalType(new GeographyType(nullable));
+ };
+
/** Returns the Flink operator table, creating it if necessary. */
public static synchronized FlinkSqlOperatorTable instance(boolean isBatchMode) {
FlinkSqlOperatorTable instance = cachedInstances.get(isBatchMode);
@@ -867,6 +878,42 @@ public SqlSyntax getSyntax() {
OperandTypes.family(SqlTypeFamily.BINARY, SqlTypeFamily.CHARACTER),
SqlFunctionCategory.STRING);
+ // GEOGRAPHY FUNCTIONS
+ public static final SqlFunction ST_GEOGFROMTEXT =
+ BuiltInSqlFunction.newBuilder()
+ .name(BuiltInFunctionDefinitions.ST_GEOGFROMTEXT.getName())
+ .returnType(GEOGRAPHY_NULLABLE_IF_ARGS)
+ .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY))
+ .category(SqlFunctionCategory.STRING)
+ .build();
+
+ public static final SqlFunction ST_GEOGFROMWKB =
+ BuiltInSqlFunction.newBuilder()
+ .name(BuiltInFunctionDefinitions.ST_GEOGFROMWKB.getName())
+ .returnType(GEOGRAPHY_NULLABLE_IF_ARGS)
+ .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY))
+ .category(SqlFunctionCategory.STRING)
+ .build();
+
+ public static final SqlFunction ST_ASTEXT =
+ BuiltInSqlFunction.newBuilder()
+ .name(BuiltInFunctionDefinitions.ST_ASTEXT.getName())
+ .returnType(VARCHAR_FORCE_NULLABLE)
+ .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY))
+ .category(SqlFunctionCategory.STRING)
+ .build();
+
+ public static final SqlFunction ST_ASWKB =
+ BuiltInSqlFunction.newBuilder()
+ .name(BuiltInFunctionDefinitions.ST_ASWKB.getName())
+ .returnType(
+ ReturnTypes.cascade(
+ ReturnTypes.explicit(SqlTypeName.VARBINARY),
+ SqlTypeTransforms.TO_NULLABLE))
+ .operandTypeChecker(OperandTypes.family(SqlTypeFamily.ANY))
+ .category(SqlFunctionCategory.STRING)
+ .build();
+
public static final SqlFunction INSTR =
new SqlFunction(
"INSTR",
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/FlinkHintStrategies.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/FlinkHintStrategies.java
index 33e6d4346dd913..5978f98017d91d 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/FlinkHintStrategies.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/hint/FlinkHintStrategies.java
@@ -82,7 +82,9 @@ public static HintStrategyTable createHintStrategyTable() {
.hintStrategy(
FlinkHints.HINT_NAME_JSON_AGGREGATE_WRAPPED,
HintStrategy.builder(HintPredicates.AGGREGATE)
- .excludedRules(WrapJsonAggFunctionArgumentsRule.INSTANCE)
+ .excludedRules(
+ WrapJsonAggFunctionArgumentsRule.AGGREGATE_INSTANCE,
+ WrapJsonAggFunctionArgumentsRule.WINDOW_AGGREGATE_INSTANCE)
.build())
// internal query hint used for alias
.hintStrategy(
diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/MergeTableAsUtil.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/MergeTableAsUtil.java
index 2b26585d6f4116..e41e0e9ed08a7f 100644
--- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/MergeTableAsUtil.java
+++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/planner/operations/converters/MergeTableAsUtil.java
@@ -28,30 +28,28 @@
import org.apache.flink.table.api.Schema;
import org.apache.flink.table.api.Schema.UnresolvedColumn;
import org.apache.flink.table.api.Schema.UnresolvedPhysicalColumn;
-import org.apache.flink.table.api.TableException;
import org.apache.flink.table.api.ValidationException;
-import org.apache.flink.table.catalog.CatalogManager;
import org.apache.flink.table.catalog.DataTypeFactory;
import org.apache.flink.table.catalog.ResolvedCatalogBaseTable;
import org.apache.flink.table.catalog.ResolvedSchema;
import org.apache.flink.table.planner.calcite.FlinkCalciteSqlValidator;
-import org.apache.flink.table.planner.calcite.FlinkPlannerImpl;
import org.apache.flink.table.planner.calcite.FlinkTypeFactory;
-import org.apache.flink.table.planner.calcite.SqlRewriterUtils;
import org.apache.flink.table.planner.operations.PlannerQueryOperation;
-import org.apache.flink.table.planner.operations.SqlNodeToOperationConversion;
import org.apache.flink.table.planner.operations.converters.SqlNodeConverter.ConvertContext;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.RowType;
+import org.apache.calcite.plan.RelOptCluster;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlIdentifier;
-import org.apache.calcite.sql.SqlLiteral;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.SqlNodeList;
-import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.validate.SqlValidator;
+import org.apache.calcite.tools.RelBuilder;
import javax.annotation.Nullable;
@@ -88,80 +86,70 @@ public MergeTableAsUtil(ConvertContext context) {
}
/**
- * Rewrites the query operation to include only the fields that may be persisted in the sink.
+ * Reshapes the query so its output columns line up with the sink's persistable columns:
+ * reordering them and filling sink columns the query does not produce with {@code NULL}.
+ * Returns the query unchanged when it already matches the sink 1:1. A sink column the query
+ * does not produce that is declared {@code NOT NULL} raises a {@link ValidationException}.
*/
public PlannerQueryOperation maybeRewriteQuery(
- CatalogManager catalogManager,
- FlinkPlannerImpl flinkPlanner,
PlannerQueryOperation origQueryOperation,
SqlNode origQueryNode,
ResolvedCatalogBaseTable sinkTable) {
- FlinkCalciteSqlValidator sqlValidator = flinkPlanner.getOrCreateSqlValidator();
- SqlRewriterUtils rewriterUtils = new SqlRewriterUtils(sqlValidator);
- FlinkTypeFactory typeFactory = (FlinkTypeFactory) sqlValidator.getTypeFactory();
-
- // Only fields that may be persisted will be included in the select query
- RowType sinkRowType =
- ((RowType) sinkTable.getResolvedSchema().toSinkRowDataType().getLogicalType());
-
- Map sourceFields =
- IntStream.range(0, origQueryOperation.getResolvedSchema().getColumnNames().size())
+ final RelNode queryRelNode = origQueryOperation.getCalciteTree();
+ final RelOptCluster cluster = queryRelNode.getCluster();
+ final RexBuilder rexBuilder = cluster.getRexBuilder();
+ final FlinkTypeFactory typeFactory = (FlinkTypeFactory) cluster.getTypeFactory();
+
+ // Only fields that may be persisted are included in the sink.
+ final RowType sinkRowType =
+ (RowType) sinkTable.getResolvedSchema().toSinkRowDataType().getLogicalType();
+
+ final List sourceColumns = origQueryOperation.getResolvedSchema().getColumnNames();
+ final Map sourceFields =
+ IntStream.range(0, sourceColumns.size())
.boxed()
- .collect(
- Collectors.toMap(
- origQueryOperation.getResolvedSchema().getColumnNames()
- ::get,
- Function.identity()));
+ .collect(Collectors.toMap(sourceColumns::get, Function.identity()));
- // assignedFields contains the new sink fields that are not present in the source
- // and that will be included in the select query
- LinkedHashMap assignedFields = new LinkedHashMap<>();
-
- // targetPositions contains the positions of the source fields that will be
- // included in the select query
- List