Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion e2e/questdb
Submodule questdb updated 87 files
+16 −0 CLAUDE.md
+1 −1 benchmarks/pom.xml
+194 −0 benchmarks/src/main/java/org/questdb/QwpTotalBufferedBytesBenchmark.java
+26 −3 ci/templates/hosted-jobs.yml
+2 −2 ci/templates/steps.yml
+4 −0 ci/test-fuzz.yml
+4 −0 ci/test-pipeline.yml
+1 −1 core/pom.xml
+2 −1 core/src/main/java/io/questdb/DynamicPropServerConfiguration.java
+22 −0 core/src/main/java/io/questdb/PropServerConfiguration.java
+1 −0 core/src/main/java/io/questdb/PropertyKey.java
+25 −0 core/src/main/java/io/questdb/cairo/CairoConfiguration.java
+5 −0 core/src/main/java/io/questdb/cairo/CairoConfigurationWrapper.java
+76 −0 core/src/main/java/io/questdb/cairo/ReaderScanProfile.java
+19 −16 core/src/main/java/io/questdb/cairo/TableReader.java
+10 −8 core/src/main/java/io/questdb/cairo/sql/PageFrameCursor.java
+27 −0 core/src/main/java/io/questdb/cairo/wal/ColumnarRowAppender.java
+54 −0 core/src/main/java/io/questdb/cairo/wal/WalColumnarRowAppender.java
+2 −1 core/src/main/java/io/questdb/cutlass/http/processors/ExportQueryProcessor.java
+92 −0 core/src/main/java/io/questdb/cutlass/line/tcp/QwpWalAppender.java
+9 −4 core/src/main/java/io/questdb/cutlass/parquet/CopyExportRequestTask.java
+9 −5 core/src/main/java/io/questdb/cutlass/parquet/SQLSerialParquetExporter.java
+52 −8 core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpColumnDef.java
+2 −2 core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpConstants.java
+8 −2 core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpFixedWidthColumnCursor.java
+5 −2 core/src/main/java/io/questdb/cutlass/qwp/protocol/QwpTableBlockCursor.java
+60 −28 core/src/main/java/io/questdb/cutlass/qwp/server/QwpWebSocketHttpProcessor.java
+29 −2 core/src/main/java/io/questdb/cutlass/qwp/server/QwpWebSocketUpgradeProcessor.java
+62 −11 core/src/main/java/io/questdb/cutlass/qwp/server/egress/QwpEgressCompressionNegotiator.java
+20 −9 core/src/main/java/io/questdb/cutlass/qwp/server/egress/QwpEgressUpgradeProcessor.java
+3 −2 core/src/main/java/io/questdb/griffin/engine/QueryProgress.java
+39 −55 core/src/main/java/io/questdb/griffin/engine/functions/groupby/SparklineGroupByFunction.java
+66 −67 core/src/main/java/io/questdb/griffin/engine/functions/groupby/TwapGroupByFunction.java
+296 −0 core/src/main/java/io/questdb/griffin/engine/groupby/SortedRunsMerge.java
+3 −2 core/src/main/java/io/questdb/griffin/engine/table/ExtraNullColumnCursorFactory.java
+3 −2 core/src/main/java/io/questdb/griffin/engine/table/SelectedRecordCursorFactory.java
+4 −10 core/src/main/java/io/questdb/griffin/engine/table/TablePageFrameCursor.java
+3 −1 core/src/main/java/io/questdb/std/FdCache.java
+33 −38 core/src/test/java/io/questdb/test/FilesTest.java
+3 −1 core/src/test/java/io/questdb/test/ServerMainQuerySmokeTest.java
+1 −0 core/src/test/java/io/questdb/test/ServerMainTest.java
+12 −3 core/src/test/java/io/questdb/test/cairo/ArrayTest.java
+4 −1 core/src/test/java/io/questdb/test/cairo/BitmapIndexTest.java
+8 −1 core/src/test/java/io/questdb/test/cairo/RssMemoryLimitTest.java
+2 −1 core/src/test/java/io/questdb/test/cairo/TableNameRegistryTest.java
+2 −1 core/src/test/java/io/questdb/test/cairo/TableReaderTailRecordCursorTest.java
+6 −3 core/src/test/java/io/questdb/test/cairo/TableReaderTest.java
+6 −2 core/src/test/java/io/questdb/test/cairo/mv/MatViewFuzzTest.java
+14 −5 core/src/test/java/io/questdb/test/cairo/o3/O3FailureTest.java
+14 −2 core/src/test/java/io/questdb/test/cairo/o3/O3MaxLagTest.java
+19 −4 core/src/test/java/io/questdb/test/cairo/o3/O3Test.java
+11 −5 core/src/test/java/io/questdb/test/cairo/parquet/O3ParquetMergeStrategyFuzzTest.java
+3 −1 core/src/test/java/io/questdb/test/cairo/wal/WalLockerFuzzTest.java
+4 −0 core/src/test/java/io/questdb/test/cairo/wal/WalWriterTest.java
+7 −2 core/src/test/java/io/questdb/test/cutlass/http/ExpParquetExportTest.java
+47 −1 core/src/test/java/io/questdb/test/cutlass/http/line/LineHttpSenderTest.java
+7 −1 core/src/test/java/io/questdb/test/cutlass/json/JsonLexerTest.java
+167 −0 core/src/test/java/io/questdb/test/cutlass/qwp/AbstractReusedServerQwpEgressTest.java
+6 −1 core/src/test/java/io/questdb/test/cutlass/qwp/QwpColumnDefTest.java
+19 −0 core/src/test/java/io/questdb/test/cutlass/qwp/QwpCursorBoundsCheckTest.java
+141 −176 core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressBindRoundTripTest.java
+1,196 −1,315 core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressBootstrapTest.java
+167 −0 core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressCompressionNegotiatorTest.java
+62 −1 core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressCompressionTest.java
+1,088 −1,195 core/src/test/java/io/questdb/test/cutlass/qwp/QwpEgressTypesExhaustiveTest.java
+119 −0 core/src/test/java/io/questdb/test/cutlass/qwp/QwpFixedWidthDecoderTest.java
+26 −6 core/src/test/java/io/questdb/test/cutlass/qwp/QwpGeoHashDecoderTest.java
+127 −4 core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpIngressOracleFuzzTest.java
+645 −3 core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpSenderE2ETest.java
+350 −0 core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpSenderFailoverBatchSizeTest.java
+92 −36 core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpSenderFuzzTest.java
+296 −0 core/src/test/java/io/questdb/test/cutlass/qwp/e2e/QwpSenderOversizeRowInBatchTest.java
+150 −1 core/src/test/java/io/questdb/test/cutlass/qwp/load/QwpRow.java
+63 −3 core/src/test/java/io/questdb/test/cutlass/qwp/udp/QwpUdpAllTypesTest.java
+5 −0 core/src/test/java/io/questdb/test/cutlass/text/ParallelCsvFileImporterTest.java
+59 −0 core/src/test/java/io/questdb/test/cutlass/websocket/QwpWebSocketUpgradeProcessorOnHeadersReadyTest.java
+44 −0 core/src/test/java/io/questdb/test/cutlass/websocket/WebSocketHandshakeTest.java
+12 −5 core/src/test/java/io/questdb/test/griffin/CheckpointTest.java
+5 −2 core/src/test/java/io/questdb/test/griffin/ParquetLateMaterializationFuzzTest.java
+155 −0 core/src/test/java/io/questdb/test/griffin/engine/functions/groupby/FirstLastUnderContentionTest.java
+283 −0 core/src/test/java/io/questdb/test/griffin/engine/functions/groupby/TwapUnsortedRunReproTest.java
+30 −37 core/src/test/java/io/questdb/test/std/ParseNanosFuzzTest.java
+6 −2 core/src/test/java/io/questdb/test/std/bytes/DirectByteSinkTest.java
+5 −0 core/src/test/java/io/questdb/test/std/str/Utf8sTest.java
+4 −2 docs/qwp/wire-egress.md
+1 −1 java-questdb-client
+1 −1 utils/pom.xml
59 changes: 59 additions & 0 deletions e2e/tests/console/tableDetails.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -1186,4 +1186,63 @@ describe("TableDetailsDrawer", () => {
cy.dropTable(TEST_TABLE_2)
})
})

describe("table with STORAGE POLICY", () => {
before(() => {
cy.loadConsoleWithAuth()
cy.createTable(TEST_TABLE)
cy.refreshSchema()
})

beforeEach(() => {
cy.intercept(
{
method: "GET",
pathname: "/exec",
query: { query: /SHOW\s+CREATE/i },
},
(req) => {
req.continue((res) => {
const row = res.body?.dataset?.[0]
if (row && typeof row[0] === "string") {
row[0] = row[0].replace(
/(\bPARTITION\s+BY\s+\w+)/i,
"$1 STORAGE POLICY(TO PARQUET 3 DAYS, DROP NATIVE 10 DAYS, DROP LOCAL 1 YEARS)",
)
}
})
},
).as("showCreate")

cy.loadConsoleWithAuth()
cy.expandTables()
})

it("hides TTL and renders the storage policy section", () => {
cy.openDetailsDrawer(TEST_TABLE)
cy.getByDataHook("table-details-tab-details").click()

cy.getByDataHook("table-details-storage-policy-section")
.should("be.visible")
.within(() => {
cy.contains("To Parquet").should("be.visible")
cy.contains("3 Days").should("be.visible")
cy.contains("Drop Native").should("be.visible")
cy.contains("10 Days").should("be.visible")
cy.contains("Drop Local").should("be.visible")
cy.contains("1 Year").should("be.visible")
})

cy.getByDataHook("table-details-details-section")
.should("be.visible")
.within(() => {
cy.contains("TTL").should("not.exist")
})
})

after(() => {
cy.loadConsoleWithAuth()
cy.dropTable(TEST_TABLE)
})
})
})
4 changes: 4 additions & 0 deletions e2e/tests/enterprise/oidc.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ describe("OIDC", () => {

cy.wait("@tokens")
cy.getEditor().should("be.visible")
cy.window()
.its("localStorage")
.invoke("getItem", "sso.username.client1")
.should("not.be.empty")

cy.reload()
cy.getEditor().should("be.visible")
Expand Down
39 changes: 39 additions & 0 deletions e2e/tests/enterprise/tableDetails.spec.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/// <reference types="cypress" />

const TEST_TABLE = "btc_trades"

describe("TableDetailsDrawer in enterprise", () => {
describe("without a STORAGE POLICY shows 'Not configured'", () => {
before(() => {
cy.loadConsoleWithAuth()
cy.createTable(TEST_TABLE)
cy.refreshSchema()
})

it("renders the section with the 'Not configured' placeholder", () => {
cy.openDetailsDrawer(TEST_TABLE)
cy.getByDataHook("table-details-tab-details").click()

cy.getByDataHook("table-details-storage-policy-section")
.should("be.visible")
.within(() => {
cy.contains("Not configured").should("be.visible")
cy.contains("To Parquet").should("not.exist")
cy.contains("Drop Native").should("not.exist")
cy.contains("Drop Local").should("not.exist")
cy.contains("Drop Remote").should("not.exist")
})

cy.getByDataHook("table-details-details-section")
.should("be.visible")
.within(() => {
cy.contains("TTL").should("not.exist")
})
})

after(() => {
cy.loadConsoleWithAuth()
cy.dropTable(TEST_TABLE)
})
})
})
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
"@monaco-editor/react": "^4.7.0",
"@phosphor-icons/react": "^2.1.10",
"@popperjs/core": "2.4.2",
"@questdb/sql-parser": "0.1.11",
"@questdb/sql-parser": "0.1.13",
"@radix-ui/react-alert-dialog": "^1.1.15",
"@radix-ui/react-context-menu": "^2.2.16",
"@radix-ui/react-dialog": "^1.1.15",
Expand Down
65 changes: 54 additions & 11 deletions scripts/run_ent_browser_tests.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,39 @@
#!/bin/bash -x

# Run it from the 'scripts' subdirectory as:
# JAVA_HOME=<your java> MVN_REPO=<your maven repo> ./run_ent_browser_tests.sh
# Example: JAVA_HOME=/opt/homebrew/opt/openjdk@17 MVN_REPO=/Users/john/.m2/repository ./run_ent_browser_tests.sh
# ./run_ent_browser_tests.sh [--cached]
# Java 25 is required (questdb-enterprise maven enforcer needs the java25+
# profile to activate). The script auto-selects JDK 25 from the system.
# Override by exporting JAVA_HOME and/or MVN_REPO before running.
# --cached: reuse the tmp/questdb-enterprise clone and maven build from the
# previous run (falls back to cloning/building if missing). Always wipes tmp/dbroot.

# Parse args
CACHED=0
for arg in "$@"; do
case "$arg" in
--cached) CACHED=1 ;;
esac
done

# Auto-select JDK 25 if JAVA_HOME isn't already set to one
if [ -z "$JAVA_HOME" ] || ! "$JAVA_HOME/bin/java" -version 2>&1 | grep -q '"25'; then
if [ -x /usr/libexec/java_home ]; then
JAVA_HOME=$(/usr/libexec/java_home -v 25 2>/dev/null)
fi
fi
if [ -z "$JAVA_HOME" ] || [ ! -x "$JAVA_HOME/bin/java" ]; then
echo "Error: could not locate JDK 25. Install one (e.g. 'brew install openjdk@25') or set JAVA_HOME." >&2
exit 1
fi
export JAVA_HOME
export PATH="$JAVA_HOME/bin:$PATH"

# Default maven local repo
if [ -z "$MVN_REPO" ]; then
MVN_REPO="$HOME/.m2/repository"
fi
export MVN_REPO

# Get the directory where this script is located
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
Expand All @@ -14,17 +45,29 @@ cd "$UI_DIR"

# Cleanup
rm -rf tmp/dbroot
rm -rf tmp/questdb-*
if [ "$CACHED" -eq 0 ]; then
rm -rf tmp/questdb-*
fi

# Clone questdb-enterprise
git clone https://github.com/questdb/questdb-enterprise.git tmp/questdb-enterprise
cd tmp/questdb-enterprise || exit 1
git submodule init
git submodule update
cd ../..
# Clone questdb-enterprise (skip if cached clone exists)
if [ -d tmp/questdb-enterprise/.git ]; then
echo "Reusing existing tmp/questdb-enterprise checkout"
else
git clone https://github.com/questdb/questdb-enterprise.git tmp/questdb-enterprise
cd tmp/questdb-enterprise || exit 1
git submodule init
git submodule update
cd ../..
fi

# Build server
mvn clean package -e -f tmp/questdb-enterprise/pom.xml -DskipTests -P build-ent-binaries 2>&1
# Build server (skip if cached classes exist)
ENT_MAIN_CLASS=tmp/questdb-enterprise/questdb-ent/target/classes/com/questdb/EntServerMain.class
CORE_MAIN_DIR=tmp/questdb-enterprise/questdb/core/target/classes
if [ "$CACHED" -eq 1 ] && [ -f "$ENT_MAIN_CLASS" ] && [ -d "$CORE_MAIN_DIR" ]; then
echo "Reusing existing maven build output"
else
mvn clean package -e -f tmp/questdb-enterprise/pom.xml -DskipTests -P build-ent-binaries 2>&1
fi

# Create dbroot
mkdir tmp/dbroot
Expand Down
97 changes: 68 additions & 29 deletions src/scenes/Schema/TableDetailsDrawer/DetailsTab.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import {
TextColumnsIcon,
ArrowSquareInIcon,
InfoIcon,
DatabaseIcon,
XCircleIcon,
} from "@phosphor-icons/react"
import { Box, Text, CopyButton } from "../../../components"
import { LiteEditor } from "../../../components/LiteEditor"
Expand All @@ -14,7 +16,7 @@ import type {
View,
Column,
} from "../../../utils/questdb/types"
import { formatTTL } from "./utils"
import { formatTTL, extractStoragePolicyClauses } from "./utils"
import { ColumnIcon } from "../Row"
import {
Section,
Expand All @@ -39,6 +41,7 @@ export interface DetailsTabProps {
ddl: string
isMatView: boolean
isView: boolean
isEnterprise: boolean
truncatedDDL: { text: string; grayedOutLines: [number, number] | null }
baseTableStatus: "Valid" | "Suspended" | "Dropped" | null
columnsExpanded: boolean
Expand Down Expand Up @@ -92,19 +95,10 @@ const BaseTableLinkButton = styled.button<{ $disabled?: boolean }>`
}
`

const DetailsGrid = styled.div`
const MetricsGrid = styled.div<{ $columns: number }>`
width: 100%;
display: grid;
grid-template-columns: repeat(3, 1fr);
gap: 0.2rem;
border-radius: 0.5rem;
overflow: hidden;
`

const MetricsGrid = styled.div`
width: 100%;
display: grid;
grid-template-columns: repeat(2, 1fr);
grid-template-columns: repeat(${({ $columns }) => $columns}, 1fr);
gap: 0.2rem;
border-radius: 0.5rem;
overflow: hidden;
Expand All @@ -121,6 +115,15 @@ const MetricCard = styled(Box).attrs<{ $background?: string }>({
$background ?? theme.color.backgroundLighter};
`

const StoragePolicyClauses = styled.div<{ $columns: number }>`
width: 100%;
display: grid;
grid-template-columns: repeat(${({ $columns }) => $columns}, 1fr);
gap: 0.2rem;
border-radius: 0.5rem;
overflow: hidden;
`

const MetricLabel = styled(Text).attrs({
color: "gray2",
size: "sm",
Expand Down Expand Up @@ -168,6 +171,7 @@ export const DetailsTab = ({
ddl,
isMatView,
isView,
isEnterprise,
truncatedDDL,
baseTableStatus,
columnsExpanded,
Expand All @@ -180,6 +184,10 @@ export const DetailsTab = ({
const theme = useTheme()
const baseTableExists =
baseTableStatus === "Valid" || baseTableStatus === "Suspended"
const storagePolicyClauses = extractStoragePolicyClauses(ddl)
const hasStoragePolicy = storagePolicyClauses.length > 0
const hasTtl = tableData.ttlValue !== 0
const showStoragePolicySection = isEnterprise || hasStoragePolicy

return (
<>
Expand Down Expand Up @@ -325,14 +333,16 @@ export const DetailsTab = ({
</SectionTitleContainer>

{isMatView && matViewData ? (
/* Matview: 2 rows × 2 columns */
<MetricsGrid>
<MetricCard $background={theme.color.backgroundDarker}>
<MetricLabel>TTL</MetricLabel>
<MetricValue>
{formatTTL(tableData.ttlValue, tableData.ttlUnit)}
</MetricValue>
</MetricCard>
/* Matview: 4 cards (2×2) when TTL is configured, 3 cards (1 row) when not. */
<MetricsGrid $columns={hasTtl ? 2 : 3}>
{hasTtl && (
<MetricCard $background={theme.color.backgroundDarker}>
<MetricLabel>TTL</MetricLabel>
<MetricValue>
{formatTTL(tableData.ttlValue, tableData.ttlUnit)}
</MetricValue>
</MetricCard>
)}
<MetricCard $background={theme.color.backgroundDarker}>
<MetricLabel>Deduplication</MetricLabel>
<MetricValue>
Expand All @@ -357,14 +367,16 @@ export const DetailsTab = ({
</MetricCard>
</MetricsGrid>
) : (
/* Table: 3 items in single row */
<DetailsGrid>
<MetricCard $background={theme.color.backgroundDarker}>
<MetricLabel>TTL</MetricLabel>
<MetricValue>
{formatTTL(tableData.ttlValue, tableData.ttlUnit)}
</MetricValue>
</MetricCard>
/* Table: 3 cards (1 row) when TTL is configured, 2 cards (1 row) when not. */
<MetricsGrid $columns={hasTtl ? 3 : 2}>
{hasTtl && (
<MetricCard $background={theme.color.backgroundDarker}>
<MetricLabel>TTL</MetricLabel>
<MetricValue>
{formatTTL(tableData.ttlValue, tableData.ttlUnit)}
</MetricValue>
</MetricCard>
)}
<MetricCard $background={theme.color.backgroundDarker}>
<MetricLabel>Deduplication</MetricLabel>
<MetricValue>
Expand All @@ -380,7 +392,34 @@ export const DetailsTab = ({
tableData.partitionBy.slice(1).toLowerCase()}
</MetricValue>
</MetricCard>
</DetailsGrid>
</MetricsGrid>
)}
</Section>
)}

{!isView && showStoragePolicySection && (
<Section data-hook="table-details-storage-policy-section">
<SectionTitleContainer>
<DatabaseIcon size="16px" weight="bold" />
<SectionTitle>Storage policy</SectionTitle>
</SectionTitleContainer>
{hasStoragePolicy ? (
<StoragePolicyClauses $columns={storagePolicyClauses.length}>
{storagePolicyClauses.map((clause) => (
<MetricCard
key={clause.action}
$background={theme.color.backgroundDarker}
>
<MetricLabel>{clause.action}</MetricLabel>
<MetricValue>{clause.duration}</MetricValue>
</MetricCard>
))}
</StoragePolicyClauses>
) : (
<Box gap="0.5rem" align="center">
<XCircleIcon size={16} weight="fill" color={theme.color.gray2} />
<Text color="gray2">Not configured</Text>
</Box>
)}
</Section>
)}
Expand Down
Loading
Loading