From e97ab4c03806572940e82951c6ba95c2e1cc7915 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Wed, 8 Apr 2026 14:31:11 -0700 Subject: [PATCH 1/3] Dremio Docker Container and Setup Script Authored by Justin Co-authored-by: justing-bq --- .../sql/odbc/tests/dremio/docker-compose.yml | 39 +++++++++++ .../tests/dremio/set_up_dremio_instance.sh | 66 +++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml create mode 100755 cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh diff --git a/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml b/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml new file mode 100644 index 000000000000..bf8f7f32ed4d --- /dev/null +++ b/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +networks: + odbc_net: + external: true + +services: + dremio: + platform: linux/x86_64 + image: dremio/dremio-oss:latest + ports: + - 9047:9047 # REST API + - 31010:31010 # JDBC/ODBC + - 32010:32010 + container_name: dremio_container + environment: + - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-8 -Dsaffron.default.nationalcharset=UTF-8 -Dsaffron.default.collation.name=UTF-8$$en_US + healthcheck: + test: curl --fail http://localhost:9047 || exit 1 + interval: 10s + timeout: 5s + retries: 30 + networks: + - odbc_net diff --git a/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh b/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh new file mode 100755 index 000000000000..8d632bb2c3e1 --- /dev/null +++ b/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# GH-48068 TODO: run remote ODBC tests on Linux + +#!/bin/bash +set -e + +HOST_URL="http://localhost:9047" +NEW_USER_URL="$HOST_URL/apiv2/bootstrap/firstuser" +LOGIN_URL="$HOST_URL/apiv2/login" +SQL_URL="$HOST_URL/api/v3/sql" + +ADMIN_USER="admin" +ADMIN_PASSWORD="admin2025" + +# Wait for Dremio to be available. +until curl -s "$NEW_USER_URL"; do + echo 'Waiting for Dremio to start...' + sleep 5 +done + +echo "" +echo 'Creating admin user...' + +# Create new admin account. +curl -X PUT "$NEW_USER_URL" \ + -H "Content-Type: application/json" \ + -d "{ \"userName\": \"$ADMIN_USER\", \"password\": \"$ADMIN_PASSWORD\" }" + +echo "" +echo "Created admin user." + +# Use admin account to login and acquire a token. +TOKEN=$(curl -s -X POST "$LOGIN_URL" \ + -H "Content-Type: application/json" \ + -d "{ \"userName\": \"$ADMIN_USER\", \"password\": \"$ADMIN_PASSWORD\" }" \ + | grep -oP '(?<="token":")[^"]+') + +SQL_QUERY="Create Table \$scratch.ODBCTest As SELECT CAST(2147483647 AS INTEGER) AS sinteger_max, CAST(9223372036854775807 AS BIGINT) AS sbigint_max, CAST(999999999 AS DECIMAL(38,0)) AS decimal_positive, CAST(3.40282347E38 AS FLOAT) AS float_max, CAST(1.7976931348623157E308 AS DOUBLE) AS double_max, CAST(true AS BOOLEAN) AS bit_true, CAST(DATE '9999-12-31' AS DATE) AS date_max, CAST(TIME '23:59:59' AS TIME) AS time_max, CAST(TIMESTAMP '9999-12-31 23:59:59' AS TIMESTAMP) AS timestamp_max;" +ESCAPED_QUERY=$(printf '%s' "$SQL_QUERY" | sed 's/"/\\"/g') + +echo "Creating \$scratch.ODBCTest table." + +# Create a new table by sending a SQL query. +curl -i -X POST "$SQL_URL" \ + -H "Authorization: _dremio$TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"sql\": \"$ESCAPED_QUERY\"}" + +echo "" +echo "Finished setting up dremio docker instance." From e09e2d1ce871d54af244c6ffcdf277ce0e3ba7ac Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" <96995091+alinaliBQ@users.noreply.github.com> Date: Wed, 8 Apr 2026 11:39:21 -0700 Subject: [PATCH 2/3] Enable Remote Testing With Dremio Instance * Disable non-odbc workflows * Install `netcat` and set `ARROW_FLIGHT_SQL_ODBC_CONN` * Attempt to enable docker network Update compose.yaml Add executable bash script * add commands for testing Dremio instance value * Change to use `host.docker.internal` * Change test to run remote test Update cpp_extra.yml * Use `localhost` as hostname * Use `dremio_container` as host * Clean up PR * Test remote instance * CI can connect to remote instance, revert "Test remote instance" This reverts commit db975bfd30490e21e21b4667f8a004f49347e0ca. * Disable ODBC Debian build * Seeing segfault at ODBC test. Can probably be solved by passing test linkage = static like we did with Ubuntu. Since we don't have capacity to work on Debian now, we can close this. --- .github/workflows/cpp_extra.yml | 6 ++++++ compose.yaml | 8 ++++++++ .../sql/odbc/tests/dremio/set_up_dremio_instance.sh | 2 -- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index 199c7e2d49fd..fe7d553fc5f2 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -369,6 +369,12 @@ jobs: with: fetch-depth: 0 submodules: recursive + - name: Create Docker Test Network + run: docker network create odbc_net + - name: Set Up Dremio Instance + run: | + docker compose -f cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml up -d + cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh - name: Cache Docker Volumes uses: actions/cache@v5 with: diff --git a/compose.yaml b/compose.yaml index 17e08ad1173e..8477eb203e46 100644 --- a/compose.yaml +++ b/compose.yaml @@ -204,6 +204,11 @@ volumes: ubuntu-ccache: name: ${ARCH}-ubuntu-${UBUNTU}-ccache +networks: + # GH-48068 for Flight SQL ODBC Testing + odbc_net: + external: true + services: ################################# C++ ####################################### @@ -509,6 +514,7 @@ services: ARROW_DEPENDENCY_SOURCE: BUNDLED ARROW_DEPENDENCY_USE_SHARED: "OFF" ARROW_FLIGHT_SQL_ODBC: "ON" + ARROW_FLIGHT_SQL_ODBC_CONN: "driver={Apache Arrow Flight SQL ODBC Driver};HOST=dremio_container;port=32010;pwd=admin2025;uid=admin;useEncryption=false;UseWideChar=true;" ARROW_GANDIVA: "OFF" ARROW_GCS: "OFF" ARROW_HDFS: "OFF" @@ -517,6 +523,8 @@ services: ARROW_S3: "OFF" ARROW_SUBSTRAIT: "OFF" # Register ODBC before running tests + networks: + - odbc_net command: > /bin/bash -c " /arrow/ci/scripts/cpp_build.sh /arrow /build && diff --git a/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh b/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh index 8d632bb2c3e1..1442a31dff1e 100755 --- a/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh +++ b/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -# GH-48068 TODO: run remote ODBC tests on Linux - #!/bin/bash set -e From 4a892cfa6ce6519ea20031fa0afc6dd66fad8e94 Mon Sep 17 00:00:00 2001 From: "Alina (Xi) Li" Date: Fri, 10 Apr 2026 13:39:57 -0700 Subject: [PATCH 3/3] Address feedback - Move `dremio` to `compose.yaml` at project root - Move shebang to first line - add `set_up_dremio_instance.sh` to pre-commit hook - Fix `SQL_QUERY` variable format --- .github/workflows/cpp_extra.yml | 2 +- .pre-commit-config.yaml | 1 + compose.yaml | 19 +++++++++ .../sql/odbc/tests/dremio/docker-compose.yml | 39 ------------------- .../tests/dremio/set_up_dremio_instance.sh | 24 +++++++++--- 5 files changed, 40 insertions(+), 45 deletions(-) delete mode 100644 cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml index fe7d553fc5f2..9902c11af016 100644 --- a/.github/workflows/cpp_extra.yml +++ b/.github/workflows/cpp_extra.yml @@ -373,7 +373,7 @@ jobs: run: docker network create odbc_net - name: Set Up Dremio Instance run: | - docker compose -f cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml up -d + docker compose up -d dremio cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh - name: Cache Docker Volumes uses: actions/cache@v5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1a1267d32cf0..0776b10e07a8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -351,6 +351,7 @@ repos: ?^cpp/examples/minimal_build/run\.sh$| ?^cpp/examples/tutorial_examples/run\.sh$| ?^cpp/src/arrow/flight/sql/odbc/install/unix/install_odbc\.sh$| + ?^cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance\.sh$| ?^dev/release/05-binary-upload\.sh$| ?^dev/release/08-binary-verify\.sh$| ?^dev/release/binary-recover\.sh$| diff --git a/compose.yaml b/compose.yaml index 8477eb203e46..492e50007410 100644 --- a/compose.yaml +++ b/compose.yaml @@ -139,6 +139,7 @@ x-hierarchy: - fedora-python - fedora-r-clang - python-sdist + - dremio - ubuntu-cpp: - ubuntu-cpp-static - ubuntu-c-glib: @@ -376,6 +377,24 @@ services: /arrow/ci/scripts/cpp_build.sh /arrow /build && /arrow/ci/scripts/cpp_test.sh /arrow /build" + dremio: + platform: linux/x86_64 + image: dremio/dremio-oss:latest + ports: + - 9047:9047 # REST API + - 31010:31010 # JDBC/ODBC + - 32010:32010 + container_name: dremio_container + environment: + - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-8 -Dsaffron.default.nationalcharset=UTF-8 -Dsaffron.default.collation.name=UTF-8$$en_US + healthcheck: + test: curl --fail http://localhost:9047 || exit 1 + interval: 10s + timeout: 5s + retries: 30 + networks: + - odbc_net + ubuntu-cpp: &ubuntu-cpp-base # Usage: # docker compose build ubuntu-cpp diff --git a/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml b/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml deleted file mode 100644 index bf8f7f32ed4d..000000000000 --- a/cpp/src/arrow/flight/sql/odbc/tests/dremio/docker-compose.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -networks: - odbc_net: - external: true - -services: - dremio: - platform: linux/x86_64 - image: dremio/dremio-oss:latest - ports: - - 9047:9047 # REST API - - 31010:31010 # JDBC/ODBC - - 32010:32010 - container_name: dremio_container - environment: - - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dsaffron.default.charset=UTF-8 -Dsaffron.default.nationalcharset=UTF-8 -Dsaffron.default.collation.name=UTF-8$$en_US - healthcheck: - test: curl --fail http://localhost:9047 || exit 1 - interval: 10s - timeout: 5s - retries: 30 - networks: - - odbc_net diff --git a/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh b/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh index 1442a31dff1e..b009588a3c20 100755 --- a/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh +++ b/cpp/src/arrow/flight/sql/odbc/tests/dremio/set_up_dremio_instance.sh @@ -1,3 +1,5 @@ +#!/bin/bash +# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,7 +17,6 @@ # specific language governing permissions and limitations # under the License. -#!/bin/bash set -e HOST_URL="http://localhost:9047" @@ -49,16 +50,29 @@ TOKEN=$(curl -s -X POST "$LOGIN_URL" \ -d "{ \"userName\": \"$ADMIN_USER\", \"password\": \"$ADMIN_PASSWORD\" }" \ | grep -oP '(?<="token":")[^"]+') -SQL_QUERY="Create Table \$scratch.ODBCTest As SELECT CAST(2147483647 AS INTEGER) AS sinteger_max, CAST(9223372036854775807 AS BIGINT) AS sbigint_max, CAST(999999999 AS DECIMAL(38,0)) AS decimal_positive, CAST(3.40282347E38 AS FLOAT) AS float_max, CAST(1.7976931348623157E308 AS DOUBLE) AS double_max, CAST(true AS BOOLEAN) AS bit_true, CAST(DATE '9999-12-31' AS DATE) AS date_max, CAST(TIME '23:59:59' AS TIME) AS time_max, CAST(TIMESTAMP '9999-12-31 23:59:59' AS TIMESTAMP) AS timestamp_max;" -ESCAPED_QUERY=$(printf '%s' "$SQL_QUERY" | sed 's/"/\\"/g') - +SQL_QUERY=" +Create Table \$scratch.ODBCTest As + SELECT CAST(2147483647 AS INTEGER) AS sinteger_max, + CAST(9223372036854775807 AS BIGINT) AS sbigint_max, + CAST(999999999 AS DECIMAL(38,0)) AS decimal_positive, + CAST(3.40282347E38 AS FLOAT) AS float_max, + CAST(1.7976931348623157E308 AS DOUBLE) AS double_max, + CAST(true AS BOOLEAN) AS bit_true, + CAST(DATE '9999-12-31' AS DATE) AS date_max, + CAST(TIME '23:59:59' AS TIME) AS time_max, + CAST(TIMESTAMP '9999-12-31 23:59:59' AS TIMESTAMP) AS timestamp_max; +" echo "Creating \$scratch.ODBCTest table." # Create a new table by sending a SQL query. curl -i -X POST "$SQL_URL" \ -H "Authorization: _dremio$TOKEN" \ -H "Content-Type: application/json" \ - -d "{\"sql\": \"$ESCAPED_QUERY\"}" + -d "$(python3 - <