From c3a1ea396f956cbe649c8e4b0319e33b1f5b855f Mon Sep 17 00:00:00 2001 From: aandres Date: Tue, 24 Mar 2026 16:06:10 +0000 Subject: [PATCH 1/5] feat: make pyarrow hot fix optional --- .github/workflows/ibis-main.yml | 15 ++++++++++----- ibis/backends/__init__.py | 9 ++++++--- ibis/backends/athena/__init__.py | 4 +++- ibis/backends/clickhouse/__init__.py | 4 +++- ibis/backends/databricks/__init__.py | 4 +++- ibis/backends/datafusion/__init__.py | 5 +++-- ibis/backends/datafusion/udfs.py | 4 +++- ibis/backends/duckdb/__init__.py | 10 +++++++--- ibis/backends/flink/__init__.py | 15 ++++++++++----- ibis/backends/impala/__init__.py | 3 ++- ibis/backends/pyspark/__init__.py | 3 ++- ibis/backends/snowflake/__init__.py | 4 +++- ibis/backends/snowflake/tests/conftest.py | 1 - ibis/formats/__init__.py | 3 ++- ibis/formats/pandas.py | 3 ++- ibis/formats/pyarrow.py | 6 +++--- ibis/util.py | 8 ++++++++ 17 files changed, 70 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ibis-main.yml b/.github/workflows/ibis-main.yml index 1a901fba17b4..99d6d2d5eefb 100644 --- a/.github/workflows/ibis-main.yml +++ b/.github/workflows/ibis-main.yml @@ -49,8 +49,9 @@ jobs: - "3.10" - "3.14" pyarrow: - - true - - false + - none + - latest + - old steps: - name: checkout uses: actions/checkout@v6 @@ -76,9 +77,13 @@ jobs: if: matrix.os == 'windows-latest' run: choco install graphviz - - name: install numpy/pandas/pyarrow - if: matrix.pyarrow - run: pip install numpy pandas pyarrow pyarrow-hotfix + - name: install numpy/pandas/pyarrow (latest) + if: matrix.pyarrow == 'latest' + run: pip install numpy pandas pyarrow + + - name: install numpy/pandas/pyarrow (old) + if: matrix.pyarrow == 'old' + run: pip install numpy pandas pyarrow==10.0.1 pyarrow-hotfix - uses: extractions/setup-just@v3 env: diff --git a/ibis/backends/__init__.py b/ibis/backends/__init__.py index 908654622508..3789ce7905e0 100644 --- a/ibis/backends/__init__.py +++ b/ibis/backends/__init__.py @@ -101,7 +101,8 @@ def _import_pyarrow(): "Exporting to arrow formats requires `pyarrow` but it is not installed" ) else: - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() return pyarrow @@ -1607,7 +1608,8 @@ class PyArrowExampleLoader(ExampleLoader): temporary_example: bool = True def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table: - import pyarrow_hotfix # noqa: F401, I001 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() import pyarrow.parquet as pq table = pq.read_table(path) @@ -1619,7 +1621,8 @@ def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table: ) def _load_csv(self, *, path: str | Path, table_name: str) -> ir.Table: - import pyarrow_hotfix # noqa: F401, I001 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() import pyarrow as pa import pyarrow.csv diff --git a/ibis/backends/athena/__init__.py b/ibis/backends/athena/__init__.py index bd392b72f701..09274476d35d 100644 --- a/ibis/backends/athena/__init__.py +++ b/ibis/backends/athena/__init__.py @@ -11,7 +11,6 @@ from typing import TYPE_CHECKING, Any import fsspec -import pyarrow_hotfix # noqa: F401 import pyathena import sqlglot as sg import sqlglot.expressions as sge @@ -25,9 +24,12 @@ import ibis.expr.types as ir from ibis import util from ibis.backends import CanCreateDatabase, NoExampleLoader, UrlFromPath +from ibis.util import apply_pyarrow_hotfix from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import AlterTable, RenameTable +apply_pyarrow_hotfix() + if TYPE_CHECKING: from collections.abc import Callable, Mapping diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index 62601bf8f4cf..aa9bb52abe3d 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -11,7 +11,6 @@ import clickhouse_connect as cc import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge import toolz @@ -26,6 +25,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util +from ibis.util import apply_pyarrow_hotfix from ibis.backends import ( BaseBackend, CanCreateDatabase, @@ -36,6 +36,8 @@ from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import C +apply_pyarrow_hotfix() + if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping from pathlib import Path diff --git a/ibis/backends/databricks/__init__.py b/ibis/backends/databricks/__init__.py index 24bbf55de565..399e2aab6448 100644 --- a/ibis/backends/databricks/__init__.py +++ b/ibis/backends/databricks/__init__.py @@ -14,7 +14,6 @@ import databricks.sql import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge @@ -27,10 +26,13 @@ import ibis.expr.types as ir from ibis import util from ibis.backends import CanCreateDatabase, PyArrowExampleLoader, UrlFromPath +from ibis.util import apply_pyarrow_hotfix from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, AlterTable, RenameTable from ibis.backends.sql.datatypes import DatabricksType +apply_pyarrow_hotfix() + if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index ddb866ba6aa2..7d9f7357cb91 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -9,7 +9,6 @@ import datafusion as df import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge @@ -35,7 +34,9 @@ from ibis.common.dispatch import lazy_singledispatch from ibis.expr.operations.udf import InputType from ibis.formats.pyarrow import PyArrowSchema, PyArrowType -from ibis.util import gen_name, normalize_filename, normalize_filenames, warn_deprecated +from ibis.util import apply_pyarrow_hotfix, gen_name, normalize_filename, normalize_filenames, warn_deprecated + +apply_pyarrow_hotfix() try: from datafusion import ExecutionContext as SessionContext diff --git a/ibis/backends/datafusion/udfs.py b/ibis/backends/datafusion/udfs.py index 7ef1fc97c9b5..2282190b6c43 100644 --- a/ibis/backends/datafusion/udfs.py +++ b/ibis/backends/datafusion/udfs.py @@ -5,10 +5,12 @@ import pyarrow as pa import pyarrow.compute as pc -import pyarrow_hotfix # noqa: F401 import ibis.common.exceptions as com import ibis.expr.datatypes as dt +from ibis.util import apply_pyarrow_hotfix + +apply_pyarrow_hotfix() def _extract_epoch_seconds(array) -> dt.int32: diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index 7e46b9bc3b24..4a025945c4a4 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -35,6 +35,9 @@ from ibis.backends.sql.compilers.base import STAR, AlterTable, C, RenameTable from ibis.common.dispatch import lazy_singledispatch from ibis.expr.operations.udf import InputType +from ibis.util import apply_pyarrow_hotfix + +apply_pyarrow_hotfix() if TYPE_CHECKING: from collections.abc import Iterable, Mapping, MutableMapping, Sequence @@ -42,7 +45,6 @@ import pandas as pd import polars as pl import pyarrow as pa - import pyarrow_hotfix # noqa: F401 import torch from fsspec import AbstractFileSystem @@ -1380,7 +1382,8 @@ def to_pyarrow_batches( The number of rows to fetch per batch """ import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() self._run_pre_execute_hooks(expr) table = expr.as_table() @@ -1422,7 +1425,8 @@ def execute( """Execute an expression.""" import pandas as pd import pyarrow.types as pat - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() from ibis.backends.duckdb.converter import DuckDBPandasData diff --git a/ibis/backends/flink/__init__.py b/ibis/backends/flink/__init__.py index a29e0bd39f68..cca5e9db92bc 100644 --- a/ibis/backends/flink/__init__.py +++ b/ibis/backends/flink/__init__.py @@ -493,7 +493,8 @@ def create_table( """ import pandas as pd import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() import ibis.expr.types as ir @@ -937,7 +938,8 @@ def insert( """ import pandas as pd import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() if isinstance(obj, ir.Table): statement = InsertSelect( @@ -982,7 +984,8 @@ def to_pyarrow( **kwargs: Any, ) -> pa.Table: import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() pyarrow_batches = iter( self.to_pyarrow_batches(expr, params=params, limit=limit, **kwargs) @@ -1009,7 +1012,8 @@ def to_pyarrow_batches( **kwargs: Any, ): import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() ibis_table = expr.as_table() @@ -1059,7 +1063,8 @@ def _from_pyflink_table_to_pyarrow_batches( chunk_size: int | None = None, ): import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() from pyflink.java_gateway import get_gateway from pyflink.table.serializers import ArrowSerializer from pyflink.table.types import create_arrow_schema diff --git a/ibis/backends/impala/__init__.py b/ibis/backends/impala/__init__.py index 1daa4a528c9e..69db257b2d1a 100644 --- a/ibis/backends/impala/__init__.py +++ b/ibis/backends/impala/__init__.py @@ -1334,7 +1334,8 @@ def to_pyarrow( **kwargs: Any, ) -> pa.Table: import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() from ibis.formats.pyarrow import PyArrowData diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index ddf7f67125af..98afb936a98a 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -1042,7 +1042,8 @@ def to_pyarrow( "PySpark in streaming mode does not support to_pyarrow" ) import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() from ibis.formats.pyarrow import PyArrowData diff --git a/ibis/backends/snowflake/__init__.py b/ibis/backends/snowflake/__init__.py index fbf45902a4e5..f9271b80d2e6 100644 --- a/ibis/backends/snowflake/__init__.py +++ b/ibis/backends/snowflake/__init__.py @@ -14,7 +14,6 @@ from urllib.request import urlcleanup, urlretrieve import pyarrow as pa -import pyarrow_hotfix # noqa: F401 import sqlglot as sg import sqlglot.expressions as sge @@ -25,6 +24,7 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util +from ibis.util import apply_pyarrow_hotfix from ibis.backends import ( CanCreateCatalog, CanCreateDatabase, @@ -37,6 +37,8 @@ from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR +apply_pyarrow_hotfix() + if TYPE_CHECKING: from collections.abc import Generator, Iterator, Mapping from urllib.parse import ParseResult diff --git a/ibis/backends/snowflake/tests/conftest.py b/ibis/backends/snowflake/tests/conftest.py index 493d75241fc3..91445fdaf7c3 100644 --- a/ibis/backends/snowflake/tests/conftest.py +++ b/ibis/backends/snowflake/tests/conftest.py @@ -10,7 +10,6 @@ from urllib.request import urlretrieve import pyarrow.parquet as pq -import pyarrow_hotfix # noqa: F401 import pytest import snowflake.connector as sc import sqlglot as sg diff --git a/ibis/formats/__init__.py b/ibis/formats/__init__.py index 0e76b1db735e..3f01d0b73c74 100644 --- a/ibis/formats/__init__.py +++ b/ibis/formats/__init__.py @@ -253,7 +253,8 @@ def to_polars(self, schema: Schema) -> pl.DataFrame: # pragma: no cover def to_pyarrow_bytes(self, schema: Schema) -> bytes: import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() data = self.to_pyarrow(schema=schema) out = pa.BufferOutputStream() diff --git a/ibis/formats/pandas.py b/ibis/formats/pandas.py index 06d79ca2dde4..2798b388668f 100644 --- a/ibis/formats/pandas.py +++ b/ibis/formats/pandas.py @@ -433,7 +433,8 @@ def to_pyarrow(self, schema: sch.Schema) -> pa.Table: from decimal import Decimal import pyarrow as pa - import pyarrow_hotfix # noqa: F401 + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() pyarrow_schema = PyArrowSchema.from_ibis(schema) diff --git a/ibis/formats/pyarrow.py b/ibis/formats/pyarrow.py index 5ce9205f15f2..61c503285894 100644 --- a/ibis/formats/pyarrow.py +++ b/ibis/formats/pyarrow.py @@ -4,13 +4,13 @@ from typing import TYPE_CHECKING, Any import pyarrow as pa -import pyarrow_hotfix # noqa: F401 - import ibis.common.exceptions as com import ibis.expr.datatypes as dt from ibis.expr.schema import Schema from ibis.formats import DataMapper, SchemaMapper, TableProxy, TypeMapper -from ibis.util import V +from ibis.util import V, apply_pyarrow_hotfix + +apply_pyarrow_hotfix() if TYPE_CHECKING: from collections.abc import Sequence diff --git a/ibis/util.py b/ibis/util.py index 892edcbcfd48..bca4c4819246 100644 --- a/ibis/util.py +++ b/ibis/util.py @@ -42,6 +42,14 @@ HORIZONTAL_ELLIPSIS = "…" +def apply_pyarrow_hotfix(): + """Apply pyarrow hotfix for CVE-2023-47248 on pyarrow < 14.0.1.""" + import pyarrow as pa + + if tuple(int(x) for x in pa.__version__.split(".")[:3]) < (14, 0, 1): + import pyarrow_hotfix # noqa: F401 + + def guid() -> str: """Return a uuid4 hexadecimal value.""" return uuid4().hex From 3d6ead8762613d4217f2632cc6556235067363dd Mon Sep 17 00:00:00 2001 From: aandres Date: Tue, 24 Mar 2026 16:22:31 +0000 Subject: [PATCH 2/5] feat: better test matrix --- .github/workflows/ibis-main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ibis-main.yml b/.github/workflows/ibis-main.yml index 99d6d2d5eefb..4759e1cc3772 100644 --- a/.github/workflows/ibis-main.yml +++ b/.github/workflows/ibis-main.yml @@ -52,6 +52,9 @@ jobs: - none - latest - old + exclude: + - python-version: "3.14" + pyarrow: old steps: - name: checkout uses: actions/checkout@v6 @@ -83,7 +86,7 @@ jobs: - name: install numpy/pandas/pyarrow (old) if: matrix.pyarrow == 'old' - run: pip install numpy pandas pyarrow==10.0.1 pyarrow-hotfix + run: pip install "numpy<2" pandas pyarrow==10.0.1 pyarrow-hotfix - uses: extractions/setup-just@v3 env: From bfe9c9a493556d468485bcd1dd7898d4e823d80a Mon Sep 17 00:00:00 2001 From: aandres Date: Tue, 24 Mar 2026 17:56:21 +0000 Subject: [PATCH 3/5] feat: lint --- ibis/backends/__init__.py | 3 +++ ibis/backends/athena/__init__.py | 2 +- ibis/backends/clickhouse/__init__.py | 2 +- ibis/backends/databricks/__init__.py | 2 +- ibis/backends/datafusion/__init__.py | 8 +++++++- ibis/backends/duckdb/__init__.py | 4 ++++ ibis/backends/flink/__init__.py | 10 ++++++++++ ibis/backends/impala/__init__.py | 2 ++ ibis/backends/pyspark/__init__.py | 2 ++ ibis/backends/snowflake/__init__.py | 2 +- ibis/formats/__init__.py | 2 ++ ibis/formats/pandas.py | 2 ++ ibis/formats/pyarrow.py | 1 + 13 files changed, 37 insertions(+), 5 deletions(-) diff --git a/ibis/backends/__init__.py b/ibis/backends/__init__.py index 3789ce7905e0..21ad1a71968c 100644 --- a/ibis/backends/__init__.py +++ b/ibis/backends/__init__.py @@ -102,6 +102,7 @@ def _import_pyarrow(): ) else: from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() return pyarrow @@ -1609,6 +1610,7 @@ class PyArrowExampleLoader(ExampleLoader): def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table: from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() import pyarrow.parquet as pq @@ -1622,6 +1624,7 @@ def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table: def _load_csv(self, *, path: str | Path, table_name: str) -> ir.Table: from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() import pyarrow as pa import pyarrow.csv diff --git a/ibis/backends/athena/__init__.py b/ibis/backends/athena/__init__.py index 09274476d35d..b739644ff788 100644 --- a/ibis/backends/athena/__init__.py +++ b/ibis/backends/athena/__init__.py @@ -24,9 +24,9 @@ import ibis.expr.types as ir from ibis import util from ibis.backends import CanCreateDatabase, NoExampleLoader, UrlFromPath -from ibis.util import apply_pyarrow_hotfix from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import AlterTable, RenameTable +from ibis.util import apply_pyarrow_hotfix apply_pyarrow_hotfix() diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index aa9bb52abe3d..6854ca3b5740 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -25,7 +25,6 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.util import apply_pyarrow_hotfix from ibis.backends import ( BaseBackend, CanCreateDatabase, @@ -35,6 +34,7 @@ from ibis.backends.clickhouse.converter import ClickHousePandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import C +from ibis.util import apply_pyarrow_hotfix apply_pyarrow_hotfix() diff --git a/ibis/backends/databricks/__init__.py b/ibis/backends/databricks/__init__.py index 399e2aab6448..7dd2fcf24c09 100644 --- a/ibis/backends/databricks/__init__.py +++ b/ibis/backends/databricks/__init__.py @@ -26,10 +26,10 @@ import ibis.expr.types as ir from ibis import util from ibis.backends import CanCreateDatabase, PyArrowExampleLoader, UrlFromPath -from ibis.util import apply_pyarrow_hotfix from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, AlterTable, RenameTable from ibis.backends.sql.datatypes import DatabricksType +from ibis.util import apply_pyarrow_hotfix apply_pyarrow_hotfix() diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index 7d9f7357cb91..fc275713e432 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -34,7 +34,13 @@ from ibis.common.dispatch import lazy_singledispatch from ibis.expr.operations.udf import InputType from ibis.formats.pyarrow import PyArrowSchema, PyArrowType -from ibis.util import apply_pyarrow_hotfix, gen_name, normalize_filename, normalize_filenames, warn_deprecated +from ibis.util import ( + apply_pyarrow_hotfix, + gen_name, + normalize_filename, + normalize_filenames, + warn_deprecated, +) apply_pyarrow_hotfix() diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index 4a025945c4a4..8fe3909a2846 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -1382,7 +1382,9 @@ def to_pyarrow_batches( The number of rows to fetch per batch """ import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() self._run_pre_execute_hooks(expr) @@ -1425,7 +1427,9 @@ def execute( """Execute an expression.""" import pandas as pd import pyarrow.types as pat + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() from ibis.backends.duckdb.converter import DuckDBPandasData diff --git a/ibis/backends/flink/__init__.py b/ibis/backends/flink/__init__.py index cca5e9db92bc..c1a8ca54fb2a 100644 --- a/ibis/backends/flink/__init__.py +++ b/ibis/backends/flink/__init__.py @@ -493,7 +493,9 @@ def create_table( """ import pandas as pd import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() import ibis.expr.types as ir @@ -938,7 +940,9 @@ def insert( """ import pandas as pd import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() if isinstance(obj, ir.Table): @@ -984,7 +988,9 @@ def to_pyarrow( **kwargs: Any, ) -> pa.Table: import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() pyarrow_batches = iter( @@ -1012,7 +1018,9 @@ def to_pyarrow_batches( **kwargs: Any, ): import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() ibis_table = expr.as_table() @@ -1063,7 +1071,9 @@ def _from_pyflink_table_to_pyarrow_batches( chunk_size: int | None = None, ): import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() from pyflink.java_gateway import get_gateway from pyflink.table.serializers import ArrowSerializer diff --git a/ibis/backends/impala/__init__.py b/ibis/backends/impala/__init__.py index 69db257b2d1a..ee1a353767e2 100644 --- a/ibis/backends/impala/__init__.py +++ b/ibis/backends/impala/__init__.py @@ -1334,7 +1334,9 @@ def to_pyarrow( **kwargs: Any, ) -> pa.Table: import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() from ibis.formats.pyarrow import PyArrowData diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index 98afb936a98a..ce1c2fe59a22 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -1042,7 +1042,9 @@ def to_pyarrow( "PySpark in streaming mode does not support to_pyarrow" ) import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() from ibis.formats.pyarrow import PyArrowData diff --git a/ibis/backends/snowflake/__init__.py b/ibis/backends/snowflake/__init__.py index f9271b80d2e6..bf8b66e8477a 100644 --- a/ibis/backends/snowflake/__init__.py +++ b/ibis/backends/snowflake/__init__.py @@ -24,7 +24,6 @@ import ibis.expr.schema as sch import ibis.expr.types as ir from ibis import util -from ibis.util import apply_pyarrow_hotfix from ibis.backends import ( CanCreateCatalog, CanCreateDatabase, @@ -36,6 +35,7 @@ from ibis.backends.snowflake.converter import SnowflakePandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR +from ibis.util import apply_pyarrow_hotfix apply_pyarrow_hotfix() diff --git a/ibis/formats/__init__.py b/ibis/formats/__init__.py index 3f01d0b73c74..02cfa179b8c0 100644 --- a/ibis/formats/__init__.py +++ b/ibis/formats/__init__.py @@ -253,7 +253,9 @@ def to_polars(self, schema: Schema) -> pl.DataFrame: # pragma: no cover def to_pyarrow_bytes(self, schema: Schema) -> bytes: import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() data = self.to_pyarrow(schema=schema) diff --git a/ibis/formats/pandas.py b/ibis/formats/pandas.py index 2798b388668f..e32440c6ee40 100644 --- a/ibis/formats/pandas.py +++ b/ibis/formats/pandas.py @@ -433,7 +433,9 @@ def to_pyarrow(self, schema: sch.Schema) -> pa.Table: from decimal import Decimal import pyarrow as pa + from ibis.util import apply_pyarrow_hotfix + apply_pyarrow_hotfix() pyarrow_schema = PyArrowSchema.from_ibis(schema) diff --git a/ibis/formats/pyarrow.py b/ibis/formats/pyarrow.py index 61c503285894..347fcda42ca7 100644 --- a/ibis/formats/pyarrow.py +++ b/ibis/formats/pyarrow.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any import pyarrow as pa + import ibis.common.exceptions as com import ibis.expr.datatypes as dt from ibis.expr.schema import Schema From 342d249897f6bed8fc6ebdabdefc457bbb16a7dd Mon Sep 17 00:00:00 2001 From: aandres Date: Thu, 26 Mar 2026 15:55:39 +0000 Subject: [PATCH 4/5] feat: apply to test files --- ibis/backends/snowflake/tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ibis/backends/snowflake/tests/conftest.py b/ibis/backends/snowflake/tests/conftest.py index 91445fdaf7c3..e04ca3349085 100644 --- a/ibis/backends/snowflake/tests/conftest.py +++ b/ibis/backends/snowflake/tests/conftest.py @@ -19,10 +19,13 @@ from ibis.backends.sql.datatypes import SnowflakeType from ibis.backends.tests.base import BackendTest from ibis.formats.pyarrow import PyArrowSchema +from ibis.util import apply_pyarrow_hotfix if TYPE_CHECKING: from ibis.backends import BaseBackend +apply_pyarrow_hotfix() + def _get_url(): if (url := os.environ.get("SNOWFLAKE_URL")) is not None: From 73b02b5899856a8fa96168e75a2fc5ccec7e7586 Mon Sep 17 00:00:00 2001 From: aandres Date: Thu, 26 Mar 2026 16:14:47 +0000 Subject: [PATCH 5/5] feat: use simple inmport --- ibis/backends/__init__.py | 13 +++++------- ibis/backends/athena/__init__.py | 4 +--- ibis/backends/clickhouse/__init__.py | 4 +--- ibis/backends/databricks/__init__.py | 4 +--- ibis/backends/datafusion/__init__.py | 4 +--- ibis/backends/datafusion/udfs.py | 4 +--- ibis/backends/duckdb/__init__.py | 12 +---------- ibis/backends/flink/__init__.py | 22 +++++---------------- ibis/backends/impala/__init__.py | 5 +---- ibis/backends/pyspark/__init__.py | 5 +---- ibis/backends/snowflake/__init__.py | 4 +--- ibis/backends/snowflake/tests/conftest.py | 4 +--- ibis/common/import_to_try_pyarrow_hotfix.py | 9 +++++++++ ibis/formats/__init__.py | 4 +--- ibis/formats/pandas.py | 4 +--- ibis/formats/pyarrow.py | 5 ++--- ibis/util.py | 8 -------- 17 files changed, 33 insertions(+), 82 deletions(-) create mode 100644 ibis/common/import_to_try_pyarrow_hotfix.py diff --git a/ibis/backends/__init__.py b/ibis/backends/__init__.py index 21ad1a71968c..872f370287d8 100644 --- a/ibis/backends/__init__.py +++ b/ibis/backends/__init__.py @@ -101,9 +101,7 @@ def _import_pyarrow(): "Exporting to arrow formats requires `pyarrow` but it is not installed" ) else: - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 return pyarrow @@ -1609,11 +1607,10 @@ class PyArrowExampleLoader(ExampleLoader): temporary_example: bool = True def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table: - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() import pyarrow.parquet as pq + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 + table = pq.read_table(path) return self.create_table( table_name, @@ -1623,12 +1620,12 @@ def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table: ) def _load_csv(self, *, path: str | Path, table_name: str) -> ir.Table: - from ibis.util import apply_pyarrow_hotfix - apply_pyarrow_hotfix() import pyarrow as pa import pyarrow.csv + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 + # The convert options lets pyarrow treat empty strings as null for # string columns, but not quoted empty strings. table = pyarrow.csv.read_csv( diff --git a/ibis/backends/athena/__init__.py b/ibis/backends/athena/__init__.py index b739644ff788..a8d193d2e0a3 100644 --- a/ibis/backends/athena/__init__.py +++ b/ibis/backends/athena/__init__.py @@ -26,9 +26,7 @@ from ibis.backends import CanCreateDatabase, NoExampleLoader, UrlFromPath from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import AlterTable, RenameTable -from ibis.util import apply_pyarrow_hotfix - -apply_pyarrow_hotfix() +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if TYPE_CHECKING: from collections.abc import Callable, Mapping diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index 6854ca3b5740..ce1b5d8c77ec 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -34,9 +34,7 @@ from ibis.backends.clickhouse.converter import ClickHousePandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import C -from ibis.util import apply_pyarrow_hotfix - -apply_pyarrow_hotfix() +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping diff --git a/ibis/backends/databricks/__init__.py b/ibis/backends/databricks/__init__.py index 7dd2fcf24c09..4f155387f6f9 100644 --- a/ibis/backends/databricks/__init__.py +++ b/ibis/backends/databricks/__init__.py @@ -29,9 +29,7 @@ from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, AlterTable, RenameTable from ibis.backends.sql.datatypes import DatabricksType -from ibis.util import apply_pyarrow_hotfix - -apply_pyarrow_hotfix() +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index fc275713e432..1ebd28b8d0d0 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -31,19 +31,17 @@ ) from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import C +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.common.dispatch import lazy_singledispatch from ibis.expr.operations.udf import InputType from ibis.formats.pyarrow import PyArrowSchema, PyArrowType from ibis.util import ( - apply_pyarrow_hotfix, gen_name, normalize_filename, normalize_filenames, warn_deprecated, ) -apply_pyarrow_hotfix() - try: from datafusion import ExecutionContext as SessionContext except ImportError: diff --git a/ibis/backends/datafusion/udfs.py b/ibis/backends/datafusion/udfs.py index 2282190b6c43..cea9eebecc8c 100644 --- a/ibis/backends/datafusion/udfs.py +++ b/ibis/backends/datafusion/udfs.py @@ -8,9 +8,7 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt -from ibis.util import apply_pyarrow_hotfix - -apply_pyarrow_hotfix() +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 def _extract_epoch_seconds(array) -> dt.int32: diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index 8fe3909a2846..5b87e36acab7 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -33,11 +33,9 @@ ) from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR, AlterTable, C, RenameTable +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.common.dispatch import lazy_singledispatch from ibis.expr.operations.udf import InputType -from ibis.util import apply_pyarrow_hotfix - -apply_pyarrow_hotfix() if TYPE_CHECKING: from collections.abc import Iterable, Mapping, MutableMapping, Sequence @@ -1383,10 +1381,6 @@ def to_pyarrow_batches( """ import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() - self._run_pre_execute_hooks(expr) table = expr.as_table() sql = self.compile(table, limit=limit, params=params) @@ -1428,10 +1422,6 @@ def execute( import pandas as pd import pyarrow.types as pat - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() - from ibis.backends.duckdb.converter import DuckDBPandasData rel = self._to_duckdb_relation(expr, params=params, limit=limit, **kwargs) diff --git a/ibis/backends/flink/__init__.py b/ibis/backends/flink/__init__.py index c1a8ca54fb2a..ded53530b501 100644 --- a/ibis/backends/flink/__init__.py +++ b/ibis/backends/flink/__init__.py @@ -494,11 +494,8 @@ def create_table( import pandas as pd import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() - import ibis.expr.types as ir + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if obj is None and schema is None: raise exc.IbisError("`schema` or `obj` is required") @@ -941,9 +938,7 @@ def insert( import pandas as pd import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if isinstance(obj, ir.Table): statement = InsertSelect( @@ -989,9 +984,7 @@ def to_pyarrow( ) -> pa.Table: import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 pyarrow_batches = iter( self.to_pyarrow_batches(expr, params=params, limit=limit, **kwargs) @@ -1019,9 +1012,7 @@ def to_pyarrow_batches( ): import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 ibis_table = expr.as_table() @@ -1071,15 +1062,12 @@ def _from_pyflink_table_to_pyarrow_batches( chunk_size: int | None = None, ): import pyarrow as pa - - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() from pyflink.java_gateway import get_gateway from pyflink.table.serializers import ArrowSerializer from pyflink.table.types import create_arrow_schema from ibis.backends.flink.datatypes import get_field_data_types + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 # Note (mehmet): Implementation of this is based on # pyflink/table/table.py: to_pandas(). diff --git a/ibis/backends/impala/__init__.py b/ibis/backends/impala/__init__.py index ee1a353767e2..dfd6efad8e5e 100644 --- a/ibis/backends/impala/__init__.py +++ b/ibis/backends/impala/__init__.py @@ -1335,10 +1335,7 @@ def to_pyarrow( ) -> pa.Table: import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() - + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowData self._run_pre_execute_hooks(expr) diff --git a/ibis/backends/pyspark/__init__.py b/ibis/backends/pyspark/__init__.py index ce1c2fe59a22..535852dbc3e5 100644 --- a/ibis/backends/pyspark/__init__.py +++ b/ibis/backends/pyspark/__init__.py @@ -1043,10 +1043,7 @@ def to_pyarrow( ) import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() - + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowData table_expr = expr.as_table() diff --git a/ibis/backends/snowflake/__init__.py b/ibis/backends/snowflake/__init__.py index bf8b66e8477a..a8e07b712983 100644 --- a/ibis/backends/snowflake/__init__.py +++ b/ibis/backends/snowflake/__init__.py @@ -35,9 +35,7 @@ from ibis.backends.snowflake.converter import SnowflakePandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import STAR -from ibis.util import apply_pyarrow_hotfix - -apply_pyarrow_hotfix() +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 if TYPE_CHECKING: from collections.abc import Generator, Iterator, Mapping diff --git a/ibis/backends/snowflake/tests/conftest.py b/ibis/backends/snowflake/tests/conftest.py index e04ca3349085..b0935452d1b5 100644 --- a/ibis/backends/snowflake/tests/conftest.py +++ b/ibis/backends/snowflake/tests/conftest.py @@ -18,14 +18,12 @@ from ibis.backends.conftest import TEST_TABLES from ibis.backends.sql.datatypes import SnowflakeType from ibis.backends.tests.base import BackendTest +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.formats.pyarrow import PyArrowSchema -from ibis.util import apply_pyarrow_hotfix if TYPE_CHECKING: from ibis.backends import BaseBackend -apply_pyarrow_hotfix() - def _get_url(): if (url := os.environ.get("SNOWFLAKE_URL")) is not None: diff --git a/ibis/common/import_to_try_pyarrow_hotfix.py b/ibis/common/import_to_try_pyarrow_hotfix.py new file mode 100644 index 000000000000..cf97a4a5dd11 --- /dev/null +++ b/ibis/common/import_to_try_pyarrow_hotfix.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +import pyarrow as pa + +if tuple(int(x) for x in pa.__version__.split(".")[:3]) < (14, 0, 1): + try: + import pyarrow_hotfix # noqa: F401 + except ImportError: + raise ImportError("pyarrow_hotfix should be installed for pyarrow<14.0.1") diff --git a/ibis/formats/__init__.py b/ibis/formats/__init__.py index 02cfa179b8c0..a4f52f2e7970 100644 --- a/ibis/formats/__init__.py +++ b/ibis/formats/__init__.py @@ -254,9 +254,7 @@ def to_polars(self, schema: Schema) -> pl.DataFrame: # pragma: no cover def to_pyarrow_bytes(self, schema: Schema) -> bytes: import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 data = self.to_pyarrow(schema=schema) out = pa.BufferOutputStream() diff --git a/ibis/formats/pandas.py b/ibis/formats/pandas.py index e32440c6ee40..8cf3d9c15f2d 100644 --- a/ibis/formats/pandas.py +++ b/ibis/formats/pandas.py @@ -434,9 +434,7 @@ def to_pyarrow(self, schema: sch.Schema) -> pa.Table: import pyarrow as pa - from ibis.util import apply_pyarrow_hotfix - - apply_pyarrow_hotfix() + from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 pyarrow_schema = PyArrowSchema.from_ibis(schema) diff --git a/ibis/formats/pyarrow.py b/ibis/formats/pyarrow.py index 347fcda42ca7..48622bf53522 100644 --- a/ibis/formats/pyarrow.py +++ b/ibis/formats/pyarrow.py @@ -7,11 +7,10 @@ import ibis.common.exceptions as com import ibis.expr.datatypes as dt +from ibis.common import import_to_try_pyarrow_hotfix # noqa: F401 from ibis.expr.schema import Schema from ibis.formats import DataMapper, SchemaMapper, TableProxy, TypeMapper -from ibis.util import V, apply_pyarrow_hotfix - -apply_pyarrow_hotfix() +from ibis.util import V if TYPE_CHECKING: from collections.abc import Sequence diff --git a/ibis/util.py b/ibis/util.py index bca4c4819246..892edcbcfd48 100644 --- a/ibis/util.py +++ b/ibis/util.py @@ -42,14 +42,6 @@ HORIZONTAL_ELLIPSIS = "…" -def apply_pyarrow_hotfix(): - """Apply pyarrow hotfix for CVE-2023-47248 on pyarrow < 14.0.1.""" - import pyarrow as pa - - if tuple(int(x) for x in pa.__version__.split(".")[:3]) < (14, 0, 1): - import pyarrow_hotfix # noqa: F401 - - def guid() -> str: """Return a uuid4 hexadecimal value.""" return uuid4().hex