feat: make pyarrow hot fix optional

.github/workflows/ibis-main.yml

@@ -49,8 +49,12 @@ jobs:
           - "3.10"
           - "3.14"
         pyarrow:
-          - true
-          - false
+          - none
+          - latest
+          - old
+        exclude:
+          - python-version: "3.14"
+            pyarrow: old
     steps:
       - name: checkout
         uses: actions/checkout@v6
@@ -76,9 +80,13 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: choco install graphviz
 
-      - name: install numpy/pandas/pyarrow
-        if: matrix.pyarrow
-        run: pip install numpy pandas pyarrow pyarrow-hotfix
+      - name: install numpy/pandas/pyarrow (latest)
+        if: matrix.pyarrow == 'latest'
+        run: pip install numpy pandas pyarrow
+
+      - name: install numpy/pandas/pyarrow (old)
+        if: matrix.pyarrow == 'old'
+        run: pip install "numpy<2" pandas pyarrow==10.0.1 pyarrow-hotfix
 
       - uses: extractions/setup-just@v3
         env:

ibis/backends/__init__.py

@@ -101,7 +101,7 @@ def _import_pyarrow():
                 "Exporting to arrow formats requires `pyarrow` but it is not installed"
             )
         else:
-            import pyarrow_hotfix  # noqa: F401
+            from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
             return pyarrow
 
@@ -1607,9 +1607,10 @@ class PyArrowExampleLoader(ExampleLoader):
     temporary_example: bool = True
 
     def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table:
-        import pyarrow_hotfix  # noqa: F401, I001
         import pyarrow.parquet as pq
 
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
+
         table = pq.read_table(path)
         return self.create_table(
             table_name,
@@ -1619,10 +1620,12 @@ def _load_parquet(self, *, path: str | Path, table_name: str) -> ir.Table:
         )
 
     def _load_csv(self, *, path: str | Path, table_name: str) -> ir.Table:
-        import pyarrow_hotfix  # noqa: F401, I001
+
         import pyarrow as pa
         import pyarrow.csv
 
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
+
         # The convert options lets pyarrow treat empty strings as null for
         # string columns, but not quoted empty strings.
         table = pyarrow.csv.read_csv(

ibis/backends/athena/__init__.py

@@ -11,7 +11,6 @@
 from typing import TYPE_CHECKING, Any
 
 import fsspec
-import pyarrow_hotfix  # noqa: F401
 import pyathena
 import sqlglot as sg
 import sqlglot.expressions as sge
@@ -27,6 +26,7 @@
 from ibis.backends import CanCreateDatabase, NoExampleLoader, UrlFromPath
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import AlterTable, RenameTable
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Mapping

ibis/backends/clickhouse/__init__.py

@@ -11,7 +11,6 @@
 
 import clickhouse_connect as cc
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 import sqlglot as sg
 import sqlglot.expressions as sge
 import toolz
@@ -35,6 +34,7 @@
 from ibis.backends.clickhouse.converter import ClickHousePandasData
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import C
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator, Mapping

ibis/backends/databricks/__init__.py

@@ -14,7 +14,6 @@
 
 import databricks.sql
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 import sqlglot as sg
 import sqlglot.expressions as sge
 
@@ -30,6 +29,7 @@
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import STAR, AlterTable, RenameTable
 from ibis.backends.sql.datatypes import DatabricksType
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable, Mapping

ibis/backends/datafusion/__init__.py

@@ -9,7 +9,6 @@
 
 import datafusion as df
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 import sqlglot as sg
 import sqlglot.expressions as sge
 
@@ -32,10 +31,16 @@
 )
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import C
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 from ibis.common.dispatch import lazy_singledispatch
 from ibis.expr.operations.udf import InputType
 from ibis.formats.pyarrow import PyArrowSchema, PyArrowType
-from ibis.util import gen_name, normalize_filename, normalize_filenames, warn_deprecated
+from ibis.util import (
+    gen_name,
+    normalize_filename,
+    normalize_filenames,
+    warn_deprecated,
+)
 
 try:
     from datafusion import ExecutionContext as SessionContext

ibis/backends/datafusion/udfs.py

@@ -5,10 +5,10 @@
 
 import pyarrow as pa
 import pyarrow.compute as pc
-import pyarrow_hotfix  # noqa: F401
 
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dt
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
 
 def _extract_epoch_seconds(array) -> dt.int32:

ibis/backends/duckdb/__init__.py

@@ -33,6 +33,7 @@
 )
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import STAR, AlterTable, C, RenameTable
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 from ibis.common.dispatch import lazy_singledispatch
 from ibis.expr.operations.udf import InputType
 
@@ -42,7 +43,6 @@
     import pandas as pd
     import polars as pl
     import pyarrow as pa
-    import pyarrow_hotfix  # noqa: F401
     import torch
     from fsspec import AbstractFileSystem
 
@@ -1380,7 +1380,6 @@ def to_pyarrow_batches(
             The number of rows to fetch per batch
         """
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
 
         self._run_pre_execute_hooks(expr)
         table = expr.as_table()
@@ -1422,7 +1421,6 @@ def execute(
         """Execute an expression."""
         import pandas as pd
         import pyarrow.types as pat
-        import pyarrow_hotfix  # noqa: F401
 
         from ibis.backends.duckdb.converter import DuckDBPandasData
 

ibis/backends/flink/__init__.py

@@ -493,9 +493,9 @@ def create_table(
         """
         import pandas as pd
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
 
         import ibis.expr.types as ir
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
         if obj is None and schema is None:
             raise exc.IbisError("`schema` or `obj` is required")
@@ -937,7 +937,8 @@ def insert(
         """
         import pandas as pd
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
         if isinstance(obj, ir.Table):
             statement = InsertSelect(
@@ -982,7 +983,8 @@ def to_pyarrow(
         **kwargs: Any,
     ) -> pa.Table:
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
         pyarrow_batches = iter(
             self.to_pyarrow_batches(expr, params=params, limit=limit, **kwargs)
@@ -1009,7 +1011,8 @@ def to_pyarrow_batches(
         **kwargs: Any,
     ):
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
         ibis_table = expr.as_table()
 
@@ -1059,12 +1062,12 @@ def _from_pyflink_table_to_pyarrow_batches(
         chunk_size: int | None = None,
     ):
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
         from pyflink.java_gateway import get_gateway
         from pyflink.table.serializers import ArrowSerializer
         from pyflink.table.types import create_arrow_schema
 
         from ibis.backends.flink.datatypes import get_field_data_types
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
         # Note (mehmet): Implementation of this is based on
         # pyflink/table/table.py: to_pandas().
 

ibis/backends/impala/__init__.py

@@ -1334,8 +1334,8 @@ def to_pyarrow(
         **kwargs: Any,
     ) -> pa.Table:
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
 
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
         from ibis.formats.pyarrow import PyArrowData
 
         self._run_pre_execute_hooks(expr)

ibis/backends/pyspark/__init__.py

@@ -1042,8 +1042,8 @@ def to_pyarrow(
                 "PySpark in streaming mode does not support to_pyarrow"
             )
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
 
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
         from ibis.formats.pyarrow import PyArrowData
 
         table_expr = expr.as_table()

ibis/backends/snowflake/__init__.py

@@ -14,7 +14,6 @@
 from urllib.request import urlcleanup, urlretrieve
 
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 import sqlglot as sg
 import sqlglot.expressions as sge
 
@@ -36,6 +35,7 @@
 from ibis.backends.snowflake.converter import SnowflakePandasData
 from ibis.backends.sql import SQLBackend
 from ibis.backends.sql.compilers.base import STAR
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
 if TYPE_CHECKING:
     from collections.abc import Generator, Iterator, Mapping

ibis/backends/snowflake/tests/conftest.py

@@ -10,7 +10,6 @@
 from urllib.request import urlretrieve
 
 import pyarrow.parquet as pq
-import pyarrow_hotfix  # noqa: F401
 import pytest
 import snowflake.connector as sc
 import sqlglot as sg
@@ -19,6 +18,7 @@
 from ibis.backends.conftest import TEST_TABLES
 from ibis.backends.sql.datatypes import SnowflakeType
 from ibis.backends.tests.base import BackendTest
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 from ibis.formats.pyarrow import PyArrowSchema
 
 if TYPE_CHECKING:

ibis/common/import_to_try_pyarrow_hotfix.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import pyarrow as pa
+
+if tuple(int(x) for x in pa.__version__.split(".")[:3]) < (14, 0, 1):
+    try:
+        import pyarrow_hotfix  # noqa: F401
+    except ImportError:
+        raise ImportError("pyarrow_hotfix should be installed for pyarrow<14.0.1")

ibis/formats/__init__.py

@@ -253,7 +253,8 @@ def to_polars(self, schema: Schema) -> pl.DataFrame:  # pragma: no cover
 
     def to_pyarrow_bytes(self, schema: Schema) -> bytes:
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
         data = self.to_pyarrow(schema=schema)
         out = pa.BufferOutputStream()

ibis/formats/pandas.py

@@ -433,7 +433,8 @@ def to_pyarrow(self, schema: sch.Schema) -> pa.Table:
         from decimal import Decimal
 
         import pyarrow as pa
-        import pyarrow_hotfix  # noqa: F401
+
+        from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 
         pyarrow_schema = PyArrowSchema.from_ibis(schema)
 

ibis/formats/pyarrow.py

@@ -4,10 +4,10 @@
 from typing import TYPE_CHECKING, Any
 
 import pyarrow as pa
-import pyarrow_hotfix  # noqa: F401
 
 import ibis.common.exceptions as com
 import ibis.expr.datatypes as dt
+from ibis.common import import_to_try_pyarrow_hotfix  # noqa: F401
 from ibis.expr.schema import Schema
 from ibis.formats import DataMapper, SchemaMapper, TableProxy, TypeMapper
 from ibis.util import V