Skip to content
Merged
Show file tree
Hide file tree
Changes from 27 commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
202d431
SNOW-2912540: translate create_temp_table=True to table_type in AST e…
sfc-gh-fpawlowski Jul 28, 2026
5ee8c25
SNOW-2912540: define ASYNC_RETRY_PATTERN and compat.OK locally
sfc-gh-fpawlowski Jul 9, 2026
203a371
SNOW-2912540: remove docs file and unnecessary comments
sfc-gh-fpawlowski Jul 22, 2026
f7819db
SNOW-2912540: decouple LocalTestOOBTelemetryService from connector.te…
sfc-gh-fpawlowski Aug 6, 2026
e4551d6
SNOW-2912540: add missing size() to LocalTestOOBTelemetryService
sfc-gh-fpawlowski Aug 20, 2026
5ed9dfd
SNOW-2912540: revert unrelated import/formatting changes
sfc-gh-fpawlowski Jul 22, 2026
6eb29a0
SNOW-2912540: inline _create_temp_stage and _create_temp_file_format …
sfc-gh-fpawlowski Aug 6, 2026
409ffc7
SNOW-2912540: add IS_V5_DRIVER constant for version-conditioned conne…
sfc-gh-fpawlowski Aug 6, 2026
a84576a
SNOW-2912540: move SecretDetector masking into snowpark-python
sfc-gh-fpawlowski Aug 21, 2026
818fab2
SNOW-2912540: define ENV_VAR_PARTNER locally
sfc-gh-fpawlowski Aug 6, 2026
1eb4bb4
SNOW-2912540: fix black formatting and pyright overload error in inli…
sfc-gh-fpawlowski Aug 17, 2026
f00d60b
SNOW-2912540: define MissingOptionalDependency/pandas/pyarrow locally…
sfc-gh-fpawlowski Aug 12, 2026
f53b5c4
SNOW-2912540: make SecretDetector move a literal 1:1 port, not a rewrite
sfc-gh-fpawlowski Aug 21, 2026
9ab3e6a
Update .pre-commit-config.yaml
sfc-gh-fpawlowski Aug 17, 2026
5812065
SNOW-2912540: fix stale gzip compression expectation in test_misc_set…
sfc-gh-fpawlowski Aug 20, 2026
d99e701
SNOW-2912540: use IS_V5_DRIVER for TelemetryData import path
sfc-gh-fpawlowski Aug 12, 2026
d3f8f6a
SNOW-2912540: revert local MissingOptionalDependency/pandas/pyarrow r…
sfc-gh-fpawlowski Aug 14, 2026
6123a91
SNOW-2912540: fix MissingPandas import to match UD PR #1151's now-cur…
sfc-gh-fpawlowski Aug 15, 2026
3320739
SNOW-2912540: fix TelemetryData import to a real, verified location o…
sfc-gh-fpawlowski Aug 15, 2026
ffae718
SNOW-2912540: remove _pandas_importer(), source pandas/installed_pand…
sfc-gh-fpawlowski Aug 15, 2026
403501e
SNOW-2912540: dedup mock/_options.py's numpy handling against _common…
sfc-gh-fpawlowski Aug 15, 2026
d47099d
SNOW-2912540: fix black formatting after removing _pandas_importer()
sfc-gh-fpawlowski Aug 17, 2026
2fb390a
SNOW-2912540: fix flake8 F401/E402 in utils.py's IS_V5_DRIVER import …
sfc-gh-fpawlowski Aug 17, 2026
320c11f
SNOW-2912540: gate ReauthenticationRequest/TelemetryClient/TelemetryF…
sfc-gh-fpawlowski Aug 21, 2026
3b28796
SNOW-2912540: fix 4 integ test files' unconditional connector.options…
sfc-gh-fpawlowski Aug 21, 2026
a53a6f3
SNOW-2912540: extract options-related imports to new _internal/option…
sfc-gh-fpawlowski Aug 21, 2026
64d0102
SNOW-2912540: import IS_V5_DRIVER from options; gate Modin TelemetryF…
cursoragent Sep 2, 2026
4a3c259
SNOW-2912540: move F401 noqa onto the IS_V5_DRIVER from-import line
cursoragent Sep 2, 2026
d40af46
SNOW-2912540: re-export IS_V5_DRIVER via assignment so flake8 sees a use
cursoragent Sep 2, 2026
8dcd947
SNOW-2912540: add missing license header to test_secret_detector.py
cursoragent Sep 2, 2026
e4abd83
Merge branch 'SNOW-2912540-decouple-oob-telemetry' into SNOW-2912540-…
cursoragent Sep 2, 2026
0576c7a
Merge branch 'create-temp-table-ast-fix-on-main' into SNOW-2912540-re…
cursoragent Sep 2, 2026
b62074c
Merge branch 'SNOW-2912540-remove-connector-shims' into SNOW-2912540-…
cursoragent Sep 2, 2026
09d51a7
Merge branch 'SNOW-2912540-inline-pandas-staging-helpers' into SNOW-2…
cursoragent Sep 2, 2026
8062f15
Merge branch 'SNOW-2912540-decouple-oob-telemetry' into SNOW-2912540-…
cursoragent Sep 2, 2026
c2b7973
Merge branch 'create-temp-table-ast-fix-on-main' into SNOW-2912540-re…
cursoragent Sep 2, 2026
e1fcfaa
Merge branch 'SNOW-2912540-remove-connector-shims' into SNOW-2912540-…
cursoragent Sep 2, 2026
a2a62a8
Merge branch 'SNOW-2912540-inline-pandas-staging-helpers' into SNOW-2…
cursoragent Sep 2, 2026
223d07f
Merge branch 'SNOW-2912540-decouple-oob-telemetry' into SNOW-2912540-…
cursoragent Sep 2, 2026
ef63724
Merge branch 'create-temp-table-ast-fix-on-main' into SNOW-2912540-re…
sfc-gh-fpawlowski Sep 3, 2026
b7cbf51
Merge branch 'SNOW-2912540-remove-connector-shims' into SNOW-2912540-…
sfc-gh-fpawlowski Sep 3, 2026
f6e0c0d
Merge branch 'SNOW-2912540-inline-pandas-staging-helpers' into SNOW-2…
sfc-gh-fpawlowski Sep 3, 2026
5725348
Merge branch 'SNOW-2912540-decouple-oob-telemetry' into SNOW-2912540-…
sfc-gh-fpawlowski Sep 3, 2026
8bcd6a9
Merge graphite-base/4282 into SNOW-2912540-secret-detector-import
sfc-gh-yuwang Sep 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 134 additions & 8 deletions src/snowflake/snowpark/_internal/analyzer/analyzer_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,13 @@
import math
import os
import re
import secrets
import tempfile
from typing import Any, Dict, List, Optional, Tuple, Union, Literal, Sequence
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, Literal, Sequence

from snowflake.connector import ProgrammingError
from snowflake.connector.cursor import SnowflakeCursor
from snowflake.connector.options import pyarrow
from snowflake.connector.pandas_tools import (
_create_temp_stage,
_create_temp_file_format,
build_location_helper,
)
from snowflake.snowpark._internal.options import pyarrow
from snowflake.snowpark._internal.analyzer.binary_plan_node import (
AsOf,
Except,
Expand Down Expand Up @@ -2212,6 +2208,136 @@ def cte_statement(queries: List[str], table_names: List[str]) -> str:
return f"{WITH}{result}"


# ---------------------------------------------------------------------------
# Pandas/Arrow staging helpers
# (ported from snowflake.connector.pandas_tools so Snowpark owns the impl)
# ---------------------------------------------------------------------------

_PANDAS_COMPRESSION_MAP: Dict[str, str] = {
"gzip": "auto",
"snappy": "snappy",
"none": "none",
}


def _pandas_quote_identifier(name: str) -> str:
return '"' + name.replace('"', '""') + '"'


def _qualify_name(
database: Optional[str],
schema: Optional[str],
name: str,
quote_identifiers: bool,
) -> str:
parts = [p for p in (database, schema, name) if p is not None]
if quote_identifiers:
parts = [_pandas_quote_identifier(p) for p in parts]
return ".".join(parts)


build_location_helper = _qualify_name


def _pandas_generate_temp_name(prefix: str) -> str:
return f"__WRITE_PANDAS_{prefix}_{secrets.token_hex(8)}"


def _pandas_create_temp_object(
cursor: SnowflakeCursor,
sql_builder: Callable[[str], Tuple[str, tuple]],
qualified_name: str,
bare_name: str,
) -> str:
sql, params = sql_builder(qualified_name)
try:
# _force_qmark_paramstyle is only on execute()'s real signature, not its
# @overload stubs, so pyright can't match any overload for this call.
cursor.execute(sql, params=params, _force_qmark_paramstyle=True) # type: ignore[call-overload]
return qualified_name
except ProgrammingError:
sql, params = sql_builder(bare_name)
cursor.execute(sql, params=params, _force_qmark_paramstyle=True) # type: ignore[call-overload]
return bare_name


def _stage_sql(
name: str,
compression: str,
binary_as_text_false: bool,
use_scoped: bool = False,
) -> Tuple[str, tuple]:
mapped = _PANDAS_COMPRESSION_MAP[compression]
temp_type = "SCOPED TEMPORARY" if use_scoped else "TEMPORARY"
fmt_opts = [f"TYPE=PARQUET COMPRESSION={mapped}"]
if binary_as_text_false:
fmt_opts.append("BINARY_AS_TEXT=FALSE")
return (
f"CREATE {temp_type} STAGE IDENTIFIER(?) FILE_FORMAT=({' '.join(fmt_opts)})",
(name,),
)


def _file_format_sql(
name: str,
compression: str,
use_logical_type_suffix: str = "",
use_scoped: bool = False,
) -> Tuple[str, tuple]:
mapped = _PANDAS_COMPRESSION_MAP[compression]
temp_type = "SCOPED TEMPORARY" if use_scoped else "TEMPORARY"
return (
f"CREATE {temp_type} FILE FORMAT IDENTIFIER(?) TYPE=PARQUET COMPRESSION={mapped}{use_logical_type_suffix}",
(name,),
)


def _create_temp_stage(
cursor: SnowflakeCursor,
database: Optional[str],
schema: Optional[str],
quote_identifiers: bool,
compression: str,
auto_create_table: bool,
overwrite: bool,
use_scoped_temp_object: bool = False,
) -> str:
name = _pandas_generate_temp_name("STAGE")
qualified = _qualify_name(database, schema, name, quote_identifiers)
return _pandas_create_temp_object(
cursor,
lambda n: _stage_sql(
n, compression, auto_create_table or overwrite, use_scoped_temp_object
),
qualified,
name,
)


def _create_temp_file_format(
cursor: SnowflakeCursor,
database: Optional[str],
schema: Optional[str],
quote_identifiers: bool,
compression: str,
sql_use_logical_type: str,
use_scoped_temp_object: bool = False,
) -> str:
name = _pandas_generate_temp_name("FILE_FORMAT")
qualified = _qualify_name(database, schema, name, quote_identifiers)
return _pandas_create_temp_object(
cursor,
lambda n: _file_format_sql(
n, compression, sql_use_logical_type, use_scoped_temp_object
),
qualified,
name,
)


# ---------------------------------------------------------------------------


def write_arrow(
cursor: SnowflakeCursor,
table: "pyarrow.Table",
Expand Down Expand Up @@ -2364,7 +2490,7 @@ def drop_object(name: str, object_type: str) -> None:
database,
schema,
quote_identifiers,
compression_map[compression],
compression,
sql_use_logical_type,
use_scoped_temp_object,
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from snowflake.snowpark._internal.data_source.drivers.base_driver import BaseDriver
from snowflake.snowpark.exceptions import SnowparkDataframeReaderException
from snowflake.snowpark.types import StructType
from snowflake.connector.options import pandas as pd
from snowflake.snowpark._internal.options import pandas as pd
import logging

logger = logging.getLogger(__name__)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from enum import Enum
import datetime
from typing import Dict, List, Callable, Any, Optional, TYPE_CHECKING
from snowflake.connector.options import pandas as pd
from snowflake.snowpark._internal.options import pandas as pd

from snowflake.snowpark._internal.analyzer.analyzer_utils import unquote_if_quoted
from snowflake.snowpark._internal.data_source.datasource_typing import (
Expand Down
5 changes: 4 additions & 1 deletion src/snowflake/snowpark/_internal/event_table_telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,13 @@
from abc import ABC
from logging import getLogger
from typing import Dict, Optional, Tuple
from snowflake.connector.options import MissingOptionalDependency, ModuleLikeObject
import snowflake.snowpark
import requests

from snowflake.snowpark._internal.options import (
MissingOptionalDependency,
ModuleLikeObject,
)
from snowflake.snowpark._internal.utils import parse_table_name

_logger = getLogger(__name__)
Expand Down
38 changes: 38 additions & 0 deletions src/snowflake/snowpark/_internal/options.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
#
# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
#

from snowflake.connector.version import VERSION as connector_version

IS_V5_DRIVER: bool = connector_version[0] >= 5

if IS_V5_DRIVER:
from snowflake.connector._common.extras import pandas # noqa: F401
from snowflake.connector._common.extras import ModuleLikeObject # noqa: F401
from snowflake.connector._common.extras import installed_pandas # noqa: F401
from snowflake.connector._common.extras import (
MissingOptionalDependency,
pyarrow,
installed_pyarrow,
)
else:
from snowflake.connector.options import pandas # noqa: F401
from snowflake.connector.options import ModuleLikeObject # noqa: F401
from snowflake.connector.options import installed_pandas # noqa: F401
from snowflake.connector.options import (
MissingOptionalDependency,
MissingPandas,
pyarrow,
)

# connector.options (v4) never exported installed_pyarrow as its own name.
installed_pyarrow: bool = not isinstance(pyarrow, MissingOptionalDependency)


def _missing_pandas() -> MissingOptionalDependency:
# v4 has no __init__ override (no-arg-subclass only); v5 deleted
# MissingPandas in favor of the positional-arg form.
if IS_V5_DRIVER:
return MissingOptionalDependency("pandas")
return MissingPandas()
14 changes: 11 additions & 3 deletions src/snowflake/snowpark/_internal/server_connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,10 @@
)

from snowflake.connector import SnowflakeConnection, connect
from snowflake.connector.constants import ENV_VAR_PARTNER, FIELD_ID_TO_NAME
from snowflake.connector.constants import FIELD_ID_TO_NAME
from snowflake.connector.cursor import ResultMetadata, SnowflakeCursor
from snowflake.connector.errors import Error, NotSupportedError, ProgrammingError
from snowflake.connector.network import ReauthenticationRequest
from snowflake.connector.options import pandas
from snowflake.snowpark._internal.options import pandas
from snowflake.snowpark._internal.analyzer.analyzer_utils import (
quote_name_without_upper_casing,
)
Expand All @@ -54,6 +53,7 @@
get_plan_telemetry_metrics,
)
from snowflake.snowpark._internal.utils import (
IS_V5_DRIVER,
create_rlock,
create_thread_local,
escape_quotes,
Expand All @@ -69,6 +69,11 @@
result_set_to_rows,
unwrap_stage_location_single_quote,
)

if IS_V5_DRIVER:
from snowflake.connector.errors import ReauthenticationRequest
else:
from snowflake.connector.network import ReauthenticationRequest
from snowflake.snowpark import context
from snowflake.snowpark.async_job import AsyncJob, _AsyncResultType
from snowflake.snowpark.query_history import QueryListener, QueryRecord
Expand All @@ -82,6 +87,9 @@

logger = getLogger(__name__)

# backward compatibility constant
ENV_VAR_PARTNER = "SF_PARTNER"

# parameters needed for usage tracking
PARAM_APPLICATION = "application"
PARAM_INTERNAL_APPLICATION_NAME = "internal_application_name"
Expand Down
19 changes: 14 additions & 5 deletions src/snowflake/snowpark/_internal/telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,6 @@
from typing import Any, Dict, List, Optional

from snowflake.connector import SnowflakeConnection
from snowflake.connector.telemetry import (
TelemetryClient as PCTelemetryClient,
TelemetryData as PCTelemetryData,
TelemetryField as PCTelemetryField,
)
from snowflake.connector.time_util import get_time_millis
from snowflake.snowpark._internal.analyzer.query_plan_analysis_utils import (
PlanState,
Expand All @@ -30,6 +25,7 @@
DescribeQueryTelemetryField,
)
from snowflake.snowpark._internal.utils import (
IS_V5_DRIVER,
get_application_name,
get_os_name,
get_python_version,
Expand All @@ -39,6 +35,19 @@
generate_random_alphanumeric,
)

if IS_V5_DRIVER:
from snowflake.connector._common.telemetry import (
TelemetryClient as PCTelemetryClient,
TelemetryData as PCTelemetryData,
TelemetryField as PCTelemetryField,
)
else:
from snowflake.connector.telemetry import (
TelemetryClient as PCTelemetryClient,
TelemetryData as PCTelemetryData,
TelemetryField as PCTelemetryField,
)

try:
import psutil

Expand Down
2 changes: 1 addition & 1 deletion src/snowflake/snowpark/_internal/type_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import snowflake.snowpark.types # type: ignore
from snowflake.connector.constants import FIELD_ID_TO_NAME
from snowflake.connector.cursor import ResultMetadata
from snowflake.connector.options import installed_pandas, pandas
from snowflake.snowpark._internal.options import installed_pandas, pandas
from snowflake.snowpark._internal.utils import quote_name
from snowflake.snowpark.row import Row
from snowflake.snowpark.types import (
Expand Down
2 changes: 1 addition & 1 deletion src/snowflake/snowpark/_internal/udf_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from packaging.requirements import Requirement

import snowflake.snowpark
from snowflake.connector.options import installed_pandas, pandas
from snowflake.snowpark._internal.options import installed_pandas, pandas
from snowflake.snowpark._internal import code_generation, type_utils
from snowflake.snowpark._internal.analyzer.datatype_mapper import to_sql, to_sql_no_cast
from snowflake.snowpark._internal.telemetry import TelemetryField
Expand Down
26 changes: 7 additions & 19 deletions src/snowflake/snowpark/_internal/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,12 +54,18 @@
from snowflake.connector.constants import FIELD_ID_TO_NAME
from snowflake.connector.cursor import ResultMetadata, SnowflakeCursor
from snowflake.connector.description import OPERATING_SYSTEM, PLATFORM
from snowflake.connector.options import MissingOptionalDependency, ModuleLikeObject
from snowflake.connector.version import VERSION as connector_version
from snowflake.snowpark._internal.error_message import SnowparkClientExceptionMessages
from snowflake.snowpark._internal.options import (
IS_V5_DRIVER, # noqa: F401 - re-exported for existing callers
MissingOptionalDependency,
ModuleLikeObject,
installed_pandas,
)
from snowflake.snowpark.row import Row
from snowflake.snowpark.version import VERSION as snowpark_version


if TYPE_CHECKING:
from snowflake.snowpark._internal.analyzer.snowflake_plan import (
SnowflakePlan,
Expand Down Expand Up @@ -240,24 +246,6 @@

SUPPORTED_TABLE_TYPES = ["temp", "temporary", "transient"]

# TODO: merge fixed pandas importer changes to connector.
def _pandas_importer(): # noqa: E302
"""Helper function to lazily import pandas and return MissingPandas if not installed."""
from snowflake.connector.options import MissingPandas

pandas = MissingPandas()
try:
pandas = importlib.import_module("pandas")
# since we enable relative imports without dots this import gives us an issues when ran from test directory
from pandas import DataFrame # NOQA
except ImportError: # pragma: no cover
pass # pragma: no cover
return pandas


pandas = _pandas_importer()
installed_pandas = not isinstance(pandas, MissingOptionalDependency)


class TempObjectType(Enum):
TABLE = "TABLE"
Expand Down
Loading
Loading