From 5d2000bec1c979247d356b7fc42258fac5733612 Mon Sep 17 00:00:00 2001 From: Ben Kogan Date: Mon, 17 Aug 2026 22:01:48 +0000 Subject: [PATCH 1/4] NO-SNOW: xfail AI complete named-placeholders test, drop expired date xfails test_dataframe_ai_complete_with_named_placeholders fails deterministically (26/26 daily precommit jobs, all clouds, through 5 pytest reruns) because AI_COMPLETE returns NULL for its prompt. Scope the xfail to that one test so the ~50 other AI tests in the file keep gating, and use strict=False so an XPASS surfaces once the underlying issue is resolved. The blanket date-based xfails in the module pytestmark and the doctest hook expired on 2026-06-04, so they were dead code masking nothing. --- src/conftest.py | 11 ----------- tests/integ/test_dataframe_ai.py | 14 ++++++++------ 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/src/conftest.py b/src/conftest.py index f33dbb32e2..5d6ec2587e 100644 --- a/src/conftest.py +++ b/src/conftest.py @@ -2,7 +2,6 @@ # # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved. # -import datetime import doctest import logging import os @@ -101,16 +100,6 @@ def pytest_collection_modifyitems(config, items): This function is used to skip some doctests on certain conditions. For example, some functions in PrPr might not be available on Azure and GCP. """ - # Mark flaky tests as xfail for a time - if datetime.date.today() <= datetime.date(2026, 6, 4): - xfail_ai = pytest.mark.xfail( - reason="dataframe_ai_functions doctests flaky due to infrastructure issues", - strict=False, - ) - for item in items: - if isinstance(item, DoctestItem) and "dataframe_ai" in item.name: - item.add_marker(xfail_ai) - host = ( conn_params.get("host") or conn_params.get("HOST") or conn_params.get("account") ) diff --git a/tests/integ/test_dataframe_ai.py b/tests/integ/test_dataframe_ai.py index fb262af083..f88a0a857a 100644 --- a/tests/integ/test_dataframe_ai.py +++ b/tests/integ/test_dataframe_ai.py @@ -2,7 +2,6 @@ # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved. # -import datetime import json import pytest from snowflake.snowpark.functions import ai_complete, ai_extract, col, lit, to_file @@ -14,14 +13,17 @@ "config.getoption('local_testing_mode', default=False)", reason="AI functions are not yet supported in local testing mode.", ), - pytest.mark.xfail( - datetime.date.today() <= datetime.date(2026, 6, 4), - reason="AI tests flaky due to infrastructure issues", - strict=False, - ), ] +@pytest.mark.xfail( + reason=( + "AI_COMPLETE returns NULL for this named-placeholder prompt in daily CI " + "(26/26 jobs on 2026-08-17). Client SQL generation is correct; sibling " + "llama3.1-8b complete tests pass. Keep this test running so XPASS is visible." + ), + strict=False, +) def test_dataframe_ai_complete_with_named_placeholders(session): """Test DataFrame.ai.complete with named placeholders.""" # Create a DataFrame with review data From 622ec2003328d2709086f7b9fe0936d271a2f397 Mon Sep 17 00:00:00 2001 From: Ben Kogan Date: Mon, 17 Aug 2026 22:12:57 +0000 Subject: [PATCH 2/4] NO-SNOW: record PROMPT() non-string argument root cause in xfail reason Reproduced on a live deployment: PROMPT() stores an integer argument as a number, and the executor only accepts text/file arguments, so the row comes back as a NULL completion. Point the xfail at that instead of "infrastructure". --- tests/integ/test_dataframe_ai.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integ/test_dataframe_ai.py b/tests/integ/test_dataframe_ai.py index f88a0a857a..a2f63e2e86 100644 --- a/tests/integ/test_dataframe_ai.py +++ b/tests/integ/test_dataframe_ai.py @@ -18,9 +18,10 @@ @pytest.mark.xfail( reason=( - "AI_COMPLETE returns NULL for this named-placeholder prompt in daily CI " - "(26/26 jobs on 2026-08-17). Client SQL generation is correct; sibling " - "llama3.1-8b complete tests pass. Keep this test running so XPASS is visible." + "Server-side PROMPT() rejects non-string arguments: the integer 'rating' " + "column is embedded in the prompt object as a number, which fails with " + "'unsupported argument type' and surfaces as a NULL completion. Casting " + "the column to VARCHAR makes it pass, so this needs a server-side fix." ), strict=False, ) From 2963463c95ba2e24f50d5ca0d524eaa56d05f236 Mon Sep 17 00:00:00 2001 From: Ben Kogan Date: Mon, 17 Aug 2026 22:19:57 +0000 Subject: [PATCH 3/4] NO-SNOW: cast numeric prompt argument to string in AI complete test PROMPT() only accepts string arguments, so the integer rating column reached the server as a number and every row came back as a NULL completion. Cast it so the test exercises named placeholders instead of tripping over arg typing. --- tests/integ/test_dataframe_ai.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tests/integ/test_dataframe_ai.py b/tests/integ/test_dataframe_ai.py index a2f63e2e86..510b9bfcc7 100644 --- a/tests/integ/test_dataframe_ai.py +++ b/tests/integ/test_dataframe_ai.py @@ -5,6 +5,7 @@ import json import pytest from snowflake.snowpark.functions import ai_complete, ai_extract, col, lit, to_file +from snowflake.snowpark.types import StringType from tests.utils import RUNNING_ON_JENKINS, TestFiles, Utils from snowflake.snowpark.exceptions import SnowparkSQLException @@ -16,15 +17,6 @@ ] -@pytest.mark.xfail( - reason=( - "Server-side PROMPT() rejects non-string arguments: the integer 'rating' " - "column is embedded in the prompt object as a number, which fails with " - "'unsupported argument type' and surfaces as a NULL completion. Casting " - "the column to VARCHAR makes it pass, so this needs a server-side fix." - ), - strict=False, -) def test_dataframe_ai_complete_with_named_placeholders(session): """Test DataFrame.ai.complete with named placeholders.""" # Create a DataFrame with review data @@ -42,7 +34,9 @@ def test_dataframe_ai_complete_with_named_placeholders(session): prompt="Analyze this {category} product review: '{review}' with rating {rating}/5. What is the sentiment?", input_columns={ "review": col("review"), - "rating": col("rating"), + # PROMPT() only accepts string arguments; a numeric column reaches the + # server as a number and fails with "unsupported argument type". + "rating": col("rating").cast(StringType()), "category": col("category"), }, output_column="sentiment_analysis", From 94660c5bbee22902033c828a670f2883756c5366 Mon Sep 17 00:00:00 2001 From: Ben Kogan Date: Mon, 17 Aug 2026 22:25:22 +0000 Subject: [PATCH 4/4] NO-SNOW: clarify why the AI complete test casts rating to string Docs allow NUMBER prompt args; the runtime currently rejects them. Make the comment match that rather than implying strings are the only supported type. --- tests/integ/test_dataframe_ai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integ/test_dataframe_ai.py b/tests/integ/test_dataframe_ai.py index 510b9bfcc7..89086099c3 100644 --- a/tests/integ/test_dataframe_ai.py +++ b/tests/integ/test_dataframe_ai.py @@ -34,8 +34,8 @@ def test_dataframe_ai_complete_with_named_placeholders(session): prompt="Analyze this {category} product review: '{review}' with rating {rating}/5. What is the sentiment?", input_columns={ "review": col("review"), - # PROMPT() only accepts string arguments; a numeric column reaches the - # server as a number and fails with "unsupported argument type". + # Docs allow NUMBER prompt args; runtime currently rejects them with + # "unsupported argument type". Cast until the server-side bug is fixed. "rating": col("rating").cast(StringType()), "category": col("category"), },