Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions tests/integ/test_dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import json
import logging
import math
import re
import time
from array import array
from collections import namedtuple
Expand Down Expand Up @@ -1679,12 +1680,24 @@ def test_cache_result_query(session):
with session.query_history() as history:
df.cache_result()

assert len(history.queries) == 2
assert "CREATE SCOPED TEMPORARY TABLE" in history.queries[0].sql_text
# connector >= 4.7.1 no longer caches database/schema from non-USE statements (e.g. CREATE TEMP TABLE).
# Filter out metadata queries (e.g. SELECT CURRENT_DATABASE()) that may appear
# in some environments before or between the expected DDL/DML statements.
filtered_queries = [
q
for q in history.queries
if not re.match(
r"^\s*SELECT\s+CURRENT_(DATABASE|SCHEMA)\s*\(\s*\)\s*$",
q.sql_text,
re.IGNORECASE,
)
]
assert len(filtered_queries) == 2
assert "CREATE SCOPED TEMPORARY TABLE" in filtered_queries[0].sql_text
assert (
"INSERT INTO" in history.queries[1].sql_text
"INSERT INTO" in filtered_queries[1].sql_text
and 'SELECT $1 AS "A", $2 AS "B" FROM VALUES (1 :: INT, 2 :: INT)'
in history.queries[1].sql_text
in filtered_queries[1].sql_text
)


Expand Down
90 changes: 54 additions & 36 deletions tests/integ/test_large_query_breakdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import logging
import os
import re
import tempfile
from unittest.mock import patch

Expand Down Expand Up @@ -32,6 +33,14 @@
)
]

_METADATA_QUERY_RE = re.compile(
r"^\s*SELECT\s+CURRENT_(DATABASE|SCHEMA)\s*\(\s*\)\s*$", re.IGNORECASE
)


def _filter_metadata_queries(queries):
return [q for q in queries if not _METADATA_QUERY_RE.match(q.sql_text)]


@pytest.fixture(autouse=True)
def large_query_df(session):
Expand Down Expand Up @@ -283,13 +292,14 @@ def test_save_as_table(session, large_query_df):
with SqlCounter(query_count=4, union_count=1, describe_count=1):
large_query_df.write.save_as_table(table_name, mode="overwrite")

assert len(history.queries) == 4
assert history.queries[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert history.queries[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert history.queries[2].sql_text.startswith(
filtered = _filter_metadata_queries(history.queries)
assert len(filtered) == 4
assert filtered[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert filtered[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert filtered[2].sql_text.startswith(
f"CREATE OR REPLACE TABLE {table_name}"
)
assert history.queries[3].sql_text.startswith("DROP TABLE If EXISTS")
assert filtered[3].sql_text.startswith("DROP TABLE If EXISTS")


def test_variable_binding(session):
Expand Down Expand Up @@ -338,21 +348,23 @@ def test_update_delete_merge(session, large_query_df):
# 3 describe call triggered due to column state extraction
with SqlCounter(query_count=4, union_count=1, describe_count=2):
t.update({"B": 0}, t.a == large_query_df.a, large_query_df)
assert len(history.queries) == 4
assert history.queries[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert history.queries[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert history.queries[2].sql_text.startswith(f"UPDATE {table_name}")
assert history.queries[3].sql_text.startswith("DROP TABLE If EXISTS")
filtered = _filter_metadata_queries(history.queries)
assert len(filtered) == 4
assert filtered[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert filtered[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert filtered[2].sql_text.startswith(f"UPDATE {table_name}")
assert filtered[3].sql_text.startswith("DROP TABLE If EXISTS")

# delete
with session.query_history() as history:
with SqlCounter(query_count=4, union_count=1, describe_count=0):
t.delete(t.a == large_query_df.a, large_query_df)
assert len(history.queries) == 4
assert history.queries[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert history.queries[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert history.queries[2].sql_text.startswith(f"DELETE FROM {table_name} USING")
assert history.queries[3].sql_text.startswith("DROP TABLE If EXISTS")
filtered = _filter_metadata_queries(history.queries)
assert len(filtered) == 4
assert filtered[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert filtered[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert filtered[2].sql_text.startswith(f"DELETE FROM {table_name} USING")
assert filtered[3].sql_text.startswith("DROP TABLE If EXISTS")

# merge
with session.query_history() as history:
Expand All @@ -362,11 +374,12 @@ def test_update_delete_merge(session, large_query_df):
t.a == large_query_df.a,
[when_matched().update({"b": large_query_df.b})],
)
assert len(history.queries) == 4
assert history.queries[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert history.queries[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert history.queries[2].sql_text.startswith(f"MERGE INTO {table_name} USING")
assert history.queries[3].sql_text.startswith("DROP TABLE If EXISTS")
filtered = _filter_metadata_queries(history.queries)
assert len(filtered) == 4
assert filtered[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert filtered[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert filtered[2].sql_text.startswith(f"MERGE INTO {table_name} USING")
assert filtered[3].sql_text.startswith("DROP TABLE If EXISTS")


def test_copy_into_location(session, large_query_df):
Expand All @@ -380,11 +393,12 @@ def test_copy_into_location(session, large_query_df):
overwrite=True,
single=True,
)
assert len(history.queries) == 4, history.queries
assert history.queries[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert history.queries[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert history.queries[2].sql_text.startswith(f"COPY INTO '{remote_file_path}'")
assert history.queries[3].sql_text.startswith("DROP TABLE If EXISTS")
filtered = _filter_metadata_queries(history.queries)
assert len(filtered) == 4, filtered
assert filtered[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert filtered[1].sql_text.startswith("CREATE SCOPED TEMPORARY TABLE")
assert filtered[2].sql_text.startswith(f"COPY INTO '{remote_file_path}'")
assert filtered[3].sql_text.startswith("DROP TABLE If EXISTS")


def test_in_with_subquery_multiple_query(session):
Expand Down Expand Up @@ -561,8 +575,9 @@ def test_optimization_skipped_with_transaction(session, large_query_df, caplog):

check_optimization_skipped_reason(patch_send, {"active transaction": 1})

assert len(history.queries) == 2, history.queries
assert history.queries[0].sql_text == "SELECT CURRENT_TRANSACTION()"
filtered_queries = _filter_metadata_queries(history.queries)
assert len(filtered_queries) == 2, filtered_queries
assert filtered_queries[0].sql_text == "SELECT CURRENT_TRANSACTION()"
assert "Skipping large query breakdown" in caplog.text
assert Utils.is_active_transaction(session)
assert session.sql("commit").collect()
Expand Down Expand Up @@ -699,15 +714,18 @@ def test_add_parent_plan_uuid_to_statement_params(session, large_query_df):
Utils.check_answer(result, [Row(1, 5999), Row(2, 5998)])

plan = large_query_df._plan
# 1 for current transaction, 1 for partition, 1 for main query, 1 for post action
assert patched_run_query.call_count == 4

for i, call in enumerate(patched_run_query.call_args_list):
if i == 0:
assert call.args[0] == "SELECT CURRENT_TRANSACTION()"
else:
assert "_statement_params" in call.kwargs
assert call.kwargs["_statement_params"]["_PLAN_UUID"] == plan.uuid
filtered_calls = [
call
for call in patched_run_query.call_args_list
if not _METADATA_QUERY_RE.match(call.args[0])
]
assert len(filtered_calls) == 4

for call in filtered_calls:
if call.args[0] == "SELECT CURRENT_TRANSACTION()":
continue
assert "_statement_params" in call.kwargs
assert call.kwargs["_statement_params"]["_PLAN_UUID"] == plan.uuid


@pytest.mark.skipif(
Expand Down
4 changes: 3 additions & 1 deletion tests/integ/test_multithreading.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,9 @@ def test_session_stage_created_once(threadsafe_session):
for _ in range(10):
executor.submit(threadsafe_session.get_session_stage)

assert patched_run_query.call_count == 1
# connector >= 4.7.1 no longer caches database/schema from non-USE statements (e.g. CREATE TEMP STAGE).
# Older connectors did, expect 3 queries for >= 4.7.1, 1 for earlier versions.
assert patched_run_query.call_count in [1, 3]


def test_action_ids_are_unique(threadsafe_session):
Expand Down
Loading