Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "datateam_moss"
version = "0.2.26"
version = "0.2.27"
description = "Functions needed for datateam MOSS (municipality of Amsterdam)"
readme = "README.md"
requires-python = ">=3.10"
Expand Down
21 changes: 16 additions & 5 deletions src/datateam_moss/spark_io_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -333,11 +333,17 @@ def create_table_from_ddl(
def add_new_columns_to_table(spark: SparkSession, full_table_name: str, table_definition: dict) -> None:
"""Voegt kolommen uit de definitie toe die nog niet in de tabel bestaan,
in één ALTER TABLE."""
existing_fields = spark.table(full_table_name).schema.fields
existing_columns = {f.name.lower() for f in existing_fields}

existing_columns = {f.name.lower() for f in spark.table(full_table_name).schema.fields}
# De SID-kolom staat niet in de definitie, maar moet wel vooraan blijven.
sid_column = next(
(f.name for f in existing_fields if f.name.lower().startswith("sid_")),
None,
)
sql_clauses = []
added_columns = []
previous_column = None
previous_column = sid_column

# Bepaal de kolommen die nog niet bestaan en de positie die ze moeten krijgen.
for col in table_definition["columns"]:
Expand All @@ -356,13 +362,18 @@ def add_new_columns_to_table(spark: SparkSession, full_table_name: str, table_de
logger.info(f"Added columns {added_columns} to table {full_table_name}")


def create_or_update_table(spark: SparkSession, full_table_name: str, table_definition: dict) -> None:
def create_or_update_table(spark: SparkSession, full_table_name: str, table_definition: dict, genereer_sid: bool = False) -> None:
"""Maakt de tabel aan als hij nog niet bestaat, en voegt anders
ontbrekende kolommen uit de definitie toe."""
if spark.catalog.tableExists(full_table_name):
add_new_columns_to_table(spark, full_table_name, table_definition)
add_new_columns_to_table(spark=spark,
full_table_name=full_table_name,
table_definition=table_definition)
else:
create_table_from_ddl(spark, full_table_name, table_definition)
create_table_from_ddl(spark=spark,
full_table_name=full_table_name,
table_definition=table_definition,
genereer_sid=genereer_sid)


def create_stringtype_dataframe_from_list(spark, data: List) -> DataFrame:
Expand Down
115 changes: 96 additions & 19 deletions tests/test_spark_io_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from unittest.mock import MagicMock

import pytest
from pyspark.sql.types import StructType, StructField, StringType, IntegerType
from pyspark.sql.types import StructType, StructField, StringType, IntegerType, LongType

from datateam_moss import spark_io_utils

Expand Down Expand Up @@ -82,11 +82,21 @@ def test_add_metadata_columns_to_dataframe_runtime_and_timestamps(spark, numeric
first_row = result.first()
assert str(first_row["m_aangemaakt_op"]).startswith("2023-05-10 08:45")


def create_spark_mock_with_schema(existing_schema: StructType) -> MagicMock:
spark = MagicMock()
spark.table.return_value.schema = existing_schema
return spark


def create_spark_mock_with_table_exists(table_exists: bool) -> MagicMock:
spark = MagicMock()
spark.catalog.tableExists.return_value = table_exists
return spark


# --- add_new_columns_to_table ---

def test_add_new_columns_to_table_adds_missing_col_after_previous_col():
spark = create_spark_mock_with_schema(StructType([
StructField("id", IntegerType()),
Expand All @@ -100,10 +110,9 @@ def test_add_new_columns_to_table_adds_missing_col_after_previous_col():

spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie)

spark.sql.assert_called_once()
query = spark.sql.call_args.args[0]
assert "ALTER TABLE db.klanten ADD COLUMNS" in query
assert "`email` string AFTER `id`" in query
spark.sql.assert_called_once_with(
"ALTER TABLE db.klanten ADD COLUMNS (\n `email` string AFTER `id`\n)"
)


def test_add_new_columns_to_table_first_col_gets_first():
Expand All @@ -115,17 +124,66 @@ def test_add_new_columns_to_table_first_col_gets_first():

spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie)

assert "`id` int FIRST" in spark.sql.call_args.args[0]
spark.sql.assert_called_once_with(
"ALTER TABLE db.klanten ADD COLUMNS (\n `id` int FIRST\n)"
)


def test_add_new_columns_to_table_first_col_after_sid():
spark = create_spark_mock_with_schema(StructType([
StructField("sid_klanten", LongType()),
StructField("naam", StringType()),
]))
definitie = {"columns": [
{"name": "id", "type": "IntegerType()"},
{"name": "naam", "type": "StringType()"},
]}

spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie)

spark.sql.assert_called_once_with(
"ALTER TABLE db.klanten ADD COLUMNS (\n `id` int AFTER `sid_klanten`\n)"
)


def test_add_new_columns_to_table_multiple_cols():
spark = create_spark_mock_with_schema(StructType([StructField("id", IntegerType())]))
definitie = {"columns": [
{"name": "id", "type": "IntegerType()"},
{"name": "naam", "type": "StringType()"},
{"name": "email", "type": "StringType()"},
]}

spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie)

spark.sql.assert_called_once_with(
"ALTER TABLE db.klanten ADD COLUMNS (\n"
" `naam` string AFTER `id`,\n"
" `email` string AFTER `naam`\n"
")"
)


def test_add_new_columns_to_table_nothing_to_add():
spark = create_spark_mock_with_schema(StructType([StructField("id", IntegerType())]))
definitie = {"columns": [{"name": "id", "type": "IntegerType()"}]} # hoofdletterongevoelig
definitie = {"columns": [{"name": "id", "type": "IntegerType()"}]}

spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie)

spark.sql.assert_not_called()


def test_add_new_columns_to_table_case_insensitive():
spark = create_spark_mock_with_schema(StructType([StructField("id", IntegerType())]))
definitie = {"columns": [{"name": "ID", "type": "IntegerType()"}]}

spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie)

spark.sql.assert_not_called()


# --- create_or_update_table ---

@pytest.fixture
def mocked_functions(monkeypatch):
"""Vervangt de twee onderliggende functies door mocks, zodat we alleen
Expand All @@ -142,29 +200,48 @@ def table_definition():
return {"columns": [{"name": "id", "type": "IntegerType()"}]}


def create_spark_mock_with_table_exists(table_exists: bool) -> MagicMock:
spark = MagicMock()
spark.catalog.tableExists.return_value = table_exists
return spark


def test_create_or_update_table_create_table_if_not_exists(mocked_functions, table_definition):
@pytest.mark.parametrize("genereer_sid", [False, True])
def test_create_or_update_table_creates_table_if_not_exists(
mocked_functions, table_definition, genereer_sid
):
create_mock, add_mock = mocked_functions
spark = create_spark_mock_with_table_exists(table_exists=False)

spark_io_utils.create_or_update_table(spark, "db.klanten", table_definition)
spark_io_utils.create_or_update_table(
spark, "db.klanten", table_definition, genereer_sid=genereer_sid
)

spark.catalog.tableExists.assert_called_once_with("db.klanten")
create_mock.assert_called_once_with(spark, "db.klanten", table_definition)
create_mock.assert_called_once_with(
spark=spark,
full_table_name="db.klanten",
table_definition=table_definition,
genereer_sid=genereer_sid,
)
add_mock.assert_not_called()


def test_create_or_update_table_add_columns_if_table_exists(mocked_functions, table_definition):
def test_create_or_update_table_genereer_sid_defaults_to_false(mocked_functions, table_definition):
create_mock, _ = mocked_functions
spark = create_spark_mock_with_table_exists(table_exists=False)

spark_io_utils.create_or_update_table(spark, "db.klanten", table_definition)

assert create_mock.call_args.kwargs["genereer_sid"] is False


def test_create_or_update_table_adds_columns_if_table_exists(mocked_functions, table_definition):
create_mock, add_mock = mocked_functions
spark = create_spark_mock_with_table_exists(table_exists=True)

spark_io_utils.create_or_update_table(spark, "db.klanten", table_definition)
spark_io_utils.create_or_update_table(
spark, "db.klanten", table_definition, genereer_sid=True
)

spark.catalog.tableExists.assert_called_once_with("db.klanten")
add_mock.assert_called_once_with(spark, "db.klanten", table_definition)
add_mock.assert_called_once_with(
spark=spark,
full_table_name="db.klanten",
table_definition=table_definition,
)
create_mock.assert_not_called()
Loading