From fff60ba94db62044d18bfe727ea86032945e34d0 Mon Sep 17 00:00:00 2001 From: Aydin Date: Tue, 6 Oct 2026 11:52:37 +0200 Subject: [PATCH] parameter toegevoegd --- pyproject.toml | 2 +- src/datateam_moss/spark_io_utils.py | 21 +++-- tests/test_spark_io_utils.py | 115 +++++++++++++++++++++++----- 3 files changed, 113 insertions(+), 25 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5d64c40..b3a87ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "datateam_moss" -version = "0.2.26" +version = "0.2.27" description = "Functions needed for datateam MOSS (municipality of Amsterdam)" readme = "README.md" requires-python = ">=3.10" diff --git a/src/datateam_moss/spark_io_utils.py b/src/datateam_moss/spark_io_utils.py index 7845c0d..0096044 100644 --- a/src/datateam_moss/spark_io_utils.py +++ b/src/datateam_moss/spark_io_utils.py @@ -333,11 +333,17 @@ def create_table_from_ddl( def add_new_columns_to_table(spark: SparkSession, full_table_name: str, table_definition: dict) -> None: """Voegt kolommen uit de definitie toe die nog niet in de tabel bestaan, in één ALTER TABLE.""" + existing_fields = spark.table(full_table_name).schema.fields + existing_columns = {f.name.lower() for f in existing_fields} - existing_columns = {f.name.lower() for f in spark.table(full_table_name).schema.fields} + # De SID-kolom staat niet in de definitie, maar moet wel vooraan blijven. + sid_column = next( + (f.name for f in existing_fields if f.name.lower().startswith("sid_")), + None, + ) sql_clauses = [] added_columns = [] - previous_column = None + previous_column = sid_column # Bepaal de kolommen die nog niet bestaan en de positie die ze moeten krijgen. for col in table_definition["columns"]: @@ -356,13 +362,18 @@ def add_new_columns_to_table(spark: SparkSession, full_table_name: str, table_de logger.info(f"Added columns {added_columns} to table {full_table_name}") -def create_or_update_table(spark: SparkSession, full_table_name: str, table_definition: dict) -> None: +def create_or_update_table(spark: SparkSession, full_table_name: str, table_definition: dict, genereer_sid: bool = False) -> None: """Maakt de tabel aan als hij nog niet bestaat, en voegt anders ontbrekende kolommen uit de definitie toe.""" if spark.catalog.tableExists(full_table_name): - add_new_columns_to_table(spark, full_table_name, table_definition) + add_new_columns_to_table(spark=spark, + full_table_name=full_table_name, + table_definition=table_definition) else: - create_table_from_ddl(spark, full_table_name, table_definition) + create_table_from_ddl(spark=spark, + full_table_name=full_table_name, + table_definition=table_definition, + genereer_sid=genereer_sid) def create_stringtype_dataframe_from_list(spark, data: List) -> DataFrame: diff --git a/tests/test_spark_io_utils.py b/tests/test_spark_io_utils.py index 2d2bfb4..26c8b66 100644 --- a/tests/test_spark_io_utils.py +++ b/tests/test_spark_io_utils.py @@ -3,7 +3,7 @@ from unittest.mock import MagicMock import pytest -from pyspark.sql.types import StructType, StructField, StringType, IntegerType +from pyspark.sql.types import StructType, StructField, StringType, IntegerType, LongType from datateam_moss import spark_io_utils @@ -82,11 +82,21 @@ def test_add_metadata_columns_to_dataframe_runtime_and_timestamps(spark, numeric first_row = result.first() assert str(first_row["m_aangemaakt_op"]).startswith("2023-05-10 08:45") + def create_spark_mock_with_schema(existing_schema: StructType) -> MagicMock: spark = MagicMock() spark.table.return_value.schema = existing_schema return spark + +def create_spark_mock_with_table_exists(table_exists: bool) -> MagicMock: + spark = MagicMock() + spark.catalog.tableExists.return_value = table_exists + return spark + + +# --- add_new_columns_to_table --- + def test_add_new_columns_to_table_adds_missing_col_after_previous_col(): spark = create_spark_mock_with_schema(StructType([ StructField("id", IntegerType()), @@ -100,10 +110,9 @@ def test_add_new_columns_to_table_adds_missing_col_after_previous_col(): spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie) - spark.sql.assert_called_once() - query = spark.sql.call_args.args[0] - assert "ALTER TABLE db.klanten ADD COLUMNS" in query - assert "`email` string AFTER `id`" in query + spark.sql.assert_called_once_with( + "ALTER TABLE db.klanten ADD COLUMNS (\n `email` string AFTER `id`\n)" + ) def test_add_new_columns_to_table_first_col_gets_first(): @@ -115,17 +124,66 @@ def test_add_new_columns_to_table_first_col_gets_first(): spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie) - assert "`id` int FIRST" in spark.sql.call_args.args[0] + spark.sql.assert_called_once_with( + "ALTER TABLE db.klanten ADD COLUMNS (\n `id` int FIRST\n)" + ) + + +def test_add_new_columns_to_table_first_col_after_sid(): + spark = create_spark_mock_with_schema(StructType([ + StructField("sid_klanten", LongType()), + StructField("naam", StringType()), + ])) + definitie = {"columns": [ + {"name": "id", "type": "IntegerType()"}, + {"name": "naam", "type": "StringType()"}, + ]} + + spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie) + + spark.sql.assert_called_once_with( + "ALTER TABLE db.klanten ADD COLUMNS (\n `id` int AFTER `sid_klanten`\n)" + ) + + +def test_add_new_columns_to_table_multiple_cols(): + spark = create_spark_mock_with_schema(StructType([StructField("id", IntegerType())])) + definitie = {"columns": [ + {"name": "id", "type": "IntegerType()"}, + {"name": "naam", "type": "StringType()"}, + {"name": "email", "type": "StringType()"}, + ]} + + spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie) + + spark.sql.assert_called_once_with( + "ALTER TABLE db.klanten ADD COLUMNS (\n" + " `naam` string AFTER `id`,\n" + " `email` string AFTER `naam`\n" + ")" + ) def test_add_new_columns_to_table_nothing_to_add(): spark = create_spark_mock_with_schema(StructType([StructField("id", IntegerType())])) - definitie = {"columns": [{"name": "id", "type": "IntegerType()"}]} # hoofdletterongevoelig + definitie = {"columns": [{"name": "id", "type": "IntegerType()"}]} + + spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie) + + spark.sql.assert_not_called() + + +def test_add_new_columns_to_table_case_insensitive(): + spark = create_spark_mock_with_schema(StructType([StructField("id", IntegerType())])) + definitie = {"columns": [{"name": "ID", "type": "IntegerType()"}]} spark_io_utils.add_new_columns_to_table(spark, "db.klanten", definitie) spark.sql.assert_not_called() + +# --- create_or_update_table --- + @pytest.fixture def mocked_functions(monkeypatch): """Vervangt de twee onderliggende functies door mocks, zodat we alleen @@ -142,29 +200,48 @@ def table_definition(): return {"columns": [{"name": "id", "type": "IntegerType()"}]} -def create_spark_mock_with_table_exists(table_exists: bool) -> MagicMock: - spark = MagicMock() - spark.catalog.tableExists.return_value = table_exists - return spark - - -def test_create_or_update_table_create_table_if_not_exists(mocked_functions, table_definition): +@pytest.mark.parametrize("genereer_sid", [False, True]) +def test_create_or_update_table_creates_table_if_not_exists( + mocked_functions, table_definition, genereer_sid +): create_mock, add_mock = mocked_functions spark = create_spark_mock_with_table_exists(table_exists=False) - spark_io_utils.create_or_update_table(spark, "db.klanten", table_definition) + spark_io_utils.create_or_update_table( + spark, "db.klanten", table_definition, genereer_sid=genereer_sid + ) spark.catalog.tableExists.assert_called_once_with("db.klanten") - create_mock.assert_called_once_with(spark, "db.klanten", table_definition) + create_mock.assert_called_once_with( + spark=spark, + full_table_name="db.klanten", + table_definition=table_definition, + genereer_sid=genereer_sid, + ) add_mock.assert_not_called() -def test_create_or_update_table_add_columns_if_table_exists(mocked_functions, table_definition): +def test_create_or_update_table_genereer_sid_defaults_to_false(mocked_functions, table_definition): + create_mock, _ = mocked_functions + spark = create_spark_mock_with_table_exists(table_exists=False) + + spark_io_utils.create_or_update_table(spark, "db.klanten", table_definition) + + assert create_mock.call_args.kwargs["genereer_sid"] is False + + +def test_create_or_update_table_adds_columns_if_table_exists(mocked_functions, table_definition): create_mock, add_mock = mocked_functions spark = create_spark_mock_with_table_exists(table_exists=True) - spark_io_utils.create_or_update_table(spark, "db.klanten", table_definition) + spark_io_utils.create_or_update_table( + spark, "db.klanten", table_definition, genereer_sid=True + ) spark.catalog.tableExists.assert_called_once_with("db.klanten") - add_mock.assert_called_once_with(spark, "db.klanten", table_definition) + add_mock.assert_called_once_with( + spark=spark, + full_table_name="db.klanten", + table_definition=table_definition, + ) create_mock.assert_not_called()