From 8e0425bf802159ec13dfc9ad1e8cc5800e8f549d Mon Sep 17 00:00:00 2001 From: Aydin Date: Wed, 17 Jun 2026 17:22:32 +0200 Subject: [PATCH 1/2] cleaning functies toegevoegd --- pyproject.toml | 2 +- src/datateam_moss/__init__.py | 3 +- src/datateam_moss/cleaning.py | 200 +++++++++++++++++++++++++ src/datateam_moss/spark_io_utils.py | 20 ++- src/datateam_moss/transform_context.py | 14 ++ 5 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 src/datateam_moss/transform_context.py diff --git a/pyproject.toml b/pyproject.toml index 2a83129..41b0722 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "datateam_moss" -version = "0.2.22" +version = "0.2.23" authors = [ { name="XX", email="x@x.nl" } ] diff --git a/src/datateam_moss/__init__.py b/src/datateam_moss/__init__.py index 9d7df91..23bdb45 100644 --- a/src/datateam_moss/__init__.py +++ b/src/datateam_moss/__init__.py @@ -10,4 +10,5 @@ from datateam_moss.spark_io_utils import * from datateam_moss.logger import * from datateam_moss.initialization import * -from datateam_moss.verwerk_bbga import * \ No newline at end of file +from datateam_moss.verwerk_bbga import * +from datateam_moss.transform_context import * \ No newline at end of file diff --git a/src/datateam_moss/cleaning.py b/src/datateam_moss/cleaning.py index 5b36b32..c8b6e43 100644 --- a/src/datateam_moss/cleaning.py +++ b/src/datateam_moss/cleaning.py @@ -1,6 +1,7 @@ # Databricks notebook source from datateam_moss.logger import get_logger from datateam_moss import spark_io_utils as siu +from datateam_moss.transform_context import TransformContext from databricks.sdk.runtime import * from pyspark.sql import functions as F @@ -284,4 +285,203 @@ def cleanse_and_prep_dataframe(df: DataFrame, table_schema: Dict, m_columns: Lis df = cast_columns_from_schema(df=df, table_schema=table_schema) + return df + +def cast_columns_to_booleans(df: DataFrame, ctx: TransformContext) -> DataFrame: + """ + Converteer alle geconfigureerde boolean-kolommen naar een Databricks-compatibel formaat. + + Args: + df: Het bron DataFrame. + ctx: De transformatie-context met schema-informatie. + """ + for column in siu.get_columns_by_type( + ctx.table_schema, + "BooleanType()", + ): + if column in df.columns: + df = df.withColumn( + column, + to_databricks_boolean_column(column), + ) + + return df + +def cast_columns_to_dates(df: DataFrame, ctx: TransformContext) -> DataFrame: + """ + Parse en formatteer alle geconfigureerde datumkolommen. + + Args: + df: Het bron DataFrame. + ctx: De transformatie-context met schema-informatie. + + """ + for column in siu.get_columns_by_type(ctx.table_schema,"DateType()"): + if column in df.columns: + try: + df = df.withColumn( + column, + F.to_date(F.col(column), "yyyy-MM-dd") + ) + + except Exception: + try: + df = parse_and_format_date( + df=df, + date_column=column, + output_format=ctx.date_output_format + ) + + except Exception as e: + logger.warning( + f"Kon datumkolom '{column}' niet converteren: {e}" + ) + + return df + + + +def cast_columns_to_timestamp( + df: DataFrame, + ctx: TransformContext, +) -> DataFrame: + """ + Converteer timestamp-kolommen van een ISO-8601 tekenreeks naar een Spark timestamp. + + Args: + df: Het bron DataFrame. + ctx: De transformatie-context met schema-informatie. + """ + for column in siu.get_columns_by_type( + ctx.table_schema, + "TimestampType()", + ): + if column in df.columns: + df = df.withColumn( + column, + F.to_timestamp( + F.col(column), + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + ), + ) + + return df + + +def convert_columns_to_dutch_timezone( + df: DataFrame, + ctx: TransformContext, +) -> DataFrame: + """ + Zet UTC-timestamps om naar de Nederlandse tijdzone (Europe/Amsterdam). + + Args: + df: Het bron DataFrame. + ctx: De transformatie-context met schema-informatie. + """ + for column in siu.get_columns_by_type( + ctx.table_schema, + "TimestampType()", + ): + if column in df.columns: + df = df.withColumn( + column, + F.from_utc_timestamp( + F.col(column), + "Europe/Amsterdam", + ), + ) + + return df + + +def cast_string_columns_to_decimal_to_integer( + df: DataFrame, + ctx: TransformContext, +) -> DataFrame: + """ + Converteer integer-kolommen via een tussenstap naar Decimal en vervolgens naar Integer. + + Dit wordt gebruikt wanneer numerieke waarden als tekenreeks worden aangeleverd. + + Args: + df: Het bron DataFrame. + ctx: De transformatie-context met schema-informatie. + """ + for column in siu.get_columns_by_type( + ctx.table_schema, + "IntegerType()", + ): + if column in df.columns: + df = df.withColumn( + column, + F.col(column) + .cast(DecimalType(10, 0)) + .cast(IntegerType()), + ) + + return df + +def cast_columns_to_decimal( + df: DataFrame, + ctx: TransformContext, +) -> DataFrame: + """ + Converteer alle geconfigureerde decimale kolommen naar het gewenste datatype. + + Args: + df: Het bron DataFrame. + ctx: De transformatie-context met schema-informatie. + """ + for column_def in ctx.table_schema.get("columns", []): + column_type = column_def.get("type") + column_name = column_def.get("name") + + if not column_type.startswith("DecimalType"): + continue + + if column_name not in df.columns: + continue + + spark_type = siu._parse_spark_type(column_type) + + if ctx.translate_comma_to_dot: + df = df.withColumn(column_name, F.translate(F.col(column_name), ",", ".")) + + df = df.withColumn(column_name, F.col(column_name).cast(spark_type)) + + return df + +def cast_schema( + df: DataFrame, + ctx: TransformContext) -> DataFrame: + """ + Converteer alle kolommen naar het type dat is opgegeven in het schema. + + Args: + df: Het bron DataFrame. + ctx: De transformatie-context met schema-informatie. + """ + + return cast_columns_from_schema(df=df, table_schema=ctx.table_schema) + +def run_cleansing_transformations( + df: DataFrame, + ctx: TransformContext, + transformations: List[Callable[[DataFrame, TransformContext], DataFrame]], +) -> DataFrame: + """ + Voer alle transformaties uit die onderdeel zijn van de ETL-pijplijn. + + Args: + df: Het bron DataFrame. + ctx: De transformatie-context. + pipeline: Geordende lijst met transformatiefuncties. + + Returns: + Het getransformeerde DataFrame nadat alle stappen zijn uitgevoerd. + """ + for transformation in transformations: + df = transformation(df, ctx) + return df \ No newline at end of file diff --git a/src/datateam_moss/spark_io_utils.py b/src/datateam_moss/spark_io_utils.py index 5298787..6b6d47b 100644 --- a/src/datateam_moss/spark_io_utils.py +++ b/src/datateam_moss/spark_io_utils.py @@ -437,4 +437,22 @@ def write_to_table( logger.info(f"Data succesvol geschreven naar tabel {target_table}.") except Exception as e: logger.error(f"Fout bij het schrijven naar tabel {target_table}: {e}") - raise # Laat de fout opnieuw opgooien om de pipeline te laten stoppen \ No newline at end of file + raise # Laat de fout opnieuw opgooien om de pipeline te laten stoppen + +def get_columns_by_type( + table_schema: Dict[str, Any], + spark_type_name: str, +) -> List[str]: + """ + Haal alle kolomnamen uit het schema op die overeenkomen met het opgegeven Spark-type. + + Args: + table_schema: Definitie van het tabelschema met een collectie van kolommen. + spark_type_name: Naam van het Spark-type waarop gefilterd wordt + (bijvoorbeeld ``"DateType()"``). + """ + return [ + c["name"] + for c in table_schema.get("columns", []) + if c["type"] == spark_type_name + ] \ No newline at end of file diff --git a/src/datateam_moss/transform_context.py b/src/datateam_moss/transform_context.py new file mode 100644 index 0000000..3f66cb9 --- /dev/null +++ b/src/datateam_moss/transform_context.py @@ -0,0 +1,14 @@ +from typing import * +from datetime import datetime +from dataclasses import dataclass + +@dataclass +class TransformContext: + """Context object die de configuratie bevat voor datatransformaties""" + + table_schema: Dict[str, Any] + m_columns: Optional[List[str]] = None + runtime: Optional[datetime] = None + rename_columns: Optional[List[Dict[str, str]]] = None + date_output_format: Optional[str] = None + translate_comma_to_dot: bool = False \ No newline at end of file From c6d46c546252e717d8e6d68e192d7f1d63f4054b Mon Sep 17 00:00:00 2001 From: Aydin Date: Fri, 19 Jun 2026 11:07:35 +0200 Subject: [PATCH 2/2] foutafhandeling aangepast --- src/datateam_moss/cleaning.py | 111 +++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 49 deletions(-) diff --git a/src/datateam_moss/cleaning.py b/src/datateam_moss/cleaning.py index c8b6e43..83159f9 100644 --- a/src/datateam_moss/cleaning.py +++ b/src/datateam_moss/cleaning.py @@ -214,6 +214,7 @@ def cast_columns_from_schema(df: DataFrame, table_schema: Dict[str, Any]) -> Dat return df +#DEPRECATED, gebruik run_cleansing_transformations def cleanse_and_prep_dataframe(df: DataFrame, table_schema: Dict, m_columns: List[str], runtime: datetime, rename_columns:List = None ) -> DataFrame: """ Leest data uit de bron tabel, plakt de extra records onderaan en schrijft deze weg naar de doeltabel. @@ -299,11 +300,13 @@ def cast_columns_to_booleans(df: DataFrame, ctx: TransformContext) -> DataFrame: ctx.table_schema, "BooleanType()", ): - if column in df.columns: - df = df.withColumn( - column, - to_databricks_boolean_column(column), - ) + if column not in df.columns: + raise KeyError(f"Column '{column}' not found in DataFrame") + + df = df.withColumn( + column, + to_databricks_boolean_column(column), + ) return df @@ -317,25 +320,29 @@ def cast_columns_to_dates(df: DataFrame, ctx: TransformContext) -> DataFrame: """ for column in siu.get_columns_by_type(ctx.table_schema,"DateType()"): - if column in df.columns: - try: - df = df.withColumn( - column, - F.to_date(F.col(column), "yyyy-MM-dd") - ) - - except Exception: - try: - df = parse_and_format_date( - df=df, - date_column=column, - output_format=ctx.date_output_format - ) - - except Exception as e: - logger.warning( - f"Kon datumkolom '{column}' niet converteren: {e}" - ) + + if column not in df.columns: + raise KeyError(f"Column '{column}' not found in DataFrame") + + try: + df = df.withColumn( + column, + F.to_date(F.col(column), "yyyy-MM-dd") + ) + + except Exception: + try: + df = parse_and_format_date( + df=df, + date_column=column, + output_format=ctx.date_output_format + ) + + except Exception as e: + raise e + logger.error( + f"Kon datumkolom '{column}' niet converteren: {e}" + ) return df @@ -356,14 +363,16 @@ def cast_columns_to_timestamp( ctx.table_schema, "TimestampType()", ): - if column in df.columns: - df = df.withColumn( - column, - F.to_timestamp( - F.col(column), - "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", - ), - ) + if column not in df.columns: + raise KeyError(f"Column '{column}' not found in DataFrame") + + df = df.withColumn( + column, + F.to_timestamp( + F.col(column), + "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", + ), + ) return df @@ -383,14 +392,16 @@ def convert_columns_to_dutch_timezone( ctx.table_schema, "TimestampType()", ): - if column in df.columns: - df = df.withColumn( - column, - F.from_utc_timestamp( - F.col(column), - "Europe/Amsterdam", - ), - ) + if column not in df.columns: + raise KeyError(f"Column '{column}' not found in DataFrame") + + df = df.withColumn( + column, + F.from_utc_timestamp( + F.col(column), + "Europe/Amsterdam", + ), + ) return df @@ -412,13 +423,15 @@ def cast_string_columns_to_decimal_to_integer( ctx.table_schema, "IntegerType()", ): - if column in df.columns: - df = df.withColumn( - column, - F.col(column) - .cast(DecimalType(10, 0)) - .cast(IntegerType()), - ) + if column not in df.columns: + raise KeyError(f"Column '{column}' not found in DataFrame") + + df = df.withColumn( + column, + F.col(column) + .cast(DecimalType(10, 0)) + .cast(IntegerType()), + ) return df @@ -441,7 +454,7 @@ def cast_columns_to_decimal( continue if column_name not in df.columns: - continue + raise KeyError(f"Column '{column_name}' not found in DataFrame") spark_type = siu._parse_spark_type(column_type) @@ -476,7 +489,7 @@ def run_cleansing_transformations( Args: df: Het bron DataFrame. ctx: De transformatie-context. - pipeline: Geordende lijst met transformatiefuncties. + transformations: Geordende lijst met transformatiefuncties. Returns: Het getransformeerde DataFrame nadat alle stappen zijn uitgevoerd.