diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 27cf00c79..1c1164e78 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ files: ^soda-[^/]+/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: check-added-large-files @@ -17,7 +17,7 @@ repos: - id: detect-private-key - id: end-of-file-fixer - repo: https://github.com/PyCQA/autoflake - rev: v2.2.1 + rev: v2.3.3 hooks: - id: autoflake args: ["--in-place", "--remove-all-unused-imports"] @@ -29,13 +29,13 @@ repos: # exclude: _models?\.py$ # args: [--py38-plus, --keep-runtime-typing] - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 9.0.0a3 hooks: - id: isort additional_dependencies: [toml] name: Sort imports using isort - - repo: https://github.com/psf/black - rev: 23.3.0 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 26.3.1 hooks: - id: black name: Run black formatter diff --git a/soda-core/src/soda_core/cli/handlers/data_source.py b/soda-core/src/soda_core/cli/handlers/data_source.py index 3c90bbf78..a9c39f6eb 100644 --- a/soda-core/src/soda_core/cli/handlers/data_source.py +++ b/soda-core/src/soda_core/cli/handlers/data_source.py @@ -23,9 +23,7 @@ def handle_create_data_source(data_source_file_path: str, data_source_type: str) try: Path(dir_name).mkdir(parents=True, exist_ok=True) with open(data_source_file_path, "w") as text_file: - text_file.write( - dedent( - """ + text_file.write(dedent(""" type: postgres name: postgres_ds connection: @@ -33,9 +31,7 @@ def handle_create_data_source(data_source_file_path: str, data_source_type: str) user: ${POSTGRES_USERNAME} password: ${POSTGRES_PASSWORD} database: your_postgres_db - """ - ).strip() - ) + """).strip()) soda_logger.info(f"{Emoticons.WHITE_CHECK_MARK} Created data source file '{data_source_file_path}'") return ExitCode.OK except Exception as exc: diff --git a/soda-core/src/soda_core/cli/handlers/soda_cloud.py b/soda-core/src/soda_core/cli/handlers/soda_cloud.py index 18542333b..23ce16db1 100644 --- a/soda-core/src/soda_core/cli/handlers/soda_cloud.py +++ b/soda-core/src/soda_core/cli/handlers/soda_cloud.py @@ -20,16 +20,12 @@ def handle_create_soda_cloud(soda_cloud_file_path: str) -> ExitCode: try: Path(dirname(soda_cloud_file_path)).mkdir(parents=True, exist_ok=True) with open(soda_cloud_file_path, "w") as text_file: - text_file.write( - dedent( - """ + text_file.write(dedent(""" soda_cloud: host: cloud.soda.io api_key_id: ${SODA_CLOUD_API_KEY_ID} api_key_secret: ${SODA_CLOUD_API_KEY_SECRET} - """ - ).strip() - ) + """).strip()) soda_logger.info(f"{Emoticons.WHITE_CHECK_MARK} Created Soda Cloud configuration file '{soda_cloud_file_path}'") return ExitCode.OK except Exception as exc: diff --git a/soda-core/src/soda_core/common/soda_cloud.py b/soda-core/src/soda_core/common/soda_cloud.py index 255b73878..728e0cf4f 100644 --- a/soda-core/src/soda_core/common/soda_cloud.py +++ b/soda-core/src/soda_core/common/soda_cloud.py @@ -1457,12 +1457,14 @@ def _build_contract_result_json_dict(contract_verification_result: ContractVerif # The scan definition name is still required on result ingestion to link to the contract # and determine if we're dealing with a default or test contract. "definitionName": _build_scan_definition_name(contract_verification_result), - "defaultDataSource": contract_verification_result.data_source.name - if contract_verification_result.data_source - else None, - "defaultDataSourceProperties": {"type": contract_verification_result.data_source.type} - if contract_verification_result.data_source - else None, + "defaultDataSource": ( + contract_verification_result.data_source.name if contract_verification_result.data_source else None + ), + "defaultDataSourceProperties": ( + {"type": contract_verification_result.data_source.type} + if contract_verification_result.data_source + else None + ), # dataTimestamp can be changed by user, this is shown in Cloud as time of a scan. # It's the timestamp used to identify the time partition, which is the slice of data that is verified. "dataTimestamp": contract_verification_result.data_timestamp, @@ -1547,9 +1549,11 @@ def _build_diagnostics_json_dict(check_result: CheckResult) -> Optional[dict]: return { # TODO: this default 0 value is here only because check.diagnostics.value is a required non-nullable field in the api. - "value": int(check_result.threshold_value) - if isinstance(check_result.threshold_value, bool) - else (check_result.threshold_value or 0), + "value": ( + int(check_result.threshold_value) + if isinstance(check_result.threshold_value, bool) + else (check_result.threshold_value or 0) + ), "fail": _build_fail_threshold(check_result), "v4": _build_v4_diagnostics_check_type_json_dict(check_result), } diff --git a/soda-core/src/soda_core/common/sql_ast.py b/soda-core/src/soda_core/common/sql_ast.py index afa972b8b..ca7781492 100644 --- a/soda-core/src/soda_core/common/sql_ast.py +++ b/soda-core/src/soda_core/common/sql_ast.py @@ -61,10 +61,8 @@ def __post_init__(self): # Check that the select contains a distinct and has multiple fields -> give a warning if isinstance(self.fields, list) and len(self.fields) > 1: if any(isinstance(field, DISTINCT) for field in self.fields): - logger.warning( - """Found DISTINCT in a SELECT statement with multiple fields. - This might have unintended consequences.""" - ) + logger.warning("""Found DISTINCT in a SELECT statement with multiple fields. + This might have unintended consequences.""") @dataclass diff --git a/soda-core/src/soda_core/common/yaml.py b/soda-core/src/soda_core/common/yaml.py index 6fff12e61..8d9292596 100644 --- a/soda-core/src/soda_core/common/yaml.py +++ b/soda-core/src/soda_core/common/yaml.py @@ -181,16 +181,13 @@ def parse(self) -> YamlObject: raise YamlParserException(f"YAML syntax error", str(location)) -class DataSourceYamlSource(YamlSource, file_type=FileType.DATA_SOURCE): - ... +class DataSourceYamlSource(YamlSource, file_type=FileType.DATA_SOURCE): ... -class SodaCloudYamlSource(YamlSource, file_type=FileType.SODA_CLOUD): - ... +class SodaCloudYamlSource(YamlSource, file_type=FileType.SODA_CLOUD): ... -class ContractYamlSource(YamlSource, file_type=FileType.CONTRACT): - ... +class ContractYamlSource(YamlSource, file_type=FileType.CONTRACT): ... class YamlValue: diff --git a/soda-core/src/soda_core/contracts/impl/check_types/schema_check.py b/soda-core/src/soda_core/contracts/impl/check_types/schema_check.py index b63eeecec..a01e77a70 100644 --- a/soda-core/src/soda_core/contracts/impl/check_types/schema_check.py +++ b/soda-core/src/soda_core/contracts/impl/check_types/schema_check.py @@ -291,9 +291,9 @@ def execute(self) -> list[Measurement]: logger.error(msg=f"Could not execute schema query {self.sql}: {e}", exc_info=True) return [] try: - metadata_columns: list[ - ColumnMetadata - ] = self.data_source_impl.sql_dialect.build_column_metadatas_from_query_result(query_result) + metadata_columns: list[ColumnMetadata] = ( + self.data_source_impl.sql_dialect.build_column_metadatas_from_query_result(query_result) + ) except Exception as e: logger.error(f"Error building column metadata from query result: {e}") return [] diff --git a/soda-core/src/soda_core/contracts/impl/contract_yaml.py b/soda-core/src/soda_core/contracts/impl/contract_yaml.py index 6ff5be23f..caba041b8 100644 --- a/soda-core/src/soda_core/contracts/impl/contract_yaml.py +++ b/soda-core/src/soda_core/contracts/impl/contract_yaml.py @@ -32,8 +32,7 @@ class ContractYamlExtension(Protocol): # Extend the contract YAML object. Can modify the state of the contract YAML. - def extend(self, contract_yaml: "ContractYaml") -> None: - ... + def extend(self, contract_yaml: "ContractYaml") -> None: ... class ContractYaml: diff --git a/soda-databricks/src/soda_databricks/model/data_source/databricks_connection_properties.py b/soda-databricks/src/soda_databricks/model/data_source/databricks_connection_properties.py index 50b4d771a..9cca51f12 100644 --- a/soda-databricks/src/soda_databricks/model/data_source/databricks_connection_properties.py +++ b/soda-databricks/src/soda_databricks/model/data_source/databricks_connection_properties.py @@ -7,8 +7,7 @@ ) -class DatabricksConnectionProperties(DataSourceConnectionProperties, ABC): - ... +class DatabricksConnectionProperties(DataSourceConnectionProperties, ABC): ... class DatabricksSharedConnectionProperties(DatabricksConnectionProperties, ABC): diff --git a/soda-postgres/src/soda_postgres/common/data_sources/postgres_data_source.py b/soda-postgres/src/soda_postgres/common/data_sources/postgres_data_source.py index 55e4a74ef..fb6fcdc44 100644 --- a/soda-postgres/src/soda_postgres/common/data_sources/postgres_data_source.py +++ b/soda-postgres/src/soda_postgres/common/data_sources/postgres_data_source.py @@ -270,8 +270,7 @@ def _build_pg_columns_metadata_query( [ COLUMN("attname", table_alias="a", field_alias="column_name"), # Normalize data type into information_schema.columns style. - RAW_SQL( - """CASE + RAW_SQL("""CASE -- arrays WHEN t.typcategory = 'A' OR t.typelem <> 0 THEN 'ARRAY' @@ -294,12 +293,10 @@ def _build_pg_columns_metadata_query( ELSE COALESCE(bt.typname, t.typname) END END AS \"data_type\" - """ - ), + """), # Extract type parameters. All a.atttypmod are offset by 4 in Postgres # varchar/char length (NULL otherwise) - RAW_SQL( - """CASE + RAW_SQL("""CASE WHEN t.typname IN ('varchar','bpchar') THEN CASE WHEN a.atttypmod > 4 THEN a.atttypmod - 4 @@ -307,11 +304,9 @@ def _build_pg_columns_metadata_query( END ELSE NULL END AS "character_maximum_length" - """ - ), + """), # numeric precision (NULL otherwise) - RAW_SQL( - """CASE + RAW_SQL("""CASE WHEN t.typname = 'numeric' THEN CASE WHEN a.atttypmod > 4 THEN ((a.atttypmod - 4) >> 16) @@ -319,11 +314,9 @@ def _build_pg_columns_metadata_query( END ELSE NULL END AS "numeric_precision" - """ - ), + """), # numeric scale (NULL otherwise) - RAW_SQL( - """CASE + RAW_SQL("""CASE WHEN t.typname = 'numeric' THEN CASE WHEN a.atttypmod > 4 THEN ((a.atttypmod - 4) & 65535) @@ -331,11 +324,9 @@ def _build_pg_columns_metadata_query( END ELSE NULL END AS "numeric_scale" - """ - ), + """), # datetime precision (NULL otherwise) - RAW_SQL( - """CASE + RAW_SQL("""CASE WHEN t.typname IN ('time','timetz','timestamp','timestamptz') THEN CASE WHEN a.atttypmod >= 0 THEN a.atttypmod @@ -343,8 +334,7 @@ def _build_pg_columns_metadata_query( END ELSE NULL END AS "datetime_precision" - """ - ), + """), COLUMN(current_database_expression, field_alias="table_catalog"), COLUMN("nspname", table_alias="n", field_alias="table_schema"), COLUMN("relname", table_alias="c", field_alias="table_name"), diff --git a/soda-snowflake/src/soda_snowflake/common/data_sources/snowflake_data_source_connection.py b/soda-snowflake/src/soda_snowflake/common/data_sources/snowflake_data_source_connection.py index 5b3559fd5..5922765c1 100644 --- a/soda-snowflake/src/soda_snowflake/common/data_sources/snowflake_data_source_connection.py +++ b/soda-snowflake/src/soda_snowflake/common/data_sources/snowflake_data_source_connection.py @@ -21,8 +21,7 @@ USER_DESCRIPTION = "Username for authentication" -class SnowflakeConnectionProperties(DataSourceConnectionProperties, ABC): - ... +class SnowflakeConnectionProperties(DataSourceConnectionProperties, ABC): ... class SnowflakeSharedConnectionProperties(SnowflakeConnectionProperties, ABC): diff --git a/soda-snowflake/tests/data_sources/test_snowflake.py b/soda-snowflake/tests/data_sources/test_snowflake.py index 32b94046d..e04c77586 100644 --- a/soda-snowflake/tests/data_sources/test_snowflake.py +++ b/soda-snowflake/tests/data_sources/test_snowflake.py @@ -92,7 +92,7 @@ # TestConnection( # connect using external browser. commented out by default because CI will hang. Uncomment to test locally # test_name="external_browser_connection", # connection_yaml_str=f""" - # type: snowflake + # type: snowflake # name: SNOWFLAKE_TEST # connection: # account: '{SNOWFLAKE_ACCOUNT}' diff --git a/soda-sqlserver/src/soda_sqlserver/common/data_sources/sqlserver_data_source.py b/soda-sqlserver/src/soda_sqlserver/common/data_sources/sqlserver_data_source.py index d12ab61c8..ad563d3df 100644 --- a/soda-sqlserver/src/soda_sqlserver/common/data_sources/sqlserver_data_source.py +++ b/soda-sqlserver/src/soda_sqlserver/common/data_sources/sqlserver_data_source.py @@ -138,9 +138,7 @@ def create_schema_if_not_exists_sql(self, prefixes: list[str], add_semicolon: bo FROM sys.schemas WHERE name = N'{schema_name}' ) EXEC('CREATE SCHEMA [{schema_name}]') - """ + ( - ";" if add_semicolon else "" - ) + """ + (";" if add_semicolon else "") def build_drop_table_sql(self, drop_table: DROP_TABLE | DROP_TABLE_IF_EXISTS, add_semicolon: bool = True) -> str: if_exists_sql: str = ( diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_aggregate_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_aggregate_check_yaml_parsing.py index 822927222..47f8b1a18 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_aggregate_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_aggregate_check_yaml_parsing.py @@ -9,7 +9,6 @@ Common fields (name, filter, threshold, metric, attributes) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_check_from_contract diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_duplicate_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_duplicate_check_yaml_parsing.py index 611f62dc1..12dbe17ce 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_duplicate_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_duplicate_check_yaml_parsing.py @@ -8,7 +8,6 @@ Common fields (name, filter, threshold, metric, attributes) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_check_from_contract, parse_column_check diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_failed_rows_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_failed_rows_check_yaml_parsing.py index 3290257ab..f923b0923 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_failed_rows_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_failed_rows_check_yaml_parsing.py @@ -11,7 +11,6 @@ Common fields (name, filter, threshold, metric, store_failed_rows) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_check_from_contract diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_freshness_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_freshness_check_yaml_parsing.py index c5ff40b33..d6a7a9161 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_freshness_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_freshness_check_yaml_parsing.py @@ -11,7 +11,6 @@ Common fields (name, filter, threshold, metric, attributes) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_check_from_contract diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_invalid_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_invalid_check_yaml_parsing.py index 06296c4f2..bf6874373 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_invalid_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_invalid_check_yaml_parsing.py @@ -12,7 +12,6 @@ Common fields (name, filter, threshold, metric, attributes) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_column_check diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_metric_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_metric_check_yaml_parsing.py index 793e86efc..30fc8ef81 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_metric_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_metric_check_yaml_parsing.py @@ -11,7 +11,6 @@ Common fields (name, filter, threshold, metric, attributes) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_check_from_contract diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_missing_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_missing_check_yaml_parsing.py index 9ecef8a23..a010cf66f 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_missing_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_missing_check_yaml_parsing.py @@ -10,7 +10,6 @@ Common fields (name, filter, threshold, metric, attributes) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_column_check diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_row_count_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_row_count_check_yaml_parsing.py index 19ed95422..85cc1d8fb 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_row_count_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_row_count_check_yaml_parsing.py @@ -7,7 +7,6 @@ Common fields (name, filter, threshold, metric, attributes) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_check_from_contract diff --git a/soda-tests/tests/unit/check_types/yaml_parsing/test_schema_check_yaml_parsing.py b/soda-tests/tests/unit/check_types/yaml_parsing/test_schema_check_yaml_parsing.py index d6e0535f9..a48e3564b 100644 --- a/soda-tests/tests/unit/check_types/yaml_parsing/test_schema_check_yaml_parsing.py +++ b/soda-tests/tests/unit/check_types/yaml_parsing/test_schema_check_yaml_parsing.py @@ -10,7 +10,6 @@ Common fields (name, qualifier, filter, attributes) are tested in test_common_check_yaml_features.py. """ - from helpers.yaml_parsing_helpers import parse_check_from_contract diff --git a/soda-tests/tests/unit/manual_soda_cloud_flow.py b/soda-tests/tests/unit/manual_soda_cloud_flow.py index bd9355bb5..8c6818067 100644 --- a/soda-tests/tests/unit/manual_soda_cloud_flow.py +++ b/soda-tests/tests/unit/manual_soda_cloud_flow.py @@ -27,8 +27,7 @@ def test_manual_soda_cloud_flow_ecommerce_orders_contract_and_results_upload_to_ schema_name: str = data_source_test_helper.dataset_prefix[1] qualified_table_name: str = f'{schema_name}."ecommerce_orders"' - contract_yaml_str: str = dedent( - f""" + contract_yaml_str: str = dedent(f""" dataset: {data_source_name}/{database_name}/{schema_name}/ecommerce_orders variables: start_timestamp: @@ -167,19 +166,16 @@ def test_manual_soda_cloud_flow_ecommerce_orders_contract_and_results_upload_to_ - Bank Transfer - Credit Card qualifier: v2 - """ - ).strip() + """).strip() print(f"contract_yaml_str: \n{contract_yaml_str}") - soda_cloud_yaml_str = dedent( - """ + soda_cloud_yaml_str = dedent(""" soda_cloud: host: ${env.SODA_CLOUD_HOST} api_key_id: ${env.SODA_CLOUD_API_KEY_ID} api_key_secret: ${env.SODA_CLOUD_API_KEY_SECRET} - """ - ).strip() + """).strip() soda_cloud: SodaCloud = SodaCloud.from_yaml_source( SodaCloudYamlSource.from_str(soda_cloud_yaml_str), provided_variable_values={} @@ -199,16 +195,13 @@ def test_manual_soda_cloud_flow_ecommerce_orders_contract_and_results_upload_to_ def drop_ecommerce_orders_table(data_source_impl: DataSourceImpl, schema: str): - data_source_impl.execute_update( - f""" + data_source_impl.execute_update(f""" drop table if exists {schema}.ecommerce_orders; - """ - ) + """) def create_ecommerce_orders_table(data_source_impl: DataSourceImpl, schema: str): - data_source_impl.execute_update( - f""" + data_source_impl.execute_update(f""" create table {schema}.ecommerce_orders ( order_id integer not null primary key, @@ -223,13 +216,11 @@ def create_ecommerce_orders_table(data_source_impl: DataSourceImpl, schema: str) vat numeric, created_at timestamp ); - """ - ) + """) def insert_ecommerce_orders_data(data_source_impl, schema: str): - data_source_impl.execute_update( - f""" + data_source_impl.execute_update(f""" insert into {schema}.ecommerce_orders (order_id, customer_id, order_date, region, product_category, quantity, price, payment_method, is_fraud, vat, created_at) values (2000, 213, '2025-05-01', 'North', 'Clothing', 2, 480.51, 'PayPal', FALSE, 21, '2025-06-18 16:05:06'), (2001, 229, '2025-05-02', 'East', 'Books', 5, 190.47, 'PayPal', TRUE, 21, '2025-06-18 16:05:06'), @@ -343,5 +334,4 @@ def insert_ecommerce_orders_data(data_source_impl, schema: str): (3048, 236, '2025-05-09', 'South', 'Books', 5, 422.19, 'PayPal', FALSE, 21, '2025-06-18 16:05:06'), (3049, 244, '2025-05-19', 'West', 'Electronics', 1, 237.75, 'Credit Card', TRUE, 21, '2025-06-18 16:05:06'), (3050, 250, '2025-05-29', 'West', 'Books', 4, 233.76, 'Bank Transfer', FALSE, 21, '2025-06-18 16:05:06'); - """ - ) + """) diff --git a/soda-tests/tests/unit/manual_test_agent_flow.py b/soda-tests/tests/unit/manual_test_agent_flow.py index 50edc801a..18729dad9 100644 --- a/soda-tests/tests/unit/manual_test_agent_flow.py +++ b/soda-tests/tests/unit/manual_test_agent_flow.py @@ -14,8 +14,7 @@ def main(): project_root_dir = __file__[: -len("/soda-core/tests/soda_core/tests/components/manual_test_agent_flow.py")] load_dotenv(f"{project_root_dir}/.env", override=True) - contract_yaml_str: str = dedent( - """ + contract_yaml_str: str = dedent(""" data_source: bus_nienu dataset_prefix: [nyc, public] dataset: bus_breakdown_and_delays @@ -26,16 +25,13 @@ def main(): valid_values: [ 'Heavy Traffic', 'Other', 'Mechanical Problem', 'Won`t Start', 'Problem Run' ] checks: - schema: - """ - ).strip() + """).strip() - soda_cloud_yaml_str = dedent( - """ + soda_cloud_yaml_str = dedent(""" soda_cloud: api_key_id: ${SODA_CLOUD_API_KEY_ID} api_key_secret: ${SODA_CLOUD_API_KEY_SECRET} - """ - ).strip() + """).strip() soda_cloud: SodaCloud = SodaCloud.from_yaml_source( SodaCloudYamlSource.from_str(soda_cloud_yaml_str), provided_variable_values={} diff --git a/soda-tests/tests/unit/test_cli_soda_cloud_handlers.py b/soda-tests/tests/unit/test_cli_soda_cloud_handlers.py index 327350610..3e02faf47 100644 --- a/soda-tests/tests/unit/test_cli_soda_cloud_handlers.py +++ b/soda-tests/tests/unit/test_cli_soda_cloud_handlers.py @@ -49,8 +49,9 @@ def test_test_soda_cloud_connection_failure(mock_soda_cloud_cls): mock_soda_cloud.test_connection.return_value = None mock_soda_cloud_cls.from_file.return_value = mock_soda_cloud - with patch("soda_core.cli.handlers.soda_cloud.Logs.has_errors", return_value=True), patch( - "soda_core.cli.handlers.soda_cloud.Logs.get_errors_str", return_value="boom" + with ( + patch("soda_core.cli.handlers.soda_cloud.Logs.has_errors", return_value=True), + patch("soda_core.cli.handlers.soda_cloud.Logs.get_errors_str", return_value="boom"), ): exit_code = handle_test_soda_cloud("sc.yaml") diff --git a/soda-tests/tests/unit/test_contract_parser_api.py b/soda-tests/tests/unit/test_contract_parser_api.py index 2c7e6a4ec..8942a31d0 100644 --- a/soda-tests/tests/unit/test_contract_parser_api.py +++ b/soda-tests/tests/unit/test_contract_parser_api.py @@ -5,9 +5,7 @@ def test_parse_relative_complete_contract(): - contract_yaml_source: ContractYamlSource = ContractYamlSource.from_str( - yaml_str=dedent_and_strip( - """ + contract_yaml_source: ContractYamlSource = ContractYamlSource.from_str(yaml_str=dedent_and_strip(""" dataset: sdf/soda_test/dev_xxx/SODATEST_test_schema_31761d69 columns: - name: id @@ -16,9 +14,7 @@ def test_parse_relative_complete_contract(): - missing: checks: - schema: - """ - ) - ) + """)) contract_yaml: ContractYaml = ContractYaml.parse( contract_yaml_source=contract_yaml_source, provided_variable_values={} @@ -35,29 +31,21 @@ def test_parse_relative_complete_contract(): def test_legacy_dataset_specification(logs: Logs): - ContractYaml.parse( - contract_yaml_source=ContractYamlSource.from_str( - """ + ContractYaml.parse(contract_yaml_source=ContractYamlSource.from_str(""" data_source: abc dataset_prefix: [a, b] dataset: dsname columns: [] - """ - ) - ) + """)) assert "Invalid dataset qualified name in 'dataset'" in logs.get_errors_str() def test_minimal_contract(): logs: Logs = Logs() - ContractYaml.parse( - contract_yaml_source=ContractYamlSource.from_str( - """ + ContractYaml.parse(contract_yaml_source=ContractYamlSource.from_str(""" dataset: a/b/c/d columns: [] - """ - ) - ) + """)) logs.remove_from_root_logger() assert not logs.get_errors_str() diff --git a/soda-tests/tests/unit/test_contract_parsing_errors.py b/soda-tests/tests/unit/test_contract_parsing_errors.py index fdd1e58e6..7cf588687 100644 --- a/soda-tests/tests/unit/test_contract_parsing_errors.py +++ b/soda-tests/tests/unit/test_contract_parsing_errors.py @@ -8,9 +8,7 @@ def test_parsing_error_wrong_type(data_source_test_helper: DataSourceTestHelper): with pytest.raises(YamlParserException): - data_source_test_helper.assert_contract_error( - dedent_and_strip( - """ + data_source_test_helper.assert_contract_error(dedent_and_strip(""" dataset: data_source: milan_nyc namespace_prefix: [soda_test, dev_tom] @@ -24,14 +22,11 @@ def test_parsing_error_wrong_type(data_source_test_helper: DataSourceTestHelper) - name: age checks: - schema: - """ - ) - ) + """)) def test_duplicate_identity_error(data_source_test_helper: DataSourceTestHelper): - errors_str: str = data_source_test_helper.assert_contract_error( - contract_yaml_str=""" + errors_str: str = data_source_test_helper.assert_contract_error(contract_yaml_str=""" dataset: the_test_ds/a/b/TBLE columns: - name: id @@ -39,8 +34,7 @@ def test_duplicate_identity_error(data_source_test_helper: DataSourceTestHelper) - missing: - missing: name: lksdfj - """ - ) + """) assert ( "Duplicate identity yaml_string.yml/id/missing. Original(yaml_string.yml[4,8]) Duplicate(yaml_string.yml[6,10])" @@ -48,14 +42,12 @@ def test_duplicate_identity_error(data_source_test_helper: DataSourceTestHelper) def test_error_duplicate_column_names(data_source_test_helper: DataSourceTestHelper): - errors_str: str = data_source_test_helper.assert_contract_error( - contract_yaml_str=f""" + errors_str: str = data_source_test_helper.assert_contract_error(contract_yaml_str=f""" dataset: the_test_ds/a/b/TBLE columns: - name: id - name: id - """ - ) + """) assert "Duplicate columns with name 'id': In yaml_string.yml at: [2,4], [3,4]" in errors_str @@ -65,12 +57,10 @@ def test_error_no_dataset(data_source_test_helper: DataSourceTestHelper): YamlParserException, match=re.escape("The YAML is missing the required 'dataset' property, in yaml_string.yml[0,0]"), ): - data_source_test_helper.assert_contract_error( - contract_yaml_str=f""" + data_source_test_helper.assert_contract_error(contract_yaml_str=f""" columns: - name: id - """ - ) + """) def test_valid_values_not_configured(data_source_test_helper: DataSourceTestHelper): diff --git a/soda-tests/tests/unit/test_contract_publication_api.py b/soda-tests/tests/unit/test_contract_publication_api.py index 443332dab..57f8f8b90 100644 --- a/soda-tests/tests/unit/test_contract_publication_api.py +++ b/soda-tests/tests/unit/test_contract_publication_api.py @@ -9,18 +9,11 @@ def test_contract_publication_fails_on_missing_soda_cloud_config(): - contract_publication_result: ContractPublicationResult = ( - ContractPublication.builder() - .with_contract_yaml_str( - f""" + contract_publication_result: ContractPublicationResult = ContractPublication.builder().with_contract_yaml_str(f""" dataset: ds/db/sch/CUSTOMERS columns: - name: id - """ - ) - .build() - .execute() - ) + """).build().execute() assert contract_publication_result.has_errors assert "Cannot publish without a Soda Cloud configuration" in contract_publication_result.logs.get_logs_str() @@ -44,14 +37,12 @@ def test_contract_publication_fails_on_missing_contract_file(): contract_publication_result: ContractPublicationResult = ( ContractPublication.builder() .with_contract_yaml_file("../soda/mydb/myschema/table.yml") - .with_soda_cloud_yaml_str( - """ + .with_soda_cloud_yaml_str(""" soda_cloud: host: host.soda.io api_key_id: id api_key_secret: secret - """ - ) + """) .with_soda_cloud(mock_cloud) .build() .execute() @@ -95,20 +86,16 @@ def test_contract_publication_returns_result_for_each_added_contract(): contract_publication_result = ( ContractPublication.builder() - .with_contract_yaml_str( - f""" + .with_contract_yaml_str(f""" dataset: test/some/schema/CUSTOMERS columns: - name: id - """ - ) - .with_contract_yaml_str( - f""" + """) + .with_contract_yaml_str(f""" dataset: test2/some/schema/CUSTOMERS2 columns: - name: id - """ - ) + """) .with_soda_cloud(mock_cloud) .build() .execute() diff --git a/soda-tests/tests/unit/test_contract_verification_api.py b/soda-tests/tests/unit/test_contract_verification_api.py index d511ce926..fabcbe8f3 100644 --- a/soda-tests/tests/unit/test_contract_verification_api.py +++ b/soda-tests/tests/unit/test_contract_verification_api.py @@ -51,15 +51,11 @@ def test_contract_provided_and_configured(): If there is no default data source configured and there is none provided in the contract, an error has to be logged """ contract_verification_session_result: ContractVerificationSessionResult = ContractVerificationSession.execute( - contract_yaml_sources=[ - ContractYamlSource.from_str( - f""" + contract_yaml_sources=[ContractYamlSource.from_str(f""" dataset: abc/CUSTOMERS columns: - name: id - """ - ) - ], + """)], ) assert "Data source 'abc' not found" in contract_verification_session_result.get_errors_str() diff --git a/soda-tests/tests/unit/test_contract_verification_logs.py b/soda-tests/tests/unit/test_contract_verification_logs.py index 0451036fd..f677e55a1 100644 --- a/soda-tests/tests/unit/test_contract_verification_logs.py +++ b/soda-tests/tests/unit/test_contract_verification_logs.py @@ -22,14 +22,12 @@ def test_split_log_lines(data_source_test_helper: DataSourceTestHelper): test_table = data_source_test_helper.ensure_test_table(test_table_specification) contract_verification_result: ContractVerificationResult = data_source_test_helper.assert_contract_pass( test_table=test_table, - contract_yaml_str=dedent_and_strip( - """ + contract_yaml_str=dedent_and_strip(""" checks: - row_count: threshold: must_be: 3 - """ - ), + """), ) log_lines = contract_verification_result.get_logs() diff --git a/soda-tests/tests/unit/test_data_source_api.py b/soda-tests/tests/unit/test_data_source_api.py index b2c74629c..9fd2e4531 100644 --- a/soda-tests/tests/unit/test_data_source_api.py +++ b/soda-tests/tests/unit/test_data_source_api.py @@ -25,8 +25,7 @@ def test_data_source_env_var_resolving(env_vars: dict): env_vars["TEST_POSTGRES_PASSWORD"] = "***" env_vars["TEST_POSTGRES_DATABASE"] = "soda_test" - data_source_yaml_source: DataSourceYamlSource = DataSourceYamlSource.from_str( - yaml_str=f""" + data_source_yaml_source: DataSourceYamlSource = DataSourceYamlSource.from_str(yaml_str=f""" type: postgres name: postgres_test_ds connection: @@ -34,8 +33,7 @@ def test_data_source_env_var_resolving(env_vars: dict): user: ${{env.TEST_POSTGRES_USERNAME}} password: '${{env.TEST_POSTGRES_PASSWORD}}' database: ${{env.TEST_POSTGRES_DATABASE}} - """ - ) + """) data_source_impl: DataSourceImpl = DataSourceImpl.from_yaml_source(data_source_yaml_source) connection_properties = data_source_impl.data_source_model.connection_properties diff --git a/soda-tests/tests/unit/test_soda_cloud.py b/soda-tests/tests/unit/test_soda_cloud.py index 03242928d..c5cdd934b 100644 --- a/soda-tests/tests/unit/test_soda_cloud.py +++ b/soda-tests/tests/unit/test_soda_cloud.py @@ -58,14 +58,12 @@ .build() ) -YAML_SOURCE: SodaCloudYamlSource = SodaCloudYamlSource.from_str( - """ +YAML_SOURCE: SodaCloudYamlSource = SodaCloudYamlSource.from_str(""" soda_cloud: host: dev.sodadata.io api_key_id: some_key_id api_key_secret: some_key_secret -""" -) +""") def test_soda_cloud_from_yaml_source_with_api_key_auth(): @@ -80,12 +78,10 @@ def test_soda_cloud_from_yaml_source_with_api_key_auth(): def test_soda_cloud_from_yaml_source_with_token_auth(): os.environ.update({"SODA_CLOUD_TOKEN": "some_token"}) - yaml_source = SodaCloudYamlSource.from_str( - """ + yaml_source = SodaCloudYamlSource.from_str(""" soda_cloud: host: dev.sodadata.io - """ - ) + """) try: soda_cloud = SodaCloud.from_yaml_source(yaml_source, provided_variable_values={}) assert not soda_cloud.api_key_id @@ -521,17 +517,11 @@ def test_publish_contract(): ] mock_cloud = MockSodaCloud(responses) - res = mock_cloud.publish_contract( - ContractYaml.parse( - ContractYamlSource.from_str( - f""" + res = mock_cloud.publish_contract(ContractYaml.parse(ContractYamlSource.from_str(f""" dataset: test/some/schema/CUSTOMERS columns: - name: id - """ - ) - ) - ) + """))) assert isinstance(res, ContractPublicationResult) @@ -554,15 +544,11 @@ def test_verify_contract_on_agent_permission_check(): mock_cloud = MockSodaCloud(responses) res = mock_cloud.verify_contract_on_agent( - ContractYaml.parse( - ContractYamlSource.from_str( - f""" + ContractYaml.parse(ContractYamlSource.from_str(f""" dataset: test/some/schema/CUSTOMERS columns: - name: id - """ - ) - ), + """)), variables={}, blocking_timeout_in_minutes=60, publish_results=False, diff --git a/soda-tests/tests/unit/test_thresholds.py b/soda-tests/tests/unit/test_thresholds.py index 96ff472f9..231f81637 100644 --- a/soda-tests/tests/unit/test_thresholds.py +++ b/soda-tests/tests/unit/test_thresholds.py @@ -13,12 +13,10 @@ def test_threshold_must_be(threshold_level): if threshold_level: threshold_level_yaml = f"level: {threshold_level}" - threshold: ThresholdImpl = parse_threshold( - f""" + threshold: ThresholdImpl = parse_threshold(f""" must_be: 0 {threshold_level_yaml} - """ - ) + """) assert threshold.get_assertion_summary("m") == "m = 0" assert not threshold.passes(-1) @@ -29,11 +27,9 @@ def test_threshold_must_be(threshold_level): def test_threshold_must_not_be(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_not_be: 0 - """ - ) + """) assert threshold.get_assertion_summary("m") == "m != 0" assert threshold.passes(-1) @@ -42,11 +38,9 @@ def test_threshold_must_not_be(): def test_threshold_greater_than(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_be_greater_than: 0 - """ - ) + """) assert threshold.get_assertion_summary("m") == "0 < m" assert not threshold.passes(-1) @@ -55,11 +49,9 @@ def test_threshold_greater_than(): def test_threshold_greater_than_or_equal(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_be_greater_than_or_equal: 0 - """ - ) + """) assert threshold.get_assertion_summary("m") == "0 <= m" assert not threshold.passes(-1) @@ -68,11 +60,9 @@ def test_threshold_greater_than_or_equal(): def test_threshold_less_than(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_be_less_than: 0 - """ - ) + """) assert threshold.get_assertion_summary("m") == "m < 0" assert threshold.passes(-1) @@ -81,11 +71,9 @@ def test_threshold_less_than(): def test_threshold_less_than_or_equal(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_be_less_than_or_equal: 0 - """ - ) + """) assert threshold.get_assertion_summary("m") == "m <= 0" assert threshold.passes(-1) @@ -94,13 +82,11 @@ def test_threshold_less_than_or_equal(): def test_threshold_between(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_be_between: greater_than_or_equal: 0 less_than_or_equal: 1 - """ - ) + """) assert threshold.get_assertion_summary("m") == "0 <= m <= 1" assert not threshold.passes(-1) @@ -110,13 +96,11 @@ def test_threshold_between(): def test_threshold_custom_inner(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_be_between: greater_than: 0 less_than_or_equal: 2 - """ - ) + """) assert threshold.get_assertion_summary("m") == "0 < m <= 2" assert not threshold.passes(-1) @@ -127,13 +111,11 @@ def test_threshold_custom_inner(): def test_threshold_not_between(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_be_not_between: greater_than_or_equal: 2 less_than_or_equal: 0 - """ - ) + """) assert threshold.get_assertion_summary("m") == "m <= 0 or 2 <= m" assert threshold.passes(-1) @@ -144,13 +126,11 @@ def test_threshold_not_between(): def test_threshold_custom_outer(): - threshold: ThresholdImpl = parse_threshold( - """ + threshold: ThresholdImpl = parse_threshold(""" must_be_not_between: less_than_or_equal: 0 greater_than: 2 - """ - ) + """) assert threshold.get_assertion_summary("m") == "m <= 0 or 2 < m" assert threshold.passes(-1) diff --git a/soda-tests/tests/unit/test_variables.py b/soda-tests/tests/unit/test_variables.py index c8345c390..de25966a9 100644 --- a/soda-tests/tests/unit/test_variables.py +++ b/soda-tests/tests/unit/test_variables.py @@ -238,8 +238,7 @@ def test_missing_variable(logs: Logs): def test_variables_example1_in_docs(logs: Logs): - contract_yaml: ContractYaml = parse_contract( - contract_yaml_str=""" + contract_yaml: ContractYaml = parse_contract(contract_yaml_str=""" dataset: postgres_adventureworks/adventureworks/${var.DATASET_SCHEMA}/${var.DATASET_PREFIX}_employee variables: @@ -250,8 +249,7 @@ def test_variables_example1_in_docs(logs: Logs): columns: - name: id - """ - ) + """) assert dedent_and_strip(contract_yaml.dataset) == "postgres_adventureworks/adventureworks/advw/dim_employee" assert "" == logs.get_errors_str() diff --git a/soda-tests/tests/unit/test_yaml_api.py b/soda-tests/tests/unit/test_yaml_api.py index 2e3c56032..56535d243 100644 --- a/soda-tests/tests/unit/test_yaml_api.py +++ b/soda-tests/tests/unit/test_yaml_api.py @@ -9,13 +9,9 @@ def test_yaml_api(logs: Logs): - yaml_source: YamlSource = YamlSource.from_str( - yaml_str=dedent_and_strip( - """ + yaml_source: YamlSource = YamlSource.from_str(yaml_str=dedent_and_strip(""" dataset: lskdjflks - """ - ) - ) + """)) yaml_object: YamlObject = yaml_source.parse() assert not logs.has_errors assert yaml_object.read_string("dataset") == "lskdjflks" @@ -23,13 +19,9 @@ def test_yaml_api(logs: Logs): def test_yaml_resolve_env_vars(env_vars: dict, logs: Logs): env_vars["DS"] = "thedataset" - yaml_source: YamlSource = YamlSource.from_str( - yaml_str=dedent_and_strip( - """ + yaml_source: YamlSource = YamlSource.from_str(yaml_str=dedent_and_strip(""" dataset: ${env.DS} - """ - ) - ) + """)) yaml_source.resolve() yaml_object: YamlObject = yaml_source.parse() assert not logs.has_errors @@ -53,12 +45,10 @@ def test_yaml_error_file_not_found(logs: Logs): def test_yaml_error_invalid_top_level_element(logs: Logs): yaml_source: YamlSource = YamlSource.from_str( - yaml_str=dedent_and_strip( - """ + yaml_str=dedent_and_strip(""" - one - two - """ - ), + """), file_path="yaml_string.yml", ) @@ -73,16 +63,12 @@ def test_yaml_error_empty_yaml_str(logs: Logs): def test_yaml_nested_level(logs: Logs): - yaml_source: YamlSource = YamlSource.from_str( - yaml_str=dedent_and_strip( - """ + yaml_source: YamlSource = YamlSource.from_str(yaml_str=dedent_and_strip(""" level_one: level_two: - type: item name: entry - """ - ) - ) + """)) yaml_object: YamlObject = yaml_source.parse() assert not logs.has_errors @@ -98,8 +84,7 @@ def test_yaml_nested_level(logs: Logs): def test_yaml_locations(): - yaml_str: str = dedent_and_strip( - """ + yaml_str: str = dedent_and_strip(""" one: one_two: - type: item @@ -107,8 +92,7 @@ def test_yaml_locations(): two: two_one: a two_two: b - """ - ) + """) logging.debug(f"\n=== YAML string ============\n{yaml_str}\n============================") yaml_source: YamlSource = YamlSource.from_str(yaml_str=yaml_str) root_object: YamlObject = yaml_source.parse() diff --git a/soda-trino/tests/data_sources/test_trino.py b/soda-trino/tests/data_sources/test_trino.py index bd7bff1b5..83d7343e5 100644 --- a/soda-trino/tests/data_sources/test_trino.py +++ b/soda-trino/tests/data_sources/test_trino.py @@ -153,7 +153,7 @@ def mock_requests_post(token_url: str, data: dict) -> Mock: # TestConnection( # real token, should work # test_name="real_jwt_token", # connection_yaml_str=f""" - # type: trino + # type: trino # name: TRINO_TEST_DS # connection: # host: '{TRINO_HOST}'