diff --git a/.coveragerc b/.coveragerc index 5cc8357..e95535c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,4 +4,4 @@ omit = *__init__* [report] -fail_under = 85 +fail_under = 90 diff --git a/dynamicio/__init__.py b/dynamicio/__init__.py index c1aa90f..844031e 100644 --- a/dynamicio/__init__.py +++ b/dynamicio/__init__.py @@ -1,5 +1,5 @@ """A package for wrapping your I/O operations.""" -# pylint: disable=abstract-method + import os from contextlib import suppress @@ -8,7 +8,7 @@ try: from importlib.metadata import PackageNotFoundError, version except ImportError: - from importlib_metadata import PackageNotFoundError, version # type: ignore + from importlib_metadata import PackageNotFoundError, version with suppress(Exception): try: @@ -23,7 +23,7 @@ os.environ["LC_CTYPE"] = "en_US.UTF" # Set your locale to a unicode-compatible one -class UnifiedIO(WithS3File, WithS3PathPrefix, WithLocalBatch, WithLocal, WithKafka, WithAthena, WithPostgres, DynamicDataIO): # type: ignore +class UnifiedIO(WithS3File, WithS3PathPrefix, WithLocalBatch, WithLocal, WithKafka, WithAthena, WithPostgres, DynamicDataIO): """A unified io composed of dynamicio.mixins.""" diff --git a/dynamicio/__main__.py b/dynamicio/__main__.py index ba3addf..61f5dbc 100644 --- a/dynamicio/__main__.py +++ b/dynamicio/__main__.py @@ -1,4 +1,6 @@ """Invokes dynamicio cli.""" + +# Application Imports from dynamicio.cli import run run() diff --git a/dynamicio/cli.py b/dynamicio/cli.py index 9497ee2..7513f49 100644 --- a/dynamicio/cli.py +++ b/dynamicio/cli.py @@ -1,4 +1,5 @@ """Implements the dynamicio Command Line Interface (CLI).""" + import argparse import glob import os @@ -8,6 +9,7 @@ import pandas as pd # type: ignore import yaml +# Application Imports from dynamicio.errors import InvalidDatasetTypeError diff --git a/dynamicio/config/io_config.py b/dynamicio/config/io_config.py index f1f6033..0066e76 100644 --- a/dynamicio/config/io_config.py +++ b/dynamicio/config/io_config.py @@ -52,6 +52,7 @@ } } """ + __all__ = ["IOConfig", "SafeDynamicResourceLoader", "SafeDynamicSchemaLoader"] import re @@ -62,6 +63,7 @@ import yaml from magic_logger import logger +# Application Imports from dynamicio.config.pydantic import BindingsYaml, IOEnvironment diff --git a/dynamicio/config/pydantic/__init__.py b/dynamicio/config/pydantic/__init__.py index 2c4be3a..4ba1a42 100644 --- a/dynamicio/config/pydantic/__init__.py +++ b/dynamicio/config/pydantic/__init__.py @@ -1,7 +1,9 @@ """Pydantic config models.""" +# Application Imports from dynamicio.config.pydantic.config import BindingsYaml from dynamicio.config.pydantic.io_resources import ( + AthenaDataEnvironment, IOEnvironment, KafkaDataEnvironment, LocalBatchDataEnvironment, @@ -9,6 +11,5 @@ PostgresDataEnvironment, S3DataEnvironment, S3PathPrefixEnvironment, - AthenaDataEnvironment ) from dynamicio.config.pydantic.table_schema import DataframeSchema, SchemaColumn diff --git a/dynamicio/config/pydantic/config.py b/dynamicio/config/pydantic/config.py index acc8772..0db3087 100644 --- a/dynamicio/config/pydantic/config.py +++ b/dynamicio/config/pydantic/config.py @@ -1,10 +1,12 @@ +"""Pydantic schema for YAML files.""" + # pylint: disable=no-member, no-self-argument, unused-argument -"""Pydantic schema for YAML files""" from typing import Mapping, MutableMapping import pydantic +# Application Imports import dynamicio.config.pydantic.io_resources as env_spec @@ -21,14 +23,16 @@ def _validate_bindings(cls, value: Mapping): if not isinstance(value, Mapping): raise ValueError(f"Bindings must be a mapping. (got {value!r} instead).") # Tell each binding its name - for (name, sub_config) in value.items(): + for name, sub_config in value.items(): if not isinstance(sub_config, MutableMapping): raise ValueError(f"Each element for the name binding must be a dict. (got {sub_config!r} instead)") sub_config["__binding_name__"] = name return value def update_config_refs(self) -> "BindingsYaml": - """Updates dynamic parts of the config: + """Updates dynamic parts of the config. + + Specifically: - Configure _parent for all `IOEnvironment`s - Replace all IOSchemaRef with actual schema objects """ diff --git a/dynamicio/config/pydantic/io_resources.py b/dynamicio/config/pydantic/io_resources.py index 209474a..81b527c 100644 --- a/dynamicio/config/pydantic/io_resources.py +++ b/dynamicio/config/pydantic/io_resources.py @@ -275,11 +275,7 @@ class PostgresDataEnvironment(IOEnvironment): class AthenaDataSubSection(BaseModel): """AWS Athena configuration section.""" - s3_staging_dir: str - region_name: str - - # Optional fields, one must be provided via YAML or mixin options - query: Optional[str] = None + s3_output: str class AthenaDataEnvironment(IOEnvironment): diff --git a/dynamicio/config/pydantic/table_schema.py b/dynamicio/config/pydantic/table_schema.py index 05bdce0..2a24889 100644 --- a/dynamicio/config/pydantic/table_schema.py +++ b/dynamicio/config/pydantic/table_schema.py @@ -1,6 +1,7 @@ +"""This module defines Config schema for data source (pandas dataframe).""" + # pylint: disable=no-member, no-self-argument, unused-argument -"""This module defines Config schema for data source (pandas dataframe)""" import enum from typing import Mapping, Sequence @@ -51,11 +52,11 @@ def is_valid_pandas_type(cls, info): @pydantic.validator("validations", pre=True) def remap_validations(cls, info): - """Remap the yaml structure of {validation_type: } to a list with validation_type as a key""" + """Remap the yaml structure of {validation_type: } to a list with validation_type as a key.""" if not isinstance(info, dict): raise ValueError(f"{info!r} should be a dict") out = [] - for (key, params) in info.items(): + for key, params in info.items(): new_el = params.copy() new_el.update({"name": key}) out.append(new_el) @@ -79,7 +80,7 @@ class DataframeSchema(pydantic.BaseModel): @pydantic.validator("columns", pre=True) def supply_column_names(cls, info): - """Tell each column its name (the key it is listed under)""" + """Tell each column its name (the key it is listed under).""" if not isinstance(info, Mapping): raise ValueError(f"{info!r} shoudl be a dict.") diff --git a/dynamicio/core.py b/dynamicio/core.py index e263d18..959dd2a 100644 --- a/dynamicio/core.py +++ b/dynamicio/core.py @@ -1,4 +1,5 @@ """Implements the DynamicDataIO class which provides functionality for data: loading; sinking, and; schema validation.""" + # pylint: disable=no-member __all__ = ["DynamicDataIO", "SCHEMA_FROM_FILE", "CASTING_WARNING_MSG"] @@ -8,7 +9,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import Any, List, Mapping, MutableMapping, Optional, Tuple -import pandas as pd # type: ignore +import pandas as pd import pydantic from magic_logger import logger @@ -100,7 +101,7 @@ def _schema_from_obj(target) -> DataframeSchema: - CAN have `schema_validations` and `schema_metrics` attributes """ col_info = {} - for (col_name, dtype) in target.schema.items(): + for col_name, dtype in target.schema.items(): col_validations = {} col_metrics = [] try: diff --git a/dynamicio/errors.py b/dynamicio/errors.py index 7c41fe8..ed49b1c 100644 --- a/dynamicio/errors.py +++ b/dynamicio/errors.py @@ -1,5 +1,7 @@ """Hosts exception implementations for different errors.""" + # pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring, super-init-not-called + __all__ = [ "DynamicIOError", "DataSourceError", diff --git a/dynamicio/metrics.py b/dynamicio/metrics.py index 109b112..cc1dbc9 100644 --- a/dynamicio/metrics.py +++ b/dynamicio/metrics.py @@ -1,4 +1,5 @@ """A module responsible for metrics generation and logging.""" + # pylint: disable=missing-function-docstring,missing-class-docstring import json import logging diff --git a/dynamicio/mixins/utils.py b/dynamicio/mixins/utils.py index 3076222..f8325cf 100644 --- a/dynamicio/mixins/utils.py +++ b/dynamicio/mixins/utils.py @@ -1,64 +1,62 @@ """Mixin utility functions.""" -# pylint: disable=no-member, protected-access, too-few-public-methods import inspect import string from contextlib import contextmanager from enum import Enum from functools import wraps -from types import FunctionType, MethodType -from typing import Any, Collection, Iterable, Mapping, MutableMapping, Optional, Union +from typing import Any, Callable, Collection, Iterable, Mapping, MutableMapping, Optional, Union from magic_logger import logger -def allow_options(options: Union[Iterable, FunctionType, MethodType]): - """Validate **options for a decorated reader function. +def allow_options(options: Union[Iterable[str], Callable]): + """Decorator to filter **kwargs passed to a function, allowing only valid ones. Args: - options: A set of valid options for a reader (e.g. `pandas.read_parquet` or `pandas.read_csv`) + options: A list of valid options or a callable that returns a list of valid options. Returns: - read_with_valid_options: The input function called with modified options. + Callable: A decorator that filters **kwargs passed to the function. """ def _filter_out_irrelevant_options(kwargs: Mapping, valid_options: Iterable): filtered_options = {} invalid_options = {} - for key_arg in kwargs.keys(): - if key_arg in valid_options: - filtered_options[key_arg] = kwargs[key_arg] + for key, val in kwargs.items(): + if key in valid_options: + filtered_options[key] = val else: - invalid_options[key_arg] = kwargs[key_arg] - if len(invalid_options) > 0: - logger.warning( - f"Options {invalid_options} were not used because they were not supported by the read or write method configured for this source. " - "Check if you expected any of those to have been used by the operation!" - ) + invalid_options[key] = val + if invalid_options: + logger.warning(f"Options {invalid_options} were not used because they are not supported by this operation. " f"Review your kwargs!") return filtered_options def read_with_valid_options(func): @wraps(func) - def _(*args, **kwargs): - if callable(options): - return func(*args, **_filter_out_irrelevant_options(kwargs, args_of(options))) - return func(*args, **_filter_out_irrelevant_options(kwargs, options)) + def wrapper(*args, **kwargs): + valid = args_of(options) if callable(options) else set(options) + return func(*args, **_filter_out_irrelevant_options(kwargs, valid)) - return _ + return wrapper return read_with_valid_options -def args_of(func): - """Retrieve allowed options for a given function. +def args_of(*funcs) -> set[str]: + """Retrieve a set of accepted keyword arguments from one or more functions. Args: - func: A function like, e.g., pd.read_csv + funcs: A list of functions to inspect. Returns: - A set of allowed options + set[str]: A set of accepted keyword arguments. """ - return set(inspect.signature(func).parameters.keys()) + allowed_args = set() + for func in funcs: + sig = inspect.signature(func) + allowed_args.update(sig.parameters.keys()) + return allowed_args def get_string_template_field_names(s: str) -> Collection[str]: # pylint: disable=C0103 @@ -67,13 +65,12 @@ def get_string_template_field_names(s: str) -> Collection[str]: # pylint: disab If `s` is not a string template, the returned `Collection` is empty. Args: - s: + s: A string which is either a template, e.g. /path/to/file/{replace_me}.h5 or just a path /path/to/file/dont_replace_me.h5 Returns: Collection[str] Example: - >>> get_string_template_field_names("abc{def}{efg}") ["def", "efg"] >>> get_string_template_field_names("{0}-{1}") @@ -102,8 +99,7 @@ def resolve_template(path: str, options: MutableMapping[str, Any]) -> str: # py str: Returns a static path replaced with the value in the options mapping. Raises: - ValueError: if any template fields in s are not named using valid Python identifiers - ValueError: if a given template field cannot be resolved in `options` + ValueError: if any template fields in s are not named using valid Python identifiers or if a given template field cannot be resolved in `options` """ fields = get_string_template_field_names(path) @@ -143,5 +139,12 @@ def pickle_protocol(protocol: Optional[int]): def get_file_type_value(file_type: Union[str, Enum]) -> str: - """Get the value of the file type.""" + """Get the value of the file type. + + Args: + file_type: The file type, which can be a string or an Enum. + + Returns: + str: The value of the file type. + """ return file_type.value if isinstance(file_type, Enum) else file_type diff --git a/dynamicio/mixins/with_athena.py b/dynamicio/mixins/with_athena.py index b00bdb1..e71f4c9 100644 --- a/dynamicio/mixins/with_athena.py +++ b/dynamicio/mixins/with_athena.py @@ -1,55 +1,56 @@ +"""This module provides mixins that support AWS Athena I/O.""" + # pylint: disable=no-member, protected-access, too-few-public-methods -"""This module provides mixins that support AWS Athena I/O.""" -import inspect -from typing import Any, MutableMapping +from typing import Any, MutableMapping, cast +import awswrangler as wr import pandas as pd -from magic_logger import logger -from pyathena import connect -from pyathena.connection import Connection -from pyathena.pandas.cursor import PandasCursor -from pyathena.pandas.result_set import AthenaPandasResultSet # Application Imports from dynamicio.config.pydantic import AthenaDataEnvironment from dynamicio.mixins.utils import allow_options -allowed_athena_options = set(inspect.signature(AthenaPandasResultSet.__init__).parameters.keys()) - {"self"} - class WithAthena: - """Handles I/O operations for AWS Athena.""" + """Handles I/O operations for AWS Athena using AWS Wrangler. + + Note: + The `__abstractmethods__ = frozenset()` is used to silence false positives from pylint, + which might wrongly assume this class has abstract methods due to NotImplementedError. + This class is *not* an abstract base class and does not use `abc.ABC`. + """ + + __abstractmethods__ = frozenset() sources_config: AthenaDataEnvironment options: MutableMapping[str, Any] def _read_from_athena(self) -> pd.DataFrame: - """Reads data from AWS Athena. + """Reads data from AWS Athena using awswrangler with validated kwargs. Expected config: - query - - s3_staging_dir - - region_name + - s3_output """ cfg = self.sources_config.athena - query = self.options.pop("query", None) - - assert query, "A 'query' must be provided for Athena read" - - conn = connect(s3_staging_dir=cfg.s3_staging_dir, region_name=cfg.region_name, cursor_class=PandasCursor) - return self._run_query(conn, query, **self.options) - - @allow_options(allowed_athena_options) - def _run_query(self, conn: Connection, query: str, **options: Any) -> pd.DataFrame: - logger.info(f"[athena] Executing query: {query}") - cursor = conn.cursor() - cursor.execute(query) - - rows = cursor.fetchall(**options) - columns = [col[0] for col in cursor.description] - return pd.DataFrame(rows, columns=columns) + raw_query = self.options.pop("query", None) + if raw_query is None: + raise ValueError("A 'query' must be provided for Athena reads") + + query = cast(str, raw_query) + # Pull config, add required positional args back + return self._run_wr_athena_query_wrapped( + sql=query, + s3_output=cfg.s3_output, + **self.options, + ) + + @staticmethod + @allow_options(wr.athena.read_sql_query) + def _run_wr_athena_query_wrapped(sql: str, s3_output: str, **options: Any) -> pd.DataFrame: + return wr.athena.read_sql_query(sql, s3_output, **options) def _write_to_athena(self, df: pd.DataFrame): """Athena does not support direct writing. Raise NotImplementedError.""" - raise NotImplementedError("Athena does not support direct writes via pandas. Consider writing to S3 or using Glue instead.") + raise NotImplementedError("Athena does not support direct writes. Use S3/Glue for persistence.") diff --git a/dynamicio/mixins/with_kafka.py b/dynamicio/mixins/with_kafka.py index 0583459..35b186b 100644 --- a/dynamicio/mixins/with_kafka.py +++ b/dynamicio/mixins/with_kafka.py @@ -1,4 +1,5 @@ """This module provides mixins that are providing Kafka I/O support.""" + # pylint: disable=no-member, protected-access, too-few-public-methods from typing import Any, Callable, Mapping, MutableMapping, Optional @@ -10,7 +11,7 @@ # Application Imports from dynamicio.config.pydantic import DataframeSchema, KafkaDataEnvironment -from dynamicio.mixins import utils +from dynamicio.mixins.utils import allow_options class WithKafka: @@ -68,8 +69,15 @@ class WithKafka: >>> {"key": "key-02", "value": {"bar": 1000, "baz": "ABC", "foo": "id_2", "id": "cm_2", "new_field": "new_value"}}, >>> {"key": "key-03", "value": {"bar": 1000, "baz": "ABC", "foo": "id_3", "id": "cm_3", "new_field": "new_value"}}, >>> ] + + Note: + The `__abstractmethods__ = frozenset()` is used to silence false positives from pylint, + which might wrongly assume this class has abstract methods due to NotImplementedError. + This class is *not* an abstract base class and does not use `abc.ABC`. """ + __abstractmethods__ = frozenset() + sources_config: KafkaDataEnvironment schema: DataframeSchema options: MutableMapping[str, Any] @@ -282,7 +290,7 @@ def populate_cls_attributes(self): if self.options.get("value_serializer") is not None: self.__value_serializer = self.options.pop("value_serializer") - @utils.allow_options(VALID_CONFIG_KEYS) + @allow_options(VALID_CONFIG_KEYS) def _get_producer(self, server: str, **options: MutableMapping[str, Any]) -> Producer: """Generate and return a Kafka Producer. diff --git a/dynamicio/mixins/with_local.py b/dynamicio/mixins/with_local.py index 419149b..0a13cb4 100644 --- a/dynamicio/mixins/with_local.py +++ b/dynamicio/mixins/with_local.py @@ -7,13 +7,15 @@ from threading import Lock from typing import Any, MutableMapping -import pandas as pd # type: ignore -from fastparquet import ParquetFile, write # type: ignore -from pyarrow.parquet import read_table, write_table # type: ignore # pylint: disable=no-name-in-module +import pandas as pd +from fastparquet import ParquetFile, write +from magic_logger import logger +from pyarrow.parquet import read_table, write_table +# Application Imports from dynamicio.config.pydantic import DataframeSchema, LocalBatchDataEnvironment, LocalDataEnvironment from dynamicio.mixins import utils -from dynamicio.mixins.utils import get_file_type_value +from dynamicio.mixins.utils import allow_options, get_file_type_value hdf_lock = Lock() @@ -32,8 +34,10 @@ def _read_from_local(self) -> pd.DataFrame: - `file_path` - `file_type` - To actually read the file, a method is dynamically invoked by name, using - "_read_{file_type}_file". + To actually read the file, a method is dynamically invoked by name, using "_read_{file_type}_file". + + Additional options: + - single_record: bool: used for json files. If True, treats the file as a single JSON object instead of a list of records. Returns: DataFrame @@ -64,7 +68,7 @@ def _write_to_local(self, df: pd.DataFrame): getattr(self, f"_write_{file_type}_file")(df, file_path, **self.options) @staticmethod - @utils.allow_options(pd.read_hdf) + @allow_options(pd.read_hdf) def _read_hdf_file(file_path: str, schema: DataframeSchema, **options: Any) -> pd.DataFrame: """Read a HDF file as a DataFrame using `pd.read_hdf`. @@ -89,7 +93,7 @@ def _read_hdf_file(file_path: str, schema: DataframeSchema, **options: Any) -> p return df @staticmethod - @utils.allow_options(pd.read_csv) + @allow_options(pd.read_csv) def _read_csv_file(file_path: str, schema: DataframeSchema, **options: Any) -> pd.DataFrame: """Read a CSV file as a DataFrame using `pd.read_csv`. @@ -106,23 +110,49 @@ def _read_csv_file(file_path: str, schema: DataframeSchema, **options: Any) -> p return pd.read_csv(file_path, **options) @staticmethod - @utils.allow_options(pd.read_json) + @allow_options([*utils.args_of(pd.read_json), *["single_record"]]) def _read_json_file(file_path: str, schema: DataframeSchema, **options: Any) -> pd.DataFrame: """Read a json file as a DataFrame using `pd.read_hdf`. All `options` are passed directly to `pd.read_hdf`. Args: - file_path: - options: + file_path: The path to the json file to be read. + options: The pandas `read_json` options. Returns: - DataFrame + DataFrame: The dataframe read from the json file. """ - df = pd.read_json(file_path, **options) - columns = [column for column in df.columns.to_list() if column in schema.column_names] - df = df[columns] - return df + user_orient = options.pop("orient", None) + user_lines = options.pop("lines", None) + + if user_orient is not None and user_orient != "records": + raise ValueError("[local-json] Unsupported orient='{user_orient}'. Only 'records' orientation is supported.") + + if user_lines is not None and user_lines is not False: + logger.warning("[local-json-read] Overriding lines=%s with lines=False for consistency with aws-wrangler expectations.", user_lines) + + if options.get("convert_dates") is True: + logger.warning("[local-json-read] Ignoring 'convert_dates=True'. Handle datetime parsing post-read.") + options.pop("convert_dates", None) + + is_single_record = options.pop("single_record", False) + df = pd.read_json(file_path, orient="records", convert_dates=False, lines=False, **options) + + # ๐Ÿงผ Check if this is a single-record json file + if is_single_record: + # Re-wrap as single dict row โ€” i.e., rehydrate the record + df = pd.DataFrame([{df.columns[0]: dict(zip(df.index, df.iloc[:, 0]))}]) + elif ( + df.shape[1] == 1 + and df.columns.dtype == "object" + and df.index.dtype == "object" + and all(isinstance(i, str) for i in df.index) + and all(isinstance(v, (str, int, float, bool, type(None))) for v in df.iloc[:, 0]) + ): + logger.warning("[local-json-read] File appears to be a single-record JSON object. Pass 'single_record=True' in options to handle this case.") + + return df[[col for col in df.columns if col in schema.column_names]] @staticmethod def _read_parquet_file(file_path: str, schema: DataframeSchema, **options: Any) -> pd.DataFrame: @@ -144,17 +174,17 @@ def _read_parquet_file(file_path: str, schema: DataframeSchema, **options: Any) return WithLocal.__read_with_pyarrow(file_path, **options) @classmethod - @utils.allow_options([*utils.args_of(pd.read_parquet), *utils.args_of(read_table)]) + @allow_options([*utils.args_of(pd.read_parquet), *utils.args_of(read_table)]) def __read_with_pyarrow(cls, file_path: str, **options: Any) -> pd.DataFrame: return pd.read_parquet(file_path, **options) @classmethod - @utils.allow_options([*utils.args_of(pd.read_parquet), *utils.args_of(ParquetFile)]) + @allow_options([*utils.args_of(pd.read_parquet), *utils.args_of(ParquetFile)]) def __read_with_fastparquet(cls, file_path: str, **options: Any) -> pd.DataFrame: return pd.read_parquet(file_path, **options) @staticmethod - @utils.allow_options([*utils.args_of(pd.DataFrame.to_hdf), *["protocol"]]) + @allow_options([*utils.args_of(pd.DataFrame.to_hdf), *["protocol"]]) def _write_hdf_file(df: pd.DataFrame, file_path: str, **options: Any): """Write a dataframe to hdf using `df.to_hdf`. @@ -176,7 +206,7 @@ def _write_hdf_file(df: pd.DataFrame, file_path: str, **options: Any): df.to_hdf(file_path, key="df", mode="w", **options) @staticmethod - @utils.allow_options(pd.DataFrame.to_csv) + @allow_options(pd.DataFrame.to_csv) def _write_csv_file(df: pd.DataFrame, file_path: str, **options: Any): """Write a dataframe as a CSV file using `df.to_csv`. @@ -190,9 +220,12 @@ def _write_csv_file(df: pd.DataFrame, file_path: str, **options: Any): df.to_csv(file_path, **options) @staticmethod - @utils.allow_options(pd.DataFrame.to_json) + @allow_options(pd.DataFrame.to_json) def _write_json_file(df: pd.DataFrame, file_path: str, **options: Any): - """Write a dataframe as a json file using `df.to_json`. + """Writes a JSON file using 'records' orientation with lines=True. + + If the user provides an unsupported `orient`, raise an error. + This mirrors wr.s3.to_json and guarantees tabular consistency. All `options` are passed directly to `df.to_json`. @@ -201,6 +234,16 @@ def _write_json_file(df: pd.DataFrame, file_path: str, **options: Any): file_path: The location where the file needs to be written. options: Options relative to writing a json file. """ + user_orient = options.pop("orient", None) + user_lines = options.pop("lines", None) + + if user_orient is not None and user_orient != "records": + raise ValueError( + f"[local-json] Unsupported orient='{user_orient}'. Only 'records' orientation is supported for tabular output (imposed for aws-wrangler consistency reasons)." + ) + if user_lines is not None and user_lines is not True: + logger.warning("[local-json-write] Overriding lines=%s with lines=True for consistency.", user_lines) + df.to_json(file_path, **options) @staticmethod @@ -219,12 +262,12 @@ def _write_parquet_file(df: pd.DataFrame, file_path: str, **options: Any): return WithLocal.__write_with_pyarrow(df, file_path, **options) @classmethod - @utils.allow_options([*utils.args_of(pd.DataFrame.to_parquet), *utils.args_of(write_table)]) + @allow_options([*utils.args_of(pd.DataFrame.to_parquet), *utils.args_of(write_table)]) def __write_with_pyarrow(cls, df: pd.DataFrame, filepath: str, **options: Any): return df.to_parquet(filepath, **options) @classmethod - @utils.allow_options([*utils.args_of(pd.DataFrame.to_parquet), *utils.args_of(write)]) + @allow_options([*utils.args_of(pd.DataFrame.to_parquet), *utils.args_of(write)]) def __write_with_fastparquet(cls, df: pd.DataFrame, filepath: str, **options: Any): return df.to_parquet(filepath, **options) @@ -283,6 +326,6 @@ def _read_from_local_batch(self) -> pd.DataFrame: dfs_to_concatenate = [] for file in files: file_to_load = os.path.join(file_path, file) - dfs_to_concatenate.append(getattr(self, f"_read_{file_type}_file")(file_to_load, self.schema, **self.options)) # type: ignore + dfs_to_concatenate.append(getattr(self, f"_read_{file_type}_file")(file_to_load, self.schema, **self.options)) return pd.concat(dfs_to_concatenate).reset_index(drop=True) diff --git a/dynamicio/mixins/with_postgres.py b/dynamicio/mixins/with_postgres.py index 1bc7c3a..949b0c3 100644 --- a/dynamicio/mixins/with_postgres.py +++ b/dynamicio/mixins/with_postgres.py @@ -7,17 +7,18 @@ from contextlib import contextmanager from typing import Any, Dict, Generator, MutableMapping, Union -import pandas as pd # type: ignore +import pandas as pd from magic_logger import logger -from sqlalchemy import BigInteger, Boolean, Column, create_engine, Date, DateTime, Float, Integer, String # type: ignore -from sqlalchemy.ext.declarative import declarative_base # type: ignore -from sqlalchemy.orm import Query # type: ignore -from sqlalchemy.orm.decl_api import DeclarativeMeta # type: ignore -from sqlalchemy.orm.session import Session as SqlAlchemySession # type: ignore -from sqlalchemy.orm.session import sessionmaker # type: ignore - +from sqlalchemy import BigInteger, Boolean, Column, Date, DateTime, Float, Integer, String, create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import Query +from sqlalchemy.orm.decl_api import DeclarativeMeta +from sqlalchemy.orm.session import Session as SqlAlchemySession +from sqlalchemy.orm.session import sessionmaker + +# Application Imports from dynamicio.config.pydantic import DataframeSchema, PostgresDataEnvironment -from dynamicio.mixins import utils +from dynamicio.mixins.utils import allow_options Session = sessionmaker(autoflush=True) @@ -127,7 +128,7 @@ def _get_table_columns(model): return tables_colums @staticmethod - @utils.allow_options(pd.read_sql) + @allow_options(pd.read_sql) def _read_database(session: SqlAlchemySession, query: Union[str, Query], **options: Any) -> pd.DataFrame: """Run `query` against active `session` and returns the result as a `DataFrame`. diff --git a/dynamicio/mixins/with_s3.py b/dynamicio/mixins/with_s3.py index c0a9ec6..b7fdd4c 100644 --- a/dynamicio/mixins/with_s3.py +++ b/dynamicio/mixins/with_s3.py @@ -1,5 +1,3 @@ -# pylint: disable=no-member, protected-access, too-few-public-methods - """This module provides mixins that are providing S3 I/O support.""" import dataclasses @@ -9,19 +7,22 @@ import urllib.parse import uuid from contextlib import contextmanager -from typing import IO, Generator, List, Optional, Union # noqa: I101 +from typing import IO, Dict, Generator, List, Optional, Union +from urllib.parse import urlparse -import boto3 # type: ignore +import awswrangler as wr +import boto3 import pandas as pd -import s3transfer.futures # type: ignore -import tables # type: ignore -from awscli.clidriver import create_clidriver # type: ignore +import s3transfer.futures +import tables +from awscli.clidriver import create_clidriver from magic_logger import logger from pandas import DataFrame, Series +# Application Imports from dynamicio.config.pydantic import DataframeSchema, S3DataEnvironment, S3PathPrefixEnvironment from dynamicio.mixins import utils, with_local -from dynamicio.mixins.utils import get_file_type_value +from dynamicio.mixins.utils import allow_options, args_of, get_file_type_value class InMemStore(pd.io.pytables.HDFStore): @@ -36,7 +37,7 @@ def __init__(self, path: str, table: tables.File, mode: str = "r"): def open(self, *_args, **_kwargs): """Open the in-memory table.""" - pd.io.pytables._tables() + pd.io.pytables._tables() # pylint: disable=protected-access self._handle = self._in_mem_table def close(self, *_args, **_kwargs): @@ -49,32 +50,71 @@ def is_open(self): class HdfIO: - """Class providing stream support for HDF tables.""" + """Provides in-memory stream support for reading and writing HDF5 tables. + + Uses PyTables to create in-memory file handles, enabling read/write + operations on HDF content without persisting to disk. + """ @contextmanager def create_file(self, label: str, mode: str, data: Optional[bytes] = None) -> Generator[tables.File, None, None]: - """Create an in-memory pytables table.""" - extra_kw = {} + """Create an in-memory HDF5 file using PyTables with optional preloaded data. + + Args: + label (str): A label used for naming the temporary in-memory file. + mode (str): File access mode ('r' for read, 'w' for write). + data (Optional[bytes]): Raw file data to preload when opening for reading. + + Yields: + tables.File: A PyTables file object representing the HDF5 structure. + """ + extra_kw = {"driver_core_backing_store": 0} if data: extra_kw["driver_core_image"] = data - file_handle = tables.File(f"{label}_{uuid.uuid4()}.h5", mode, title=label, root_uep="/", filters=None, driver="H5FD_CORE", driver_core_backing_store=0, **extra_kw) + + file_name = f"{label}_{uuid.uuid4()}.h5" + file_handle = tables.File(file_name, mode, title=label, root_uep="/", filters=None, driver="H5FD_CORE", **extra_kw) + try: yield file_handle finally: file_handle.close() - def load(self, fobj: IO[bytes], label: str = "unknown_file.h5") -> Union[DataFrame, Series]: - """Load the dataframe from an file-like object.""" + def load(self, fobj: IO[bytes], label: str = "unknown_file.h5", options: Optional[Dict] = None) -> Union[DataFrame, Series]: + """Load a DataFrame or Series from an in-memory HDF5 file-like object. + + Args: + fobj (IO[bytes]): A file-like object containing the HDF5 data. + label (str): A logical name for the file (used in metadata). + options (Optional[dict]): Optional keyword arguments to pass to `pd.read_hdf`. + + Returns: + Union[DataFrame, Series]: The object read from the HDF file. + """ + options = options or {} + with self.create_file(label, mode="r", data=fobj.read()) as file_handle: - return pd.read_hdf(InMemStore(label, file_handle)) + return pd.read_hdf(InMemStore(label, file_handle), **options) + + def save(self, df: DataFrame, fobj: IO[bytes], label: str = "unknown_file.h5", options: Optional[Dict] = None) -> None: + """Save a DataFrame to a file-like object as an HDF5 structure. - def save(self, df: DataFrame, fobj: IO[bytes], label: str = "unknown_file.h5", options: Optional[dict] = None): - """Load the dataframe to a file-like object.""" - if not options: - options = {} - with self.create_file(label, mode="w", data=fobj.read()) as file_handle: + Args: + df (DataFrame): The DataFrame to store. + fobj (IO[bytes]): The target file-like object. + label (str): A logical name used for the in-memory file. + options (Optional[dict]): Optional keyword arguments to pass to `HDFStore.put`. + You can also include a `key` (defaults to 'df'). + + Notes: + Data is first written to an in-memory PyTables structure, then streamed into the provided file-like object. + """ + options = options or {} + key = options.pop("key", "df") + + with self.create_file(label, mode="w") as file_handle: store = InMemStore(path=label, table=file_handle, mode="w") - store.put(key="df", value=df, **options) + store.put(key=key, value=df, **options) fobj.write(file_handle.get_file_image()) @@ -304,139 +344,163 @@ def _iter_s3_files(self, s3_prefix: str, file_ext: Optional[str] = None, max_mem yield handle.fobj -class WithS3File(with_local.WithLocal): - """Handles I/O operations for AWS S3. - - All files are persisted to disk first using boto3 as this has proven to be faster than reading them into memory. - Note that reading things into memory is available for csv, json and parquet types only. Unfortunately, until support - for generic buffer is added to read_hdf, we need to download and persists the file to disk first anyway. +class WithS3File: + """Handles I/O operations for AWS S3 using in-memory streaming for CSV, JSON, Parquet, and HDF files. - Options: - no_disk_space: If `True`, then s3fs + fsspec will be used to read data directly into memory. + For CSV, JSON, and Parquet, AWS Data Wrangler is used for efficient direct reads from S3. + For HDF files, content is streamed into memory using boto3 and then loaded via PyTables. """ - sources_config: S3DataEnvironment # type: ignore + sources_config: S3DataEnvironment schema: DataframeSchema - boto3_client = boto3.client("s3") - - @contextmanager - def _s3_named_file_reader(self, s3_bucket: str, s3_key: str) -> Generator: - """Contextmanager to abstract reading different file types in S3. + def _read_from_s3_file(self) -> pd.DataFrame: + """Read a file from an S3 bucket as a `DataFrame`. - This implementation saves the downloaded data to a temporary file. + The configuration object is expected to have the following keys: + - `bucket` + - `file_path` + - `file_type` - Args: - s3_bucket: The S3 bucket from where to read the file. - s3_key: The file-path to the target file to be read. + To actually read the file, a method is dynamically invoked by name, using "_read_{file_type}_file". Returns: - The local file path from where the file can be read, once it has been downloaded there by the boto3.client. - + DataFrame """ - with tempfile.NamedTemporaryFile("wb") as target_file: - # Download the file from S3 - self.boto3_client.download_fileobj(s3_bucket, s3_key, target_file) - # Yield local file path to body of `with` statement - target_file.flush() - yield target_file + s3_config = self.sources_config.s3 + file_type = get_file_type_value(s3_config.file_type) + options = getattr(self, "options", {}) + s3_path = f"s3://{s3_config.bucket}/{utils.resolve_template(s3_config.file_path, options)}" - @contextmanager - def _s3_reader(self, s3_bucket: str, s3_key: str) -> Generator[io.BytesIO, None, None]: - """Contextmanager to abstract reading different file types in S3. + logger.info(f"[s3] Started downloading: {s3_path}") - This implementation only retains data in-memory, avoiding creating any temp files. + return getattr(self, f"_read_s3_{file_type}_file")(s3_path, self.schema, **options) - Args: - s3_bucket: The S3 bucket from where to read the file. - s3_key: The file-path to the target file to be read. + @staticmethod + @allow_options(wr.s3.read_parquet) + def _read_s3_parquet_file(s3_path: str, schema: DataframeSchema, **kwargs) -> pd.DataFrame: + return wr.s3.read_parquet(path=s3_path, columns=(list(schema.columns.keys())), **kwargs) - Returns: - The local file path from where the file can be read, once it has been downloaded there by the boto3.client. + @staticmethod + @allow_options(args_of(wr.s3.read_csv, pd.read_csv)) + def _read_s3_csv_file(s3_path: str, schema: DataframeSchema, **kwargs) -> pd.DataFrame: + return wr.s3.read_csv(path=s3_path, usecols=(list(schema.columns.keys())), **kwargs) - """ - fobj = io.BytesIO() - # Download the file from S3 - self.boto3_client.download_fileobj(s3_bucket, s3_key, fobj) - # Yield the buffer - fobj.seek(0) - yield fobj + @staticmethod + @allow_options(args_of(wr.s3.read_json, pd.read_json)) + def _read_s3_json_file(s3_path: str, schema: DataframeSchema, **kwargs) -> pd.DataFrame: + orient = kwargs.pop("orient", None) + lines = kwargs.pop("lines", None) - @contextmanager - def _s3_writer(self, s3_bucket: str, s3_key: str) -> Generator[IO[bytes], None, None]: - """Contextmanager to abstract loading different file types to S3. + if orient is not None and orient != "records": + raise ValueError("[s3-json] Unsupported orient='{orient}'. Only 'records' orientation is supported.") - Args: - s3_bucket: The S3 bucket to upload the file to. - s3_key: The file-path where the target file should be uploaded to. + if lines is not None and lines is not True: + logger.warning("[s3-json-read] Overriding lines=%s with lines=True for aws-wrangler consistency.", lines) - Returns: - The local file path where to actually write the file, to be read and uploaded by boto3.client. - """ - fobj = io.BytesIO() - yield fobj + if kwargs.get("convert_dates") is True: + logger.warning("[s3-json-read] Ignoring 'convert_dates=True'. Handle datetime parsing post-read.") + kwargs.pop("convert_dates", None) + + raw_df = wr.s3.read_json(path=s3_path, orient="records", lines=True, **kwargs) + + return raw_df[[col for col in raw_df.columns if col in schema.columns]] + + @staticmethod + @allow_options(pd.read_hdf) + def _read_s3_hdf_file(s3_path: str, schema: DataframeSchema, **kwargs) -> pd.DataFrame: + parsed = urlparse(s3_path) + bucket = parsed.netloc + file_path = parsed.path.lstrip("/") + + fobj = io.BytesIO() # Stream file directly into memory (no disk), unlike tempfile or open(...) which write to disk + boto3.client("s3").download_fileobj(bucket, file_path, fobj) fobj.seek(0) - self.boto3_client.upload_fileobj(fobj, s3_bucket, s3_key, ExtraArgs={"ACL": "bucket-owner-full-control"}) - def _read_from_s3_file(self) -> pd.DataFrame: - """Read a file from an S3 bucket as a `DataFrame`. + df = HdfIO().load(fobj, options=kwargs) - The configuration object is expected to have the following keys: - - `bucket` - - `file_path` - - `file_type` + return df[list(schema.columns.keys())] if schema and schema.columns else df - To actually read the file, a method is dynamically invoked by name, using "_read_{file_type}_file". + def _write_to_s3_file(self, df: pd.DataFrame): + """Write a DataFrame to S3 based on the config's file type and path. - Returns: - DataFrame + The appropriate writer function is dynamically resolved via the file type: + `_write_parquet_file`, `_write_csv_file`, `_write_json_file`, or `_write_hdf_file`. """ s3_config = self.sources_config.s3 file_type = get_file_type_value(s3_config.file_type) - file_path = utils.resolve_template(s3_config.file_path, self.options) - bucket = s3_config.bucket + options = getattr(self, "options", {}) + s3_path = f"s3://{s3_config.bucket}/{utils.resolve_template(s3_config.file_path, options)}" + + logger.info(f"[s3] Started uploading: {s3_path}") + getattr(self, f"_write_s3_{file_type}_file")(df, s3_path, **options) + logger.info(f"[s3] Finished uploading: {s3_path}") + + @staticmethod + @allow_options(wr.s3.to_parquet) + def _write_s3_parquet_file(df: pd.DataFrame, s3_path: str, **kwargs): + if kwargs.pop("dataset", False): + raise ValueError( + "[s3-parquet] dataset=True is not supported in the WithS3File mixin. Use a file path, not a directory. " + "Use WithS3PathPrefix if you need partitioned writes or directory-style datasets." + ) - logger.info(f"[s3] Started downloading: s3://{s3_config.bucket}/{file_path}") - if self.options.pop("no_disk_space", None): - no_disk_space_rv = None - if file_type in ["csv", "json", "parquet"]: - no_disk_space_rv = getattr(self, f"_read_{file_type}_file")(f"s3://{s3_config.bucket}/{file_path}", self.schema, **self.options) # type: ignore - elif file_type == "hdf": - with self._s3_reader(s3_bucket=bucket, s3_key=file_path) as fobj: # type: ignore - no_disk_space_rv = HdfIO().load(fobj) # type: ignore - else: - raise NotImplementedError(f"Unsupported file type {file_type!r}.") - if no_disk_space_rv is not None: - return no_disk_space_rv - with self._s3_named_file_reader(s3_bucket=bucket, s3_key=file_path) as target_file: # type: ignore - return getattr(self, f"_read_{file_type}_file")(target_file.name, self.schema, **self.options) # type: ignore + if s3_path.endswith("/"): + raise ValueError("[s3-parquet] Parquet output path must be a file, not a directory (e.g., 's3://bucket/data.parquet').") - def _write_to_s3_file(self, df: pd.DataFrame): - """Write a dataframe to s3 based on the {file_type} of the config_io configuration. + wr.s3.to_parquet(df=df, path=s3_path, dataset=False, **kwargs) - The configuration object is expected to have two keys: + @staticmethod + @allow_options(args_of(wr.s3.to_csv, pd.DataFrame.to_csv)) + def _write_s3_csv_file(df: pd.DataFrame, s3_path: str, **kwargs): + if kwargs.pop("dataset", False): + raise ValueError( + "[s3-csv] dataset=True is not supported in the WithS3File mixin. Use a file path, not a directory. " + "Use WithS3PathPrefix if you need partitioned writes or directory-style datasets." + ) - - `file_path` - - `file_type` + if s3_path.endswith("/"): + raise ValueError("[s3-csv] CSV output path must be a file, not a directory (e.g., 's3://bucket/data.csv').") - To actually write the file, a method is dynamically invoked by name, using "_write_{file_type}_file". + wr.s3.to_csv(df=df, path=s3_path, index=False, **kwargs) - Args: - df: The dataframe to be written out - """ - s3_config = self.sources_config.s3 - bucket = s3_config.bucket - file_path = utils.resolve_template(s3_config.file_path, self.options) - file_type = get_file_type_value(s3_config.file_type) + @staticmethod + @allow_options(args_of(wr.s3.to_json, pd.DataFrame.to_json)) + def _write_s3_json_file(df: pd.DataFrame, s3_path: str, **kwargs): + if kwargs.pop("dataset", False): + raise ValueError( + "[s3-json] dataset=True is not supported in the WithS3File mixin. Use a file path, not a directory. " + "Use WithS3PathPrefix if you need partitioned writes or directory-style datasets." + ) - logger.info(f"[s3] Started uploading: s3://{bucket}/{file_path}") - if file_type in ["csv", "json", "parquet"]: - getattr(self, f"_write_{file_type}_file")(df, f"s3://{bucket}/{file_path}", **self.options) # type: ignore - elif file_type == "hdf": - hdf_options = dict(self.options) - pickle_protocol = hdf_options.pop("pickle_protocol", None) - with self._s3_writer(s3_bucket=s3_config.bucket, s3_key=file_path) as target_file, utils.pickle_protocol(protocol=pickle_protocol): - HdfIO().save(df, target_file, hdf_options) # type: ignore - else: - raise ValueError(f"File type: {file_type} not supported!") - logger.info(f"[s3] Finished uploading: s3://{bucket}/{file_path}") + if s3_path.endswith("/"): + raise ValueError("[s3-json] JSON output path must be a file, not a directory (e.g., 's3://bucket/data.json').") + + user_orient = kwargs.pop("orient", None) + user_lines = kwargs.pop("lines", None) + + if user_orient is not None and user_orient != "records": + raise ValueError(f"[s3-json] Unsupported orient='{user_orient}'. Only 'records' orientation is supported.") + + if user_lines is not None and user_lines is not True: + logger.warning(f"[s3-json] Overriding lines={user_lines} with lines=True for JSON serialization.") + + wr.s3.to_json(df=df, path=s3_path, orient="records", lines=True, index=False, **kwargs) + + @staticmethod + @allow_options(pd.HDFStore.put) + def _write_s3_hdf_file(df: pd.DataFrame, s3_path: str, **kwargs): + """Write a DataFrame to S3 as an HDF5 file, using in-memory streaming.""" + parsed = urlparse(s3_path) + bucket = parsed.netloc + key = parsed.path.lstrip("/") + + # Separate protocol and HDF put options + pickle_protocol = kwargs.pop("pickle_protocol", None) + + fobj = io.BytesIO() + with utils.pickle_protocol(protocol=pickle_protocol): + HdfIO().save(df, fobj, options=kwargs) + + fobj.seek(0) + boto3.client("s3").upload_fileobj(fobj, bucket, key, ExtraArgs={"ACL": "bucket-owner-full-control"}) diff --git a/dynamicio/validations.py b/dynamicio/validations.py index 7650571..d224bb9 100644 --- a/dynamicio/validations.py +++ b/dynamicio/validations.py @@ -1,4 +1,5 @@ """Implements the Validator class responsible for various generic data validations and metrics generation.""" + import operator from typing import Callable, NamedTuple, Set diff --git a/poetry.lock b/poetry.lock index 34f58fa..0db7eb8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -65,23 +65,61 @@ tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "awscli" -version = "1.38.29" +version = "1.38.38" description = "Universal Command Line Environment for AWS." optional = false python-versions = ">=3.8" files = [ - {file = "awscli-1.38.29-py3-none-any.whl", hash = "sha256:a13c6a9c6a29b48a0b7bb2a0432a400aeabbac158f430a5bd6d1b5f717d441a8"}, - {file = "awscli-1.38.29.tar.gz", hash = "sha256:1f4176e606a40a6353aefc5f823801e56805b0634a9797d039188a9ef590e070"}, + {file = "awscli-1.38.38-py3-none-any.whl", hash = "sha256:1255c115108cf7d7612d66dc48a8db189b713d5a93a1fe9710d5005bc0e27230"}, + {file = "awscli-1.38.38.tar.gz", hash = "sha256:4a85cd26c6ec20253b99f5cc736e849ab4deaf11ce1e13fe88129598f4c90b85"}, ] [package.dependencies] -botocore = "1.37.29" +botocore = "1.37.38" colorama = ">=0.2.5,<0.4.7" -docutils = ">=0.10,<0.17" +docutils = ">=0.18.1,<=0.19" PyYAML = ">=3.10,<6.1" rsa = ">=3.1.2,<4.8" s3transfer = ">=0.11.0,<0.12.0" +[[package]] +name = "awswrangler" +version = "3.11.0" +description = "Pandas on AWS." +optional = false +python-versions = "<4.0,>=3.9" +files = [ + {file = "awswrangler-3.11.0-py3-none-any.whl", hash = "sha256:ca7a1027a33a8c8fcc140e30b5fb284e20e8ee2f74069e0e84d8a4d5cc788a12"}, + {file = "awswrangler-3.11.0.tar.gz", hash = "sha256:bdd05a96907a9d71896b80ec109b2fbc700427ebc2e3e6ed11a0490e135242a6"}, +] + +[package.dependencies] +boto3 = ">=1.20.32,<2.0.0" +botocore = ">=1.23.32,<2.0.0" +numpy = {version = ">=1.26,<3.0", markers = "python_version >= \"3.10\""} +packaging = ">=21.1,<25.0" +pandas = ">=1.2.0,<3.0.0" +pyarrow = {version = ">=8.0.0,<19.0.0", markers = "python_version < \"3.13\""} +setuptools = {version = "*", markers = "python_version >= \"3.12\""} +typing-extensions = ">=4.4.0,<5.0.0" + +[package.extras] +deltalake = ["deltalake (>=0.18.0,<0.24.0)"] +geopandas = ["geopandas (>=0.13.2,<0.14.0)", "geopandas (>=0.14.1,<0.15.0)"] +gremlin = ["aiohttp (>=3.9.0,<4.0.0)", "async-timeout (>=4.0.3,<6.0.0)", "gremlinpython (>=3.7.1,<4.0.0)", "requests (>=2.0.0,<3.0.0)"] +modin = ["modin (>=0.31,<0.33)"] +mysql = ["pymysql (>=1.0.0,<2.0.0)"] +opencypher = ["requests (>=2.0.0,<3.0.0)"] +openpyxl = ["openpyxl (>=3.0.0,<4.0.0)"] +opensearch = ["jsonpath-ng (>=1.5.3,<2.0.0)", "opensearch-py (>=2.0.0,<3.0.0)", "requests-aws4auth (>=1.1.1,<2.0.0)"] +oracle = ["oracledb (>=1,<3)"] +postgres = ["pg8000 (>=1.29.0,<2.0.0)"] +progressbar = ["progressbar2 (>=4.0.0,<5.0.0)"] +ray = ["ray[data,default] (>=2.30.0,<2.38.0)"] +redshift = ["redshift-connector (>=2.0.0,<3.0.0)"] +sparql = ["SPARQLWrapper (>=2.0.0,<3.0.0)", "requests (>=2.0.0,<3.0.0)"] +sqlserver = ["pyodbc (>=4,<6)"] + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -183,17 +221,17 @@ py-cpuinfo = "*" [[package]] name = "boto3" -version = "1.37.29" +version = "1.37.38" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.37.29-py3-none-any.whl", hash = "sha256:869979050e2cf6f5461503e0f1c8f226e47ec02802e88a2210f085ec22485945"}, - {file = "boto3-1.37.29.tar.gz", hash = "sha256:5702e38356b93c56ed2a27e17f7664d791f1fe2eafd58ae6ab3853b2804cadd2"}, + {file = "boto3-1.37.38-py3-none-any.whl", hash = "sha256:b6d42803607148804dff82389757827a24ce9271f0583748853934c86310999f"}, + {file = "boto3-1.37.38.tar.gz", hash = "sha256:88c02910933ab7777597d1ca7c62375f52822e0aa1a8e0c51b2598a547af42b2"}, ] [package.dependencies] -botocore = ">=1.37.29,<1.38.0" +botocore = ">=1.37.38,<1.38.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.11.0,<0.12.0" @@ -202,13 +240,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.37.29" +version = "1.37.38" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.37.29-py3-none-any.whl", hash = "sha256:092c41e346df37a8d7cf60a799791f8225ad3a5ba7cda749047eb31d1440b9c5"}, - {file = "botocore-1.37.29.tar.gz", hash = "sha256:728c1ef3b66a0f79bc08008a59f6fd6bef2a0a0195e5b3b9e9bef255df519890"}, + {file = "botocore-1.37.38-py3-none-any.whl", hash = "sha256:23b4097780e156a4dcaadfc1ed156ce25cb95b6087d010c4bb7f7f5d9bc9d219"}, + {file = "botocore-1.37.38.tar.gz", hash = "sha256:c3ea386177171f2259b284db6afc971c959ec103fa2115911c4368bea7cbbc5d"}, ] [package.dependencies] @@ -591,105 +629,118 @@ coverage = "*" [[package]] name = "cramjam" -version = "2.9.1" +version = "2.10.0" description = "Thin Python bindings to de/compression algorithms in Rust" optional = false python-versions = ">=3.8" files = [ - {file = "cramjam-2.9.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:8e82464d1e00fbbb12958999b8471ba5e9f3d9711954505a0a7b378762332e6f"}, - {file = "cramjam-2.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d2df8a6511cc08ef1fccd2e0c65e2ebc9f57574ec8376052a76851af5398810"}, - {file = "cramjam-2.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:21ea784e6c3f1843d3523ae0f03651dd06058b39eeb64beb82ee3b100fa83662"}, - {file = "cramjam-2.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e0c5d98a4e791f0bbd0ffcb7dae879baeb2dcc357348a8dc2be0a8c10403a2a"}, - {file = "cramjam-2.9.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e076fd87089197cb61117c63dbe7712ad5eccb93968860eb3bae09b767bac813"}, - {file = "cramjam-2.9.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6d86b44933aea0151e4a2e1e6935448499849045c38167d288ca4c59d5b8cd4e"}, - {file = "cramjam-2.9.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7eb032549dec897b942ddcf80c1cdccbcb40629f15fc902731dbe6362da49326"}, - {file = "cramjam-2.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf29b4def86ec503e329fe138842a9b79a997e3beb6c7809b05665a0d291edff"}, - {file = "cramjam-2.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a36adf7d13b7accfa206e1c917f08924eb905b45aa8e62176509afa7b14db71e"}, - {file = "cramjam-2.9.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:cf4ea758d98b6fad1b4b2d808d0de690d3162ac56c26968aea0af6524e3eb736"}, - {file = "cramjam-2.9.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:4826d6d81ea490fa7a3ae7a4b9729866a945ffac1f77fe57b71e49d6e1b21efd"}, - {file = "cramjam-2.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:335103317475bf992953c58838152a4761fc3c87354000edbfc4d7e57cf05909"}, - {file = "cramjam-2.9.1-cp310-cp310-win32.whl", hash = "sha256:258120cb1e3afc3443f756f9de161ed63eed56a2c31f6093e81c571c0f2dc9f6"}, - {file = "cramjam-2.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:c60e5996aa02547d12bc2740d44e90e006b0f93100f53206f7abe6732ad56e69"}, - {file = "cramjam-2.9.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b9db1debe48060e41a5b91af9193c524e473c57f6105462c5524a41f5aabdb88"}, - {file = "cramjam-2.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f6f18f0242212d3409d26ce3874937b5b979cebd61f08b633a6ea893c32fc7b6"}, - {file = "cramjam-2.9.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:b5b1cd7d39242b2b903cf09cd4696b3a6e04dc537ffa9f3ac8668edae76eecb6"}, - {file = "cramjam-2.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a47de0a68f5f4d9951250ef5af31f2a7228132caa9ed60994234f7eb98090d33"}, - {file = "cramjam-2.9.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e13c9a697881e5e38148958612dc6856967f5ff8cd7bba5ff751f2d6ac020aa4"}, - {file = "cramjam-2.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba560244bc1335b420b74e91e35f9d4e7f307a3be3a4603ce0f0d7e15a0acdf0"}, - {file = "cramjam-2.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d47fd41ce260cf4f0ff0e788de961fab9e9c6844a05ce55d06ce31e06107bdc"}, - {file = "cramjam-2.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84d154fbadece82935396eb6bcb502085d944d2fd13b07a94348364344370c2c"}, - {file = "cramjam-2.9.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:038df668ffb94d64d67b6ecc59cbd206745a425ffc0402897dde12d89fa6a870"}, - {file = "cramjam-2.9.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:4125d8cd86fa08495d310e80926c2f0563f157b76862e7479f9b2cf94823ea0c"}, - {file = "cramjam-2.9.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4206ebdd1d1ef0f3f86c8c2f7c426aa4af6094f4f41e274601fd4c4569f37454"}, - {file = "cramjam-2.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ab687bef5c493732b9a4ab870542ee43f5eae0025f9c684c7cb399c3a85cb380"}, - {file = "cramjam-2.9.1-cp311-cp311-win32.whl", hash = "sha256:dda7698b6d7caeae1047adafebc4b43b2a82478234f6c2b45bc3edad854e0600"}, - {file = "cramjam-2.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:872b00ff83e84bcbdc7e951af291ebe65eed20b09c47e7c4af21c312f90b796f"}, - {file = "cramjam-2.9.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:79417957972553502b217a0093532e48893c8b4ca30ccc941cefe9c72379df7c"}, - {file = "cramjam-2.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce2b94117f373defc876f88e74e44049a9969223dbca3240415b71752d0422fb"}, - {file = "cramjam-2.9.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:67040e0fd84404885ec716a806bee6110f9960c3647e0ef1670aab3b7375a70a"}, - {file = "cramjam-2.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bedb84e068b53c944bd08dcb501fd00d67daa8a917922356dd559b484ce7eab"}, - {file = "cramjam-2.9.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:06e3f97a379386d97debf08638a78b3d3850fdf6124755eb270b54905a169930"}, - {file = "cramjam-2.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11118675e9c7952ececabc62f023290ee4f8ecf0bee0d2c7eb8d1c402ee9769d"}, - {file = "cramjam-2.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b7de6b61b11545570e4d6033713f3599525efc615ee353a822be8f6b0c65b77"}, - {file = "cramjam-2.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57ca8f3775324a9de3ee6f05ca172687ba258c0dea79f7e3a6b4112834982f2a"}, - {file = "cramjam-2.9.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9847dd6f288f1c56359f52acb48ff2df848ff3e3bff34d23855bbcf7016427cc"}, - {file = "cramjam-2.9.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d1248dfa7f151e893ce819670f00879e4b7650b8d4c01279ce4f12140d68dd2"}, - {file = "cramjam-2.9.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9da6d970281083bae91b914362de325414aa03c01fc806f6bb2cc006322ec834"}, - {file = "cramjam-2.9.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1c33bc095db5733c841a102b8693062be5db8cdac17b9782ebc00577c6a94480"}, - {file = "cramjam-2.9.1-cp312-cp312-win32.whl", hash = "sha256:9e9193cd4bb57e7acd3af24891526299244bfed88168945efdaa09af4e50720f"}, - {file = "cramjam-2.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:15955dd75e80f66c1ea271167a5347661d9bdc365f894a57698c383c9b7d465c"}, - {file = "cramjam-2.9.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5a7797a2fff994fc5e323f7a967a35a3e37e3006ed21d64dcded086502f482af"}, - {file = "cramjam-2.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d51b9b140b1df39a44bff7896d98a10da345b7d5f5ce92368d328c1c2c829167"}, - {file = "cramjam-2.9.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:07ac76b7f992556e7aa910244be11ece578cdf84f4d5d5297461f9a895e18312"}, - {file = "cramjam-2.9.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d90a72608c7550cd7eba914668f6277bfb0b24f074d1f1bd9d061fcb6f2adbd6"}, - {file = "cramjam-2.9.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56495975401b1821dbe1f29cf222e23556232209a2fdb809fe8156d120ca9c7f"}, - {file = "cramjam-2.9.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b695259e71fde6d5be66b77a4474523ced9ffe9fe8a34cb9b520ec1241a14d3"}, - {file = "cramjam-2.9.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab1e69dc4831bbb79b6d547077aae89074c83e8ad94eba1a3d80e94d2424fd02"}, - {file = "cramjam-2.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:440b489902bfb7a26d3fec1ca888007615336ff763d2a32a2fc40586548a0dbf"}, - {file = "cramjam-2.9.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:217fe22b41f8c3dce03852f828b059abfad11d1344a1df2f43d3eb8634b18d75"}, - {file = "cramjam-2.9.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:95f3646ddc98af25af25d5692ae65966488a283813336ea9cf41b22e542e7c0d"}, - {file = "cramjam-2.9.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:6b19fc60ead1cae9795a5b359599da3a1c95d38f869bdfb51c441fd76b04e926"}, - {file = "cramjam-2.9.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:8dc5207567459d049696f62a1fdfb220f3fe6aa0d722285d44753e12504dac6c"}, - {file = "cramjam-2.9.1-cp313-cp313-win32.whl", hash = "sha256:fbfe35929a61b914de9e5dbacde0cfbba86cbf5122f9285a24c14ed0b645490b"}, - {file = "cramjam-2.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:06068bd191a82ad4fc1ac23d6f8627fb5e37ec4be0431711b9a2dbacaccfeddb"}, - {file = "cramjam-2.9.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a2ca4d3c683d28d3217821029eb08d3487d5043d7eb455df11ff3cacfd4c916"}, - {file = "cramjam-2.9.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:008b49b455b396acc5459dfb06fb9d56049c4097ee8e590892a4d3da9a711da3"}, - {file = "cramjam-2.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:45c18cc13156e8697a8d3f9e57e49a69b00e14a103196efab0893fae1a5257f8"}, - {file = "cramjam-2.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d14a0efb21e0fec0631bcd66040b06e6a0fe10825f3aacffded38c1c978bdff9"}, - {file = "cramjam-2.9.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3f815fb0eba625af45139af4f90f5fc2ddda61b171c2cc3ab63d44b40c5c7768"}, - {file = "cramjam-2.9.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:04828cbfad7384f06a4a7d0d927c3e85ef11dc5a40b9cf5f3e29ac4e23ecd678"}, - {file = "cramjam-2.9.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0944a7c3a78f940c06d1b29bdce91a17798d80593dd01ebfeb842761e48a8b5"}, - {file = "cramjam-2.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec769e5b16251704502277a1163dcf2611551452d7590ff4cc422b7b0367fc96"}, - {file = "cramjam-2.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3ba79c7d2cc5adb897b690c05dd9b67c4d401736d207314b99315f7be3cd94fd"}, - {file = "cramjam-2.9.1-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:d35923fb5411bde30b53c0696dff8e24c8a38b010b89544834c53f4462fd71df"}, - {file = "cramjam-2.9.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:da0cc0efdbfb8ee2361f89f38ded03d11678f37e392afff7a97b09c55dadfc83"}, - {file = "cramjam-2.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f89924858712b8b936f04f3d690e72825a3e5127a140b434c79030c1c5a887ce"}, - {file = "cramjam-2.9.1-cp38-cp38-win32.whl", hash = "sha256:5925a738b8478f223ab9756fc794e3cabd5917fd7846f66adcf1d5fc2bf9864c"}, - {file = "cramjam-2.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:b7ac273498a2c6772d67707e101b74014c0d9413bb4711c51d8ec311de59b4b1"}, - {file = "cramjam-2.9.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:af39006faddfc6253beb93ca821d544931cfee7f0177b99ff106dfd8fd6a2cd8"}, - {file = "cramjam-2.9.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b3291be0d3f73d5774d69013be4ab33978c777363b5312d14f62f77817c2f75a"}, - {file = "cramjam-2.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1539fd758f0e57fad7913cebff8baaee871bb561ddf6fa710a427b74da6b6778"}, - {file = "cramjam-2.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff362f68bd68ac0eccb445209238d589bba728fb6d7f2e9dc199e0ec3a61d6e0"}, - {file = "cramjam-2.9.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:23b9786d1d17686fb8d600ade2a19374c7188d4b8867efa9af0d8274a220aec7"}, - {file = "cramjam-2.9.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8bc9c2c748aaf91863d89c4583f529c1c709485c94f8dfeb3ee48662d88e3258"}, - {file = "cramjam-2.9.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd0fa9a0e7f18224b6d2d1d69dbdc3aecec80ef1393c59244159b131604a4395"}, - {file = "cramjam-2.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ceef6e09ee22457997370882aa3c69de01e6dd0aaa2f953e1e87ad11641d042"}, - {file = "cramjam-2.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1376f6fdbf0b30712413a0b4e51663a4938ae2f6b449f8e4635dbb3694db83cf"}, - {file = "cramjam-2.9.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:342fb946f8d3e9e35b837288b03ab23cfbe0bb5a30e582ed805ef79706823a96"}, - {file = "cramjam-2.9.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a237064a6e2c2256c9a1cf2beb7c971382190c0f1eb2e810e02e971881756132"}, - {file = "cramjam-2.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53145fc9f2319c1245d4329e1da8cfacd6e35e27090c07c0b9d453ae2bbdac3e"}, - {file = "cramjam-2.9.1-cp39-cp39-win32.whl", hash = "sha256:8a9f52c27292c21457f43c4ce124939302a9acfb62295e7cda8667310563a5a3"}, - {file = "cramjam-2.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:8097ee39b61c86848a443c0b25b2df1de6b331fd512b20836a4f5cfde51ab255"}, - {file = "cramjam-2.9.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:86824c695688fcd06c5ac9bbd3fea9bdfb4cca194b1e706fbf11a629df48d2b4"}, - {file = "cramjam-2.9.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:27571bfa5a5d618604696747d0dc1d2a99b5906c967c8dee53c13a7107edfde6"}, - {file = "cramjam-2.9.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb01f6e38719818778144d3165a89ea1ad9dc58c6342b7f20aa194c70f34cbd1"}, - {file = "cramjam-2.9.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b5cef5cf40725fe64592af9ec163e7389855077700678a1d94bec549403a74d"}, - {file = "cramjam-2.9.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ac48b978aa0675f62b642750e798c394a64d25ce852e4e541f69bef9a564c2f0"}, - {file = "cramjam-2.9.1.tar.gz", hash = "sha256:336cc591d86cbd225d256813779f46624f857bc9c779db126271eff9ddc524ae"}, + {file = "cramjam-2.10.0-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:26c44f17938cf00a339899ce6ea7ba12af7b1210d707a80a7f14724fba39869b"}, + {file = "cramjam-2.10.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:ce208a3e4043b8ce89e5d90047da16882456ea395577b1ee07e8215dce7d7c91"}, + {file = "cramjam-2.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2c24907c972aca7b56c8326307e15d78f56199852dda1e67e4e54c2672afede4"}, + {file = "cramjam-2.10.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f25db473667774725e4f34e738d644ffb205bf0bdc0e8146870a1104c5f42e4a"}, + {file = "cramjam-2.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51eb00c72d4a93e4a2ddcc751ba2a7a1318026247e80742866912ec82b39e5ce"}, + {file = "cramjam-2.10.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:def47645b1b970fd97f063da852b0ddc4f5bdee9af8d5b718d9682c7b828d89d"}, + {file = "cramjam-2.10.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:42dcd7c83104edae70004a8dc494e4e57de4940e3019e5d2cbec2830d5908a85"}, + {file = "cramjam-2.10.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0744e391ea8baf0ddea5a180b0aa71a6a302490c14d7a37add730bf0172c7c6"}, + {file = "cramjam-2.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5018c7414047f640b126df02e9286a8da7cc620798cea2b39bac79731c2ee336"}, + {file = "cramjam-2.10.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4b201aacc7a06079b063cfbcf5efe78b1e65c7279b2828d06ffaa90a8316579d"}, + {file = "cramjam-2.10.0-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:5264ac242697fbb1cfffa79d0153cbc4c088538bd99d60cfa374e8a8b83e2bb5"}, + {file = "cramjam-2.10.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e193918c81139361f3f45db19696d31847601f2c0e79a38618f34d7bff6ee704"}, + {file = "cramjam-2.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22a7ab05c62b0a71fcd6db4274af1508c5ea039a43fb143ac50a62f86e6f32f7"}, + {file = "cramjam-2.10.0-cp310-cp310-win32.whl", hash = "sha256:2464bdf0e2432e0f07a834f48c16022cd7f4648ed18badf52c32c13d6722518c"}, + {file = "cramjam-2.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:73b6ffc8ffe6546462ccc7e34ca3acd9eb3984e1232645f498544a7eab6b8aca"}, + {file = "cramjam-2.10.0-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:fb73ee9616e3efd2cf3857b019c66f9bf287bb47139ea48425850da2ae508670"}, + {file = "cramjam-2.10.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:acef0e2c4d9f38428721a0ec878dee3fb73a35e640593d99c9803457dbb65214"}, + {file = "cramjam-2.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b21b1672814ecce88f1da76635f0483d2d877d4cb8998db3692792f46279bf1"}, + {file = "cramjam-2.10.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7699d61c712bc77907c48fe63a21fffa03c4dd70401e1d14e368af031fde7c21"}, + {file = "cramjam-2.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3484f1595eef64cefed05804d7ec8a88695f89086c49b086634e44c16f3d4769"}, + {file = "cramjam-2.10.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38fba4594dd0e2b7423ef403039e63774086ebb0696d9060db20093f18a2f43e"}, + {file = "cramjam-2.10.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b07fe3e48c881a75a11f722e1d5b052173b5e7c78b22518f659b8c9b4ac4c937"}, + {file = "cramjam-2.10.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3596b6ceaf85f872c1e56295c6ec80bb15fdd71e7ed9e0e5c3e654563dcc40a2"}, + {file = "cramjam-2.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1c03360c1760f8608dc5ce1ddd7e5491180765360cae8104b428d5f86fbe1b9"}, + {file = "cramjam-2.10.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3e0b70fe7796b63b87cb7ebfaad0ebaca7574fdf177311952f74b8bda6522fb8"}, + {file = "cramjam-2.10.0-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:d61a21e4153589bd53ffe71b553f93f2afbc8fb7baf63c91a83c933347473083"}, + {file = "cramjam-2.10.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:91ab85752a08dc875a05742cfda0234d7a70fadda07dd0b0582cfe991911f332"}, + {file = "cramjam-2.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c6afff7e9da53afb8d11eae27a20ee5709e2943b39af6c949b38424d0f271569"}, + {file = "cramjam-2.10.0-cp311-cp311-win32.whl", hash = "sha256:adf484b06063134ae604d4fc826d942af7e751c9d0b2fcab5bf1058a8ebe242b"}, + {file = "cramjam-2.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:9e20ebea6ec77232cd12e4084c8be6d03534dc5f3d027d365b32766beafce6c3"}, + {file = "cramjam-2.10.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0acb17e3681138b48300b27d3409742c81d5734ec39c650a60a764c135197840"}, + {file = "cramjam-2.10.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:647553c44cf6b5ce2d9b56e743cc1eab886940d776b36438183e807bb5a7a42b"}, + {file = "cramjam-2.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5c52805c7ccb533fe42d3d36c91d237c97c3b6551cd6b32f98b79eeb30d0f139"}, + {file = "cramjam-2.10.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:337ceb50bde7708b2a4068f3000625c23ceb1b2497edce2e21fd08ef58549170"}, + {file = "cramjam-2.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c071765bdd5eefa3b2157a61e84d72e161b63f95eb702a0133fee293800a619"}, + {file = "cramjam-2.10.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b40d46d2aa566f8e3def953279cce0191e47364b453cda492db12a84dd97f78"}, + {file = "cramjam-2.10.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c7bab3703babb93c9dd4444ac9797d01ec46cf521e247d3319bfb292414d053"}, + {file = "cramjam-2.10.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ba19308b8e19cdaadfbf47142f52b705d2cbfb8edd84a8271573e50fa7fa022d"}, + {file = "cramjam-2.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de3e4be5aa71b73c2640c9b86e435ec033592f7f79787937f8342259106a63ae"}, + {file = "cramjam-2.10.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:11c5ef0c70d6bdd8e1d8afed8b0430709b22decc3865eb6c0656aa00117a7b3d"}, + {file = "cramjam-2.10.0-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:86b29e349064821ceeb14d60d01a11a0788f94e73ed4b3a5c3f9fac7aa4e2cd7"}, + {file = "cramjam-2.10.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:2c7008bb54bdc5d130c0e8581925dfcbdc6f0a4d2051de7a153bfced9a31910f"}, + {file = "cramjam-2.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a94fe7024137ed8bf200308000d106874afe52ff203f852f43b3547eddfa10e"}, + {file = "cramjam-2.10.0-cp312-cp312-win32.whl", hash = "sha256:ce11be5722c9d433c5e1eb3980f16eb7d80828b9614f089e28f4f1724fc8973f"}, + {file = "cramjam-2.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:a01e89e99ba066dfa2df40fe99a2371565f4a3adc6811a73c8019d9929a312e8"}, + {file = "cramjam-2.10.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8bb0b6aaaa5f37091e05d756a3337faf0ddcffe8a68dbe8a710731b0d555ec8f"}, + {file = "cramjam-2.10.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:27b2625c0840b9a5522eba30b165940084391762492e03b9d640fca5074016ae"}, + {file = "cramjam-2.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4ba90f7b8f986934f33aad8cc029cf7c74842d3ecd5eda71f7531330d38a8dc4"}, + {file = "cramjam-2.10.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6655d04942f7c02087a6bba4bdc8d88961aa8ddf3fb9a05b3bad06d2d1ca321b"}, + {file = "cramjam-2.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7dda9be2caf067ac21c4aa63497833e0984908b66849c07aaa42b1cfa93f5e1c"}, + {file = "cramjam-2.10.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:afa36aa006d7692718fce427ecb276211918447f806f80c19096a627f5122e3d"}, + {file = "cramjam-2.10.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d46fd5a9e8eb5d56eccc6191a55e3e1e2b3ab24b19ab87563a2299a39c855fd7"}, + {file = "cramjam-2.10.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3012564760394dff89e7a10c5a244f8885cd155aec07bdbe2d6dc46be398614"}, + {file = "cramjam-2.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2d216ed4aca2090eabdd354204ae55ed3e13333d1a5b271981543696e634672"}, + {file = "cramjam-2.10.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:44c2660ee7c4c269646955e4e40c2693f803fbad12398bb31b2ad00cfc6027b8"}, + {file = "cramjam-2.10.0-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:636a48e2d01fe8d7955e9523efd2f8efce55a0221f3b5d5b4bdf37c7ff056bf1"}, + {file = "cramjam-2.10.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:44c15f6117031a84497433b5f55d30ee72d438fdcba9778fec0c5ca5d416aa96"}, + {file = "cramjam-2.10.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:76e4e42f2ecf1aca0a710adaa23000a192efb81a2aee3bcc16761f1777f08a74"}, + {file = "cramjam-2.10.0-cp313-cp313-win32.whl", hash = "sha256:5b34f4678d386c64d3be402fdf67f75e8f1869627ea2ec4decd43e828d3b6fba"}, + {file = "cramjam-2.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:88754dd516f0e2f4dd242880b8e760dc854e917315a17fe3fc626475bea9b252"}, + {file = "cramjam-2.10.0-cp38-cp38-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:645827af834a64145ba4b06f703342b2dbe1d40d1a48fb04e82373bd95cf68e2"}, + {file = "cramjam-2.10.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:570c81f991033e624874475ade96b601f1db2c51b3e69c324072adcfb23ef5aa"}, + {file = "cramjam-2.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:06ad4a8b368d30ded1d932d9eed647962fbe44923269185a6bbd5e0d11cc39ab"}, + {file = "cramjam-2.10.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bcedda2ef2560e6e62cac03734ab1ad28616206b4d4f2d138440b4f43e18c395"}, + {file = "cramjam-2.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68362d87372a90b9717536238c81d74d7feb4a14392ac239ceb61c1c199a9bac"}, + {file = "cramjam-2.10.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff7b95bd299c9360e7cb8d226002d58e2917f594ea5af0373efc713f896622b9"}, + {file = "cramjam-2.10.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2742eea6e336961167c5b6a2393fa04d54bdb10980f0d60ea36ed0a824e9a20"}, + {file = "cramjam-2.10.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8695857e0b0b5289fabb6c200b95e2b18d8575551ddd9d50746b3d78b6fb5aa8"}, + {file = "cramjam-2.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac5a8a3ef660e6869a7761cd0664223eb546b2d17e9121c8ab0ad46353635611"}, + {file = "cramjam-2.10.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6d86c1e2006fe82a8679ed851c2462a6019b57255b3902d16ac35df4a37f6cdd"}, + {file = "cramjam-2.10.0-cp38-cp38-musllinux_1_1_armv7l.whl", hash = "sha256:a094ca72440364bc1d0a793555875e515b0d7cc0eef171f4cd49c7e4855ba06e"}, + {file = "cramjam-2.10.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:05793857773ec62101edf2c0d22d8edc955707727124f637d2f6cc138e5f97aa"}, + {file = "cramjam-2.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b8dee2e4a402dac2df110e7b02fae49507a63b44b6fd91350cf069f31545a925"}, + {file = "cramjam-2.10.0-cp38-cp38-win32.whl", hash = "sha256:001fc2572adc655406fb899087f57a740e58a800b05acdccac8bf5759b617d90"}, + {file = "cramjam-2.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:9cadef44f5ad4c5b4d06ba3c28464d70241a40539c0343b1821ba43102b6a9fc"}, + {file = "cramjam-2.10.0-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:967f5f0f22bf5dba4e4d7abe9594b28f5da95606225a50555926ff6e975d84dd"}, + {file = "cramjam-2.10.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:260732e3b5c56d6182586f3a7fc5e3f3641b27bfbad5883e8d8e292af85a6870"}, + {file = "cramjam-2.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eafdc9d1721afcb4be9d20b980b61d404a592c19067197976a4077f52727bd1a"}, + {file = "cramjam-2.10.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:28a13c0317e71121b2059ffa8beefa2b185be241c52f740f6eb261f0067186db"}, + {file = "cramjam-2.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3e0067ae3513e4cbd0efbabbe5a2bcfa2c2d4bddc67188eeb0751b9a02fdb7"}, + {file = "cramjam-2.10.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:112638a4cdf806509d2d2661cb519d239d731bd5fd2e95f211c48ac0f0deeab5"}, + {file = "cramjam-2.10.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ddbf6a3d3def7ae46638ebf87d7746ccebf22f885a87884ac24d97943af3f30"}, + {file = "cramjam-2.10.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a2923b8cd2fcbd22e0842decb66bf925a9e95bda165490d037c355e5df8fef68"}, + {file = "cramjam-2.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ab6f36c772109c974890eafff2a841ddbf38ea1293b01a778b28f26089a890d"}, + {file = "cramjam-2.10.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:17dda15edf256362edb30dcb1d5ecdcd727d946c6be0d1b130e736f3f49487dc"}, + {file = "cramjam-2.10.0-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:92fd6e784ade210c3522bc627b3938821d12fac52acefe4d6630460e243e28de"}, + {file = "cramjam-2.10.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a120fc0514c9ed9a4051d040ddd36176241d4f54c4a37d8e4f3d29ac9bdb4c3a"}, + {file = "cramjam-2.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a71ab695a16c6d5aeae1f02fcc37fbd1ae876e8fb339337aca187012a3d6c0a2"}, + {file = "cramjam-2.10.0-cp39-cp39-win32.whl", hash = "sha256:61b7f3c81e5e9015e73e5f423706b2f5e85a07ce79dea35645fad93505ff06cf"}, + {file = "cramjam-2.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:0d27fe3e316f9ae7fe1367b6daf0ffc993c1c66edae588165ac0f41f91a5a6b1"}, + {file = "cramjam-2.10.0-pp310-pypy310_pp73-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77192bc1a9897ecd91cf977a5d5f990373e35a8d028c9141c8c3d3680a4a4cd7"}, + {file = "cramjam-2.10.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:50b59e981f219d6840ac43cda8e885aff1457944ddbabaa16ac047690bfd6ad1"}, + {file = "cramjam-2.10.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d84581c869d279fab437182d5db2b590d44975084e8d50b164947f7aaa2c5f25"}, + {file = "cramjam-2.10.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:04f54bea9ce39c440d1ac6901fe4d647f9218dd5cd8fe903c6fe9c42bf5e1f3b"}, + {file = "cramjam-2.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cddd12ee5a2ef4100478db7f5563a9cdb8bc0a067fbd8ccd1ecdc446d2e6a41a"}, + {file = "cramjam-2.10.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:35bcecff38648908a4833928a892a1e7a32611171785bef27015107426bc1d9d"}, + {file = "cramjam-2.10.0-pp311-pypy311_pp73-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:1e826469cfbb6dcd5b967591e52855073267835229674cfa3d327088805855da"}, + {file = "cramjam-2.10.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1a200b74220dcd80c2bb99e3bfe1cdb1e4ed0f5c071959f4316abd65f9ef1e39"}, + {file = "cramjam-2.10.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:2e419b65538786fc1f0cf776612262d4bf6c9449983d3fc0d0acfd86594fe551"}, + {file = "cramjam-2.10.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf1321a40da930edeff418d561dfb03e6d59d5b8ab5cbab1c4b03ff0aa4c6d21"}, + {file = "cramjam-2.10.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a04376601c8f9714fb3a6a0a1699b85aab665d9d952a2a31fb37cf70e1be1fba"}, + {file = "cramjam-2.10.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2c1eb6e6c3d5c1cc3f7c7f8a52e034340a3c454641f019687fa94077c05da5c2"}, + {file = "cramjam-2.10.0.tar.gz", hash = "sha256:e821dd487384ae8004e977c3b13135ad6665ccf8c9874e68441cad1146e66d8a"}, ] [package.extras] -dev = ["black (==22.3.0)", "hypothesis", "numpy", "pytest (>=5.30)", "pytest-benchmark", "pytest-xdist"] +dev = ["black (==22.3.0)", "hypothesis (<6.123.0)", "numpy", "pytest (>=5.30)", "pytest-benchmark", "pytest-xdist"] [[package]] name = "cryptography" @@ -750,13 +801,13 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "dill" -version = "0.3.9" +version = "0.4.0" description = "serialize all of Python" optional = false python-versions = ">=3.8" files = [ - {file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"}, - {file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"}, + {file = "dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049"}, + {file = "dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0"}, ] [package.extras] @@ -776,13 +827,13 @@ files = [ [[package]] name = "docutils" -version = "0.16" +version = "0.19" description = "Docutils -- Python Documentation Utilities" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.7" files = [ - {file = "docutils-0.16-py2.py3-none-any.whl", hash = "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af"}, - {file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"}, + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, ] [[package]] @@ -1013,84 +1064,66 @@ trusted-deps = ["Click (==8.0.3)", "arrow (==1.2.1)", "sh (==1.14.2)"] [[package]] name = "greenlet" -version = "3.1.1" +version = "3.2.0" description = "Lightweight in-process concurrent programming" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"}, - {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"}, - {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"}, - {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"}, - {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"}, - {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"}, - {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"}, - {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"}, - {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"}, - {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"}, - {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"}, - {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"}, - {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"}, - {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"}, - {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"}, - {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"}, - {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"}, - {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de"}, - {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa"}, - {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af"}, - {file = "greenlet-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798"}, - {file = "greenlet-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef"}, - {file = "greenlet-3.1.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1"}, - {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd"}, - {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7"}, - {file = "greenlet-3.1.1-cp38-cp38-win32.whl", hash = "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef"}, - {file = "greenlet-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d"}, - {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"}, - {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"}, - {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"}, - {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"}, - {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"}, - {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"}, + {file = "greenlet-3.2.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:b7a7b7f2bad3ca72eb2fa14643f1c4ca11d115614047299d89bc24a3b11ddd09"}, + {file = "greenlet-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60e77242e38e99ecaede853755bbd8165e0b20a2f1f3abcaa6f0dceb826a7411"}, + {file = "greenlet-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f32d7c70b1c26844fd0e4e56a1da852b493e4e1c30df7b07274a1e5a9b599e"}, + {file = "greenlet-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97bc1be4bad83b70d8b8627ada6724091af41139616696e59b7088f358583b9"}, + {file = "greenlet-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23f56a0103deb5570c8d6a0bb4ddf8a7a28931973ad7ed7a883460a67e599b32"}, + {file = "greenlet-3.2.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2919b126eeb63ca5fa971501cd20cd6cdb5522369a8e39548bbc73a3e10b8b41"}, + {file = "greenlet-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:844acfd479ee380f3810415e682c9ee941725fb90b45e139bb7fd6f85c6c9a30"}, + {file = "greenlet-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b986f1a6467710e7ffeeeac1777da0318c95bbfcc467acbd0bd35abc775f558"}, + {file = "greenlet-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:29449a2b82ed7ce11f8668c31ef20d31e9d88cd8329eb933098fab5a8608a93a"}, + {file = "greenlet-3.2.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b99de16560097b9984409ded0032f101f9555e1ab029440fc6a8b5e76dbba7ac"}, + {file = "greenlet-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0bc5776ac2831c022e029839bf1b9d3052332dcf5f431bb88c8503e27398e31"}, + {file = "greenlet-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1dcb1108449b55ff6bc0edac9616468f71db261a4571f27c47ccf3530a7f8b97"}, + {file = "greenlet-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a68a25a08f51fc8b66b113d1d9863ee123cdb0e8f1439aed9fc795cd6f85cf"}, + {file = "greenlet-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fee6f518868e8206c617f4084a83ad4d7a3750b541bf04e692dfa02e52e805d"}, + {file = "greenlet-3.2.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6fad8a9ca98b37951a053d7d2d2553569b151cd8c4ede744806b94d50d7f8f73"}, + {file = "greenlet-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e14541f9024a280adb9645143d6a0a51fda6f7c5695fd96cb4d542bb563442f"}, + {file = "greenlet-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7f163d04f777e7bd229a50b937ecc1ae2a5b25296e6001445e5433e4f51f5191"}, + {file = "greenlet-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:39801e633a978c3f829f21022501e7b0c3872683d7495c1850558d1a6fb95ed0"}, + {file = "greenlet-3.2.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:7d08b88ee8d506ca1f5b2a58744e934d33c6a1686dd83b81e7999dfc704a912f"}, + {file = "greenlet-3.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58ef3d637c54e2f079064ca936556c4af3989144e4154d80cfd4e2a59fc3769c"}, + {file = "greenlet-3.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33ea7e7269d6f7275ce31f593d6dcfedd97539c01f63fbdc8d84e493e20b1b2c"}, + {file = "greenlet-3.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e61d426969b68b2170a9f853cc36d5318030494576e9ec0bfe2dc2e2afa15a68"}, + {file = "greenlet-3.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04e781447a4722e30b4861af728cb878d73a3df79509dc19ea498090cea5d204"}, + {file = "greenlet-3.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b2392cc41eeed4055978c6b52549ccd9effd263bb780ffd639c0e1e7e2055ab0"}, + {file = "greenlet-3.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:430cba962c85e339767235a93450a6aaffed6f9c567e73874ea2075f5aae51e1"}, + {file = "greenlet-3.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5e57ff52315bfc0c5493917f328b8ba3ae0c0515d94524453c4d24e7638cbb53"}, + {file = "greenlet-3.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:211a9721f540e454a02e62db7956263e9a28a6cf776d4b9a7213844e36426333"}, + {file = "greenlet-3.2.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b86a3ccc865ae601f446af042707b749eebc297928ea7bd0c5f60c56525850be"}, + {file = "greenlet-3.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:144283ad88ed77f3ebd74710dd419b55dd15d18704b0ae05935766a93f5671c5"}, + {file = "greenlet-3.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5be69cd50994b8465c3ad1467f9e63001f76e53a89440ad4440d1b6d52591280"}, + {file = "greenlet-3.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:47aeadd1e8fbdef8fdceb8fb4edc0cbb398a57568d56fd68f2bc00d0d809e6b6"}, + {file = "greenlet-3.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18adc14ab154ca6e53eecc9dc50ff17aeb7ba70b7e14779b26e16d71efa90038"}, + {file = "greenlet-3.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8622b33d8694ec373ad55050c3d4e49818132b44852158442e1931bb02af336"}, + {file = "greenlet-3.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:e8ac9a2c20fbff3d0b853e9ef705cdedb70d9276af977d1ec1cde86a87a4c821"}, + {file = "greenlet-3.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:cd37273dc7ca1d5da149b58c8b3ce0711181672ba1b09969663905a765affe21"}, + {file = "greenlet-3.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:8a8940a8d301828acd8b9f3f85db23069a692ff2933358861b19936e29946b95"}, + {file = "greenlet-3.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee59db626760f1ca8da697a086454210d36a19f7abecc9922a2374c04b47735b"}, + {file = "greenlet-3.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7154b13ef87a8b62fc05419f12d75532d7783586ad016c57b5de8a1c6feeb517"}, + {file = "greenlet-3.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:199453d64b02d0c9d139e36d29681efd0e407ed8e2c0bf89d88878d6a787c28f"}, + {file = "greenlet-3.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0010e928e1901d36625f21d008618273f9dda26b516dbdecf873937d39c9dff0"}, + {file = "greenlet-3.2.0-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6005f7a86de836a1dc4b8d824a2339cdd5a1ca7cb1af55ea92575401f9952f4c"}, + {file = "greenlet-3.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:17fd241c0d50bacb7ce8ff77a30f94a2d0ca69434ba2e0187cf95a5414aeb7e1"}, + {file = "greenlet-3.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:7b17a26abc6a1890bf77d5d6b71c0999705386b00060d15c10b8182679ff2790"}, + {file = "greenlet-3.2.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:397b6bbda06f8fe895893d96218cd6f6d855a6701dc45012ebe12262423cec8b"}, + {file = "greenlet-3.2.0-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:4174fa6fa214e8924cedf332b6f2395ba2b9879f250dacd3c361b2fca86f58af"}, + {file = "greenlet-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6017a4d430fad5229e397ad464db504ae70cb7b903757c4688cee6c25d6ce8d8"}, + {file = "greenlet-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:78b721dfadc60e3639141c0e1f19d23953c5b4b98bfcaf04ce40f79e4f01751c"}, + {file = "greenlet-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fd2583024ff6cd5d4f842d446d001de4c4fe1264fdb5f28ddea28f6488866df"}, + {file = "greenlet-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da3bd464c2cc411b723e3d4afc27b13c219ac077ba897bac88443ae45f5ec"}, + {file = "greenlet-3.2.0-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2688b3bd3198cc4bad7a79648a95fee088c24a0f6abd05d3639e6c3040ded015"}, + {file = "greenlet-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1cf89e2d92bae0d7e2d6093ce0bed26feeaf59a5d588e3984e35fcd46fc41090"}, + {file = "greenlet-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b3538711e7c0efd5f7a8fc1096c4db9598d6ed99dc87286b31e4ce9f8a8da67"}, + {file = "greenlet-3.2.0-cp39-cp39-win32.whl", hash = "sha256:ce531d7c424ef327a391de7a9777a6c93a38e1f89e18efa903a1c4ba11f85905"}, + {file = "greenlet-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7b162de2fb61b4c7f4b5d749408bf3280cae65db9b5a6aaf7f922ac829faa67c"}, + {file = "greenlet-3.2.0.tar.gz", hash = "sha256:1d2d43bd711a43db8d9b9187500e6432ddb4fafe112d082ffabca8660a9e01a7"}, ] [package.extras] @@ -1099,13 +1132,13 @@ test = ["objgraph", "psutil"] [[package]] name = "identify" -version = "2.6.9" +version = "2.6.10" description = "File identification library for Python" optional = false python-versions = ">=3.9" files = [ - {file = "identify-2.6.9-py2.py3-none-any.whl", hash = "sha256:c98b4322da415a8e5a70ff6e51fbc2d2932c015532d77e9f8537b4ba7813b150"}, - {file = "identify-2.6.9.tar.gz", hash = "sha256:d40dfe3142a1421d8518e3d3985ef5ac42890683e32306ad614a29490abeb6bf"}, + {file = "identify-2.6.10-py2.py3-none-any.whl", hash = "sha256:5f34248f54136beed1a7ba6a6b5c4b6cf21ff495aac7c359e1ef831ae3b8ab25"}, + {file = "identify-2.6.10.tar.gz", hash = "sha256:45e92fd704f3da71cc3880036633f48b4b7265fd4de2b57627cb157216eb7eb8"}, ] [package.extras] @@ -1288,48 +1321,25 @@ type = ["pygobject-stubs", "pytest-mypy", "shtab", "types-pywin32"] [[package]] name = "lazy-object-proxy" -version = "1.10.0" +version = "1.11.0" description = "A fast and thorough lazy object proxy." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "lazy-object-proxy-1.10.0.tar.gz", hash = "sha256:78247b6d45f43a52ef35c25b5581459e85117225408a4128a3daf8bf9648ac69"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:855e068b0358ab916454464a884779c7ffa312b8925c6f7401e952dcf3b89977"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab7004cf2e59f7c2e4345604a3e6ea0d92ac44e1c2375527d56492014e690c3"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc0d2fc424e54c70c4bc06787e4072c4f3b1aa2f897dfdc34ce1013cf3ceef05"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e2adb09778797da09d2b5ebdbceebf7dd32e2c96f79da9052b2e87b6ea495895"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1f711e2c6dcd4edd372cf5dec5c5a30d23bba06ee012093267b3376c079ec83"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-win32.whl", hash = "sha256:76a095cfe6045c7d0ca77db9934e8f7b71b14645f0094ffcd842349ada5c5fb9"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:b4f87d4ed9064b2628da63830986c3d2dca7501e6018347798313fcf028e2fd4"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fec03caabbc6b59ea4a638bee5fce7117be8e99a4103d9d5ad77f15d6f81020c"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02c83f957782cbbe8136bee26416686a6ae998c7b6191711a04da776dc9e47d4"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:009e6bb1f1935a62889ddc8541514b6a9e1fcf302667dcb049a0be5c8f613e56"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75fc59fc450050b1b3c203c35020bc41bd2695ed692a392924c6ce180c6f1dc9"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:782e2c9b2aab1708ffb07d4bf377d12901d7a1d99e5e410d648d892f8967ab1f"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-win32.whl", hash = "sha256:edb45bb8278574710e68a6b021599a10ce730d156e5b254941754a9cc0b17d03"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:e271058822765ad5e3bca7f05f2ace0de58a3f4e62045a8c90a0dfd2f8ad8cc6"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e98c8af98d5707dcdecc9ab0863c0ea6e88545d42ca7c3feffb6b4d1e370c7ba"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:952c81d415b9b80ea261d2372d2a4a2332a3890c2b83e0535f263ddfe43f0d43"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80b39d3a151309efc8cc48675918891b865bdf742a8616a337cb0090791a0de9"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e221060b701e2aa2ea991542900dd13907a5c90fa80e199dbf5a03359019e7a3"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:92f09ff65ecff3108e56526f9e2481b8116c0b9e1425325e13245abfd79bdb1b"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-win32.whl", hash = "sha256:3ad54b9ddbe20ae9f7c1b29e52f123120772b06dbb18ec6be9101369d63a4074"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:127a789c75151db6af398b8972178afe6bda7d6f68730c057fbbc2e96b08d282"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4ed0518a14dd26092614412936920ad081a424bdcb54cc13349a8e2c6d106a"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ad9e6ed739285919aa9661a5bbed0aaf410aa60231373c5579c6b4801bd883c"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fc0a92c02fa1ca1e84fc60fa258458e5bf89d90a1ddaeb8ed9cc3147f417255"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0aefc7591920bbd360d57ea03c995cebc204b424524a5bd78406f6e1b8b2a5d8"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5faf03a7d8942bb4476e3b62fd0f4cf94eaf4618e304a19865abf89a35c0bbee"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-win32.whl", hash = "sha256:e333e2324307a7b5d86adfa835bb500ee70bfcd1447384a822e96495796b0ca4"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:cb73507defd385b7705c599a94474b1d5222a508e502553ef94114a143ec6696"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366c32fe5355ef5fc8a232c5436f4cc66e9d3e8967c01fb2e6302fd6627e3d94"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2297f08f08a2bb0d32a4265e98a006643cd7233fb7983032bd61ac7a02956b3b"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18dd842b49456aaa9a7cf535b04ca4571a302ff72ed8740d06b5adcd41fe0757"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:217138197c170a2a74ca0e05bddcd5f1796c735c37d0eee33e43259b192aa424"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a3a87cf1e133e5b1994144c12ca4aa3d9698517fe1e2ca82977781b16955658"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-win32.whl", hash = "sha256:30b339b2a743c5288405aa79a69e706a06e02958eab31859f7f3c04980853b70"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:a899b10e17743683b293a729d3a11f2f399e8a90c73b089e29f5d0fe3509f0dd"}, - {file = "lazy_object_proxy-1.10.0-pp310.pp311.pp312.pp38.pp39-none-any.whl", hash = "sha256:80fa48bd89c8f2f456fc0765c11c23bf5af827febacd2f523ca5bc1893fcc09d"}, + {file = "lazy_object_proxy-1.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:132bc8a34f2f2d662a851acfd1b93df769992ed1b81e2b1fda7db3e73b0d5a18"}, + {file = "lazy_object_proxy-1.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:01261a3afd8621a1accb5682df2593dc7ec7d21d38f411011a5712dcd418fbed"}, + {file = "lazy_object_proxy-1.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:090935756cc041e191f22f4f9c7fd4fe9a454717067adf5b1bbd2ce3046b556e"}, + {file = "lazy_object_proxy-1.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:76ec715017f06410f57df442c1a8d66e6b5f7035077785b129817f5ae58810a4"}, + {file = "lazy_object_proxy-1.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9a9f39098e93a63618a79eef2889ae3cf0605f676cd4797fdfd49fcd7ddc318b"}, + {file = "lazy_object_proxy-1.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ee13f67f4fcd044ef27bfccb1c93d39c100046fec1fad6e9a1fcdfd17492aeb3"}, + {file = "lazy_object_proxy-1.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fd4c84eafd8dd15ea16f7d580758bc5c2ce1f752faec877bb2b1f9f827c329cd"}, + {file = "lazy_object_proxy-1.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:d2503427bda552d3aefcac92f81d9e7ca631e680a2268cbe62cd6a58de6409b7"}, + {file = "lazy_object_proxy-1.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:0613116156801ab3fccb9e2b05ed83b08ea08c2517fdc6c6bc0d4697a1a376e3"}, + {file = "lazy_object_proxy-1.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bb03c507d96b65f617a6337dedd604399d35face2cdf01526b913fb50c4cb6e8"}, + {file = "lazy_object_proxy-1.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28c174db37946f94b97a97b579932ff88f07b8d73a46b6b93322b9ac06794a3b"}, + {file = "lazy_object_proxy-1.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:d662f0669e27704495ff1f647070eb8816931231c44e583f4d0701b7adf6272f"}, + {file = "lazy_object_proxy-1.11.0-py3-none-any.whl", hash = "sha256:a56a5093d433341ff7da0e89f9b486031ccd222ec8e52ec84d0ec1cdc819674b"}, + {file = "lazy_object_proxy-1.11.0.tar.gz", hash = "sha256:18874411864c9fbbbaa47f9fc1dd7aea754c86cfde21278ef427639d1dd78e9c"}, ] [[package]] @@ -1414,13 +1424,13 @@ files = [ [[package]] name = "mako" -version = "1.3.9" +version = "1.3.10" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." optional = false python-versions = ">=3.8" files = [ - {file = "Mako-1.3.9-py3-none-any.whl", hash = "sha256:95920acccb578427a9aa38e37a186b1e43156c87260d7ba18ca63aa4c7cbd3a1"}, - {file = "mako-1.3.9.tar.gz", hash = "sha256:b5d65ff3462870feec922dbccf38f6efb44e5714d7b593a656be86663d8600ac"}, + {file = "mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59"}, + {file = "mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28"}, ] [package.dependencies] @@ -1433,17 +1443,17 @@ testing = ["pytest"] [[package]] name = "markdown" -version = "3.7" +version = "3.8" description = "Python implementation of John Gruber's Markdown." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803"}, - {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, + {file = "markdown-3.8-py3-none-any.whl", hash = "sha256:794a929b79c5af141ef5ab0f2f642d0f7b1872981250230e72682346f7cc90dc"}, + {file = "markdown-3.8.tar.gz", hash = "sha256:7df81e63f0df5c4b24b7d156eb81e4690595239b7d70937d0409f1b0de319c6f"}, ] [package.extras] -docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.5)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] +docs = ["mdx_gh_links (>=0.2)", "mkdocs (>=1.6)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] testing = ["coverage", "pyyaml"] [[package]] @@ -2143,6 +2153,7 @@ files = [ {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:bb89f0a835bcfc1d42ccd5f41f04870c1b936d8507c6df12b7737febc40f0909"}, {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f0c2d907a1e102526dd2986df638343388b94c33860ff3bbe1384130828714b1"}, {file = "psycopg2_binary-2.9.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f8157bed2f51db683f31306aa497311b560f2265998122abe1dce6428bd86567"}, + {file = "psycopg2_binary-2.9.10-cp313-cp313-win_amd64.whl", hash = "sha256:27422aa5f11fbcd9b18da48373eb67081243662f9b46e6fd07c3eb46e4535142"}, {file = "psycopg2_binary-2.9.10-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:eb09aa7f9cecb45027683bb55aebaaf45a0df8bf6de68801a6afdc7947bb09d4"}, {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b73d6d7f0ccdad7bc43e6d34273f70d587ef62f824d7261c4ae9b8b1b6af90e8"}, {file = "psycopg2_binary-2.9.10-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce5ab4bf46a211a8e924d307c1b1fcda82368586a19d0a24f8ae166f5c784864"}, @@ -2189,53 +2200,53 @@ files = [ [[package]] name = "pyarrow" -version = "19.0.1" +version = "18.1.0" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.9" files = [ - {file = "pyarrow-19.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:fc28912a2dc924dddc2087679cc8b7263accc71b9ff025a1362b004711661a69"}, - {file = "pyarrow-19.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:fca15aabbe9b8355800d923cc2e82c8ef514af321e18b437c3d782aa884eaeec"}, - {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad76aef7f5f7e4a757fddcdcf010a8290958f09e3470ea458c80d26f4316ae89"}, - {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d03c9d6f2a3dffbd62671ca070f13fc527bb1867b4ec2b98c7eeed381d4f389a"}, - {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:65cf9feebab489b19cdfcfe4aa82f62147218558d8d3f0fc1e9dea0ab8e7905a"}, - {file = "pyarrow-19.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:41f9706fbe505e0abc10e84bf3a906a1338905cbbcf1177b71486b03e6ea6608"}, - {file = "pyarrow-19.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:c6cb2335a411b713fdf1e82a752162f72d4a7b5dbc588e32aa18383318b05866"}, - {file = "pyarrow-19.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:cc55d71898ea30dc95900297d191377caba257612f384207fe9f8293b5850f90"}, - {file = "pyarrow-19.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:7a544ec12de66769612b2d6988c36adc96fb9767ecc8ee0a4d270b10b1c51e00"}, - {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0148bb4fc158bfbc3d6dfe5001d93ebeed253793fff4435167f6ce1dc4bddeae"}, - {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f24faab6ed18f216a37870d8c5623f9c044566d75ec586ef884e13a02a9d62c5"}, - {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4982f8e2b7afd6dae8608d70ba5bd91699077323f812a0448d8b7abdff6cb5d3"}, - {file = "pyarrow-19.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:49a3aecb62c1be1d822f8bf629226d4a96418228a42f5b40835c1f10d42e4db6"}, - {file = "pyarrow-19.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:008a4009efdb4ea3d2e18f05cd31f9d43c388aad29c636112c2966605ba33466"}, - {file = "pyarrow-19.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:80b2ad2b193e7d19e81008a96e313fbd53157945c7be9ac65f44f8937a55427b"}, - {file = "pyarrow-19.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee8dec072569f43835932a3b10c55973593abc00936c202707a4ad06af7cb294"}, - {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d5d1ec7ec5324b98887bdc006f4d2ce534e10e60f7ad995e7875ffa0ff9cb14"}, - {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3ad4c0eb4e2a9aeb990af6c09e6fa0b195c8c0e7b272ecc8d4d2b6574809d34"}, - {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d383591f3dcbe545f6cc62daaef9c7cdfe0dff0fb9e1c8121101cabe9098cfa6"}, - {file = "pyarrow-19.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b4c4156a625f1e35d6c0b2132635a237708944eb41df5fbe7d50f20d20c17832"}, - {file = "pyarrow-19.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:5bd1618ae5e5476b7654c7b55a6364ae87686d4724538c24185bbb2952679960"}, - {file = "pyarrow-19.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:e45274b20e524ae5c39d7fc1ca2aa923aab494776d2d4b316b49ec7572ca324c"}, - {file = "pyarrow-19.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:d9dedeaf19097a143ed6da37f04f4051aba353c95ef507764d344229b2b740ae"}, - {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ebfb5171bb5f4a52319344ebbbecc731af3f021e49318c74f33d520d31ae0c4"}, - {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a21d39fbdb948857f67eacb5bbaaf36802de044ec36fbef7a1c8f0dd3a4ab2"}, - {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:99bc1bec6d234359743b01e70d4310d0ab240c3d6b0da7e2a93663b0158616f6"}, - {file = "pyarrow-19.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1b93ef2c93e77c442c979b0d596af45e4665d8b96da598db145b0fec014b9136"}, - {file = "pyarrow-19.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:d9d46e06846a41ba906ab25302cf0fd522f81aa2a85a71021826f34639ad31ef"}, - {file = "pyarrow-19.0.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:c0fe3dbbf054a00d1f162fda94ce236a899ca01123a798c561ba307ca38af5f0"}, - {file = "pyarrow-19.0.1-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:96606c3ba57944d128e8a8399da4812f56c7f61de8c647e3470b417f795d0ef9"}, - {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f04d49a6b64cf24719c080b3c2029a3a5b16417fd5fd7c4041f94233af732f3"}, - {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a9137cf7e1640dce4c190551ee69d478f7121b5c6f323553b319cac936395f6"}, - {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:7c1bca1897c28013db5e4c83944a2ab53231f541b9e0c3f4791206d0c0de389a"}, - {file = "pyarrow-19.0.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:58d9397b2e273ef76264b45531e9d552d8ec8a6688b7390b5be44c02a37aade8"}, - {file = "pyarrow-19.0.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:b9766a47a9cb56fefe95cb27f535038b5a195707a08bf61b180e642324963b46"}, - {file = "pyarrow-19.0.1-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:6c5941c1aac89a6c2f2b16cd64fe76bcdb94b2b1e99ca6459de4e6f07638d755"}, - {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd44d66093a239358d07c42a91eebf5015aa54fccba959db899f932218ac9cc8"}, - {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:335d170e050bcc7da867a1ed8ffb8b44c57aaa6e0843b156a501298657b1e972"}, - {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:1c7556165bd38cf0cd992df2636f8bcdd2d4b26916c6b7e646101aff3c16f76f"}, - {file = "pyarrow-19.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:699799f9c80bebcf1da0983ba86d7f289c5a2a5c04b945e2f2bcf7e874a91911"}, - {file = "pyarrow-19.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8464c9fbe6d94a7fe1599e7e8965f350fd233532868232ab2596a71586c5a429"}, - {file = "pyarrow-19.0.1.tar.gz", hash = "sha256:3bf266b485df66a400f282ac0b6d1b500b9d2ae73314a153dbe97d6d5cc8a99e"}, + {file = "pyarrow-18.1.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c"}, + {file = "pyarrow-18.1.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4"}, + {file = "pyarrow-18.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f443122c8e31f4c9199cb23dca29ab9427cef990f283f80fe15b8e124bcc49b"}, + {file = "pyarrow-18.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a03da7f2758645d17b7b4f83c8bffeae5bbb7f974523fe901f36288d2eab71"}, + {file = "pyarrow-18.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ba17845efe3aa358ec266cf9cc2800fa73038211fb27968bfa88acd09261a470"}, + {file = "pyarrow-18.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:3c35813c11a059056a22a3bef520461310f2f7eea5c8a11ef9de7062a23f8d56"}, + {file = "pyarrow-18.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9736ba3c85129d72aefa21b4f3bd715bc4190fe4426715abfff90481e7d00812"}, + {file = "pyarrow-18.1.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:eaeabf638408de2772ce3d7793b2668d4bb93807deed1725413b70e3156a7854"}, + {file = "pyarrow-18.1.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:3b2e2239339c538f3464308fd345113f886ad031ef8266c6f004d49769bb074c"}, + {file = "pyarrow-18.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f39a2e0ed32a0970e4e46c262753417a60c43a3246972cfc2d3eb85aedd01b21"}, + {file = "pyarrow-18.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31e9417ba9c42627574bdbfeada7217ad8a4cbbe45b9d6bdd4b62abbca4c6f6"}, + {file = "pyarrow-18.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:01c034b576ce0eef554f7c3d8c341714954be9b3f5d5bc7117006b85fcf302fe"}, + {file = "pyarrow-18.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:f266a2c0fc31995a06ebd30bcfdb7f615d7278035ec5b1cd71c48d56daaf30b0"}, + {file = "pyarrow-18.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:d4f13eee18433f99adefaeb7e01d83b59f73360c231d4782d9ddfaf1c3fbde0a"}, + {file = "pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d"}, + {file = "pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee"}, + {file = "pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992"}, + {file = "pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54"}, + {file = "pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33"}, + {file = "pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30"}, + {file = "pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99"}, + {file = "pyarrow-18.1.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b"}, + {file = "pyarrow-18.1.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2"}, + {file = "pyarrow-18.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191"}, + {file = "pyarrow-18.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa"}, + {file = "pyarrow-18.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c"}, + {file = "pyarrow-18.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c"}, + {file = "pyarrow-18.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181"}, + {file = "pyarrow-18.1.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc"}, + {file = "pyarrow-18.1.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386"}, + {file = "pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324"}, + {file = "pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8"}, + {file = "pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9"}, + {file = "pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba"}, + {file = "pyarrow-18.1.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:0b331e477e40f07238adc7ba7469c36b908f07c89b95dd4bd3a0ec84a3d1e21e"}, + {file = "pyarrow-18.1.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:2c4dd0c9010a25ba03e198fe743b1cc03cd33c08190afff371749c52ccbbaf76"}, + {file = "pyarrow-18.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f97b31b4c4e21ff58c6f330235ff893cc81e23da081b1a4b1c982075e0ed4e9"}, + {file = "pyarrow-18.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a4813cb8ecf1809871fd2d64a8eff740a1bd3691bbe55f01a3cf6c5ec869754"}, + {file = "pyarrow-18.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:05a5636ec3eb5cc2a36c6edb534a38ef57b2ab127292a716d00eabb887835f1e"}, + {file = "pyarrow-18.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:73eeed32e724ea3568bb06161cad5fa7751e45bc2228e33dcb10c614044165c7"}, + {file = "pyarrow-18.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052"}, + {file = "pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73"}, ] [package.extras] @@ -2300,13 +2311,13 @@ files = [ [[package]] name = "pydantic" -version = "2.11.2" +version = "2.11.3" description = "Data validation using Python type hints" optional = false python-versions = ">=3.9" files = [ - {file = "pydantic-2.11.2-py3-none-any.whl", hash = "sha256:7f17d25846bcdf89b670a86cdfe7b29a9f1c9ca23dee154221c9aa81845cfca7"}, - {file = "pydantic-2.11.2.tar.gz", hash = "sha256:2138628e050bd7a1e70b91d4bf4a91167f4ad76fdb83209b107c8d84b854917e"}, + {file = "pydantic-2.11.3-py3-none-any.whl", hash = "sha256:a082753436a07f9ba1289c6ffa01cd93db3548776088aa917cc43b63f68fa60f"}, + {file = "pydantic-2.11.3.tar.gz", hash = "sha256:7471657138c16adad9322fe3070c0116dd6c3ad8d649300e3cbdfe91f4db4ec3"}, ] [package.dependencies] @@ -2782,13 +2793,13 @@ fsspec = ">=0.6.0" [[package]] name = "s3transfer" -version = "0.11.4" +version = "0.11.5" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.8" files = [ - {file = "s3transfer-0.11.4-py3-none-any.whl", hash = "sha256:ac265fa68318763a03bf2dc4f39d5cbd6a9e178d81cc9483ad27da33637e320d"}, - {file = "s3transfer-0.11.4.tar.gz", hash = "sha256:559f161658e1cf0a911f45940552c696735f5c74e64362e515f333ebed87d679"}, + {file = "s3transfer-0.11.5-py3-none-any.whl", hash = "sha256:757af0f2ac150d3c75bc4177a32355c3862a98d20447b69a0161812992fe0bd4"}, + {file = "s3transfer-0.11.5.tar.gz", hash = "sha256:8c8aad92784779ab8688a61aefff3e28e9ebdce43142808eaa3f0b0f402f68b7"}, ] [package.dependencies] @@ -3040,6 +3051,10 @@ files = [ {file = "tables-3.10.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e36ce9f10471c69c1f0b06c6966de762558a35d62592c55df7994a8019adaf0c"}, {file = "tables-3.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f233e78cc9fa4157ec4c3ef2abf01a731fe7969bc6ed73539e5f4cd3b94c98b2"}, {file = "tables-3.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:34357d2f2f75843a44e6fe54d1f11fc2e35a8fd3cb134df3d3362cff78010adb"}, + {file = "tables-3.10.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6fc5b46a4f359249c3ab9a0a0a2448d7e680e68cffd63fdf3fb7171781edd46e"}, + {file = "tables-3.10.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2ecabd7f459d40b7f9f5256850dd5f43773fda7b789f827de92c3d26df1e320f"}, + {file = "tables-3.10.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40a4ee18f3c9339d9dd8fd3777c75cda5768f2ff347064a2796f59161a190af8"}, + {file = "tables-3.10.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757c6ea257c174af8036cf8f273ede756bbcd6db5ac7e2a4d64e788b0f371152"}, {file = "tables-3.10.1.tar.gz", hash = "sha256:4aa07ac734b9c037baeaf44aec64ec902ad247f57811b59f30c4e31d31f126cf"}, ] @@ -3225,13 +3240,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.13.1" +version = "4.13.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.13.1-py3-none-any.whl", hash = "sha256:4b6cf02909eb5495cfbc3f6e8fd49217e6cc7944e145cdda8caa3734777f9e69"}, - {file = "typing_extensions-4.13.1.tar.gz", hash = "sha256:98795af00fb9640edec5b8e31fc647597b4691f099ad75f469a2616be1a76dff"}, + {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, + {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, ] [[package]] @@ -3261,13 +3276,13 @@ files = [ [[package]] name = "urllib3" -version = "2.3.0" +version = "2.4.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" files = [ - {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, - {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, + {file = "urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813"}, + {file = "urllib3-2.4.0.tar.gz", hash = "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466"}, ] [package.extras] @@ -3436,4 +3451,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "f1e78f1b60e0305bddc8c9bbef7120c9f5bd7ad49082439dcfab331ec5bcbb8c" +content-hash = "4a8881c5af0c6fa67fe60aa19d9d3ac68e4c655851788326f1b97480542ae407" diff --git a/pyproject.toml b/pyproject.toml index 625aea9..c288f20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dynamicio" -version = "0.0.0" # dummy, will be overridden by git tag +version = "8.1.0" # dummy, will be overridden by git tag description = "Panda's wrapper for IO operations" authors = [ "Christos Hadjinikolis ", @@ -17,45 +17,46 @@ classifiers = [ ] [tool.poetry.dependencies] -python = ">=3.10,<3.13" +PyYAML = ">=5.4.1" +SQLAlchemy = "~1.4.11" awscli = ">=1.22.24" +awswrangler = ">=3.0.0" boto3 = ">=1.20.24" -pyathena = "*" +confluent-kafka = "~2.4.0" fastparquet = ">=0.8.0" fsspec = "==2022.11.0" -confluent-kafka = "~2.4.0" +libcst = "==1.5.1" logzero = ">=1.7.0" magic-logger = ">=1.0.2" -numpy = ">=1.24.0,<2.0.0" no_implicit_optional = "==1.4.0" +numpy = ">=1.24.0,<2.0.0" pandas = ">=1.5.1" psycopg2-binary = "~2.9.3" pyarrow = ">=10.0.1" +pyathena = "*" pydantic = ">=1.9.2,<3" +python = ">=3.10,<3.13" python-json-logger = "~2.0.1" -PyYAML = ">=5.4.1" s3fs = "==0.4.2" simplejson = "~3.17.2" -SQLAlchemy = "~1.4.11" tables = ">=3.10.1" -libcst = "==1.5.1" [tool.poetry.group.dev.dependencies] black = "==24.3.0" coverage-badge = "==1.1.0" +flake8 = "^7.2.0" flake8-import-order = "*" flake8-print = "*" flake8-tidy-imports = "*" -flake8 = "^7.2.0" gitlint = "==0.17.0" mock = "==4.0.3" pandas-stubs = "==2.0.3.230814" pre-commit = "==2.20.0" pydocstyle = "==6.1.1" pylint = "==2.15.5" +pytest = "==7.2.0" pytest-asyncio = "==0.20.2" pytest-cov = "==4.0.0" -pytest = "==7.2.0" tox = "==3.27.1" types-PyYAML = "==6.0.12.2" types-setuptools = "==65.5.0.3" @@ -89,9 +90,9 @@ style = "pep440" fix-shallow-repository = true [tool.black] -py38 = true +target-version = ["py38"] line-length = 185 -include = '\\.(pyi?)$' +include = '\.pyi?$' exclude = ''' ( /( diff --git a/tests/mocking/io.py b/tests/mocking/io.py index 5ec1d41..3a18b4a 100644 --- a/tests/mocking/io.py +++ b/tests/mocking/io.py @@ -15,6 +15,14 @@ class ReadS3IO(UnifiedIO): schema = {"id": "int64"} +class ReadS3JsonOrientRecordsIO(UnifiedIO): + schema = {"data": "object"} + + +class ReadS3JsonOrientRecordsAltIO(UnifiedIO): + schema = {"release": "object", "timestamp": "int64"} + + class ReadMockS3CsvIO(UnifiedIO): schema = SCHEMA_FROM_FILE @@ -77,10 +85,14 @@ class ReadS3JsonIO(UnifiedIO): schema = {"id": "int64", "foo_name": "object", "bar": "int64"} -class WriteS3ParquetIO(UnifiedIO): +class WriteS3IO(UnifiedIO): schema = {"col_1": "int64", "col_2": "object"} +class WriteS3JsonOrientRecordsIO(UnifiedIO): + schema = {"release": "object", "timestamp": "int64"} + + class WriteS3ParquetExternalIO(UnifiedIO): schema = { "bar": "int64", diff --git a/tests/mocking/models.py b/tests/mocking/models.py index 849ee2e..020b03b 100644 --- a/tests/mocking/models.py +++ b/tests/mocking/models.py @@ -1,4 +1,5 @@ """A module for defining sql_alchemy models.""" + # pylint: disable=too-few-public-methods, R0801, C0104 __all__ = ["ERModel", "PgModel"] diff --git a/tests/resources/data/input/multi_row_json.json b/tests/resources/data/input/multi_row_json.json new file mode 100644 index 0000000..a327c7c --- /dev/null +++ b/tests/resources/data/input/multi_row_json.json @@ -0,0 +1,10 @@ +[ + { + "release": "feb09", + "timestamp": 1614268643313 + }, + { + "release": "feb10", + "timestamp": 1614268643313 + } +] \ No newline at end of file diff --git a/tests/resources/data/input/single_row_json.json b/tests/resources/data/input/single_row_json.json new file mode 100644 index 0000000..b9e10a6 --- /dev/null +++ b/tests/resources/data/input/single_row_json.json @@ -0,0 +1,6 @@ +{ + "data":{ + "release":"feb09", + "timestamp":1614268643313 + } +} \ No newline at end of file diff --git a/tests/resources/definitions/external.yaml b/tests/resources/definitions/external.yaml index 415287f..91ee3fd 100644 --- a/tests/resources/definitions/external.yaml +++ b/tests/resources/definitions/external.yaml @@ -8,8 +8,7 @@ READ_FROM_ATHENA: CLOUD: type: athena athena: - s3_staging_dir: s3://test-staging/ - region_name: eu-west-1 + s3_output: s3://test-staging/ READ_MOCK_S3_CSV: LOCAL: diff --git a/tests/resources/definitions/input.yaml b/tests/resources/definitions/input.yaml index 3eabaca..ed77b3a 100644 --- a/tests/resources/definitions/input.yaml +++ b/tests/resources/definitions/input.yaml @@ -40,6 +40,26 @@ READ_FROM_S3_JSON: file_path: "[[ MOCK_KEY ]]" file_type: "json" +S3_PANDAS_READER_CONSISTENCY: + LOCAL: + type: "local" + local: + file_path: "[[ TEST_RESOURCES ]]/data/input/{file_name}.json" + file_type: "json" + CLOUD: + type: "s3_file" + s3: + bucket: "[[ MOCK_BUCKET ]]" + file_path: "[[ MOCK_KEY ]]" + file_type: "json" + +CHECK_JSON_READS_RAW_TIMESTAMPS: + LOCAL: + type: "local" + local: + file_path: "[[ TEST_RESOURCES ]]/data/input/multi_row_json.json" + file_type: "json" + READ_FROM_S3_HDF: LOCAL: type: "local" @@ -317,5 +337,4 @@ READ_FROM_ATHENA: CLOUD: type: athena athena: - s3_staging_dir: s3://test-staging/ - region_name: eu-west-1 + s3_output: s3://test-staging/ diff --git a/tests/test_core.py b/tests/test_core.py index 9ec2936..8e70681 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -3,6 +3,7 @@ import logging import os import time +from tempfile import NamedTemporaryFile from typing import Mapping, Tuple from unittest.mock import patch @@ -28,9 +29,12 @@ ReadS3CsvWithWrongSchemaIO, ReadS3DataWithFalseTypes, ReadS3ParquetIO, + ReadS3ParquetWithDifferentCastableDTypeIO, + ReadS3ParquetWithDifferentNonCastableDTypeIO, SubclassMissingMixin, WriteS3CsvIO, WriteS3CsvWithSchema, + WriteS3IO, WriteS3ParquetExternalIO, ) @@ -186,7 +190,7 @@ def test_key_error_is_thrown_for_missing_schema_if_unified_io_subclass_assigns_s with pytest.raises(SchemaNotFoundError): ReadMockS3CsvIO(source_config=read_mock_s3_cloud_config) - @pytest.mark.integration + @pytest.mark.unit def test_schema_validations_are_applied_for_an_io_class_with_a_schema_definition(self, valid_dataframe): # Given df = valid_dataframe @@ -203,7 +207,7 @@ def test_schema_validations_are_applied_for_an_io_class_with_a_schema_definition # Then assert io_instance == return_value - @pytest.mark.integration + @pytest.mark.unit def test_log_metrics_from_schema_are_applied_for_an_io_class_with_a_schema_definition(self, caplog, valid_dataframe): # Given df = valid_dataframe @@ -235,7 +239,7 @@ def test_log_metrics_from_schema_are_applied_for_an_io_class_with_a_schema_defin and (getattr(caplog.records[9], "message") == '{"message": "METRIC", "dataset": "READ_FROM_S3_CSV", "column": "bar", "metric": "Variance", "value": 0.0}') ) - @pytest.mark.integration + @pytest.mark.unit def test_schema_validations_errors_are_thrown_for_each_validation_if_df_does_not_map_to_schema_definition(self, invalid_dataframe): # Given df = invalid_dataframe @@ -249,7 +253,7 @@ def test_schema_validations_errors_are_thrown_for_each_validation_if_df_does_not with pytest.raises(SchemaValidationError): ReadS3CsvIO(source_config=s3_csv_cloud_config).validate_from_schema(df) - @pytest.mark.integration + @pytest.mark.unit def test_schema_validations_exception_message_is_a_dict_with_all_violated_validations(self, invalid_dataframe, expected_messages): # Given df = invalid_dataframe @@ -266,7 +270,7 @@ def test_schema_validations_exception_message_is_a_dict_with_all_violated_valida # Then assert _exception.message.keys() == expected_messages # pylint: disable=no-member - @pytest.mark.integration + @pytest.mark.unit def test_local_writers_only_write_out_castable_columns_according_to_the_io_schema_case_float64_to_int64_id(self, dataset_with_more_columns_than_dictated_in_schema): # Given @@ -932,7 +936,7 @@ def test_a_custom_validate_method_can_be_used_to_override_the_default_abstract_o finally: os.remove(s3_parquet_with_some_bool_col_local_config.local.file_path) - @pytest.mark.integration + @pytest.mark.unit def test_show_casting_warnings_flag_default_value_prevents_showing_casting_logs(self, caplog): # Given s3_csv_cloud_config = IOConfig( @@ -949,7 +953,7 @@ def test_show_casting_warnings_flag_default_value_prevents_showing_casting_logs( # Then assert len(caplog.records) == 0 - @pytest.mark.integration + @pytest.mark.unit def test_show_casting_warnings_flag_allows_casting_logs_to_be_printed_if_set_to_true(self, caplog): # Given s3_csv_cloud_config = IOConfig( @@ -966,6 +970,90 @@ def test_show_casting_warnings_flag_allows_casting_logs_to_be_printed_if_set_to_ # Then assert getattr(caplog.records[0], "message") == "Expected: 'float64' dtype for READ_S3_DATA_WITH_FALSE_TYPES['id'], found 'int64'" + @pytest.mark.unit + def test_write_applies_schema_before_invoking_writer(self): + # Given + # WRITE_TO_S3_PARQUET: + # LOCAL: + # ... + # CLOUD: + # type: "s3_file" + # s3: + # bucket: "[[ MOCK_BUCKET ]]" + # file_path: "test/write_some_parquet.parquet" + # file_type: "parquet" + input_df = pd.DataFrame.from_dict({"col_1": [3, 2, 1], "col_2": ["a", "b", "c"], "col_3": ["a", "b", "c"]}) + + s3_parquet_cloud_config = IOConfig( + path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key="WRITE_TO_S3_PARQUET") + + # When + # class WriteS3IO(DynamicDataIO): + # schema = {"col_1": "int64", "col_2": "object"} + # + # @staticmethod + # def validate(df: pd.DataFrame): + # pass + with patch.object(dynamicio.mixins.with_s3.wr.s3, "to_parquet") as mock__wr_s3_parquet_writer, patch.object(WriteS3IO, "_apply_schema") as mock__apply_schema: + with NamedTemporaryFile(delete=False) as temp_file: + mock__wr_s3_parquet_writer.return_value = temp_file + WriteS3IO(source_config=s3_parquet_cloud_config).write(input_df) + + # Then + mock__apply_schema.assert_called() + mock__wr_s3_parquet_writer.assert_called() + + @pytest.mark.unit + def test_columns_data_type_error_exception_is_not_generated_if_column_dtypes_can_be_casted_to_the_expected_dtypes(self, expected_s3_parquet_df): + # Given + s3_parquet_cloud_config = IOConfig( + path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key="READ_FROM_S3_PARQUET") + + # When + with patch.object(dynamicio.mixins.with_s3.wr.s3, "read_parquet") as mock__wr_s3_parquet_reader: + mock__wr_s3_parquet_reader.return_value = expected_s3_parquet_df + ReadS3ParquetWithDifferentCastableDTypeIO(source_config=s3_parquet_cloud_config).read() + + assert True, "No exception was raised" + + @pytest.mark.unit + @patch.object(dynamicio.mixins.with_s3.wr.s3, "read_parquet") + def test_columns_data_type_error_exception_is_generated_if_column_dtypes_dont_map_to_the_expected_dtypes(self, mock__wr_s3_parquet_reader, expected_s3_parquet_df): + """ + ------------------------------ Captured log call ------------------------------- + + WARNING ...:dataio.py:273 Expected: 'float64' dtype for column: 'id', found: 'int64' instead. + WARNING ...:dataio.py:273 Expected: 'int64' dtype for column: 'foo_name', found: 'object' instead. + ERROR ...:dataio.py:277 Tried casting column: 'foo_name' to 'int64' from 'object', but failed. + + =========================== short test summary info ============================ + + FAILED ...:test_columns_data_type_error_exception_is_generated_if_column_dtypes_dont_map_to_the_expected_dtypes + + ============================== 1 failed in 0.48s =============================== + + """ + # Given + dataframe_returned = expected_s3_parquet_df + mock__wr_s3_parquet_reader.return_value = dataframe_returned + + s3_parquet_cloud_config = IOConfig( + path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key="READ_FROM_S3_PARQUET") + + # When/Then + with pytest.raises(ColumnsDataTypeError): + ReadS3ParquetWithDifferentNonCastableDTypeIO(source_config=s3_parquet_cloud_config).read() + mock__wr_s3_parquet_reader.assert_called() + class TestAsyncCoreIO: @pytest.mark.unit diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..f2b5dae --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,77 @@ +# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring, too-many-public-methods, too-few-public-methods +import pytest + +# Application Imports +from dynamicio.errors import ( + ColumnsDataTypeError, + DataSourceError, + DynamicIOError, + InvalidDatasetTypeError, + MissingSchemaDefinition, + NonUniqueIdColumnError, + NotExpectedCategoricalValue, + NullValueInColumnError, + SchemaNotFoundError, + SchemaValidationError, +) + + +class TestErrors: + + @pytest.mark.unit + @pytest.mark.parametrize( + "exception_cls, init_arg, expected_str", + [ + # โœ… Custom __str__ formatting + (SchemaNotFoundError, "test-source", "Schema not specified in the provided source: test-source "), + (MissingSchemaDefinition, "test-class", "The resource definition for this class is missing a schema definition: test-class"), + (InvalidDatasetTypeError, "my_file.avro", "Dataset: my_file.avro provided is not amongst the supported types (parquet, json, csv, h5) handled by dynamicio."), + ], + ) + def test_custom_exceptions_format_output(self, exception_cls, init_arg, expected_str): + # Given + ex = exception_cls(init_arg) + + # When + result_str = str(ex) + + # Then + assert result_str == expected_str + assert isinstance(ex, DynamicIOError) + + @pytest.mark.unit + @pytest.mark.parametrize( + "exception_cls", + [ + # โœ… Subclasses without custom formatting + SchemaValidationError, + DataSourceError, + ColumnsDataTypeError, + NonUniqueIdColumnError, + NullValueInColumnError, + NotExpectedCategoricalValue, + ], + ) + def test_simple_dynamicio_exceptions_with_message(self, exception_cls): + # Given + ex = exception_cls("Something went wrong") + + # When + result_str = str(ex) + + # Then + assert result_str == "Something went wrong" + assert ex.message == "Something went wrong" + assert isinstance(ex, DynamicIOError) + + @pytest.mark.unit + def test_dynamicio_base_exception_without_message(self): + # Given + ex = DynamicIOError() + + # When + result_str = str(ex) + + # Then + assert ex.message is None + assert result_str == "" diff --git a/tests/test_mixins/test_athena_mixin.py b/tests/test_mixins/test_athena_mixin.py index fbb4ad2..1ef3a5b 100644 --- a/tests/test_mixins/test_athena_mixin.py +++ b/tests/test_mixins/test_athena_mixin.py @@ -13,14 +13,10 @@ @pytest.mark.unit class TestAthenaIO: @pytest.mark.unit - @patch("dynamicio.mixins.with_athena.connect") - def test_read_from_athena_with_query(self, mock_connect, test_df): - # Given: mock return values - mock_cursor = mock_connect.return_value.cursor.return_value - mock_cursor.execute.return_value = None # execute returns None - mock_cursor.fetchall.return_value = test_df - mock_cursor.description = [("id", None), ("foo", None), ("bar", None), ("baz", None)] - + @patch("dynamicio.mixins.with_athena.wr.athena.read_sql_query") + def test_read_from_athena_with_query(self, mock_wr_read, test_df): + # Given + mock_wr_read.return_value = test_df athena_config = IOConfig( path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", @@ -28,11 +24,11 @@ def test_read_from_athena_with_query(self, mock_connect, test_df): ).get(source_key="READ_FROM_ATHENA") # When - ReadAthenaIO(source_config=athena_config, query="SELECT * FROM dummy").read() + df = ReadAthenaIO(source_config=athena_config, query="SELECT * FROM dummy").read() - # Then: assert calls - mock_cursor.execute.assert_called_once_with("SELECT * FROM dummy") - mock_cursor.fetchall.assert_called_once() + # Then + mock_wr_read.assert_called_once() + assert df.equals(test_df) @pytest.mark.unit def test_read_from_athena_raises_when_query_missing_raises_error(self): @@ -44,18 +40,14 @@ def test_read_from_athena_raises_when_query_missing_raises_error(self): ).get(source_key="READ_FROM_ATHENA") # When / Then - with pytest.raises(AssertionError, match="A 'query' must be provided for Athena read"): + with pytest.raises(ValueError, match="A 'query' must be provided for Athena reads"): ReadAthenaIO(source_config=athena_config).read() @pytest.mark.unit - @patch("dynamicio.mixins.with_athena.connect") - def test_read_from_athena_with_query_and_options(self, mock_connect, test_df): - # Given: mock return values - mock_cursor = mock_connect.return_value.cursor.return_value - mock_cursor.execute.return_value = None # execute returns None - mock_cursor.fetchall.return_value = test_df - mock_cursor.description = [("id", None), ("foo", None), ("bar", None), ("baz", None)] - + @patch("dynamicio.mixins.with_athena.wr.athena.read_sql_query") + def test_read_from_athena_with_query_and_valid_options(self, mock_wr_read, test_df): + # Given + mock_wr_read.return_value = test_df athena_config = IOConfig( path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", @@ -63,21 +55,18 @@ def test_read_from_athena_with_query_and_options(self, mock_connect, test_df): ).get(source_key="READ_FROM_ATHENA") # When - ReadAthenaIO(source_config=athena_config, query="SELECT * FROM dummy", chunksize=1000).read() + df = ReadAthenaIO(source_config=athena_config, query="SELECT * FROM dummy", ctas_approach=True).read() # Then - mock_cursor.execute.assert_called_once_with("SELECT * FROM dummy") - mock_cursor.fetchall.assert_called_with(chunksize=1000) + _, kwargs = mock_wr_read.call_args + assert "ctas_approach" in kwargs and kwargs["ctas_approach"] is True + assert df.equals(test_df) @pytest.mark.unit - @patch("dynamicio.mixins.with_athena.connect") - def test_read_from_athena_filters_invalid_options(self, mock_connect, test_df): - # Given: mock return values - mock_cursor = mock_connect.return_value.cursor.return_value - mock_cursor.execute.return_value = None # execute returns None - mock_cursor.fetchall.return_value = test_df - mock_cursor.description = [("id", None), ("foo", None), ("bar", None), ("baz", None)] - + @patch("dynamicio.mixins.with_athena.wr.athena.read_sql_query") + def test_read_from_athena_filters_invalid_options(self, mock_wr_read, test_df): + # Given + mock_wr_read.return_value = test_df athena_config = IOConfig( path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", @@ -85,8 +74,9 @@ def test_read_from_athena_filters_invalid_options(self, mock_connect, test_df): ).get(source_key="READ_FROM_ATHENA") # When - ReadAthenaIO(source_config=athena_config, query="SELECT * FROM dummy", foo="bar").read() + df = ReadAthenaIO(source_config=athena_config, query="SELECT * FROM dummy", foo="bar").read() # Then - mock_cursor.execute.assert_called_once_with("SELECT * FROM dummy") - mock_cursor.fetchall.assert_called_with() + call_kwargs = mock_wr_read.call_args.kwargs + assert "foo" not in call_kwargs + assert df.equals(test_df) diff --git a/tests/test_mixins/test_local_mixins.py b/tests/test_mixins/test_local_mixins.py index bfca973..0e3349e 100644 --- a/tests/test_mixins/test_local_mixins.py +++ b/tests/test_mixins/test_local_mixins.py @@ -9,6 +9,7 @@ import pandas as pd import pytest +# Application Imports import dynamicio from dynamicio.config import IOConfig from tests import constants @@ -24,13 +25,15 @@ ReadS3DataWithLessColumnsIO, ReadS3HdfIO, ReadS3JsonIO, + ReadS3JsonOrientRecordsAltIO, + ReadS3JsonOrientRecordsIO, ReadS3ParquetIO, TemplatedFile, WriteKafkaIO, WritePostgresIO, WriteS3CsvIO, WriteS3HdfIO, - WriteS3ParquetIO, + WriteS3IO, ) from tests.mocking.models import ERModel @@ -151,6 +154,51 @@ def test_read_json_pandas_reader_will_only_filter_out_columns_not_in_schema(self # Then assert expected_df_with_less_columns.equals(s3_json_df) + @pytest.mark.unit + def test_warning_is_logged_when_single_record_detected_but_option_not_passed(self, caplog): + # Given + caplog.set_level("WARNING") + local_json_config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), + env_identifier="LOCAL", + dynamic_vars=constants, + ).get(source_key="S3_PANDAS_READER_CONSISTENCY") + + # When + df = ReadS3JsonOrientRecordsIO(source_config=local_json_config, file_name="single_row_json").read() + + # Then + assert "File appears to be a single-record JSON object" in caplog.text + assert isinstance(df, pd.DataFrame) + assert "data" in df.columns # assuming schema = {"data": "object"} + + @pytest.mark.unit + def test_read_json_does_not_parse_dates_by_default(self): + # Given + # [ + # { + # "release": "feb09", + # "timestamp": 1614268643313 + # }, + # { + # "release": "feb10", + # "timestamp": 1614268643313 + # } + # ] + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), + env_identifier="LOCAL", + dynamic_vars=constants, + ).get(source_key="CHECK_JSON_READS_RAW_TIMESTAMPS") + + # When + df = ReadS3JsonOrientRecordsAltIO(source_config=config).read() + + # Then + assert df["timestamp"].dtype == "int64" + assert isinstance(df["timestamp"].iloc[0], (int, np.integer)) + assert df["timestamp"].iloc[0] == 1614268643313 + @pytest.mark.unit def test_read_hdf_pandas_reader_will_only_filter_out_columns_not_in_schema(self, expected_df_with_less_columns): # Given @@ -223,7 +271,7 @@ def test_a_local_json_file_is_loaded_when_io_config_is_initialised_with_local_en ).get(source_key="READ_FROM_S3_JSON") # When - options = {"orient": "columns"} + options = {"orient": "records"} s3_json_df = ReadS3JsonIO(source_config=s3_json_local_config, **options).read() # Then @@ -448,7 +496,7 @@ def test_write_resolves_file_path_if_templated_for_some_output_data(self): pd.testing.assert_frame_equal(df, called_with_df) assert called_with_file_path == config.local.file_path.format(file_name_to_replace="some_csv_to_read") - @pytest.mark.integration + @pytest.mark.unit def test_local_writers_only_write_out_castable_columns_according_to_the_io_schema_case_float64_to_int64_id( self, ): @@ -464,13 +512,13 @@ def test_local_writers_only_write_out_castable_columns_according_to_the_io_schem ).get(source_key="WRITE_TO_S3_PARQUET") # When - # class WriteS3ParquetIO(DynamicDataIO): + # class WriteS3IO(DynamicDataIO): # schema = {"col_1": "int64", "col_2": "object"} # # @staticmethod # def validate(df: pd.DataFrame): # pass - write_s3_io = WriteS3ParquetIO(source_config=s3_parquet_local_config) + write_s3_io = WriteS3IO(source_config=s3_parquet_local_config) write_s3_io.write(input_df) # # Then @@ -483,7 +531,7 @@ def test_local_writers_only_write_out_castable_columns_according_to_the_io_schem finally: os.remove(s3_parquet_local_config.local.file_path) - @pytest.mark.integration + @pytest.mark.unit def test_local_writers_only_write_out_columns_in_a_provided_io_schema(self): # Given @@ -496,13 +544,13 @@ def test_local_writers_only_write_out_columns_in_a_provided_io_schema(self): ).get(source_key="WRITE_TO_S3_PARQUET") # When - # class WriteS3ParquetIO(DynamicDataIO): + # class WriteS3IO(DynamicDataIO): # schema = {"col_1": "int64", "col_2": "object"} # # @staticmethod # def validate(df: pd.DataFrame): # pass - write_s3_io = WriteS3ParquetIO(source_config=s3_parquet_local_config) + write_s3_io = WriteS3IO(source_config=s3_parquet_local_config) write_s3_io.write(input_df) # Then @@ -523,7 +571,7 @@ def test_pyarrow_is_used_as_backend_parquet(self): # Then assert implementation.__class__.__name__ == "PyArrowImpl" - @pytest.mark.integration + @pytest.mark.unit def test_write_parquet_file_is_called_with_additional_pyarrow_args(self): # Given @@ -544,13 +592,13 @@ def test_write_parquet_file_is_called_with_additional_pyarrow_args(self): # When with patch.object(dynamicio.mixins.with_local.pd.DataFrame, "to_parquet") as mocked__to_parquet: - write_s3_io = WriteS3ParquetIO(source_config=s3_parquet_local_config, **to_parquet_kwargs) + write_s3_io = WriteS3IO(source_config=s3_parquet_local_config, **to_parquet_kwargs) write_s3_io.write(input_df) # Then mocked__to_parquet.assert_called_once_with(os.path.join(constants.TEST_RESOURCES, "data/processed/write_some_parquet.parquet"), **to_parquet_kwargs) - @pytest.mark.integration + @pytest.mark.unit @patch.object(dynamicio.mixins.with_local.pd, "read_parquet") def test_read_parquet_file_is_called_with_additional_pyarrow_args(self, mock__read_parquet): @@ -629,7 +677,7 @@ def test_write_with_pyarrow_is_called_as_default_when_no_engine_option_is_provid # When with patch.object(dynamicio.mixins.with_local.WithLocal, "_WithLocal__write_with_pyarrow") as mocked__write_with_pyarrow: - WriteS3ParquetIO(config).write(input_df) + WriteS3IO(config).write(input_df) # Then mocked__write_with_pyarrow.assert_called() @@ -647,7 +695,7 @@ def test_write_with_pyarrow_is_called_when_engine_option_is_set_to_pyarrow(self) # When with patch.object(dynamicio.mixins.with_local.WithLocal, "_WithLocal__write_with_pyarrow") as mocked__write_with_pyarrow: - WriteS3ParquetIO(config, engine="pyarrow").write(input_df) + WriteS3IO(config, engine="pyarrow").write(input_df) # Then mocked__write_with_pyarrow.assert_called() @@ -665,7 +713,7 @@ def test_write_with_fastparquet_is_called_when_engine_option_is_set_to_fastparqu # When with patch.object(dynamicio.mixins.with_local.WithLocal, "_WithLocal__write_with_fastparquet") as mocked__write_with_fastparquet: - WriteS3ParquetIO(config, engine="fastparquet").write(input_df) + WriteS3IO(config, engine="fastparquet").write(input_df) # Then mocked__write_with_fastparquet.assert_called() @@ -727,6 +775,42 @@ def dummy_to_hdf(*args, **kwargs): # pylint: disable=unused-argument assert duration >= 0.2 +class TestConsistencyBetweenPandasAndWrangler: + + @pytest.mark.unit + @pytest.mark.parametrize( + "file_name, is_single_record, wrangler_df, IOClass", + [ + # โœ… Supported: Single record + ("single_row_json", True, pd.DataFrame([{"data": {"release": "feb09", "timestamp": 1614268643313}}]), ReadS3JsonOrientRecordsIO), + # โœ… Supported: Multi-records + ("multi_row_json", False, pd.DataFrame([{"release": "feb09", "timestamp": 1614268643313}, {"release": "feb10", "timestamp": 1614268643313}]), ReadS3JsonOrientRecordsAltIO), + ], + ) + def test_pandas_read_json_returns_the_same_df_as_wrangler_read_json(self, file_name, is_single_record, wrangler_df, IOClass): + # Given + pandas_config = IOConfig( + path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + env_identifier="LOCAL", + dynamic_vars=constants, + ).get(source_key="S3_PANDAS_READER_CONSISTENCY") + + wrangler_config = IOConfig( + path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key="S3_PANDAS_READER_CONSISTENCY") + + # When + pandas_df = IOClass(source_config=pandas_config, file_name=file_name, single_record=is_single_record).read() + with patch.object(dynamicio.mixins.with_s3.wr.s3, "read_json") as mock__wr_s3_json_reader: + mock__wr_s3_json_reader.return_value = wrangler_df + wrangler_df = IOClass(source_config=wrangler_config).read() + + # Then + pd.testing.assert_frame_equal(pandas_df, wrangler_df) + + class TestBatchLocal: @pytest.mark.unit def test_multiple_files_are_loaded_when_batch_local_type_is_used_for_parquet(self, expected_s3_parquet_df): diff --git a/tests/test_mixins/test_postgres_mixins.py b/tests/test_mixins/test_postgres_mixins.py index 62447db..e8ce46d 100644 --- a/tests/test_mixins/test_postgres_mixins.py +++ b/tests/test_mixins/test_postgres_mixins.py @@ -1,23 +1,35 @@ # pylint: disable=no-member, missing-module-docstring, missing-class-docstring, missing-function-docstring, too-many-public-methods, too-few-public-methods, protected-access, C0103, C0302, R0801 import os -from unittest.mock import ANY, patch +from unittest.mock import ANY, MagicMock, patch import pandas as pd import pytest from sqlalchemy.sql.base import ImmutableColumnCollection +# Application Imports from dynamicio import WithPostgres from dynamicio.config import IOConfig from tests import constants -from tests.mocking.io import ( - ReadPostgresIO, - WriteExtendedPostgresIO, - WritePostgresIO, -) +from tests.mocking.io import ReadPostgresIO, WriteExtendedPostgresIO, WritePostgresIO from tests.mocking.models import ERModel, PgModel class TestPostgresIO: + + @pytest.mark.unit + @patch.object(pd.DataFrame, "to_sql") + def test_write_to_postgres_using_replace_strategy(self, mock_to_sql): + # Given + session = MagicMock() + df = pd.DataFrame({"id": [1], "val": ["a"]}) + + # When + WithPostgres._write_to_database(session, "dummy_table", df, is_truncate_and_append=False) + + # Then + mock_to_sql.assert_called_once() + session.commit.assert_called_once() + @pytest.mark.unit def test_when_reading_from_postgres_with_env_as_cloud_get_table_columns_returns_valid_list_of_columns_for_a_model(self, expected_columns): # Given @@ -29,6 +41,7 @@ def test_when_reading_from_postgres_with_env_as_cloud_get_table_columns_returns_ # When columns = ReadPostgresIO(source_config=pg_cloud_config)._get_table_columns(ERModel) # pylint: disable=protected-access + # Then assert columns == expected_columns diff --git a/tests/test_mixins/test_s3_mixins.py b/tests/test_mixins/test_s3_mixins.py index 32c3ee5..084f656 100644 --- a/tests/test_mixins/test_s3_mixins.py +++ b/tests/test_mixins/test_s3_mixins.py @@ -3,178 +3,189 @@ import shutil from tempfile import NamedTemporaryFile from unittest import mock -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pandas as pd import pydantic import pytest import yaml +# Application Imports import dynamicio.mixins.with_local import dynamicio.mixins.with_s3 from dynamicio.config import IOConfig -from dynamicio.errors import ColumnsDataTypeError from tests import constants -from tests.constants import TEST_RESOURCES from tests.mocking.io import ( ReadS3CsvIO, ReadS3HdfIO, + ReadS3IO, ReadS3JsonIO, + ReadS3JsonOrientRecordsAltIO, + ReadS3JsonOrientRecordsIO, ReadS3ParquetIO, ReadS3ParquetWEmptyFilesIO, - ReadS3ParquetWithDifferentCastableDTypeIO, - ReadS3ParquetWithDifferentNonCastableDTypeIO, ReadS3ParquetWithLessColumnsIO, TemplatedFile, - WriteS3CsvIO, - WriteS3HdfIO, - WriteS3JsonIO, - WriteS3ParquetIO, + WriteS3IO, + WriteS3JsonOrientRecordsIO, ) class TestS3FileIO: @pytest.mark.unit def test_read_resolves_file_path_if_templated(self): - # source data read from: "[[ TEST_RESOURCES ]]/data/input/some_csv_to_read.parquet" - config = IOConfig( + # Given + # TEMPLATED_FILE_PATH: + # LOCAL: + # ... + # CLOUD: + # type: "s3_file" + # s3: + # bucket: "[[ MOCK_BUCKET ]]" + # file_path: "path/to/{file_name_to_replace}.csv" + # file_type: "csv" + cloud_config = IOConfig( path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), env_identifier="CLOUD", dynamic_vars=constants, ).get(source_key="TEMPLATED_FILE_PATH") file_path = f"{constants.TEST_RESOURCES}/data/input/some_csv_to_read.csv" + expected_df = pd.read_csv(file_path) # When - with patch.object(dynamicio.mixins.with_local.WithLocal, "_read_csv_file") as mock__read_csv_file, patch.object( - dynamicio.mixins.with_s3.WithS3File, "_s3_named_file_reader" - ) as mock_s3_reader: - with open(file_path, "r") as file: # pylint: disable=unspecified-encoding - mock_s3_reader.return_value = file - io_obj = TemplatedFile(source_config=config, file_name_to_replace="some_csv_to_read") - final_schema = io_obj.schema - io_obj.read() + # Patch the actual S3 read method used by WithS3File + with patch.object(dynamicio.mixins.with_s3.WithS3File, "_read_s3_csv_file", return_value=expected_df) as mock__read_csv_file: + io_obj = TemplatedFile(source_config=cloud_config, file_name_to_replace="some_csv_to_read") + final_schema = io_obj.schema + io_obj.read() - mock__read_csv_file.assert_called_once_with(file_path, final_schema) + # Then + mock__read_csv_file.assert_called_once_with("s3://mock-bucket/path/to/some_csv_to_read.csv", final_schema) @pytest.mark.unit def test_write_resolves_file_path_if_templated(self): # Given - # source data read from: "[[ TEST_RESOURCES ]]/data/input/some_csv_to_read.parquet" - config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + # TEMPLATED_FILE_PATH: + # LOCAL: + # ... + # CLOUD: + # type: "s3_file" + # s3: + # bucket: "[[ MOCK_BUCKET ]]" + # file_path: "path/to/{file_name_to_replace}.csv" + # file_type: "csv" + cloud_config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", dynamic_vars=constants, ).get(source_key="TEMPLATED_FILE_PATH") # When - with patch.object(dynamicio.mixins.with_local.WithLocal, "_write_csv_file") as mock__write_csv_file: - df = pd.read_csv(os.path.join(TEST_RESOURCES, "data/input/some_csv_to_read.csv")) - TemplatedFile(source_config=config, file_name_to_replace="some_csv_to_read").write(df) + file_path = os.path.join(constants.TEST_RESOURCES, "data/input/some_csv_to_read.csv") + df = pd.read_csv(file_path) + + # Patch S3 write method instead of local one + with patch.object(dynamicio.mixins.with_s3.WithS3File, "_write_s3_csv_file") as mock__write_csv_file: + TemplatedFile(source_config=cloud_config, file_name_to_replace="some_csv_to_read").write(df) # Then args, _ = mock__write_csv_file.call_args - assert "s3://mock-bucket/path/to/some_csv_to_read.csv" == args[1] + assert args[0].equals(df) + assert args[1] == "s3://mock-bucket/path/to/some_csv_to_read.csv" @pytest.mark.unit @patch.object(dynamicio.mixins.with_s3.WithS3File, "_read_from_s3_file") def test_read_from_s3_file_is_called_for_loading_a_file_with_env_as_cloud_s3(self, mock__read_from_s3_file, expected_s3_csv_df): # Given mock__read_from_s3_file.return_value = expected_s3_csv_df - s3_csv_cloud_config = IOConfig( + cloud_config = IOConfig( path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), env_identifier="CLOUD", dynamic_vars=constants, ).get(source_key="READ_FROM_S3_CSV") # When - ReadS3CsvIO(source_config=s3_csv_cloud_config).read() + ReadS3CsvIO(source_config=cloud_config).read() # Then mock__read_from_s3_file.assert_called() @pytest.mark.unit - def test_s3_reader_is_not_called_for_loading_a_parquet_with_env_as_cloud_s3_and_type_as_parquet_and_no_disk_space_flag(self): + @patch("dynamicio.mixins.with_s3.boto3.client") + def test_boto3_client_is_used_for_loading_a_hdf_with_env_as_cloud_s3_and_type_as_hdf(self, mock_boto3_client, expected_s3_hdf_file_path): # Given - s3_parquet_cloud_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + cloud_config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="READ_FROM_S3_PARQUET") + ).get(source_key="READ_FROM_S3_HDF") - file_path = f"{constants.TEST_RESOURCES}/data/input/some_csv_to_read.csv" + def mock_download_fobj(_, __, fobj): + with open(expected_s3_hdf_file_path, "rb") as fin: + shutil.copyfileobj(fin, fobj) + + mock_boto3_client.return_value.download_fileobj.side_effect = mock_download_fobj # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_reader") as mock_s3_reader, patch.object( - dynamicio.mixins.with_s3.WithS3File, "_read_parquet_file" - ) as mock_read_parquet_file: - with open(file_path, "r") as file: # pylint: disable=unspecified-encoding - mock_s3_reader.return_value = file - ReadS3ParquetIO(source_config=s3_parquet_cloud_config, no_disk_space=True).read() + ReadS3HdfIO(source_config=cloud_config).read() # Then - mock_s3_reader.assert_not_called() - mock_read_parquet_file.assert_called() + mock_boto3_client.assert_called() @pytest.mark.unit - def test_s3_reader_is_called_for_loading_a_hdf_with_env_as_cloud_s3_and_type_as_hdf(self, expected_s3_hdf_file_path, expected_s3_hdf_df): + @patch("dynamicio.mixins.with_s3.wr.s3.read_csv") + def test_wrangler_csv_reader_is_used_for_loading_a_csv_with_env_as_cloud_s3_and_file_type_csv(self, mock_wrangler_csv_reader, expected_s3_csv_df): # Given - s3_hdf_cloud_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + cloud_config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="READ_FROM_S3_HDF") - - # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "boto3_client") as mock__boto3_client: + ).get(source_key="READ_FROM_S3_CSV") - def mock_download_fobj(s3_bucket, s3_key, target_file): # pylint: disable=unused-argument - with open(expected_s3_hdf_file_path, "rb") as fin: - shutil.copyfileobj(fin, target_file) + mock_wrangler_csv_reader.return_value = expected_s3_csv_df - mock__boto3_client.download_fileobj.side_effect = mock_download_fobj - loaded_hdf_pd = ReadS3HdfIO(source_config=s3_hdf_cloud_config, no_disk_space=True).read() + # When + ReadS3CsvIO(source_config=cloud_config).read() # Then - pd.testing.assert_frame_equal(loaded_hdf_pd, expected_s3_hdf_df) + mock_wrangler_csv_reader.assert_called() @pytest.mark.unit - def test_s3_reader_is_not_called_for_loading_a_json_with_env_as_cloud_s3_and_type_as_json_and_no_disk_space_flag(self): + @patch("dynamicio.mixins.with_s3.wr.s3.read_parquet") + def test_wrangler_parquet_reader_is_used_for_loading_a_csv_with_env_as_cloud_s3_and_file_type_parquet(self, mock_wrangler_parquet_reader, expected_s3_parquet_df): # Given - s3_json_cloud_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + cloud_config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="READ_FROM_S3_JSON") + ).get(source_key="READ_FROM_S3_PARQUET") + + mock_wrangler_parquet_reader.return_value = expected_s3_parquet_df # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_reader") as mock__s3_reader, patch.object( - dynamicio.mixins.with_s3.WithS3File, "_read_json_file" - ) as mock__read_json_file: - ReadS3JsonIO(source_config=s3_json_cloud_config, no_disk_space=True).read() + ReadS3ParquetIO(source_config=cloud_config).read() # Then - mock__s3_reader.assert_not_called() - mock__read_json_file.assert_called() + mock_wrangler_parquet_reader.assert_called() @pytest.mark.unit - def test_s3_reader_is_not_called_for_loading_a_csv_with_env_as_cloud_s3_and_type_as_csv_and_no_disk_space_flag(self): + @patch("dynamicio.mixins.with_s3.wr.s3.read_json") + def test_wrangler_json_reader_is_used_for_loading_a_csv_with_env_as_cloud_s3_and_file_type_json(self, mock_wrangler_json_reader, expected_s3_json_df): # Given - s3_csv_cloud_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + cloud_config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="READ_FROM_S3_CSV") + ).get(source_key="READ_FROM_S3_JSON") + + mock_wrangler_json_reader.return_value = expected_s3_json_df # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_reader") as mock__s3_reader, patch.object( - dynamicio.mixins.with_s3.WithS3File, "_read_csv_file" - ) as mock__read_csv_file: - ReadS3CsvIO(source_config=s3_csv_cloud_config, no_disk_space=True).read() + ReadS3JsonIO(source_config=cloud_config).read() # Then - mock__s3_reader.assert_not_called() - mock__read_csv_file.assert_called() + mock_wrangler_json_reader.assert_called() @pytest.mark.unit def test_ValueError_is_raised_if_file_path_missing_from_config(self, tmp_path): @@ -193,6 +204,7 @@ def test_ValueError_is_raised_if_file_path_missing_from_config(self, tmp_path): "CLOUD": { "type": "s3_file", "s3": {"bucket": "[[ MOCK_BUCKET ]]", "file_type": "csv"}, + # The file path should have been defined here... }, "schema": {"file_path": "[[ TEST_RESOURCES ]]/schemas/read_from_s3_csv.yaml"}, } @@ -208,211 +220,451 @@ def test_ValueError_is_raised_if_file_path_missing_from_config(self, tmp_path): ) @pytest.mark.unit - def test_s3_writers_only_validate_schema_prior_writing_out_the_dataframe(self): + @patch.object(dynamicio.mixins.with_s3.WithS3File, "_write_to_s3_file") + def test_s3_writer_is_called_for_writing_a_file_with_env_is_set_to_cloud_s3(self, mock__write_to_s3_file): # Given - input_df = pd.DataFrame.from_dict({"col_1": [3, 2, 1], "col_2": ["a", "b", "c"], "col_3": ["a", "b", "c"]}) + df = pd.DataFrame.from_dict({"id": [3, 2, 1, 0], "foo_name": ["a", "b", "c", "d"], "bar": [1, 2, 3, 4]}) - s3_parquet_cloud_config = IOConfig( + s3_json_local_config = IOConfig( path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="WRITE_TO_S3_PARQUET") + ).get(source_key="WRITE_TO_S3_JSON") # When - # class WriteS3ParquetIO(DynamicDataIO): - # schema = {"col_1": "int64", "col_2": "object"} - # - # @staticmethod - # def validate(df: pd.DataFrame): - # pass - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_writer") as mock__s3_writer, patch.object(WriteS3ParquetIO, "_apply_schema") as mock__apply_schema, patch.object( - WriteS3ParquetIO, "_write_parquet_file" - ) as mock__write_parquet_file: - with NamedTemporaryFile(delete=False) as temp_file: - mock__s3_writer.return_value = temp_file - WriteS3ParquetIO(source_config=s3_parquet_cloud_config).write(input_df) + ReadS3HdfIO(source_config=s3_json_local_config).write(df) # Then - mock__apply_schema.assert_called() - mock__write_parquet_file.assert_called() + mock__write_to_s3_file.assert_called() @pytest.mark.unit - def test_columns_data_type_error_exception_is_not_generated_if_column_dtypes_can_be_casted_to_the_expected_dtypes(self, expected_s3_parquet_df): + def test_wrangler_to_parquet_is_called_for_writing_a_parquet_with_env_as_cloud_s3_and_file_type_as_parquet(self): # Given - s3_parquet_cloud_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + # WRITE_TO_S3_PARQUET: + # LOCAL: + # ... + # CLOUD: + # type: "s3_file" + # s3: + # bucket: "[[ MOCK_BUCKET ]]" + # file_path: "test/write_some_parquet.parquet" + # file_type: "parquet" + df = pd.DataFrame.from_dict({"col_1": [3, 2, 1, 0], "col_2": ["a", "b", "c", "d"]}) + + cloud_config = IOConfig( + path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="READ_FROM_S3_PARQUET") + ).get(source_key="WRITE_TO_S3_PARQUET") # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_read_parquet_file") as mock__read_parquet_file, patch.object( - dynamicio.mixins.with_s3.WithS3File, "_s3_named_file_reader" - ): - mock__read_parquet_file.return_value = expected_s3_parquet_df - ReadS3ParquetWithDifferentCastableDTypeIO(source_config=s3_parquet_cloud_config).read() + with patch.object(dynamicio.mixins.with_s3.wr.s3, "to_parquet") as mock__wr_s3_parquet_writer: + with NamedTemporaryFile(delete=False) as temp_file: + mock__wr_s3_parquet_writer.return_value = temp_file + WriteS3IO(source_config=cloud_config).write(df) - assert True, "No exception was raised" + # Then + mock__wr_s3_parquet_writer.assert_called() @pytest.mark.unit - @patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_named_file_reader") - @patch.object(dynamicio.mixins.with_s3.WithS3File, "_read_parquet_file") - def test_columns_data_type_error_exception_is_generated_if_column_dtypes_dont_map_to_the_expected_dtypes(self, mock__s3_reader, moc__read_parquet_file, expected_s3_parquet_df): - """ - ------------------------------ Captured log call ------------------------------- - - WARNING ...:dataio.py:273 Expected: 'float64' dtype for column: 'id', found: 'int64' instead. - WARNING ...:dataio.py:273 Expected: 'int64' dtype for column: 'foo_name', found: 'object' instead. - ERROR ...:dataio.py:277 Tried casting column: 'foo_name' to 'int64' from 'object', but failed. - - =========================== short test summary info ============================ - - FAILED ...:test_columns_data_type_error_exception_is_generated_if_column_dtypes_dont_map_to_the_expected_dtypes - - ============================== 1 failed in 0.48s =============================== - - """ + def test_wrangler_to_csv_is_called_for_writing_a_csv_with_env_as_cloud_s3_and_file_type_as_csv(self): # Given - dataframe_returned = expected_s3_parquet_df - mock__s3_reader.return_value = dataframe_returned - - s3_parquet_cloud_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), - env_identifier="CLOUD", - dynamic_vars=constants, - ).get(source_key="READ_FROM_S3_PARQUET") - - # When/Then - with pytest.raises(ColumnsDataTypeError): - ReadS3ParquetWithDifferentNonCastableDTypeIO(source_config=s3_parquet_cloud_config).read() - moc__read_parquet_file.assert_called() + # WRITE_TO_S3_CSV: + # LOCAL: + # ... + # CLOUD: + # type: "s3_file" + # s3: + # bucket: "[[ MOCK_BUCKET ]]" + # file_path: "test/write_some_csv.csv" + # file_type: "csv" + df = pd.DataFrame.from_dict({"col_1": [3, 2, 1, 0], "col_2": ["a", "b", "c", "d"]}) - @pytest.mark.unit - def test_read_parquet_file_is_called_while_s3_reader_is_not_for_loading_a_parquet_with_env_as_cloud_s3_and_type_as_parquet_with_no_disk_space_option( - self, - ): - # Given - s3_parquet_cloud_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml")), + cloud_config = IOConfig( + path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="READ_FROM_S3_PARQUET") + ).get(source_key="WRITE_TO_S3_CSV") # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_reader") as mock__s3_reader, patch.object( - dynamicio.mixins.with_local.WithLocal, "_read_parquet_file" - ) as mock__read_parquet_file: - ReadS3ParquetIO(source_config=s3_parquet_cloud_config, no_disk_space=True).read() + with patch.object(dynamicio.mixins.with_s3.wr.s3, "to_csv") as mock__wr_s3_csv_writer: + with NamedTemporaryFile(delete=False) as temp_file: + mock__wr_s3_csv_writer.return_value = temp_file + WriteS3IO(source_config=cloud_config).write(df) # Then - mock__s3_reader.assert_not_called() - mock__read_parquet_file.assert_called() + mock__wr_s3_csv_writer.assert_called() @pytest.mark.unit - @patch.object(dynamicio.mixins.with_s3.WithS3File, "_write_to_s3_file") - def test_s3_writer_is_called_for_writing_a_file_with_env_is_set_to_cloud_s3(self, mock__write_to_s3_file): + def test_wrangler_to_json_is_called_for_writing_a_json_with_env_as_cloud_s3_and_file_type_as_json(self): # Given - df = pd.DataFrame.from_dict({"id": [3, 2, 1, 0], "foo_name": ["a", "b", "c", "d"], "bar": [1, 2, 3, 4]}) + # WRITE_TO_S3_JSON: + # LOCAL: + # ... + # CLOUD: + # type: "s3_file" + # s3: + # bucket: "[[ MOCK_BUCKET ]]" + # file_path: "test/write_some_json.json" + # file_type: "json" + df = pd.DataFrame.from_dict({"col_1": [3, 2, 1, 0], "col_2": ["a", "b", "c", "d"]}) - s3_json_local_config = IOConfig( + cloud_config = IOConfig( path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), env_identifier="CLOUD", dynamic_vars=constants, ).get(source_key="WRITE_TO_S3_JSON") # When - ReadS3HdfIO(source_config=s3_json_local_config).write(df) + with patch.object(dynamicio.mixins.with_s3.wr.s3, "to_json") as mock__wr_s3_json_writer: + with NamedTemporaryFile(delete=False) as temp_file: + mock__wr_s3_json_writer.return_value = temp_file + WriteS3IO(source_config=cloud_config).write(df) # Then - mock__write_to_s3_file.assert_called() + mock__wr_s3_json_writer.assert_called() @pytest.mark.unit - def test_write_parquet_file_is_called_for_writing_a_parquet_with_env_as_cloud_s3_and_type_as_s3(self): + def test_boto3_client_is_used_for_uploading_a_hdf_with_env_as_cloud_s3_and_file_type_as_hdf(self): # Given + # WRITE_TO_S3_HDF: + # LOCAL: + # ... + # CLOUD: + # type: "s3_file" + # s3: + # bucket: "[[ MOCK_BUCKET ]]" + # file_path: "test/write_some_h5.h5" + # file_type: "hdf" df = pd.DataFrame.from_dict({"col_1": [3, 2, 1, 0], "col_2": ["a", "b", "c", "d"]}) - s3_parquet_local_config = IOConfig( + cloud_config = IOConfig( path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="WRITE_TO_S3_PARQUET") + ).get( + source_key="WRITE_TO_S3_HDF" + ) # โœ… fix: this should be HDF not JSON + + mock_boto3_client = MagicMock() + mock_upload = mock_boto3_client.upload_fileobj # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_writer") as mock__s3_writer, patch.object( - dynamicio.mixins.with_local.WithLocal, "_write_parquet_file" - ) as mock__write_parquet_file: - with NamedTemporaryFile(delete=False) as temp_file: - mock__s3_writer.return_value = temp_file - WriteS3ParquetIO(source_config=s3_parquet_local_config).write(df) + with patch("dynamicio.mixins.with_s3.boto3.client", return_value=mock_boto3_client): + WriteS3IO(source_config=cloud_config).write(df) # Then - mock__write_parquet_file.assert_called() + mock_upload.assert_called() + + +class TestAllowedArgsAreConfiguredCorrectlyForWithS3File: @pytest.mark.unit - def test_write_csv_file_is_called_for_writing_a_parquet_with_env_as_cloud_s3_and_type_as_csv(self): + @pytest.mark.parametrize( + "source_key, patch_target, input_options, expected_kwargs", + [ + ("READ_FROM_S3_PARQUET", "read_parquet", {"ignore_empty": True, "invalid_opt": True}, {"ignore_empty": True}), + ( + "READ_FROM_S3_CSV", + "read_csv", + {"compression": "gzip", "dataset": True, "skipinitialspace": True, "invalid_opt": 123}, + {"compression": "gzip", "dataset": True, "skipinitialspace": True}, + ), + ("READ_FROM_S3_JSON", "read_json", {"version_id": "1.0.1", "orient": "records", "invalid": "nope"}, {"version_id": "1.0.1", "orient": "records"}), + ], + ) + def test_wr_s3_readers_accept_only_valid_options(self, source_key, patch_target, input_options, expected_kwargs, expected_s3_csv_df): # Given - df = pd.DataFrame.from_dict({"id": [3, 2, 1, 0], "foo_name": ["a", "b", "c", "d"], "bar": [1, 2, 3, 4]}) - - s3_csv_local_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), + # We provide options from a combination of kwargs from both aws-wrangler and pandas readers + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), env_identifier="CLOUD", dynamic_vars=constants, - ).get(source_key="WRITE_TO_S3_CSV") + ).get(source_key=source_key) # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_writer") as mock__s3_writer, patch.object( - dynamicio.mixins.with_local.WithLocal, "_write_csv_file" - ) as mock__write_csv_file: - with NamedTemporaryFile(delete=False) as temp_file: - mock__s3_writer.return_value = temp_file - WriteS3CsvIO(source_config=s3_csv_local_config).write(df) + with patch.object(dynamicio.mixins.with_s3.wr.s3, patch_target, return_value=expected_s3_csv_df) as mock_reader: + ReadS3IO(source_config=config, **input_options).read() # Then - mock__write_csv_file.assert_called() + call_kwargs = mock_reader.call_args.kwargs + for k, v in expected_kwargs.items(): + assert call_kwargs[k] == v + assert all(k not in call_kwargs for k in input_options if k not in expected_kwargs) @pytest.mark.unit - def test_write_json_file_is_called_for_writing_a_parquet_with_env_as_cloud_s3_and_type_as_json(self): + @pytest.mark.parametrize( + "source_key, patch_target, input_options, expected_options", + [ + ("WRITE_TO_S3_PARQUET", "to_parquet", {"compression": "snappy", "invalid_opt": True}, {"compression": "snappy", "dataset": False}), + ( + "WRITE_TO_S3_CSV", + "to_csv", + {"concurrent_partitioning": True, "compression": "gzip", "invalid_opt": 123}, + {"concurrent_partitioning": True, "compression": "gzip", "index": False}, + ), + ("WRITE_TO_S3_JSON", "to_json", {"mode": "overwrite", "date_format": "iso", "invalid": "nope"}, {"mode": "overwrite", "date_format": "iso"}), + ], + ) + def test_wr_s3_writers_accept_only_valid_options(self, source_key, patch_target, input_options, expected_options): + df = pd.DataFrame({"col_1": [1, 2], "col_2": ["a", "b"]}) + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml"), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key=source_key) + + with patch.object(dynamicio.mixins.with_s3.wr.s3, patch_target) as mock_writer: + WriteS3IO(source_config=config, **input_options).write(df) + + call_kwargs = mock_writer.call_args.kwargs + for k, v in expected_options.items(): + assert call_kwargs[k] == v + assert all(k not in call_kwargs for k in input_options if k not in expected_options) + + @pytest.mark.unit + @pytest.mark.parametrize( + "input_options, raw_df_data, expected_df, raises_exception, io_class", + [ + # โœ… Supported: Single record json + # Sample Json Input: + # { + # "data": { + # "release": "feb09", + # "timestamp": 1614268643313 + # } + # } + ( + {"orient": "records"}, + pd.DataFrame([{"data": {"release": "current", "timestamp": 1744281068}}]), # raw_wrangler_json_read_df + pd.DataFrame([{"data": {"release": "current", "timestamp": 1744281068}}]), # expected_df + False, + ReadS3JsonOrientRecordsIO, + ), + # โœ… Supported: Multiple records + # Sample Json Input: + # [ + # { "release": "feb09", "timestamp": 1614268643313 }, + # { "release": "feb10", "timestamp": 1614268643313 } + # ] + ( + {"orient": "records"}, + pd.DataFrame( + [ + {"release": "feb09", "timestamp": 1614268643313}, + {"release": "feb10", "timestamp": 1614268643313}, + ] + ), + pd.DataFrame( + [ + {"release": "feb09", "timestamp": 1614268643313}, + {"release": "feb10", "timestamp": 1614268643313}, + ] + ), + False, + ReadS3JsonOrientRecordsAltIO, + ), + # โŒ Unsupported: index (should raise) + ( + {"orient": "index"}, + pd.DataFrame([{"data": {"release": "current", "timestamp": 1744281068}}]), + None, + True, + ReadS3JsonOrientRecordsIO, + ), + # โŒ Unsupported: columns (should raise) + ( + {"orient": "columns"}, + pd.DataFrame([{"data": {"release": "current", "timestamp": 1744281068}}]), + None, + True, + ReadS3JsonOrientRecordsIO, + ), + # โŒ Unsupported: values (should raise) + ( + {"orient": "values"}, + pd.DataFrame([{"data": {"release": "current", "timestamp": 1744281068}}]), + None, + True, + ReadS3JsonOrientRecordsIO, + ), + # โŒ Unsupported: split (should raise) + ( + {"orient": "split"}, + pd.DataFrame([{"data": {"release": "current", "timestamp": 1744281068}}]), + None, + True, + ReadS3JsonOrientRecordsIO, + ), + ], + ) + def test_json_reader_applies_postprocessing_for_unsupported_orientations(self, input_options, raw_df_data, expected_df, raises_exception, io_class): # Given - df = pd.DataFrame.from_dict({"col_1": [3, 2, 1, 0], "col_2": ["a", "b", "c", "d"]}) + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key="READ_FROM_S3_JSON") - s3_json_local_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), + # When + with patch("dynamicio.mixins.with_s3.wr.s3.read_json", return_value=raw_df_data) as mock_reader: + if raises_exception: + with pytest.raises(ValueError): + io_class(source_config=config, **input_options).read() + else: + df = io_class(source_config=config, **input_options).read() + call_kwargs = mock_reader.call_args.kwargs + assert call_kwargs["orient"] == "records" + assert call_kwargs["lines"] is True + pd.testing.assert_frame_equal(df, expected_df) + + @pytest.mark.unit + @pytest.mark.parametrize( + "input_options, io_class, should_raise, expected_warning", + [ + # โœ… Supported: records + ({"orient": "records", "lines": True}, WriteS3JsonOrientRecordsIO, False, None), + # โœ… Supported: records with overridden lines + ({"orient": "records", "lines": False}, WriteS3JsonOrientRecordsIO, False, "[s3-json] Overriding lines=False with lines=True for JSON serialization."), + # โŒ Unsupported: index + ({"orient": "index", "lines": True}, WriteS3JsonOrientRecordsIO, True, None), + # โŒ Unsupported: values + ({"orient": "values", "lines": True}, WriteS3JsonOrientRecordsIO, True, None), + # โŒ Unsupported: split + ({"orient": "split", "lines": True}, WriteS3JsonOrientRecordsIO, True, None), + ], + ) + def test_json_writer_enforces_orient_restrictions(self, input_options, io_class, should_raise, expected_warning, caplog): + df_input = pd.DataFrame([{"release": "feb09", "timestamp": 1614268643313}]) + + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml"), env_identifier="CLOUD", dynamic_vars=constants, ).get(source_key="WRITE_TO_S3_JSON") - # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_writer") as mock__s3_writer, patch.object( - dynamicio.mixins.with_local.WithLocal, "_write_json_file" - ) as mock__write_json_file: - with NamedTemporaryFile(delete=False) as temp_file: - mock__s3_writer.return_value = temp_file - WriteS3JsonIO(source_config=s3_json_local_config).write(df) + with patch("dynamicio.mixins.with_s3.wr.s3.to_json") as mock_writer: + if should_raise: + with pytest.raises(ValueError): + io_class(source_config=config, **input_options).write(df_input) + else: + io_class(source_config=config, **input_options).write(df_input) + + # Check that the `orient` and `lines` passed to wr.s3.to_json are correct + call_kwargs = mock_writer.call_args.kwargs + assert call_kwargs["orient"] == "records" + assert call_kwargs["lines"] is True + assert call_kwargs["index"] is False + + if expected_warning: + assert expected_warning in caplog.text + + @pytest.mark.unit + def test_hdf_reader_accepts_only_valid_options(self, expected_s3_hdf_file_path): + # Given + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/input.yaml"), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key="READ_FROM_S3_HDF") + + def mock_download_fobj(_, __, fobj): + with open(expected_s3_hdf_file_path, "rb") as fin: + fobj.write(fin.read()) + + mock_boto3 = MagicMock() + mock_boto3.download_fileobj.side_effect = mock_download_fobj + + with patch("dynamicio.mixins.with_s3.boto3.client", return_value=mock_boto3): + with patch("dynamicio.mixins.with_s3.pd.read_hdf") as mock_read_hdf: + mock_read_hdf.return_value = pd.DataFrame({"id": [1, 2]}) + + # When + ReadS3IO(source_config=config, invalid_opt=True, start=0).read() # Then - mock__write_json_file.assert_called() + call_kwargs = mock_read_hdf.call_args.kwargs + assert "start" in call_kwargs + assert "invalid_opt" not in call_kwargs @pytest.mark.unit - def test_write_hdf_file_is_called_for_writing_a_parquet_with_env_as_cloud_s3_and_type_as_hdf(self): + def test_hdf_writer_accepts_only_valid_options(self): # Given - df = pd.DataFrame.from_dict({"col_1": [3, 2, 1, 0], "col_2": ["a", "b", "c", "d"]}) - s3_hdf_local_config = IOConfig( - path_to_source_yaml=(os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml")), + df = pd.DataFrame({"col_1": [1, 2], "col_2": ["x", "y"]}) + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml"), env_identifier="CLOUD", dynamic_vars=constants, ).get(source_key="WRITE_TO_S3_HDF") - # When - with patch.object(dynamicio.mixins.with_s3.WithS3File, "_s3_writer") as mock__s3_writer: - with NamedTemporaryFile(delete=False) as temp_file: - mock__s3_writer.return_value = temp_file - WriteS3HdfIO(source_config=s3_hdf_local_config).write(df) + mock_boto3 = MagicMock() + with patch("dynamicio.mixins.with_s3.boto3.client", return_value=mock_boto3): + with patch("dynamicio.mixins.with_s3.HdfIO.save") as mock_save: + mock_save.return_value = None + + # When + WriteS3IO(source_config=config, key="my_key", invalid_opt="nope").write(df) # Then - assert os.stat(temp_file.name).st_size == 1064192, "Confirm that the output file size did not change" + call_kwargs = mock_save.call_args.kwargs["options"] + assert call_kwargs.get("key") == "my_key" + assert "invalid_opt" not in call_kwargs + + @pytest.mark.unit + @pytest.mark.parametrize( + "source_key,file_type", + [ + ("WRITE_TO_S3_PARQUET", "parquet"), + ("WRITE_TO_S3_CSV", "csv"), + ("WRITE_TO_S3_JSON", "json"), + ], + ) + def test_dataset_true_raises_value_error(self, source_key, file_type): + # Given + df = pd.DataFrame({"col_1": [1], "col_2": ["a"]}) + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml"), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key=source_key) + + # When/Then + # Patch wr.s3.to_{file_type} to prevent actual call + patch_target = f"{dynamicio.mixins.with_s3.wr.s3.__name__}.to_{file_type}" + with patch(patch_target): + with pytest.raises(ValueError, match=rf"\[s3-{file_type}\] dataset=True is not supported.*"): + WriteS3IO(source_config=config, dataset=True).write(df) + + @pytest.mark.unit + @pytest.mark.parametrize( + "source_key,file_type", + [ + ("WRITE_TO_S3_PARQUET", "parquet"), + ("WRITE_TO_S3_CSV", "csv"), + ("WRITE_TO_S3_JSON", "json"), + ], + ) + def test_s3_path_as_directory_raises_value_error(self, source_key, file_type): + # Given + df = pd.DataFrame({"col_1": [1], "col_2": ["a"]}) + config = IOConfig( + path_to_source_yaml=os.path.join(constants.TEST_RESOURCES, "definitions/processed.yaml"), + env_identifier="CLOUD", + dynamic_vars=constants, + ).get(source_key=source_key) + + # Force file_path to end with slash + config.s3.file_path = "test/some_dir/" + + # Patch wr.s3.to_{file_type} to prevent actual call + patch_target = f"{dynamicio.mixins.with_s3.wr.s3.__name__}.to_{file_type}" + with patch(patch_target): + with pytest.raises(ValueError, match=rf"\[s3-{file_type}\] .*must be a file, not a directory.*"): + WriteS3IO(source_config=config).write(df) class TestS3PathPrefixIO: + @pytest.mark.unit def test_error_is_raised_if_path_prefix_missing_from_config(self, tmp_path): tmp_yaml = tmp_path / "test.yaml" @@ -456,7 +708,7 @@ def test_ValueError_is_raised_if_partition_cols_missing_from_options_when_upload # When / Then with pytest.raises(ValueError): - WriteS3ParquetIO(source_config=s3_parquet_cloud_config).write(input_df) + WriteS3IO(source_config=s3_parquet_cloud_config).write(input_df) @pytest.mark.unit def test_error_is_raised_if_file_type_not_parquet_when_uploading(self, tmp_path): @@ -515,13 +767,13 @@ def test_write_to_s3_path_prefix_is_called_for_uploading_to_a_path_prefix_with_e ).get(source_key="WRITE_TO_S3_PATH_PREFIX_PARQUET") # When - WriteS3ParquetIO(source_config=s3_parquet_cloud_config).write(input_df) + WriteS3IO(source_config=s3_parquet_cloud_config).write(input_df) # Then mock__write_to_s3_path_prefix.assert_called() @pytest.mark.unit - @patch.object(WriteS3ParquetIO, "_write_parquet_file") + @patch.object(WriteS3IO, "_write_parquet_file") # pylint: disable=unused-argument def test_awscli_runner_is_called_with_correct_s3_path_and_aws_command_when_uploading_a_path_prefix_with_env_as_cloud_s3(self, mock__write_parquet_file, mock_temporary_directory): # Given @@ -534,7 +786,7 @@ def test_awscli_runner_is_called_with_correct_s3_path_and_aws_command_when_uploa # When with patch.object(dynamicio.mixins.with_s3, "awscli_runner") as mocked__awscli_runner: - WriteS3ParquetIO(source_config=s3_parquet_cloud_config, partition_cols="col_2").write(input_df) + WriteS3IO(source_config=s3_parquet_cloud_config, partition_cols="col_2").write(input_df) # Then mocked__awscli_runner.assert_called_with( diff --git a/tests/test_mixins/test_mixin_utils.py b/tests/test_mixins/test_utils.py similarity index 88% rename from tests/test_mixins/test_mixin_utils.py rename to tests/test_mixins/test_utils.py index c511d8b..40daa7f 100644 --- a/tests/test_mixins/test_mixin_utils.py +++ b/tests/test_mixins/test_utils.py @@ -5,6 +5,7 @@ import pytest +# Application Imports from dynamicio.config import IOConfig from dynamicio.mixins.utils import allow_options, args_of, get_file_type_value, get_string_template_field_names, resolve_template from tests import constants @@ -78,7 +79,7 @@ def magic_function(arg_a: str, arg_b: int, arg_c: bool) -> bool: # Then assert options == {"arg_a", "arg_b", "arg_c"} - @pytest.mark.integration + @pytest.mark.unit def test_allow_options_can_use_iterable_returned_from_args_of_to_filter_out_invalid_options( self, ): @@ -99,7 +100,7 @@ def mock_method(**options: Any): # Then assert options == ["arg_a", "arg_b", "arg_c"] - @pytest.mark.integration + @pytest.mark.unit def test_allow_options_does_not_filter_out_valid_args_when_they_are_passed_as_args_and_not_as_kwargs( self, ): @@ -125,7 +126,7 @@ def mock_method(schema: "str", **options: Any): captured = self.capsys.readouterr() assert (captured.out == "schema\n") and (options == ["A", 1, True]) - @pytest.mark.integration # This is an integration test as it uses `allow_options()` after `args_of()` + @pytest.mark.unit # This is an integration test as it uses `allow_options()` after `args_of()` def test_when_reading_locally_or_from_s3_invalid_options_are_ignored(self, expected_s3_csv_df): # Given invalid_option = "INVALID_OPTION" @@ -141,7 +142,7 @@ def test_when_reading_locally_or_from_s3_invalid_options_are_ignored(self, expec # Then assert expected_s3_csv_df.equals(s3_csv_df) - @pytest.mark.integration + @pytest.mark.unit def test_when_reading_locally_or_from_s3_valid_options_are_considered(self, expected_s3_csv_df): # Given # VALID OPTION: dtype=None @@ -157,6 +158,30 @@ def test_when_reading_locally_or_from_s3_valid_options_are_considered(self, expe # Then assert expected_s3_csv_df.equals(s3_csv_df) + def test_args_of_combines_args_from_multiple_functions(self): + # Given + def a(_: int): + pass + + def b(__: str): + pass + + # When/Then + assert args_of(a, b) == {"_", "__"} + + def test_invalid_options_emit_warning_log(self, caplog): + # Given + @allow_options(["a", "b"]) + def method(**_): + pass + + # When + with caplog.at_level("WARNING"): + method(a=1, b=2, x="invalid") + + # Then + assert "Options {'x': 'invalid'} were not used" in caplog.text + class DummyFileType(str, Enum): PARQUET = "parquet" diff --git a/tests/test_table_schema.py b/tests/test_table_schema.py index f43606f..1d390f2 100644 --- a/tests/test_table_schema.py +++ b/tests/test_table_schema.py @@ -1,15 +1,19 @@ -# pylint: disable=missing-module-docstring, missing-function-docstring +# pylint: disable=missing-module-docstring, missing-class-docstring, missing-function-docstring, too-many-public-methods, too-few-public-methods import pytest from pydantic import ValidationError +# Application Imports from dynamicio.config.pydantic.table_schema import SchemaColumn -def test_schema_column_accepts_valid_pandas_types(): - column = SchemaColumn(name="test", type="int64") - assert column.data_type == "int64" +class TestSchemaColumn: + @pytest.mark.unit + def test_schema_column_accepts_valid_pandas_types(self): + column = SchemaColumn(name="test", type="int64") + assert column.data_type == "int64" -def test_schema_column_rejects_invalid_pandas_types(): - with pytest.raises(ValidationError): - SchemaColumn(name="test", type="not a type") + @pytest.mark.unit + def test_schema_column_rejects_invalid_pandas_types(self): + with pytest.raises(ValidationError): + SchemaColumn(name="test", type="not a type") diff --git a/tests/test_validations.py b/tests/test_validations.py index c66f935..882f5c7 100644 --- a/tests/test_validations.py +++ b/tests/test_validations.py @@ -3,6 +3,7 @@ import pytest +# Application Imports from dynamicio.config import IOConfig from dynamicio.validations import ( has_acceptable_percentage_of_nulls, @@ -399,7 +400,7 @@ def test_returns_false_if_any_column_values_are_above_the_threshold(self, input_ class TestIsBetween: - @pytest.mark.integration + @pytest.mark.unit def test_returns_true_if_all_column_values_are_between_upper_and_lower_bounds(self, input_df): # Given df = input_df @@ -410,7 +411,7 @@ def test_returns_true_if_all_column_values_are_between_upper_and_lower_bounds(se # Then assert validation.valid is True and validation.value == 0 and validation.message == "All values of TEST[weight_a] is between 4 and 10 thresholds" - @pytest.mark.integration + @pytest.mark.unit def test_returns_false_if_any_column_values_are_below_the_lower_bound(self, input_df): # Given df = input_df @@ -421,7 +422,7 @@ def test_returns_false_if_any_column_values_are_below_the_lower_bound(self, inpu # Then assert not validation.valid and validation.value == 0.5 and validation.message == "5 cell values for TEST[weight_a] are either below 6 or above 10" - @pytest.mark.integration + @pytest.mark.unit def test_returns_false_if_any_column_values_are_above_the_upper_bound(self, input_df): # Given df = input_df @@ -432,7 +433,7 @@ def test_returns_false_if_any_column_values_are_above_the_upper_bound(self, inpu # Then assert not validation.valid and validation.value == 0.3 and validation.message == "3 cell values for TEST[weight_a] are either below 4 or above 8" - @pytest.mark.integration + @pytest.mark.unit def test_returns_true_if_all_column_values_are_within_bounds_bounds_included(self, input_df): # Given df = input_df @@ -443,7 +444,7 @@ def test_returns_true_if_all_column_values_are_within_bounds_bounds_included(sel # Then assert validation.valid is True and validation.value == 0 and validation.message == "All values of TEST[weight_a] is between 5 and 9 thresholds" - @pytest.mark.integration + @pytest.mark.unit def test_returns_true_if_all_column_values_are_between_upper_and_lower_bounds_irrespective_of_nulls(self, input_df): # Given df = input_df