From 602d82f05b8c4e7be89856abb407fc7eb4843bfd Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 24 Aug 2026 21:53:31 +0100 Subject: [PATCH 001/107] Route feature metadata persistence through to_json() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_save_training_record` hand-rolled the `features.feature_metadata` JSON with a literal five-key dict, so `SparkFeatureMetadata.to_json()` was dead on the training path. Any field added to the dataclass would have been dropped silently at persistence, and the model would then score with a different feature set than it trained on — a silent train/score skew rather than a loud failure. Three changes, no behaviour change: - `_save_training_record` now calls `artifacts.feature_metadata.to_json()`, and `to_json` iterates the dataclass fields instead of naming them, so it stays complete by construction. Field order is declaration order, so the emitted JSON is byte-identical for existing models. - `from_json` ignores unknown keys instead of raising. It did `cls(**data)`, so a record written by a newer DQX version made an older reader fail outright; it now falls back to defaults for fields it does not recognise. - New `apply_feature_engineering_from_metadata` collapses the five call sites that each hand-passed `frequency_maps` + `onehot_categories` + `categorical_cardinality_threshold` back out of a `SparkFeatureMetadata` (`core.py` x3, `training_service.py`, `feature_prep.py`). Future metadata fields thread through one function rather than five. Groundwork for #1484. --- src/databricks/labs/dqx/anomaly/core.py | 31 ++------ .../labs/dqx/anomaly/feature_prep.py | 10 +-- .../labs/dqx/anomaly/training_service.py | 26 ++----- .../labs/dqx/anomaly/transformers.py | 71 +++++++++++++++---- 4 files changed, 73 insertions(+), 65 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/core.py b/src/databricks/labs/dqx/anomaly/core.py index 46c659197..51531d37b 100644 --- a/src/databricks/labs/dqx/anomaly/core.py +++ b/src/databricks/labs/dqx/anomaly/core.py @@ -29,7 +29,7 @@ ColumnTypeClassifier, SparkFeatureMetadata, apply_feature_engineering, - reconstruct_column_infos, + apply_feature_engineering_from_metadata, ) from databricks.labs.dqx.config import AnomalyParams, IsolationForestConfig from databricks.labs.dqx.errors import ComputationError, InvalidParameterError @@ -164,14 +164,8 @@ def score_with_model( Feature engineering is applied in Spark before the pandas UDF. This enables distributed inference across the Spark cluster. """ - column_infos = reconstruct_column_infos(feature_metadata) - - engineered_df, updated_metadata = apply_feature_engineering( - df.select(*feature_cols), - column_infos, - categorical_cardinality_threshold=feature_metadata.categorical_cardinality_threshold, - frequency_maps=feature_metadata.categorical_frequency_maps, - onehot_categories=feature_metadata.onehot_categories, + engineered_df, updated_metadata = apply_feature_engineering_from_metadata( + df.select(*feature_cols), feature_metadata ) engineered_feature_cols = updated_metadata.engineered_feature_names @@ -207,14 +201,8 @@ def score_with_ensemble_models( models: list[Pipeline], df: DataFrame, feature_cols: list[str], feature_metadata: SparkFeatureMetadata ) -> DataFrame: """Score DataFrame using an ensemble of models and return mean scores.""" - column_infos = reconstruct_column_infos(feature_metadata) - - engineered_df, updated_metadata = apply_feature_engineering( - df.select(*feature_cols), - column_infos, - categorical_cardinality_threshold=feature_metadata.categorical_cardinality_threshold, - frequency_maps=feature_metadata.categorical_frequency_maps, - onehot_categories=feature_metadata.onehot_categories, + engineered_df, updated_metadata = apply_feature_engineering_from_metadata( + df.select(*feature_cols), feature_metadata ) engineered_feature_cols = updated_metadata.engineered_feature_names @@ -407,12 +395,5 @@ def prepare_engineered_pandas(train_df: DataFrame, feature_metadata: SparkFeatur Returns: Pandas DataFrame with engineered features """ - column_infos_reconstructed = reconstruct_column_infos(feature_metadata) - engineered_train_df, _ = apply_feature_engineering( - train_df, - column_infos_reconstructed, - categorical_cardinality_threshold=feature_metadata.categorical_cardinality_threshold, - frequency_maps=feature_metadata.categorical_frequency_maps, - onehot_categories=feature_metadata.onehot_categories, - ) + engineered_train_df, _ = apply_feature_engineering_from_metadata(train_df, feature_metadata) return engineered_train_df.toPandas() diff --git a/src/databricks/labs/dqx/anomaly/feature_prep.py b/src/databricks/labs/dqx/anomaly/feature_prep.py index 7dbe97db8..c89e1eea7 100644 --- a/src/databricks/labs/dqx/anomaly/feature_prep.py +++ b/src/databricks/labs/dqx/anomaly/feature_prep.py @@ -8,7 +8,7 @@ from databricks.labs.dqx.anomaly.transformers import ( ColumnTypeInfo, SparkFeatureMetadata, - apply_feature_engineering, + apply_feature_engineering_from_metadata, reconstruct_column_infos, ) from databricks.labs.dqx.errors import InvalidParameterError @@ -46,12 +46,8 @@ def apply_feature_engineering_for_scoring( cols_to_select = list(dict.fromkeys([*feature_cols, *merge_columns, *(passthrough_columns or [])])) - engineered_df, _ = apply_feature_engineering( - df.select(*cols_to_select), - column_infos, - categorical_cardinality_threshold=feature_metadata.categorical_cardinality_threshold, - frequency_maps=feature_metadata.categorical_frequency_maps, - onehot_categories=feature_metadata.onehot_categories, + engineered_df, _ = apply_feature_engineering_from_metadata( + df.select(*cols_to_select), feature_metadata, column_infos=column_infos ) return engineered_df diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index 8ab6a931b..d9b54b2e4 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -6,7 +6,6 @@ """ import collections.abc -import json import logging from copy import deepcopy from datetime import datetime @@ -35,8 +34,7 @@ from databricks.labs.dqx.anomaly.training_strategies import AnomalyTrainingStrategy, IsolationForestTrainingStrategy from databricks.labs.dqx.anomaly.transformers import ( SparkFeatureMetadata, - apply_feature_engineering, - reconstruct_column_infos, + apply_feature_engineering_from_metadata, ) from databricks.labs.dqx.anomaly.segment_utils import build_segment_name from databricks.labs.dqx.anomaly.types import AnomalyTrainingContext, TrainingArtifacts @@ -135,14 +133,7 @@ def _compute_post_training_metadata( feature_metadata: SparkFeatureMetadata, ) -> dict[str, dict[str, float]]: """Compute baseline statistics after training for drift detection.""" - column_infos_for_stats = reconstruct_column_infos(feature_metadata) - engineered_train_df, _ = apply_feature_engineering( - train_df, - column_infos_for_stats, - categorical_cardinality_threshold=feature_metadata.categorical_cardinality_threshold, - frequency_maps=feature_metadata.categorical_frequency_maps, - onehot_categories=feature_metadata.onehot_categories, - ) + engineered_train_df, _ = apply_feature_engineering_from_metadata(train_df, feature_metadata) baseline_stats = compute_baseline_statistics(engineered_train_df, feature_metadata.engineered_feature_names) return baseline_stats @@ -426,15 +417,10 @@ def _save_training_record( segment_by: list[str] | None, ) -> None: """Save training record to registry table.""" - feature_metadata_json = json.dumps( - { - "column_infos": artifacts.feature_metadata.column_infos, - "categorical_frequency_maps": artifacts.feature_metadata.categorical_frequency_maps, - "onehot_categories": artifacts.feature_metadata.onehot_categories, - "engineered_feature_names": artifacts.feature_metadata.engineered_feature_names, - "categorical_cardinality_threshold": artifacts.feature_metadata.categorical_cardinality_threshold, - } - ) + # Must go through to_json() rather than hand-rolling the payload: it is the single + # writer of this column, so any field added to SparkFeatureMetadata is persisted + # here automatically instead of being silently dropped. + feature_metadata_json = artifacts.feature_metadata.to_json() record = AnomalyModelRecord( identity=ModelIdentity( diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index a27b013a2..01adb067f 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -7,10 +7,11 @@ """ import json +import logging import re import sys import threading -from dataclasses import dataclass +from dataclasses import dataclass, fields from io import StringIO from typing import Any @@ -37,6 +38,8 @@ from databricks.labs.dqx.telemetry import get_tables_from_spark_plan from databricks.labs.dqx.utils import get_table_primary_keys +logger = logging.getLogger(__name__) + # Serialize stdout capture so concurrent column analysis is thread-safe _EXPLAIN_CAPTURE_LOCK = threading.Lock() @@ -72,22 +75,32 @@ class SparkFeatureMetadata: categorical_cardinality_threshold: int = 20 # Threshold used for categorical encoding def to_json(self) -> str: - """Serialize to JSON for storage.""" - return json.dumps( - { - "column_infos": self.column_infos, - "categorical_frequency_maps": self.categorical_frequency_maps, - "onehot_categories": self.onehot_categories, - "engineered_feature_names": self.engineered_feature_names, - "categorical_cardinality_threshold": self.categorical_cardinality_threshold, - } - ) + """Serialize to JSON for storage. + + Every field of the dataclass is persisted. This is the single writer of the + ``features.feature_metadata`` column — do not hand-roll the payload elsewhere, or a + newly added field is silently dropped at persistence and the model scores with a + different feature set than it trained on. + """ + return json.dumps({f.name: getattr(self, f.name) for f in fields(self)}) @classmethod def from_json(cls, json_str: str) -> "SparkFeatureMetadata": - """Deserialize from JSON.""" + """Deserialize from JSON. + + Unknown keys are ignored rather than raising, so a model trained by a newer DQX + version stays readable by an older one: the reader falls back to the defaults for + fields it does not know about instead of failing to load the model at all. + """ data = json.loads(json_str) - return cls(**data) + known_fields = {f.name for f in fields(cls)} + unknown = set(data) - known_fields + if unknown: + logger.debug( + f"Ignoring unknown feature metadata keys {sorted(unknown)}; " + "this model was likely trained by a newer version of DQX." + ) + return cls(**{k: v for k, v in data.items() if k in known_fields}) def _spark_type_for_category(category: str) -> T.DataType: @@ -819,3 +832,35 @@ def apply_feature_engineering( ) return result_df, metadata + + +def apply_feature_engineering_from_metadata( + df: DataFrame, + feature_metadata: SparkFeatureMetadata, + column_infos: list[ColumnTypeInfo] | None = None, +) -> tuple[DataFrame, SparkFeatureMetadata]: + """Re-apply the transformations recorded in *feature_metadata* to a new DataFrame. + + This is the derived ("scoring") mode of :func:`apply_feature_engineering`: rather than + computing encodings from the data, it replays the ones a model was trained with. Every + caller that scores, or that recomputes engineered features for an already-trained model, + should go through here so that a newly persisted metadata field has to be threaded in one + place instead of five. + + Args: + df: DataFrame to transform. + feature_metadata: Metadata persisted at training time. + column_infos: Column infos, reconstructed from *feature_metadata* when omitted. + + Returns: + The engineered DataFrame and the metadata describing it. The returned metadata's + ``engineered_feature_names`` reflects the columns that actually survived on *df*, + which is what the scoring UDF must be handed. + """ + return apply_feature_engineering( + df, + column_infos if column_infos is not None else reconstruct_column_infos(feature_metadata), + categorical_cardinality_threshold=feature_metadata.categorical_cardinality_threshold, + frequency_maps=feature_metadata.categorical_frequency_maps, + onehot_categories=feature_metadata.onehot_categories, + ) From faea0c699795faa2f92c4224ea5308ea41d08325 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 24 Aug 2026 21:56:13 +0100 Subject: [PATCH 002/107] Correct anomaly detection docs that misdescribe behaviour Four documentation defects found while reading the anomaly module for #1484. Each was verified against the code; no behaviour changes here. - `drift_threshold`'s docstring claimed "default 3.0, None to disable". The signature default is `None`, and `None` disables drift detection outright (`drift.py:227` gates on `is not None`), so the stated default was the exact opposite of the real one. The reference docs already had this right. - Drift was described as watching the "score distribution". It compares the *input feature* distributions against the per-column baseline statistics recorded at training and reports which columns drifted (`compute_drift_score`); it never looks at anomaly scores. Corrected in the check table, the parameter reference, and the guide. Also documented the 1,000-row floor below which drift is skipped as too noisy to judge. - `ensemble_size` was documented as if it always applied. Segmented training passes `allow_ensemble=False`, so it trains exactly one model per segment regardless, which also means `confidence_std` is unavailable for segmented models. Said so on the parameter, in the reference table, and in the guide. - The guide advised "100+ rows per segment" without mentioning that segments below **10** rows are skipped entirely, leaving their rows unscored with no error. Documented the hard threshold alongside the guidance. Groundwork for #1484. --- docs/dqx/docs/guide/row_anomaly_detection/index.mdx | 6 +++--- docs/dqx/docs/reference/quality_checks.mdx | 6 +++--- src/databricks/labs/dqx/anomaly/check_funcs.py | 4 +++- src/databricks/labs/dqx/config.py | 3 +++ 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index ae850d4c9..74e270497 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -307,7 +307,7 @@ Isolation Forest measures how "easy" it is to isolate a data point; anomalies ar ### Output structure and options -The `_dq_info` column is an array of structs (one element per dataset-level check that produces info; for row anomaly, one per `has_no_row_anomalies` check). Use `severity_percentile` for threshold decisions; raw `score` is for diagnostics only. Enable `drift_threshold` (for example `3.0`) to get warnings when the scoring distribution shifts from training so you know when to retrain. +The `_dq_info` column is an array of structs (one element per dataset-level check that produces info; for row anomaly, one per `has_no_row_anomalies` check). Use `severity_percentile` for threshold decisions; raw `score` is for diagnostics only. Enable `drift_threshold` (for example `3.0`) to get warnings when the distribution of the input features shifts away from the training baseline, so you know when to retrain. Anomalous records can be identified using the standard reporting columns (`_errors` and `_warnings`). The `_dq_info[0].anomaly.is_anomaly` field provides additional detail for in-depth analysis and is set to `False` for records that are not anomalous. @@ -402,14 +402,14 @@ Use row anomaly detection when you want to catch unusual combinations across col
Q: How much training data do I really need? -See the **Training data requirements** tip under Quick start. In short: 1,000+ rows preferred; quality and coverage matter more than quantity. For segmented models, use at least 100+ rows per segment. Ensure training data includes all realistic values for categorical columns (regions, types, etc.). +See the **Training data requirements** tip under Quick start. In short: 1,000+ rows preferred; quality and coverage matter more than quantity. For segmented models, use at least 100+ rows per segment; segments with fewer than **10** rows are skipped entirely and get no model, so rows in them are left unscored. Ensure training data includes all realistic values for categorical columns (regions, types, etc.) — a categorical value absent from training is not scored reliably.
Q: How often should I retrain? **Retrain when**: -- Drift warnings appear (distribution changed). Enable `drift_threshold=3.0` to get warnings when retraining is needed +- Drift warnings appear (the input feature distribution changed). Enable `drift_threshold=3.0` to get warnings when retraining is needed - Business logic changes (new products, pricing, processes) - Seasonality shifts (quarterly/annual patterns) - Major data pipeline changes diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 720ca77c1..4507f90ae 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -1963,7 +1963,7 @@ You can also define your own custom dataset-level checks (see [Creating custom c | `has_no_gaps_per_time_window` | Dataset check that flags gaps in a time series, i.e. time windows of a given size that contain no rows between windows that do. The violation is reported on the boundary row before each interior gap. | `column`: timestamp or date column (can be a string column name or a column expression); `window_minutes`: size of the time window in minutes that defines the expected data grain (for example 1440 for daily); `group_by`: optional list of columns or column expressions to detect gaps independently within each group; `trailing_gap`: (optional) if `true`, also flags the last present window (per group) when it ends more than one window before `curr_timestamp`, so missing recent data is caught at the tail of the series (defaults to `false`); `curr_timestamp`: (optional) current timestamp column used to anchor trailing-gap detection, only used when `trailing_gap` is `true` (if not provided, current_timestamp() function is used) | | `has_valid_schema` | Schema check that validates whether the DataFrame schema matches an expected schema. In non-strict mode, validates that all expected columns exist with compatible types (allows extra columns). In strict mode, validates exact schema match (same columns, same order, same types) for all columns by default or for all columns specified in `columns`. This check is applied at the dataset level and reports schema violations for all rows in the DataFrame when incompatibilities are detected. All columns in the `exclude_columns` list will be ignored even if the column is present in the `columns` list. | `expected_schema`: (optional) expected schema as a DDL string (e.g., "id INT, name STRING") or StructType object; `ref_df_name`: (optional) name of the reference DataFrame to load the schema from (dictionary of DataFrames can be passed when applying checks); `ref_table`: (optional) fully qualified reference table name to load the schema from (e.g. "catalog.schema.table"); exactly one of `expected_schema`, `ref_df_name`, or `ref_table` must be provided; `columns`: (optional) list of columns to validate (if not provided, all columns are considered); `strict`: (optional) whether to perform strict schema validation (default: False) - False: validates that all expected columns exist with compatible types, True: validates exact schema match; `exclude_columns`: (optional) list of columns to ignore during validation (if not provided, all columns are considered); | | `has_no_outliers` | Checks whether the values in the input column contain any outliers. This function implements a median absolute deviation (MAD) algorithm to find outliers. | `column`: column of type numeric to check (can be a string column name or a column expression); | -| `has_no_row_anomalies` | Flags rows that are anomalous according to a trained ML model. The model learns "normal" patterns from your training data; at check time each row is scored (severity percentile 0–100) and optionally enriched with SHAP contributions. Requires a model trained with the anomaly engine first. See [Row Anomaly Detection](#row-anomaly-detection) below for training, full parameters, and usage. | `model_name`: fully qualified model name (e.g. catalog.schema.model_name); `registry_table`: fully qualified registry table (e.g. catalog.schema.model_registry); `threshold`: (optional) severity percentile threshold (default 95); `drift_threshold`: (optional) warn when score distribution drifts from training (None = off); `enable_contributions`: (optional) add SHAP per-feature contributions to `_dq_info` (default True; set False to skip the SHAP cost); `enable_confidence_std`: (optional) add ensemble score std to `_dq_info` (default False); `enable_ai_explanation`: (optional) add an LLM-generated explanation to `_dq_info` (default True; degrades to null if contributions are off or no serving endpoint is reachable); `ai_explanation_llm_model_config`: (optional) Databricks Model Serving endpoint config for the explanation; `redact_columns`: (optional) feature/segment names to keep out of the LLM prompt; `max_groups`: (optional) cap on LLM calls per run (default 500). See [Row Anomaly Detection](/docs/reference/quality_checks#row-anomaly-detection) section for full parameter details. | +| `has_no_row_anomalies` | Flags rows that are anomalous according to a trained ML model. The model learns "normal" patterns from your training data; at check time each row is scored (severity percentile 0–100) and optionally enriched with SHAP contributions. Requires a model trained with the anomaly engine first. See [Row Anomaly Detection](#row-anomaly-detection) below for training, full parameters, and usage. | `model_name`: fully qualified model name (e.g. catalog.schema.model_name); `registry_table`: fully qualified registry table (e.g. catalog.schema.model_registry); `threshold`: (optional) severity percentile threshold (default 95); `drift_threshold`: (optional) warn when the input feature distribution drifts from the training baseline (None = off, the default); `enable_contributions`: (optional) add SHAP per-feature contributions to `_dq_info` (default True; set False to skip the SHAP cost); `enable_confidence_std`: (optional) add ensemble score std to `_dq_info` (default False); `enable_ai_explanation`: (optional) add an LLM-generated explanation to `_dq_info` (default True; degrades to null if contributions are off or no serving endpoint is reachable); `ai_explanation_llm_model_config`: (optional) Databricks Model Serving endpoint config for the explanation; `redact_columns`: (optional) feature/segment names to keep out of the LLM prompt; `max_groups`: (optional) cap on LLM calls per run (default 500). See [Row Anomaly Detection](/docs/reference/quality_checks#row-anomaly-detection) section for full parameter details. | | `are_polygons_mutually_disjoint` | Checks whether the polygons in a geometry column are mutually disjoint. Polygons sharing an edge or boundary are considered intersecting. Nulls and invalid geometries are excluded from the check. Requires Databricks runtime 17.1 or above. | `column`: column to check (can be a string column name or a column expression), must contain polygon or multipolygon geometries | | `is_geo_contains` | Checks if the reference geometry contains each column geometry using `st_contains` with meter-level precision. A geometry A *contains* B when B lies entirely within the interior of A with no boundary points of B on the boundary of A. Points on the shared boundary are not considered contained — use `is_geo_covers` for boundary-inclusive checks. When a convert flag is set to `True`, `try_to_geometry` is applied to parse the input from any supported format (WKT, WKB, EWKT, EWKB). Null values are skipped. Requires Databricks runtime 17.1 or above. | `column`: column to check (can be a string column name or a column expression); `reference_geometry`: reference geometry as a literal WKT/WKB/EWKT/EWKB string or bytes value, or a `Column` expression (e.g. `F.col('col_name')`) — a plain string is always treated as a literal, not a column name; `convert_column`: when `True`, applies `try_to_geometry` to convert the column values to GEOMETRY (default `False`); `convert_reference_geometry`: when `True`, applies `try_to_geometry` to convert the reference geometry to GEOMETRY (default `False`) | | `is_geo_covers` | Checks if the reference geometry covers each column geometry. When `precise=True`, uses `st_covers` for exact computation — A *covers* B when every point of B lies within A, including boundary points. When `precise=False` (default), approximates coverage using H3 cell indexing: all hexagonal cells of the column geometry must exist in the H3 cells of the reference geometry. Edge membership is not supported by H3 — geometries near boundaries may be misclassified. Higher `resolution` values give finer precision at the cost of more cells. Null values are skipped; in approximate mode invalid (unparseable) geometries are also skipped rather than flagged — use `is_geometry` to flag invalid values. Requires Databricks runtime 17.1 or above. | `column`: column to check (can be a string column name or a column expression); `reference_geometry`: reference geometry as a literal WKT/WKB/EWKT/EWKB string or bytes value, or a `Column` expression (e.g. `F.col('col_name')`) — a plain string is always treated as a literal, not a column name (bytes/WKB only supported when `precise=True`); `precise`: when `True`, uses exact `st_covers`; when `False` (default), uses H3 approximation and requires `resolution`; `resolution`: H3 resolution integer (0–15) or a column — required when `precise=False`; higher values give finer precision at the cost of more cells; `convert_column`: when `True`, applies `try_to_geometry` to the column (only used in precise mode, default `False`); `convert_reference_geometry`: when `True`, applies `try_to_geometry` to the reference geometry (only used in precise mode, default `False`) | @@ -3512,7 +3512,7 @@ Pass an `AnomalyParams` object to the `params` argument to customize training be | `sample_fraction` | float | 0.3 | Fraction of data to sample for training (30%). Reduce for faster training on large datasets. | | `max_rows` | int | 1,000,000 | Maximum rows to use for training. Caps memory usage for very large datasets. | | `train_ratio` | float | 0.8 | Train/validation split ratio (80% train, 20% validation). | -| `ensemble_size` | int or None | 3 | Number of models in ensemble. Set to None for single model. More models = more robust but slower training. | +| `ensemble_size` | int or None | 3 | Number of models in ensemble. Set to None for single model. More models = more robust but slower training. Applies to a single global model only: **segmented training always trains one model per segment and ignores this setting**, so `confidence_std` is unavailable for segmented models. | #### IsolationForestConfig (Algorithm Parameters) @@ -3622,7 +3622,7 @@ checks = [ - `registry_table`: Registry table (required, fully qualified Unity Catalog table name: `catalog.schema.table`). - `threshold`: Severity percentile threshold (0–100, default 95). - `row_filter`: Optional SQL expression to filter rows before scoring. -- `drift_threshold`: Optional float (e.g. 3.0) to enable drift detection; default None (disabled). When set, a warning is emitted if the scoring distribution at check time deviates from training. A value of 3.0 corresponds to roughly 3-sigma deviation from training statistics. See the section below for more details. +- `drift_threshold`: Optional float (e.g. 3.0) to enable drift detection; default None (disabled). When set, a warning is emitted if the distribution of the *input features* at check time deviates from the baseline statistics recorded at training; the warning names the drifted columns. It does not look at the anomaly scores themselves. Batches smaller than 1,000 rows are skipped, because per-column statistics are too noisy to compare at that size. A value of 3.0 corresponds to roughly 3-sigma deviation from training statistics. See the section below for more details. - `enable_contributions`: Include per-feature contributions in `_dq_info[0].anomaly.contributions` (default `True`). SHAP is computed only for anomalous rows (severity at or above the threshold), so the cost scales with the number of anomalies rather than the table size; non-anomalous rows get a `null` map. Set `False` to skip the SHAP computation entirely (which also disables AI explanations). See the section below and [Schema of _dq_info](/docs/guide/row_anomaly_detection#schema-of-the-info-column-_dq_info) for field details. - `enable_confidence_std`: Include `confidence_std` for ensembles (default `False`). Useful when using ensemble training. - `enable_ai_explanation`: Add an LLM-generated plain-language explanation in `_dq_info[0].anomaly.ai_explanation` (default `True`). Uses the SHAP contributions as input — if `enable_contributions=False`, explanations are disabled with a warning (not an error). The LLM call runs inside Spark via the SQL `ai_query` function against a Databricks Model Serving endpoint — no extra dependency, but it requires Databricks serverless compute or Databricks Runtime 15.4 LTS or above (where `ai_query` is available). If `ai_query` is unavailable or the endpoint isn't reachable, explanations are skipped with a warning and scoring still completes. See the **AI Explanations** section below. diff --git a/src/databricks/labs/dqx/anomaly/check_funcs.py b/src/databricks/labs/dqx/anomaly/check_funcs.py index 6a3f8b6b9..5dfb7e07e 100644 --- a/src/databricks/labs/dqx/anomaly/check_funcs.py +++ b/src/databricks/labs/dqx/anomaly/check_funcs.py @@ -165,7 +165,9 @@ def has_no_row_anomalies( row_filter: Optional SQL expression (e.g. \"region = 'US'\"). Only rows matching this expression are scored; others are left in the output with null anomaly result. Auto-injected from the check filter. - drift_threshold: Drift detection threshold (default 3.0, None to disable). + drift_threshold: Drift detection threshold, in standard deviations of the training + baseline (default None, which disables drift detection). Set a positive value + such as 3.0 to enable it. enable_contributions: Include SHAP feature contributions for explainability (default True). Per-feature contributions are added to _dq_info for anomalous rows only (severity at or above the threshold; other rows get a null map), so the SHAP cost scales with the number diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index f9f6a7e2a..f648a3c8e 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -205,6 +205,9 @@ class AnomalyParams: - Confidence scores via standard deviation - Better generalization Performance: Optimized ensemble scoring makes this negligible overhead. + Note: this applies to a single global model only. Segmented training always + trains exactly one model per segment and ignores ``ensemble_size``, so + confidence scores are not available for segmented models. algorithm_config: Isolation Forest parameters (contamination, num_trees, seed). feature_engineering: Feature engineering parameters (temporal features, scaling, etc.). """ From c53c6698d189be4997aff97dd0be0a25bbd4ef74 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 13:59:22 +0100 Subject: [PATCH 003/107] Name the segment-training thresholds in one module The numbers governing per-segment training were literals spread across the training service. Collecting them in `anomaly/group_config.py` gives each one a name and a reason, and makes the ceiling below reviewable as policy rather than as a magic number. MAX_SEGMENT_MODELS is the cost fuse: a 90-segment run was measured at 70 minutes without completing. MIN_ROWS_TO_TRAIN_SEGMENT and MIN_ROWS_PER_SEGMENT separate "too small to fit at all" from "small enough to warn about", which were previously the same literal used for two different questions. --- .../labs/dqx/anomaly/group_config.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/databricks/labs/dqx/anomaly/group_config.py diff --git a/src/databricks/labs/dqx/anomaly/group_config.py b/src/databricks/labs/dqx/anomaly/group_config.py new file mode 100644 index 000000000..de4c43f12 --- /dev/null +++ b/src/databricks/labs/dqx/anomaly/group_config.py @@ -0,0 +1,37 @@ +"""Thresholds governing per-segment model training. + +These were previously inline literals spread across the profiler and the training service. Naming +them in one place makes the policy reviewable. + +All of them now guard the legacy ``segment_by`` path — the only path that trains one model per +group, and therefore the only one whose cost grows with the group count. ``baseline_by`` trains a +single pooled model however many groups there are, so it needs no ceiling. + +See https://github.com/databrickslabs/dqx/issues/1484 for the measurements behind them. +""" + +# Hard ceiling on the number of per-segment models a single training run will attempt. Segmented +# training does not ensemble, so cost is linear in the segment count: 90 segments measured roughly +# 88 minutes, and a 90-segment run was cancelled after 70 minutes without finishing. Overridable +# via ``AnomalyParams.max_segment_models``. +MAX_SEGMENT_MODELS = 50 + +# Below this many rows per segment on average, a per-segment model is calibrated on too small a +# sample to be trustworthy. Severity is a percentile of each model's own training scores, so a thin +# sample yields a fragile threshold rather than an obviously bad one. Used by the profiler to warn. +MIN_ROWS_PER_SEGMENT = 100 + +# Segment count above which training logs a slow-training warning. Retained at its historical value +# so existing runs keep warning where they used to; MAX_SEGMENT_MODELS is the ceiling that actually +# stops a run. +SEGMENT_COUNT_WARN_THRESHOLD = 100 + +# A segment with fewer rows than this is skipped and gets no model at all, so its rows come back +# unscored. Distinct from MIN_ROWS_PER_SEGMENT, which is the average below which per-segment +# modelling is merely discouraged. +MIN_ROWS_TO_TRAIN_SEGMENT = 10 + +# Upper bound on the distinct values a column may have to be *recommended* as a grouping by +# auto-discovery. Conservative on purpose: a wide grouping is fine for baseline_by, which trains one +# model, but auto-discovery's recommendation is also what the legacy segmented path would consume. +MAX_AUTO_GROUP_COUNT = 20 From 6bc0e1bbcfb8dddd4922e7d01345dbef0a3a22e4 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:00:32 +0100 Subject: [PATCH 004/107] Validate baseline columns before they can corrupt a key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five checks, each guarding a failure that is silent rather than loud if it gets through: the columns must exist; they must not also be feature columns; their types must render identically in Python and Spark; the list must not be empty when declared; and it must not duplicate an entry. The type restriction is the one with teeth. Floating-point and decimal columns are rejected because Python's `str()` and Spark's `cast("string")` disagree on them, which would produce one key at training and a different key at scoring — and that mismatch does not raise, it just misses every baseline lookup and silently falls back to the global baseline. --- src/databricks/labs/dqx/anomaly/validation.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/databricks/labs/dqx/anomaly/validation.py b/src/databricks/labs/dqx/anomaly/validation.py index bd004584a..f7de2a56d 100644 --- a/src/databricks/labs/dqx/anomaly/validation.py +++ b/src/databricks/labs/dqx/anomaly/validation.py @@ -5,10 +5,12 @@ (e.g. sklearn version mismatch). Registry types and persistence live in model_registry. """ +import collections import collections.abc import warnings import sklearn from pyspark.sql import DataFrame, SparkSession +from pyspark.sql import types as T from databricks.labs.dqx.anomaly.model_config import AnomalyModelRecord from databricks.labs.dqx.anomaly.transformers import ColumnTypeClassifier @@ -54,6 +56,64 @@ def validate_columns( return warnings_list +#: Group column types whose Spark ``cast("string")`` is guaranteed to match Python's ``str()``. +#: Floating-point and decimal types are excluded deliberately: the two disagree on formatting +#: (Spark renders 1.0 as "1.0" but 1e-7 differently from Python), and a group key that differs +#: between training and scoring misses every baseline lookup silently rather than failing. +_ALLOWED_GROUP_COLUMN_TYPES = ( + T.StringType, + T.ByteType, + T.ShortType, + T.IntegerType, + T.LongType, + T.BooleanType, + T.DateType, +) + + +def validate_baseline_columns( + df: DataFrame, baseline_by: list[str] | None, columns: collections.abc.Iterable[str] +) -> None: + """Validate declared group columns. + + Group columns identify the comparison basis; they are not features. They must therefore + exist, must not double as feature columns, and must have a type whose string rendering is + stable between Spark and Python. + """ + if not baseline_by: + return + + duplicates = [name for name, count in collections.Counter(baseline_by).items() if count > 1] + if duplicates: + raise InvalidParameterError(f"baseline_by contains duplicate columns: {duplicates}.") + + schema_fields = {field.name: field.dataType for field in df.schema.fields} + missing = [name for name in baseline_by if name not in schema_fields] + if missing: + raise InvalidParameterError(f"baseline_by columns not found in DataFrame: {missing}. Available: {df.columns}.") + + overlap = sorted(set(baseline_by) & set(columns)) + if overlap: + raise InvalidParameterError( + f"Columns {overlap} are used both as features and as baseline_by columns. A group column " + "defines the basis a metric is compared against, so it cannot also be one of the " + "metrics being compared. Remove them from one or the other." + ) + + unsupported = { + name: schema_fields[name].simpleString() + for name in baseline_by + if not isinstance(schema_fields[name], _ALLOWED_GROUP_COLUMN_TYPES) + } + if unsupported: + raise InvalidParameterError( + f"baseline_by columns have unsupported types: {unsupported}. Group columns must be string, " + "integral, boolean or date. Floating-point and decimal columns are rejected because " + "Spark and Python format them differently, which would make a row's group key differ " + "between training and scoring. Cast to string or bucket the value first." + ) + + def _validate_float_range( value: float, *, From 2cdeaa44e637bed766e7ddc83a143324a7c15840 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:00:34 +0100 Subject: [PATCH 005/107] Compute one baseline key, in Python and in Spark, and pin their agreement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Training persists a baseline under a key built in Python; scoring looks it up with a key built in Spark. A disagreement between the two does not raise — every lookup simply misses, falls back to the global baseline, and produces a model that appears to work while conditioning on nothing. That makes this the most expensive invariant in the feature to get wrong, so both halves and the test that pins them land together. `_as_spark_string` exists because `str()` is not a drop-in for Spark's `cast("string")`. Booleans are the case that bites: Spark renders `true`/`false`, Python renders `True`/`False`. The first real Spark run of this code asserted `'true\x1f42' == 'True\x1f42'` and failed, which is precisely the class of bug this contract exists to catch — and a stale unit test had been pinning the wrong behaviour. Floating-point and decimal baseline columns are rejected rather than shimmed: their two renderings diverge in ways no small helper can reconcile. The separator is ASCII unit separator, a control character, so ordinary categorical data cannot contain it. It is a separator and not an escaping scheme, and the docstring says so: data that genuinely contains \x1f can still collide. Deliberately not solved by length-prefixing, which would buy protection against pathological data by widening the one invariant most expensive to get wrong. A test pins the documented limitation so nobody mistakes it for collision-proofing. --- .../labs/dqx/anomaly/segment_utils.py | 90 ++++++++++++++++++- tests/unit/test_anomaly_group_key.py | 88 ++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_anomaly_group_key.py diff --git a/src/databricks/labs/dqx/anomaly/segment_utils.py b/src/databricks/labs/dqx/anomaly/segment_utils.py index 8928aac25..c9743c529 100644 --- a/src/databricks/labs/dqx/anomaly/segment_utils.py +++ b/src/databricks/labs/dqx/anomaly/segment_utils.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from typing import Any -from pyspark.sql import Column +from pyspark.sql import Column, DataFrame import pyspark.sql.functions as F @@ -20,6 +20,94 @@ def build_segment_name(segment_values: Mapping[str, Any] | None) -> str: return "_".join(f"{key}={value}" for key, value in canonical_values.items()) +# Separator between group column values in a composite group key. ASCII unit separator: a +# control character, so ordinary categorical data (names, codes, countries) cannot contain it and +# two different group tuples cannot collide onto one key. +# +# Known limitation: this is a separator, not an escaping scheme. Data that genuinely contains +# \x1f inside a group value can still collide — ("x\x1fy", "z") and ("x", "y\x1fz") produce the +# same key. Deliberately not solved by length-prefixing: the Python and Spark halves of this key +# must agree exactly, and adding length arithmetic to that contract buys protection against +# pathological data at the cost of widening the one invariant most expensive to get wrong. +BASELINE_KEY_SEPARATOR = "\x1f" + +# Stand-in for a NULL group value. NULLs must map to a real key rather than propagating, +# or every row in a group with a missing dimension silently loses its baseline. +BASELINE_KEY_NULL = "MISSING" + +# The single column every stage reads the group key from. See :func:`with_baseline_key`. +BASELINE_KEY_COLUMN = "__dqx_baseline_key" + + +def _as_spark_string(value: Any) -> str: + """Render *value* the way Spark's ``cast("string")`` does. + + Python's ``str()`` is not a drop-in for Spark's cast, and the differences are silent: + + * **Booleans.** Spark renders ``true``/``false``; Python renders ``True``/``False``. Found by + the integration test that pins Python/Spark key agreement, and it is exactly the failure this + contract exists to catch — a capitalised key persisted at training would miss every lookup at + scoring and fall back to the global baseline without raising. + * Integrals, strings and dates already agree, which is why + :func:`~databricks.labs.dqx.anomaly.validation.validate_baseline_columns` permits those types and + rejects floating-point and decimal ones, where the two renderings diverge in ways no small + shim can reconcile. + """ + if value is None: + return BASELINE_KEY_NULL + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def build_baseline_key(group_values: Mapping[str, Any] | None) -> str: + """Build the group key for a row, in Python. + + Must agree exactly with :func:`baseline_key_column`, which computes the same key in Spark. + Training persists baselines under keys produced by one and scoring looks them up with the + other, so a disagreement does not fail — it silently misses every lookup and falls back to + the global baseline, producing a model that appears to work and conditions on nothing. + Their agreement is pinned by an integration test, which is how the boolean rendering + difference handled in :func:`_as_spark_string` was found. + + Values are ordered by column name so the key does not depend on the order the caller + happened to list the group columns in. + """ + if not group_values: + return "" + ordered = sorted(group_values.items(), key=lambda item: str(item[0])) + return BASELINE_KEY_SEPARATOR.join(_as_spark_string(value) for _key, value in ordered) + + +def baseline_key_column(baseline_by: list[str]) -> Column: + """Build the group key for every row, in Spark. + + The Spark half of the contract described on :func:`build_baseline_key`. This side is the source of + truth — it is what runs at scoring time — so :func:`_as_spark_string` is written to match + ``cast("string")`` rather than the other way round. + """ + parts = [F.coalesce(F.col(name).cast("string"), F.lit(BASELINE_KEY_NULL)) for name in sorted(baseline_by, key=str)] + return F.concat_ws(BASELINE_KEY_SEPARATOR, *parts) + + +def with_baseline_key(df: DataFrame, baseline_by: list[str]) -> DataFrame: + """Ensure *df* carries :data:`BASELINE_KEY_COLUMN`, computing it only when absent. + + Idempotent on purpose, and the point is *availability*, not agreement: every stage calls + :func:`baseline_key_column`, so two Spark-side computations cannot disagree with each other. + What they can do is run on a frame that no longer has the columns to compute from. + + Feature engineering drops the raw group columns so they cannot reach the sklearn pipeline or + the inferred MLflow signature. Stages downstream of it therefore have the grouping only if + something carried it forward. Training-time severity calibration is downstream — it runs on the + scored engineered frame — and rebuilding the key there raised + ``UNRESOLVED_COLUMN`` on the first grouped model ever trained against real Spark. + """ + if not baseline_by or BASELINE_KEY_COLUMN in df.columns: + return df + return df.withColumn(BASELINE_KEY_COLUMN, baseline_key_column(baseline_by)) + + def build_segment_filter(segment_values: dict[str, str] | None) -> Column | None: """Build Spark filter expression for a segment's values. diff --git a/tests/unit/test_anomaly_group_key.py b/tests/unit/test_anomaly_group_key.py new file mode 100644 index 000000000..065487ac9 --- /dev/null +++ b/tests/unit/test_anomaly_group_key.py @@ -0,0 +1,88 @@ +"""Unit tests for the Python half of the group key. + +The group key is the riskiest invariant in group conditioning: training persists baselines under +keys built by :func:`build_baseline_key` and scoring looks them up with keys built by +:func:`baseline_key_column` in Spark. A disagreement does not raise — every lookup simply misses and +falls back to the global baseline, giving a model that trains and scores cleanly while +conditioning on nothing at all. + +These tests pin the Python side. Parity with the Spark side needs a session and so lives in +``tests/integration_anomaly/test_anomaly_group_relative_features.py``. +""" + +from databricks.labs.dqx.anomaly.segment_utils import ( + BASELINE_KEY_NULL, + BASELINE_KEY_SEPARATOR, + build_baseline_key, +) + + +def test_single_group_column(): + assert build_baseline_key({"country": "DE"}) == "DE" + + +def test_composite_key_joins_with_the_separator(): + key = build_baseline_key({"country": "DE", "product": "casino"}) + assert key == f"DE{BASELINE_KEY_SEPARATOR}casino" + + +def test_key_is_independent_of_declaration_order(): + """Ordering by column name means the caller's argument order cannot change the key.""" + assert build_baseline_key({"country": "DE", "product": "casino"}) == build_baseline_key( + {"product": "casino", "country": "DE"} + ) + + +def test_nulls_become_a_real_key_rather_than_propagating(): + """A missing dimension must still land in a group, or its rows lose their baseline.""" + key = build_baseline_key({"country": None, "product": "casino"}) + assert key == f"{BASELINE_KEY_NULL}{BASELINE_KEY_SEPARATOR}casino" + + +def test_integral_values_are_stringified(): + """Integral baseline columns are allowed, so their rendering must be defined.""" + assert build_baseline_key({"store_id": 42}) == "42" + assert build_baseline_key({"store_id": -7}) == "-7" + + +def test_booleans_render_the_way_spark_does_not_the_way_python_does(): + """Regression: this assertion used to read ``== "True"`` and was wrong. + + Spark's ``cast("string")`` renders booleans lowercase; Python's ``str(True)`` capitalises. The + Spark side is the source of truth because it is what runs at scoring time, so the Python half + must match it. Found by the integration test that pins the two halves against a real session — + before the fix, training persisted ``True\\x1f42`` and scoring looked up ``true\\x1f42``, missing + every baseline and silently falling back to the global one. + """ + assert build_baseline_key({"is_vip": True}) == "true" + assert build_baseline_key({"is_vip": False}) == "false" + + +def test_empty_grouping_is_the_empty_key(): + assert build_baseline_key(None) == "" + assert build_baseline_key({}) == "" + + +def test_printable_punctuation_in_values_does_not_collide(): + """The ambiguity a printable separator would have: ("x_y", "z") versus ("x", "y_z"). + + These collide under a "_" separator and must not collide here. This is what choosing a + control character buys — real categorical data contains underscores, dashes and colons. + """ + for punctuation in ("_", "-", ":", "|", "=", ",", " "): + left = build_baseline_key({"a": f"x{punctuation}y", "b": "z"}) + right = build_baseline_key({"a": "x", "b": f"y{punctuation}z"}) + assert left != right, f"collision on {punctuation!r}" + + +def test_separator_inside_a_value_is_a_documented_collision(): + """Pins the known limitation rather than pretending it does not exist. + + ``BASELINE_KEY_SEPARATOR`` is a separator, not an escaping scheme, so data that genuinely + contains the control character can collide. Accepted deliberately: escaping would widen the + Python/Spark parity contract, which is the invariant most expensive to get wrong. If this + ever needs fixing, this test is the one that should start failing. + """ + left = build_baseline_key({"a": f"x{BASELINE_KEY_SEPARATOR}y", "b": "z"}) + right = build_baseline_key({"a": "x", "b": f"y{BASELINE_KEY_SEPARATOR}z"}) + assert left == right From 07cae7e1f1f0c7dc755a6016319454ed80d12c91 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:00:53 +0100 Subject: [PATCH 006/107] Judge each metric against its own group's baseline, via baseline_by MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes databrickslabs/dqx#1484. A value can be perfectly ordinary for the table and badly wrong for its own group. Row anomaly detection had no scalable way to express that: the only grouping mechanism was `segment_by`, which trains one model per group, so conditioning cost one model per group and a discovered 90-way grouping became an hours-long run. `baseline_by` adds, for each numeric metric, its deviation from that group's baseline: rel = signed_log(value) - signed_log(group_median(value)) on a **single** model, whatever the group count. Baselines are computed once at training, persisted in the feature metadata, and broadcast-joined at scoring. The log-ratio form is stable when a baseline is near zero and symmetric for halving versus doubling. The raw metric is kept alongside, so a globally absurd value stays detectable even where it is ordinary for its group. Severity is calibrated per baseline group as well, since a score that is extreme for one group is unremarkable for another, with the global calibration as the fallback for groups that have none. A row whose group was never seen at training is reported rather than scored. The alternatives are both silent and both wrong: a high-cardinality frequency map returns 0.0 for an unseen key, which reads as "perfectly normal", and a one-hot encoding leaves every indicator at zero, which reads as extreme. `is_new_baseline` and `new_baseline_key` name the case instead. This widens the `_dq_info` struct, so appending to an existing results table needs `mergeSchema`. Three properties worth calling out, because they are what makes this safe to enable: * **Baseline columns are never features.** They are the basis of comparison, not a metric being compared, so they are excluded before the sklearn pipeline and the inferred MLflow signature see anything. `prepare_training_features` projects to the feature list explicitly; the key column rides through as a passthrough for the calibration stage, which runs downstream of where the raw group columns are dropped. * **`segment_by` keeps its exact behaviour.** `_resolve_grouping` clears `baseline_by` on that path. Leaving both set would compute relative features *inside* each segment, where the key is constant because the frame is already filtered, while `compute_config_hash` is built from `segment_by` alone and would not change — an identical hash with a different feature list, which is a silent train/score hazard. * **Models trained before this change score identically.** Every new metadata field defaults to empty, so the relative transform returns immediately and `engineered_feature_names` is byte-identical to what it was. There is deliberately no strategy selector and no heterogeneity gate. `baseline_by` declared means relative features, always. The evidence for that choice, including the measurement that removing the gate costs nothing, is in `benchmarks/anomaly_conditioning/`. Also folded in here, because `config.py` and `training_service.py` carry it in the same hunks: **the segment model ceiling now errors rather than warns.** Above the ceiling the previous behaviour was to log a warning and continue, which meant a 90-segment run started training 90 models and was measured at 70 minutes without completing. A warning that precedes an hour of unusable work is not a warning. It now raises, and the message names the two ways forward — segment more coarsely, or raise `max_segment_models` explicitly — because an error that only says no is a worse experience than the warning it replaces. `max_group_models` is renamed `max_segment_models`. Its only consumer is the legacy `segment_by` path, the only path that trains N models, so it is a cost fuse on that path rather than a general tuning knob. The name now says which mechanism it bounds. `AnomalyParams` cannot import from the anomaly package — `anomaly/__init__.py` raises without the anomaly extra — so the default is a literal, kept in step with `MAX_SEGMENT_MODELS` by a unit test rather than by hope. --- .../labs/dqx/anomaly/anomaly_engine.py | 27 +- .../labs/dqx/anomaly/anomaly_info_schema.py | 6 + .../labs/dqx/anomaly/anomaly_workflow.py | 1 + .../labs/dqx/anomaly/check_funcs.py | 11 + src/databricks/labs/dqx/anomaly/core.py | 92 +++++- .../labs/dqx/anomaly/feature_prep.py | 6 +- src/databricks/labs/dqx/anomaly/profiler.py | 13 +- .../labs/dqx/anomaly/scoring_config.py | 11 + .../labs/dqx/anomaly/scoring_run.py | 106 ++++++- .../labs/dqx/anomaly/scoring_utils.py | 274 ++++++++++++++++-- .../labs/dqx/anomaly/training_service.py | 141 ++++++++- .../labs/dqx/anomaly/transformers.py | 183 +++++++++++- src/databricks/labs/dqx/anomaly/types.py | 10 +- src/databricks/labs/dqx/config.py | 19 +- tests/unit/test_anomaly_configs.py | 12 + .../test_anomaly_group_calibration_points.py | 66 +++++ tests/unit/test_anomaly_transformers.py | 94 ++++++ 17 files changed, 998 insertions(+), 74 deletions(-) create mode 100644 tests/unit/test_anomaly_group_calibration_points.py diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index 45f801939..948524b78 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -64,6 +64,7 @@ def train( params: AnomalyParams | None = None, exclude_columns: list[str] | None = None, expected_anomaly_rate: float = 0.02, + baseline_by: list[str] | None = None, ) -> str: """ Train row anomaly detection model(s) with intelligent auto-discovery. @@ -83,7 +84,19 @@ def train( registry_table: Registry table (REQUIRED). Must be fully qualified Unity Catalog table as 'catalog.schema.table'. columns: Columns to use for row anomaly detection (auto-discovered if omitted). - segment_by: Segment columns (auto-discovered if both columns and segment_by omitted). + baseline_by: Columns identifying the group a row belongs to, so a metric is judged + against its own group's baseline rather than against the whole table. Each + numeric metric gains its deviation from that baseline as an extra feature on + a single pooled model, so the cost does not grow with the group count. This + is what catches a value that is unremarkable across the table but wrong for + its own group. Cannot be combined with `segment_by`. + segment_by: Legacy. Trains one model per group instead. Kept for compatibility, and not + recommended: on the Server Machine Dataset per-group models were the worst + of three configurations, with one entity producing 15,963 false positives + on 28,392 normal rows, and cost is linear in the group count (90 groups + measures roughly 88 minutes, capped by `params.max_segment_models`). Prefer + `baseline_by`. Auto-discovered when both `columns` and `segment_by` are + omitted, in which case the discovered grouping is used as `baseline_by`. params: Optional anomaly parameters for tuning training behavior. exclude_columns: Columns to exclude from training (e.g., IDs, labels, ground truth). Exclusions always take precedence over `columns` if both are provided. @@ -143,6 +156,17 @@ def train( registry_table="catalog.schema.dqx_anomaly_models", columns=["revenue", "transactions"], ) + + # Judge each row against its own group rather than the whole table. With few + # groups this trains one model each; with many it switches to group-relative + # features, which keeps one model however many groups there are. + anomaly_engine.train( + df, + model_name="catalog.schema.regional_model", + registry_table="catalog.schema.dqx_anomaly_models", + columns=["event_count"], + baseline_by=["country", "product"], + ) """ training_service = AnomalyTrainingService(self.spark) context = training_service.build_context( @@ -154,6 +178,7 @@ def train( params=params, exclude_columns=exclude_columns, expected_anomaly_rate=expected_anomaly_rate, + baseline_by=baseline_by, ) log_telemetry(self.ws, "anomaly_num_features", str(len(context.columns))) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py index b4351e789..1c4a99076 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py @@ -39,5 +39,11 @@ StructField("contributions", MapType(StringType(), DoubleType()), True), StructField("confidence_std", DoubleType(), True), StructField("ai_explanation", ai_explanation_struct_schema, True), + # True when the row's group was never seen in training, so its score and severity are + # null rather than guessed. Appended at the end: existing named-field queries such as + # _dq_info[0].anomaly.score keep working, but a Delta table already holding _dq_info + # needs mergeSchema on append because the struct is now wider. + StructField("is_new_baseline", BooleanType(), True), + StructField("new_baseline_key", StringType(), True), ] ) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_workflow.py b/src/databricks/labs/dqx/anomaly/anomaly_workflow.py index f4b6a15d0..8286e63ab 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_workflow.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_workflow.py @@ -46,6 +46,7 @@ def train_model(self, ctx: WorkflowContext) -> None: df=df, columns=anomaly_config.columns, segment_by=anomaly_config.segment_by, + baseline_by=anomaly_config.baseline_by, model_name=model_name, registry_table=registry_table, ) diff --git a/src/databricks/labs/dqx/anomaly/check_funcs.py b/src/databricks/labs/dqx/anomaly/check_funcs.py index 5dfb7e07e..b65cf747e 100644 --- a/src/databricks/labs/dqx/anomaly/check_funcs.py +++ b/src/databricks/labs/dqx/anomaly/check_funcs.py @@ -148,12 +148,23 @@ def has_no_row_anomalies( - _dq_info[0].anomaly.contributions: SHAP contributions as percentages (0–100); populated only for anomalous rows, null otherwise - _dq_info[0].anomaly.confidence_std: Ensemble std (if requested) + - _dq_info[0].anomaly.is_new_baseline: True when the row's group was absent from training, + in which case score and severity_percentile are null + - _dq_info[0].anomaly.new_baseline_key: The unrecognised group key, for unseen rows Notes: DQX always scores using the columns the model was trained on. DQX aligns scored rows back to the input using an internal row id and removes it before returning. Segmentation is inferred from the trained model configuration. + Rows whose group was never seen in training are reported (`is_new_baseline`) but are **not** + flagged as violations: neither categorical encoder can represent an unseen value honestly + — one-hot makes it look maximally normal, frequency encoding maximally extreme — so DQX + cannot judge the row, and "could not judge" is not the same claim as "is anomalous". If an + unrecognised group value is itself a problem worth failing on, that is a membership + question rather than an anomaly one: use `foreign_key` or `is_in_list` on the group column + against your set of known values, which is the check built for it. + Args: model_name: Model name (REQUIRED). Provide the fully qualified model name in catalog.schema.table format returned from train(). diff --git a/src/databricks/labs/dqx/anomaly/core.py b/src/databricks/labs/dqx/anomaly/core.py index 51531d37b..d0b9de0fc 100644 --- a/src/databricks/labs/dqx/anomaly/core.py +++ b/src/databricks/labs/dqx/anomaly/core.py @@ -25,6 +25,7 @@ from sklearn.pipeline import Pipeline from sklearn.preprocessing import RobustScaler +from databricks.labs.dqx.anomaly.segment_utils import BASELINE_KEY_COLUMN, with_baseline_key from databricks.labs.dqx.anomaly.transformers import ( ColumnTypeClassifier, SparkFeatureMetadata, @@ -42,6 +43,18 @@ SCORE_QUANTILE_KEYS = ["p00", "p01", "p05", "p10", "p25", "p50", "p75", "p90", "p95", "p99", "p100"] +def _with_baseline_columns(columns: list[str], baseline_by: list[str] | None) -> list[str]: + """Union *columns* with *baseline_by*, preserving order and dropping duplicates. + + Group columns are not features, but they must survive every narrowing ``select`` on the way + to feature engineering, or the group-relative transform has nothing to compute a baseline + from. Feature engineering drops them again before the sklearn pipeline sees anything. + """ + if not baseline_by: + return columns + return list(dict.fromkeys([*columns, *baseline_by])) + + def sample_df(df: DataFrame, columns: list[str], params: AnomalyParams) -> tuple[DataFrame, int, bool]: """Sample DataFrame for training. @@ -54,6 +67,7 @@ def sample_df(df: DataFrame, columns: list[str], params: AnomalyParams) -> tuple Tuple of (sampled DataFrame, row count, truncated flag) """ fraction = params.sample_fraction if params.sample_fraction is not None else DEFAULT_SAMPLE_FRACTION + columns = _with_baseline_columns(columns, params.baseline_by) missing_cols = [c for c in columns if c not in df.columns] if missing_cols: raise InvalidParameterError(f"Columns not found in DataFrame: {missing_cols}") @@ -92,7 +106,9 @@ def prepare_training_features( max_engineered_features=fe_config.max_engineered_features, ) - feature_df = train_df.select(*feature_columns) + # Group columns ride along for the relative transform but are never classified as features: + # analyze_columns sees only feature_columns. + feature_df = train_df.select(*_with_baseline_columns(feature_columns, params.baseline_by)) column_infos, _ = classifier.analyze_columns(feature_df, feature_columns) engineered_df, feature_metadata = apply_feature_engineering( @@ -100,9 +116,14 @@ def prepare_training_features( column_infos, categorical_cardinality_threshold=fe_config.categorical_cardinality_threshold, frequency_maps=None, + baseline_by=params.baseline_by, ) - train_pandas = engineered_df.toPandas() + # Project to the feature list explicitly. The engineered frame also carries the baseline key + # column, which is a grouping record rather than a feature: ``fit_sklearn_model`` fits the + # pipeline on every column of this frame, so a passthrough reaching here would be trained on + # and would land in the inferred MLflow signature. + train_pandas = engineered_df.select(*feature_metadata.engineered_feature_names).toPandas() return train_pandas, feature_metadata @@ -165,7 +186,7 @@ def score_with_model( This enables distributed inference across the Spark cluster. """ engineered_df, updated_metadata = apply_feature_engineering_from_metadata( - df.select(*feature_cols), feature_metadata + df.select(*_with_baseline_columns(feature_cols, feature_metadata.baseline_by)), feature_metadata ) engineered_feature_cols = updated_metadata.engineered_feature_names @@ -202,7 +223,7 @@ def score_with_ensemble_models( ) -> DataFrame: """Score DataFrame using an ensemble of models and return mean scores.""" engineered_df, updated_metadata = apply_feature_engineering_from_metadata( - df.select(*feature_cols), feature_metadata + df.select(*_with_baseline_columns(feature_cols, feature_metadata.baseline_by)), feature_metadata ) engineered_feature_cols = updated_metadata.engineered_feature_names @@ -268,31 +289,82 @@ def compute_validation_metrics( def compute_score_quantiles( model: Pipeline, df: DataFrame, feature_cols: list[str], feature_metadata: SparkFeatureMetadata ) -> dict[str, float]: - """Compute score quantiles from the training score distribution.""" + """Compute score quantiles from the training score distribution. + + Also populates ``feature_metadata.baseline_score_quantiles`` when the model is grouped, so + scoring can calibrate severity against each group's own distribution. + """ if df.count() == 0: return {} scored = score_with_model(model, df, feature_cols, feature_metadata) - scores_df = scored.select(F.col("anomaly_score").alias("score")) - quantiles = scores_df.approxQuantile("score", SCORE_QUANTILE_PROBS, 0.01) - - return dict(zip(SCORE_QUANTILE_KEYS, quantiles, strict=False)) + return _quantiles_from_scored(scored, feature_metadata) def compute_score_quantiles_ensemble( models: list[Pipeline], df: DataFrame, feature_cols: list[str], feature_metadata: SparkFeatureMetadata ) -> dict[str, float]: - """Compute score quantiles using ensemble mean scores.""" + """Compute score quantiles using ensemble mean scores. + + Also populates ``feature_metadata.baseline_score_quantiles`` when the model is grouped. + """ if df.count() == 0: return {} scored = score_with_ensemble_models(models, df, feature_cols, feature_metadata) + return _quantiles_from_scored(scored, feature_metadata) + + +def _quantiles_from_scored(scored: DataFrame, feature_metadata: SparkFeatureMetadata) -> dict[str, float]: + """Derive the global score quantiles, and the per-group ones as a side effect. + + Note the cost: for a grouped model this walks the scored frame twice, and since ``.cache()`` + is unavailable on serverless the scoring UDF runs for each walk. Paid at training time only, + and only when ``baseline_by`` is set — an ungrouped model behaves exactly as before. + """ scores_df = scored.select(F.col("anomaly_score").alias("score")) quantiles = scores_df.approxQuantile("score", SCORE_QUANTILE_PROBS, 0.01) + if feature_metadata.baseline_by: + feature_metadata.baseline_score_quantiles = compute_baseline_score_quantiles( + scored, feature_metadata.baseline_by + ) + return dict(zip(SCORE_QUANTILE_KEYS, quantiles, strict=False)) +def compute_baseline_score_quantiles(scored_df: DataFrame, baseline_by: list[str]) -> dict[str, dict[str, float]]: + """Compute the score quantiles of each group from an already-scored DataFrame. + + Takes scores rather than a model so the training set is scored once and reused for both the + global and the per-group calibration. + + ``percentile_approx`` in one ``groupBy`` rather than ``approxQuantile`` per group: the latter + is a driver-side call and would mean one Spark job per group, which is the cost pattern this + whole change exists to avoid. + """ + if not baseline_by: + return {} + + quantile_exprs = [ + F.percentile_approx(F.col("anomaly_score"), prob).alias(key) + for prob, key in zip(SCORE_QUANTILE_PROBS, SCORE_QUANTILE_KEYS, strict=True) + ] + # Read the key feature engineering already computed rather than rebuilding it from the raw + # group columns, which this frame no longer has: it is the scored *engineered* frame, and + # feature engineering drops the group columns before the model sees them. + grouped = with_baseline_key(scored_df, baseline_by).groupBy(BASELINE_KEY_COLUMN).agg(*quantile_exprs) + + result: dict[str, dict[str, float]] = {} + for row in grouped.collect(): + quantiles = {key: float(row[key]) for key in SCORE_QUANTILE_KEYS if row[key] is not None} + # A group missing any quantile cannot be interpolated over, so it is left out entirely + # and falls back to the global calibration rather than being half-calibrated. + if len(quantiles) == len(SCORE_QUANTILE_KEYS): + result[row[BASELINE_KEY_COLUMN]] = quantiles + return result + + def compute_baseline_statistics(train_df: DataFrame, columns: list[str]) -> dict[str, dict[str, float]]: """Compute baseline distribution statistics for drift detection. diff --git a/src/databricks/labs/dqx/anomaly/feature_prep.py b/src/databricks/labs/dqx/anomaly/feature_prep.py index c89e1eea7..4c6d1b45a 100644 --- a/src/databricks/labs/dqx/anomaly/feature_prep.py +++ b/src/databricks/labs/dqx/anomaly/feature_prep.py @@ -44,7 +44,11 @@ def apply_feature_engineering_for_scoring( "Ensure the anomaly check is applied to the same DataFrame instance." ) - cols_to_select = list(dict.fromkeys([*feature_cols, *merge_columns, *(passthrough_columns or [])])) + # Group columns must survive this select or the group-relative transform has no basis to + # compute against; feature engineering drops them again before the model sees anything. + cols_to_select = list( + dict.fromkeys([*feature_cols, *feature_metadata.baseline_by, *merge_columns, *(passthrough_columns or [])]) + ) engineered_df, _ = apply_feature_engineering_from_metadata( df.select(*cols_to_select), feature_metadata, column_infos=column_infos diff --git a/src/databricks/labs/dqx/anomaly/profiler.py b/src/databricks/labs/dqx/anomaly/profiler.py index 1b63a23cc..7957c2b9c 100644 --- a/src/databricks/labs/dqx/anomaly/profiler.py +++ b/src/databricks/labs/dqx/anomaly/profiler.py @@ -25,6 +25,11 @@ TimestampType, ) +from databricks.labs.dqx.anomaly.group_config import ( + MAX_AUTO_GROUP_COUNT, + MAX_SEGMENT_MODELS, + MIN_ROWS_PER_SEGMENT, +) from databricks.labs.dqx.profiling_utils import compute_exact_distinct_counts, compute_null_and_distinct_counts logger = logging.getLogger(__name__) @@ -245,12 +250,12 @@ def _calculate_total_segments( # Warn if segments are too granular relative to data size avg_rows_per_segment = total_count / segment_count if segment_count > 0 else 0 - if segment_count > 50: + if segment_count > MAX_SEGMENT_MODELS: warnings.append( f"Detected {segment_count} total segments, training may be slow. " "Consider filtering or using coarser segmentation." ) - elif avg_rows_per_segment < 100: + elif avg_rows_per_segment < MIN_ROWS_PER_SEGMENT: warnings.append( f"Detected {segment_count} segments with only ~{int(avg_rows_per_segment)} rows per segment on average. " f"Models may be unreliable. Consider reducing segmentation or using more data (total rows: {total_count})." @@ -297,10 +302,10 @@ def _select_segment_columns( # Only consider columns with 2-20 distinct values (not 50) # Ensure at least 100 rows per segment on average meets_segment_criteria = ( - 2 <= distinct_count <= 20 # More conservative upper bound + 2 <= distinct_count <= MAX_AUTO_GROUP_COUNT # More conservative upper bound and null_rate < 0.1 and not is_id_column - and (total_count / distinct_count) >= 100 # At least 100 rows per segment + and (total_count / distinct_count) >= MIN_ROWS_PER_SEGMENT ) is_high_cardinality = distinct_count > 50 diff --git a/src/databricks/labs/dqx/anomaly/scoring_config.py b/src/databricks/labs/dqx/anomaly/scoring_config.py index 3b30c7b6b..93fd1617c 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_config.py +++ b/src/databricks/labs/dqx/anomaly/scoring_config.py @@ -65,6 +65,17 @@ class ScoringConfig: # eligible segments — total LLM calls stay <= *max_groups* regardless of segment # count. max_groups: int = 500 + # Whether a row whose group was absent from training counts as a violation. Such a row gets a + # null score and severity because neither encoder can represent an unseen category honestly, + # so `severity >= threshold` is null and the verdict has to be chosen rather than computed. + # + # Deliberately not exposed on has_no_row_anomalies. "Is this group value one I recognise?" is a + # set-membership question, and DQX already has foreign_key / is_in_list for exactly that — a + # flag here would duplicate a better-suited check while pushing the anomaly check past the + # argument count the project holds itself to. Kept as an internal seam so the behaviour is + # testable and reachable programmatically; False keeps "could not judge" distinct from + # "is anomalous", and is_new_baseline reports the fact either way. + flag_unseen_baseline_as_violation: bool = False output_columns: ScoringOutputColumns = field(default_factory=ScoringOutputColumns) @property diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index 2ee30465e..0e6fd667d 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -25,14 +25,20 @@ probe_endpoint_reachable, ) from databricks.labs.dqx.anomaly.scoring_utils import ( + add_baseline_severity_percentile_column, add_info_column, add_severity_percentile_column, apply_row_filter, create_null_scored_dataframe, join_filtered_results_back, + mark_unseen_baselines, + null_out_unseen_baseline_scores, + permissive_quantile_points, + UnseenGroupContext, ) -from databricks.labs.dqx.anomaly.scoring_config import ScoringConfig +from databricks.labs.dqx.anomaly.scoring_config import SEVERITY_QUANTILE_KEYS, ScoringConfig from databricks.labs.dqx.anomaly.segment_utils import build_segment_filter +from databricks.labs.dqx.anomaly.transformers import SparkFeatureMetadata from databricks.labs.dqx.anomaly.single_model_scorer import ( score_with_sklearn_model, score_with_sklearn_model_local, @@ -79,6 +85,59 @@ def _warn_if_max_groups_below_segments(config: ScoringConfig, num_eligible_segme ) +def _known_group_keys(parsed_metadata: SparkFeatureMetadata) -> list[str]: + """Group keys the model actually saw in training. + + Read from the persisted baselines rather than from the per-group quantiles, because the + baselines exist for every grouped model while the quantiles are dropped for groups whose + calibration was incomplete — a group with a baseline was seen, whatever its calibration. + """ + keys: set[str] = set() + for baselines in parsed_metadata.baseline_medians.values(): + keys.update(baselines) + return sorted(keys) + + +def _group_quantile_points( + parsed_metadata: SparkFeatureMetadata, +) -> dict[str, list[tuple[float, float]]]: + """Reshape persisted per-group quantiles into interpolation points, per group. + + A group missing any quantile key is dropped rather than partially interpolated: it falls back + to the global calibration, which is a defined answer, where a gap in the points would not be. + """ + return { + key: [(percentile, quantiles[quantile_key]) for percentile, quantile_key in SEVERITY_QUANTILE_KEYS] + for key, quantiles in parsed_metadata.baseline_score_quantiles.items() + if all(quantile_key in quantiles for _, quantile_key in SEVERITY_QUANTILE_KEYS) + } + + +def _add_severity( + scored_df: DataFrame, + config: ScoringConfig, + parsed_metadata: SparkFeatureMetadata, + group_quantile_points: dict[str, list[tuple[float, float]]], + global_quantile_points: list[tuple[float, float]], +) -> DataFrame: + """Calibrate severity per group where the model has per-group quantiles, globally otherwise.""" + if parsed_metadata.baseline_by and group_quantile_points: + return add_baseline_severity_percentile_column( + scored_df, + score_col=config.score_col, + severity_col=config.severity_col, + baseline_by=parsed_metadata.baseline_by, + group_quantile_points=group_quantile_points, + fallback_quantile_points=global_quantile_points, + ) + return add_severity_percentile_column( + scored_df, + score_col=config.score_col, + severity_col=config.severity_col, + quantile_points=global_quantile_points, + ) + + def score_global_model( df: DataFrame, record: AnomalyModelRecord, @@ -115,7 +174,13 @@ def score_global_model( if record.features.feature_metadata is None: raise InvalidParameterError(f"Model {record.identity.model_name} missing feature_metadata") - quantile_points = extract_quantile_points(record) + global_quantile_points = extract_quantile_points(record) + parsed_metadata = SparkFeatureMetadata.from_json(record.features.feature_metadata) + group_quantile_points = _group_quantile_points(parsed_metadata) + # The scorers use these only to decide which rows are worth a SHAP call. With per-group + # calibration the bound differs by group, so the gate takes the per-percentile minimum: + # permissive, and add_info_column re-masks contributions on the real severity anyway. + quantile_points = permissive_quantile_points(group_quantile_points, global_quantile_points) if config.driver_only: scored_df = ( score_ensemble_models_local( @@ -174,11 +239,27 @@ def score_global_model( if config.enable_contributions and "anomaly_contributions" in scored_df.columns: scored_df = scored_df.withColumnRenamed("anomaly_contributions", config.contributions_col) - scored_df = add_severity_percentile_column( + # Mark unseen groups before severity, then null the score in place afterwards: severity is + # interpolated from the score, so nulling first would leave severity computed from nothing. + unseen_col = "__dqx_is_new_group" + group_key_col = "__dqx_row_group_key" + scored_df = mark_unseen_baselines( + scored_df, + parsed_metadata.baseline_by, + _known_group_keys(parsed_metadata), + unseen_col=unseen_col, + group_key_col=group_key_col, + ) + + scored_df = _add_severity(scored_df, config, parsed_metadata, group_quantile_points, global_quantile_points) + + scored_df = null_out_unseen_baseline_scores( scored_df, + unseen_col=unseen_col, score_col=config.score_col, severity_col=config.severity_col, - quantile_points=quantile_points, + contributions_col=config.contributions_col if config.enable_contributions else None, + score_std_col=config.score_std_col, ) if config.enable_ai_explanation: @@ -194,18 +275,20 @@ def score_global_model( scored_df, config.model_name, config.threshold, + output_columns=config.output_columns, info_col_name=config.info_col, segment_values=None, enable_contributions=config.enable_contributions, enable_confidence_std=config.enable_confidence_std, ai_explanation_col=config.ai_explanation_col if config.enable_ai_explanation else None, - score_col=config.score_col, - score_std_col=config.score_std_col, - contributions_col=config.contributions_col, - severity_col=config.severity_col, + unseen=UnseenGroupContext( + unseen_col=unseen_col, + group_key_col=group_key_col, + flag_as_violation=config.flag_unseen_baseline_as_violation, + ), ) - internal_to_remove = [config.score_std_col, config.severity_col] + internal_to_remove = [config.score_std_col, config.severity_col, unseen_col, group_key_col] if config.enable_contributions: internal_to_remove.append(config.contributions_col) if config.enable_ai_explanation: @@ -326,15 +409,12 @@ def score_single_segment( segment_scored, config.model_name, config.threshold, + output_columns=config.output_columns, info_col_name=config.info_col, segment_values=segment_model.segmentation.segment_values, enable_contributions=config.enable_contributions, enable_confidence_std=config.enable_confidence_std, ai_explanation_col=config.ai_explanation_col if config.enable_ai_explanation else None, - score_col=config.score_col, - score_std_col=config.score_std_col, - contributions_col=config.contributions_col, - severity_col=config.severity_col, ) return segment_scored diff --git a/src/databricks/labs/dqx/anomaly/scoring_utils.py b/src/databricks/labs/dqx/anomaly/scoring_utils.py index 90afe8344..98dc62541 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_utils.py +++ b/src/databricks/labs/dqx/anomaly/scoring_utils.py @@ -1,7 +1,9 @@ """Anomaly scoring helpers: DataFrame/schema builders, row filter, join, reserved column checks.""" +from dataclasses import dataclass + import pyspark.sql.functions as F -from pyspark.sql import DataFrame +from pyspark.sql import Column, DataFrame from pyspark.sql.types import ( DoubleType, MapType, @@ -11,7 +13,8 @@ ) from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema, anomaly_info_struct_schema -from databricks.labs.dqx.anomaly.segment_utils import canonicalize_segment_values +from databricks.labs.dqx.anomaly.scoring_config import ScoringOutputColumns +from databricks.labs.dqx.anomaly.segment_utils import canonicalize_segment_values, baseline_key_column from databricks.labs.dqx.errors import InvalidParameterError from databricks.labs.dqx.utils import safe_filter_expr from databricks.labs.dqx.schema.dq_info_schema import ( @@ -60,19 +63,31 @@ def create_null_scored_dataframe( return result.withColumn(info_col_name, build_dq_info_struct(anomaly=null_anomaly_info)) +@dataclass(frozen=True) +class UnseenGroupContext: + """Where to find the unseen-group verdict, and what it should mean. + + Bundled rather than passed as three more parameters to ``add_info_column``: they are only ever + meaningful together, and the alternative is a twelve-plus argument signature nobody can read. + """ + + unseen_col: str + group_key_col: str + flag_as_violation: bool = False + + def add_info_column( df: DataFrame, model_name: str, threshold: float, - info_col_name: str, + *, + output_columns: ScoringOutputColumns | None = None, + info_col_name: str | None = None, segment_values: dict[str, str] | None = None, enable_contributions: bool = False, enable_confidence_std: bool = False, ai_explanation_col: str | None = None, - score_col: str = "anomaly_score", - score_std_col: str = "anomaly_score_std", - contributions_col: str = "anomaly_contributions", - severity_col: str = "severity_percentile", + unseen: UnseenGroupContext | None = None, ) -> DataFrame: """Add info struct column with anomaly metadata. @@ -80,26 +95,45 @@ def add_info_column( df: Scored DataFrame with anomaly_score, prediction, etc. model_name: Name of the model used for scoring. threshold: Threshold used for row anomaly detection. - info_col_name: Name for the info struct column (collision-safe UUID name expected). + output_columns: Internal column names to read scores, severity, contributions and std + from, and where to write the info struct. Defaults to the standard names. + info_col_name: Overrides ``output_columns.info`` when given (collision-safe UUID name). segment_values: Segment values if model is segmented (None for global models). enable_contributions: Whether anomaly_contributions are available (0–100 percent). enable_confidence_std: Whether anomaly_score_std is available. ai_explanation_col: Optional column name carrying the pre-computed AI explanation struct. When provided and present on df, it is packaged into _dq_info. - score_col: Column name for anomaly scores (internal, collision-safe). - score_std_col: Column name for ensemble std scores (internal, collision-safe). - contributions_col: Column name for SHAP contributions (internal, collision-safe, 0–100 percent). - severity_col: Column name for severity percentile (internal, collision-safe). + unseen: Where the unseen-group verdict lives and whether it counts as a violation. None + means the model is not grouped, so no row is unseen. Returns: DataFrame with info column added. """ + output_columns = output_columns or ScoringOutputColumns() + score_col = output_columns.score + score_std_col = output_columns.score_std + contributions_col = output_columns.contributions + severity_col = output_columns.severity + info_col_name = info_col_name or output_columns.info + # Build anomaly info struct + unseen_col = unseen.unseen_col if unseen else None + group_key_col = unseen.group_key_col if unseen else None + flag_unseen_baseline_as_violation = unseen.flag_as_violation if unseen else False + unseen_expr = F.col(unseen_col) if unseen_col and unseen_col in df.columns else F.lit(False) + + # An unseen group has a null severity, so `severity >= threshold` is null rather than False. + # Decide it explicitly: flag_unseen_baseline_as_violation says whether "we cannot judge this row" + # should count as a violation, which is a policy question only the caller can answer. + is_anomaly = F.when(unseen_expr, F.lit(bool(flag_unseen_baseline_as_violation))).otherwise( + F.col(severity_col) >= F.lit(threshold) + ) + anomaly_info_fields = { "check_name": F.lit("has_no_row_anomalies"), "score": F.round(F.col(score_col), 3), "severity_percentile": F.round(F.col(severity_col), 1), - "is_anomaly": F.col(severity_col) >= F.lit(threshold), + "is_anomaly": is_anomaly, "threshold": F.lit(threshold), "model": F.lit(model_name), } @@ -136,6 +170,16 @@ def add_info_column( else: anomaly_info_fields["ai_explanation"] = F.lit(None).cast(ai_explanation_struct_schema) + # Surface the unseen-group verdict, and the key that was not recognised so the caller can act + # on it (retrain, or investigate an unexpected dimension value) without re-deriving it. + anomaly_info_fields["is_new_baseline"] = unseen_expr + if group_key_col and group_key_col in df.columns: + anomaly_info_fields["new_baseline_key"] = F.when(unseen_expr, F.col(group_key_col)).otherwise( + F.lit(None).cast(StringType()) + ) + else: + anomaly_info_fields["new_baseline_key"] = F.lit(None).cast(StringType()) + anomaly_info = F.struct(*[value.alias(key) for key, value in anomaly_info_fields.items()]).cast( anomaly_info_struct_schema ) @@ -163,29 +207,202 @@ def add_severity_percentile_column( if not quantile_points: return df.withColumn(severity_col, F.lit(None).cast(DoubleType())) - # Ensure points are sorted by percentile points = sorted(quantile_points, key=lambda p: p[0]) - score_expr = F.col(score_col) + expr = _piecewise_severity_expr(F.col(score_col), [(p, F.lit(float(q))) for p, q in points]) + return df.withColumn(severity_col, expr) - # Handle null scores + +def _piecewise_severity_expr(score_expr: Column, points: list[tuple[float, Column]]) -> Column: + """Map a score onto 0–100 by piecewise linear interpolation between *points*. + + The score bounds are Columns rather than floats so the same interpolation serves both the + global calibration, where each bound is a literal, and per-group calibration, where each + bound is a column read from a broadcast lookup of that row's group. + + Args: + score_expr: The score to map. + points: ``(percentile, score bound)`` pairs, ordered by percentile. + """ expr = F.when(score_expr.isNull(), F.lit(None).cast(DoubleType())) prev_p, prev_q = points[0] - expr = expr.when(score_expr <= F.lit(prev_q), F.lit(float(prev_p))) + expr = expr.when(score_expr <= prev_q, F.lit(float(prev_p))) for current_p, current_q in points[1:]: - if current_q == prev_q: - interpolated = F.lit(float(current_p)) - else: - interpolated = F.lit(float(prev_p)) + ( - (score_expr - F.lit(prev_q)) * (float(current_p) - float(prev_p)) / (float(current_q) - float(prev_q)) - ) - expr = expr.when(score_expr <= F.lit(current_q), interpolated) + span = current_q - prev_q + # A degenerate segment (equal bounds) would divide by zero. It means every score in this + # band sits on one point, so the upper percentile is the answer outright. + interpolated = F.when(span == F.lit(0.0), F.lit(float(current_p))).otherwise( + F.lit(float(prev_p)) + ((score_expr - prev_q) * F.lit(float(current_p) - float(prev_p)) / span) + ) + expr = expr.when(score_expr <= current_q, interpolated) prev_p, prev_q = current_p, current_q - expr = expr.otherwise(F.lit(float(prev_p))) + return expr.otherwise(F.lit(float(prev_p))) - return df.withColumn(severity_col, expr) + +def add_baseline_severity_percentile_column( + df: DataFrame, + *, + score_col: str, + severity_col: str, + baseline_by: list[str], + group_quantile_points: dict[str, list[tuple[float, float]]], + fallback_quantile_points: list[tuple[float, float]], +) -> DataFrame: + """Add a severity percentile calibrated against each row's own group. + + A raw anomaly score is not comparable across groups: severity is a percentile of a score + distribution, and each group has its own. Calibrating per group is what makes "severity 97" + mean the same thing in a high-volume group as in a quiet one. + + Implemented as a broadcast join of a ``(group key, p00 … p100)`` lookup plus one piecewise + expression over those columns — deliberately *not* a nested + ``when(group_key == lit(k), )`` chain, which at 90 groups and 11 points is + around a thousand branches of generated code and blows up compilation. + + Groups absent from *group_quantile_points* fall back to the global calibration, so a group + seen at scoring but not at training still gets a severity rather than a null. + """ + if not group_quantile_points: + return add_severity_percentile_column( + df, score_col=score_col, severity_col=severity_col, quantile_points=fallback_quantile_points + ) + + percentiles = [p for p, _ in sorted(fallback_quantile_points, key=lambda point: point[0])] + fallback_by_percentile = dict(fallback_quantile_points) + quantile_cols = {p: f"__dqx_group_q{int(p * 100):05d}" for p in percentiles} + + group_key_col = "__dqx_severity_group_key" + df = df.withColumn(group_key_col, baseline_key_column(baseline_by)) + + schema = StructType( + [StructField(group_key_col, StringType(), False)] + + [StructField(quantile_cols[p], DoubleType(), True) for p in percentiles] + ) + rows = [] + for key, points in group_quantile_points.items(): + by_percentile = dict(points) + rows.append((key, *[by_percentile.get(p) for p in percentiles])) + lookup_df = df.sparkSession.createDataFrame(rows, schema=schema) + + df = df.join(F.broadcast(lookup_df), on=group_key_col, how="left") + + bounds = [(p, F.coalesce(F.col(quantile_cols[p]), F.lit(float(fallback_by_percentile[p])))) for p in percentiles] + df = df.withColumn(severity_col, _piecewise_severity_expr(F.col(score_col), bounds)) + + return df.drop(group_key_col, *quantile_cols.values()) + + +#: Above this many known group keys, membership is tested with a broadcast anti-join rather than +#: an `isin` list. An `isin` of thousands of literals bloats the query plan. +_MAX_ISIN_GROUP_KEYS = 200 + + +def mark_unseen_baselines( + df: DataFrame, + baseline_by: list[str], + known_group_keys: list[str], + *, + unseen_col: str, + group_key_col: str, +) -> DataFrame: + """Add a boolean *unseen_col*, True where the row's group was absent from training. + + Both categorical encoders mishandle an unseen value, in opposite directions, and neither + tells the caller: + + - One-hot emits all zeros. Because each one-hot column is 0 for most training rows, an + all-zeros row looks like the majority on *every* axis, so axis-aligned splits cannot see + that no category is set at all. Result: a false negative — the most normal-looking score + in the table. + - Frequency encoding coalesces the miss to 0.0, which sits *below every frequency seen in + training*. Result: a false positive, a fabricated extreme. This is the branch taken above + the cardinality threshold, so it is the one that fires on wide groupings. + + Either way the score is not meaningful, so the honest answer is to flag it rather than emit a + number. Uses ``isin`` for a modest number of known keys and a broadcast left-join for many, + since an ``isin`` over thousands of literals bloats the query plan. + """ + if not baseline_by: + return df.withColumn(unseen_col, F.lit(False)).withColumn(group_key_col, F.lit(None).cast(StringType())) + + df = df.withColumn(group_key_col, baseline_key_column(baseline_by)) + + if not known_group_keys: + # A grouped model with no recorded groups cannot judge membership; treating every row as + # unseen would null the whole frame, so trust the score instead. + return df.withColumn(unseen_col, F.lit(False)) + + if len(known_group_keys) <= _MAX_ISIN_GROUP_KEYS: + return df.withColumn(unseen_col, ~F.col(group_key_col).isin(known_group_keys)) + + known_col = "__dqx_known_group_key" + known_df = df.sparkSession.createDataFrame( + [(key,) for key in known_group_keys], + schema=StructType([StructField(known_col, StringType(), False)]), + ) + joined = df.join(F.broadcast(known_df), F.col(group_key_col) == F.col(known_col), "left") + return joined.withColumn(unseen_col, F.col(known_col).isNull()).drop(known_col) + + +def null_out_unseen_baseline_scores( + df: DataFrame, + *, + unseen_col: str, + score_col: str, + severity_col: str, + contributions_col: str | None, + score_std_col: str | None, +) -> DataFrame: + """Null the score, severity and contributions of unseen-group rows, in place. + + Deliberately not routed through ``create_null_scored_dataframe``: that builds the anomaly + struct as ``lit(None).cast(schema)``, a wholly null struct, which cannot carry + ``is_new_baseline``. Overwriting the individual columns keeps the struct intact so the flag + survives to the caller. + """ + unseen = F.col(unseen_col) + df = df.withColumn(score_col, F.when(unseen, F.lit(None).cast(DoubleType())).otherwise(F.col(score_col))) + df = df.withColumn(severity_col, F.when(unseen, F.lit(None).cast(DoubleType())).otherwise(F.col(severity_col))) + if contributions_col and contributions_col in df.columns: + df = df.withColumn( + contributions_col, + F.when(unseen, F.lit(None).cast(MapType(StringType(), DoubleType()))).otherwise(F.col(contributions_col)), + ) + if score_std_col and score_std_col in df.columns: + df = df.withColumn( + score_std_col, F.when(unseen, F.lit(None).cast(DoubleType())).otherwise(F.col(score_std_col)) + ) + return df + + +def permissive_quantile_points( + group_quantile_points: dict[str, list[tuple[float, float]]], + fallback_quantile_points: list[tuple[float, float]], +) -> list[tuple[float, float]]: + """Per-percentile *minimum* score bound across all groups. + + Used only to decide which rows are worth computing SHAP contributions for. Taking the + minimum makes the gate deliberately permissive: a row that any group would consider + anomalous passes it. That is safe because the observable contract is re-enforced downstream + by ``add_info_column``, which masks contributions on severity against the real threshold — + so a permissive gate can only cost a little wasted SHAP, while a strict one would silently + drop contributions from genuinely anomalous rows in the groups with the widest score ranges. + """ + if not group_quantile_points: + return fallback_quantile_points + + minima: dict[float, float] = {} + for points in group_quantile_points.values(): + for percentile, bound in points: + existing = minima.get(percentile) + minima[percentile] = bound if existing is None else min(existing, bound) + + for percentile, bound in fallback_quantile_points: + minima.setdefault(percentile, bound) + + return sorted(minima.items()) def create_udf_schema(enable_contributions: bool) -> StructType: @@ -236,7 +453,10 @@ def join_filtered_results_back( agg_exprs = [ F.max(score_col).alias(score_col), - F.max_by(info_col, score_col).alias(info_col), + # max_by returns null when every score in the group is null, which would throw away the + # info struct for rows that are deliberately unscored — an unseen group carries a null + # score but still has is_new_baseline to report. Fall back to any non-null info in that case. + F.coalesce(F.max_by(info_col, score_col), F.first(info_col, ignorenulls=True)).alias(info_col), ] scored_subset_unique = scored_subset.groupBy(*merge_columns).agg(*agg_exprs) diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index d9b54b2e4..b8452e14c 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -30,6 +30,10 @@ SegmentationConfig, TrainingMetadata, ) +from databricks.labs.dqx.anomaly.group_config import ( + MIN_ROWS_TO_TRAIN_SEGMENT, + SEGMENT_COUNT_WARN_THRESHOLD, +) from databricks.labs.dqx.anomaly.profiler import auto_discover_columns from databricks.labs.dqx.anomaly.training_strategies import AnomalyTrainingStrategy, IsolationForestTrainingStrategy from databricks.labs.dqx.anomaly.transformers import ( @@ -41,6 +45,7 @@ from databricks.labs.dqx.anomaly.validation import ( validate_columns, validate_fully_qualified_name, + validate_baseline_columns, validate_spark_version, validate_training_params, ) @@ -106,12 +111,26 @@ def apply_expected_anomaly_rate_if_default_contamination( @staticmethod def _get_and_validate_segments( - df: DataFrame, segment_by: list[str] + df: DataFrame, segment_by: list[str], params: AnomalyParams ) -> tuple[int, collections.abc.Iterator[dict[str, Any]]]: - """Get distinct segments and validate count.""" + """Get distinct segments and validate count. + + Raises above ``params.max_segment_models``. One model is trained per segment and + segmented training does not ensemble, so cost is linear in the segment count: 90 + segments measures roughly 88 minutes. Warning and proceeding meant a run could spend + hours registering thousands of models behind a single log line, which is why this is + an error rather than a warning. + """ segments_df = df.select(*segment_by).distinct() segment_count = segments_df.count() - if segment_count > 100: + if segment_count > params.max_segment_models: + raise InvalidParameterError( + f"Segmenting by {segment_by} produces {segment_count} segments, above the limit of " + f"{params.max_segment_models}. One model is trained per segment, so this run would train " + f"{segment_count} models (roughly {segment_count} minutes). Either segment more coarsely, " + f"or raise the limit with AnomalyParams(max_segment_models={segment_count})." + ) + if segment_count > SEGMENT_COUNT_WARN_THRESHOLD: logger.warning( f"Training {segment_count} segments may be slow. Consider coarser segmentation or explicit segment_by." ) @@ -192,6 +211,94 @@ def _resolve_columns_and_filtered_df( return None, df.select(*remaining) return None, df + def _discover_columns_and_grouping( + self, + df_filtered: DataFrame, + columns: list[str] | None, + declared_baseline_by: list[str] | None, + segment_by: list[str] | None, + ) -> tuple[list[str], list[str] | None, list[str] | None]: + """Fill in whichever of the feature columns and the grouping the caller left unspecified. + + Returns ``(columns, baseline_by, segment_by)``. + """ + if columns is None: + columns, discovered_segments = self._perform_auto_discovery(df_filtered, segment_by) + if declared_baseline_by: + # A declared baseline column is the basis metrics are compared against, not a + # metric. Auto-discovery does not know that, so drop them here rather than making + # the caller reconcile a list they never wrote. + columns = [c for c in columns if c not in declared_baseline_by] + elif segment_by is None and discovered_segments: + # Route a discovered grouping to baseline_by, not segment_by. Assigning it to + # segment_by would pin it to one model per group, which is how a discovered 90-way + # grouping became an hours-long run — and would now hit the segment ceiling and fail + # outright rather than using the mechanism that scales. + return columns, discovered_segments, None + else: + segment_by = discovered_segments + return columns, declared_baseline_by, segment_by + + if declared_baseline_by is None and segment_by is None: + # Grouping discovery used to be reachable only when the columns were discovered too, so + # naming your feature columns silently gave up any chance of conditioning. Those are + # independent questions. Costs one extra profiling pass for callers who pass explicit + # columns and no grouping. + return columns, self._discover_baseline_columns(df_filtered), None + + return columns, declared_baseline_by, segment_by + + @staticmethod + def _discover_baseline_columns(df_filtered: DataFrame) -> list[str] | None: + """Discover a baseline grouping when the caller named feature columns but no grouping. + + Kept separate from ``_perform_auto_discovery`` so that discovering a grouping does not + require also discovering the feature columns. + """ + profile = auto_discover_columns(df_filtered) + if not profile.recommended_segments: + return None + logger.info( + f"Auto-detected {len(profile.recommended_segments)} baseline columns: " + f"{profile.recommended_segments} ({profile.segment_count} total groups)" + ) + return profile.recommended_segments + + @staticmethod + def _resolve_grouping( + baseline_by: list[str] | None, + segment_by: list[str] | None, + ) -> tuple[list[str] | None, list[str] | None]: + """Reconcile ``baseline_by`` against the legacy ``segment_by``. + + Returns ``(baseline_by, effective_segment_by)``. Exactly one of the two is ever populated: + + * ``segment_by`` — the legacy path, one model per segment. **Must** clear ``baseline_by``. + Leaving both set would compute baseline-relative features *inside* each segment, where the + baseline key is constant because ``_train_segmented`` has already filtered the frame, while + ``compute_config_hash`` is built from ``segment_by`` alone and would not change. A model + whose feature list moved but whose config hash did not is a silent train/score hazard, and + it breaks the byte-identical-feature-list guarantee that makes already-trained models safe. + * ``baseline_by`` — one pooled model fed each metric's deviation from its own group's + baseline. + + There is no strategy to choose any more: on SMD, per-group models were the worst of three + configurations (PR-AUC 0.1416 against 0.1499 pooled and 0.1536 relative) with one entity + producing 15,963 false positives on 28,392 normal rows, so ``segment_by`` survives for + compatibility rather than as a recommendation. See databrickslabs/dqx#1484. + """ + if baseline_by is not None and segment_by is not None: + raise InvalidParameterError( + "Pass either baseline_by or segment_by, not both. segment_by is the legacy name for " + "training one model per group; baseline_by judges each metric against its own " + "group's baseline using a single model." + ) + + if segment_by is not None: + return None, segment_by + + return baseline_by, None + def build_context( self, df: DataFrame, @@ -203,6 +310,7 @@ def build_context( params: AnomalyParams | None, exclude_columns: list[str] | None, expected_anomaly_rate: float, + baseline_by: list[str] | None = None, ) -> AnomalyTrainingContext: """Build training context with all validated inputs.""" validate_spark_version(self._spark) @@ -221,20 +329,28 @@ def build_context( if invalid: raise InvalidParameterError(f"exclude_columns contains columns not in DataFrame: {invalid}") + params = AnomalyParams() if params is None else params + validate_training_params(params, expected_anomaly_rate) + declared_baseline_by = baseline_by if baseline_by is not None else params.baseline_by + columns, df_filtered = self._resolve_columns_and_filtered_df(df, columns, exclude_list) auto_discovery_used = columns is None - if columns is None: - columns, segment_by = self._perform_auto_discovery(df_filtered, segment_by) + columns, declared_baseline_by, segment_by = self._discover_columns_and_grouping( + df_filtered, columns, declared_baseline_by, segment_by + ) if not columns: raise InvalidParameterError("No columns provided or auto-discovered. Provide columns explicitly.") - params = AnomalyParams() if params is None else params - validate_training_params(params, expected_anomaly_rate) validation_warnings = validate_columns(df, columns, params) for warning in validation_warnings: logger.warning(warning) + validate_baseline_columns(df, declared_baseline_by, columns) + baseline_by, segment_by = self._resolve_grouping(declared_baseline_by, segment_by) + if baseline_by: + logger.info(f"Judging each metric against its own group's baseline, grouped by {baseline_by}") + self._prepare_training_config( model_name=model_name, registry_table=registry_table, @@ -243,6 +359,10 @@ def build_context( ) params = self.apply_expected_anomaly_rate_if_default_contamination(params, expected_anomaly_rate) + # Already a deepcopy, so recording the resolved grouping here cannot leak back to the + # caller's params. Downstream feature engineering reads baseline_by off params, because + # every narrowing select is already handed params and nothing else. + params.baseline_by = baseline_by return AnomalyTrainingContext( spark=self._spark, @@ -256,6 +376,7 @@ def build_context( expected_anomaly_rate=expected_anomaly_rate, exclude_columns=exclude_columns, auto_discovery_used=auto_discovery_used, + baseline_by=baseline_by, ) def train(self, context: AnomalyTrainingContext) -> str: @@ -344,7 +465,9 @@ def _train_segmented(self, context: AnomalyTrainingContext) -> str: """Train separate models for each segment.""" if context.segment_by is None: raise InvalidParameterError("segment_by is required for segmented training") - segment_count, segment_iterator = self._get_and_validate_segments(context.df_filtered, context.segment_by) + segment_count, segment_iterator = self._get_and_validate_segments( + context.df_filtered, context.segment_by, context.params + ) model_uris = [] skipped_segments = [] failed_segments: list[tuple[str, str]] = [] @@ -358,7 +481,7 @@ def _train_segmented(self, context: AnomalyTrainingContext) -> str: segment_df = segment_df.filter(segment_df[col_name] == val) sampled_df, row_count, _ = sample_df(segment_df, context.columns, context.params) - if row_count < 10: + if row_count < MIN_ROWS_TO_TRAIN_SEGMENT: skipped_segments.append(segment_name) continue diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 01adb067f..dc4451bb5 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -11,11 +11,11 @@ import re import sys import threading -from dataclasses import dataclass, fields +from dataclasses import dataclass, field, fields from io import StringIO from typing import Any -from pyspark.sql import DataFrame +from pyspark.sql import Column, DataFrame from pyspark.sql import functions as F from pyspark.sql import types as T from pyspark.sql.functions import ( @@ -34,6 +34,7 @@ ) from pyspark.sql.types import DoubleType, TimestampType +from databricks.labs.dqx.anomaly.segment_utils import BASELINE_KEY_COLUMN, with_baseline_key from databricks.labs.dqx.errors import ComputationError, InvalidParameterError from databricks.labs.dqx.telemetry import get_tables_from_spark_plan from databricks.labs.dqx.utils import get_table_primary_keys @@ -73,6 +74,17 @@ class SparkFeatureMetadata: onehot_categories: dict[str, list[str]] # col_name -> [distinct_values] for OneHot encoding engineered_feature_names: list[str] # Final feature names after engineering categorical_cardinality_threshold: int = 20 # Threshold used for categorical encoding + # Group conditioning. Every field defaults to empty, so a model trained before these existed + # deserializes with no grouping, the relative transform returns immediately, and + # engineered_feature_names is byte-identical to what it was — such models score exactly as + # they did before. See databrickslabs/dqx#1484. + baseline_by: list[str] = field(default_factory=list) # Columns forming the group key + baseline_medians: dict[str, dict[str, float]] = field(default_factory=dict) # metric -> (group key -> median) + global_medians: dict[str, float] = field(default_factory=dict) # metric -> median, for unseen groups + # Per-group score calibration: group key -> (quantile key -> score bound). Lives here rather + # than in the typed training.score_quantiles map column, which would need a + # registry migration to hold a nested map; training.score_quantiles stays the global fallback. + baseline_score_quantiles: dict[str, dict[str, float]] = field(default_factory=dict) def to_json(self) -> str: """Serialize to JSON for storage. @@ -742,12 +754,132 @@ def _process_numeric_columns( return transformed_df +def _signed_log1p(column: Column) -> Column: + """``signum(x) * log1p(abs(x))`` — a log-like transform defined over all reals. + + Plain ``log1p`` is NaN for ``x <= -1``, so it is only safe for counts and shares. Any signed + metric (profit, balance, delta) would silently produce NaN features. This form is monotone + over the whole real line and identical to ``log1p`` for ``x >= 0``. + """ + return F.signum(column) * F.log1p(F.abs(column)) + + +def _process_baseline_relative_features( + transformed_df: DataFrame, + numeric_cols: list[ColumnTypeInfo], + baseline_by: list[str], + is_training: bool, + baseline_medians: dict[str, dict[str, float]], + global_medians: dict[str, float], + engineered_features: list[str], +) -> DataFrame: + """Append each numeric metric's deviation from its own group's baseline. + + ``rel = signed_log(value) - signed_log(group_median(value))``. The log-ratio form is stable + when a baseline is near zero and symmetric for halving versus doubling; the raw metric is + kept alongside, so globally absurd values stay detectable even where they are ordinary for + their group. + + Follows ``_apply_frequency_encoding``'s shape exactly: compute and persist while training, + broadcast-join and coalesce the miss while scoring. A row whose group was never trained on + falls back to the global baseline, which makes it look ordinary rather than extreme — the + conservative direction, and the case a caller can detect explicitly via ``is_new_baseline``. + + Must run last. ``engineered_feature_names`` is positional: the sklearn pipeline is handed + columns in this order, so features may only ever be appended at the tail. Inserting a + transform before this one would silently reorder an already-trained model's inputs. + """ + if not baseline_by or not numeric_cols: + return transformed_df + + metrics = [c.name for c in numeric_cols] + group_key_col = BASELINE_KEY_COLUMN + transformed_df = with_baseline_key(transformed_df, baseline_by) + + if is_training: + computed_group, computed_global = _compute_baseline_medians(transformed_df, metrics, group_key_col) + baseline_medians.update(computed_group) + global_medians.update(computed_global) + + for metric in metrics: + feature_name = f"{metric}_rel_baseline" + baselines = baseline_medians.get(metric, {}) + global_baseline = global_medians.get(metric, 0.0) + + if not baselines: + # No baseline for this metric at all: the deviation is undefined, so emit a constant + # rather than a fabricated signal. Still appended, to keep the feature list stable. + transformed_df = transformed_df.withColumn(feature_name, lit(0.0)) + engineered_features.append(feature_name) + continue + + baseline_col = f"__dqx_{metric}_baseline" + lookup_df = _baseline_lookup_df(transformed_df, baselines, group_key_col, baseline_col) + transformed_df = transformed_df.join(broadcast(lookup_df), on=group_key_col, how="left") + resolved_baseline = coalesce(col(baseline_col), lit(global_baseline)) + transformed_df = transformed_df.withColumn( + feature_name, _signed_log1p(col(metric)) - _signed_log1p(resolved_baseline) + ) + transformed_df = transformed_df.drop(baseline_col) + engineered_features.append(feature_name) + + # The key column deliberately survives: it is the frame's only remaining record of the + # grouping once the raw group columns are dropped, and training-time severity calibration + # runs on the scored frame, downstream of here. It is not a feature — see the explicit + # projection in ``prepare_training_features``, which is what keeps it out of the model. + return transformed_df + + +def _compute_baseline_medians( + df: DataFrame, metrics: list[str], group_key_col: str +) -> tuple[dict[str, dict[str, float]], dict[str, float]]: + """Compute each metric's per-group median, plus a global median for unseen groups. + + One ``groupBy`` over all metrics rather than one per metric, and one global aggregation. + ``percentile_approx`` rather than an exact median: the baseline only has to be + representative, and an exact median would need a full sort per group. + """ + group_exprs = [F.percentile_approx(col(m), 0.5).alias(m) for m in metrics] + group_rows = df.groupBy(group_key_col).agg(*group_exprs).collect() + + baseline_medians: dict[str, dict[str, float]] = {m: {} for m in metrics} + for row in group_rows: + key = row[group_key_col] + for metric in metrics: + value = row[metric] + if value is not None: + baseline_medians[metric][key] = float(value) + + global_row = df.agg(*[F.percentile_approx(col(m), 0.5).alias(m) for m in metrics]).first() + global_medians: dict[str, float] = {} + if global_row is not None: + for metric in metrics: + value = global_row[metric] + global_medians[metric] = float(value) if value is not None else 0.0 + + return baseline_medians, global_medians + + +def _baseline_lookup_df(df: DataFrame, baselines: dict[str, float], group_key_col: str, baseline_col: str) -> DataFrame: + """Build a broadcastable ``(group key, baseline)`` lookup for one metric.""" + schema = T.StructType( + [ + T.StructField(group_key_col, T.StringType(), False), + T.StructField(baseline_col, T.DoubleType(), False), + ] + ) + return df.sparkSession.createDataFrame(list(baselines.items()), schema=schema) + + def apply_feature_engineering( df: DataFrame, column_infos: list[ColumnTypeInfo], categorical_cardinality_threshold: int = 20, frequency_maps: dict[str, dict[str, float]] | None = None, onehot_categories: dict[str, list[str]] | None = None, + baseline_by: list[str] | None = None, + baseline_medians: dict[str, dict[str, float]] | None = None, + global_medians: dict[str, float] | None = None, ) -> tuple[DataFrame, SparkFeatureMetadata]: """ Apply feature engineering transformations in Spark (distributed). @@ -756,13 +888,18 @@ def apply_feature_engineering( - DataFrame with engineered numeric features - Metadata for reconstructing transformations during scoring - Transformations applied: + Transformations applied, in this order — the order is part of the contract, because + ``engineered_feature_names`` is positional and the sklearn pipeline is handed columns in it: 1. Categorical: OneHot (low-card) or Frequency encoding (high-card) 2. Datetime: Extract hour_sin/cos, dow_sin/cos, month_sin/cos, is_weekend 3. Boolean: Map to 0/1 4. Numeric: Keep as-is - 5. Null indicators: Add column_is_null for columns with nulls - 6. Imputation: Fill nulls with 0 (numeric), "MISSING" (categorical), epoch (datetime), 0 (boolean) + 5. Group-relative: deviation of each numeric metric from its own group's baseline + 6. Null indicators: Add column_is_null for columns with nulls + 7. Imputation: Fill nulls with 0 (numeric), "MISSING" (categorical), epoch (datetime), 0 (boolean) + + New transforms must be appended at the end, never inserted: inserting one shifts the feature + positions an already-trained model expects. Args: df: Input DataFrame with original columns @@ -770,12 +907,21 @@ def apply_feature_engineering( categorical_cardinality_threshold: Threshold for OneHot vs Frequency encoding frequency_maps: Pre-computed frequency maps (for scoring). If None, compute from df (for training). onehot_categories: Pre-computed OneHot distinct values (for scoring). If None, compute from df (for training). + baseline_by: Columns forming the group key. Empty disables group-relative features entirely, + which is what makes a pre-grouping model's feature list byte-identical. + baseline_medians: Pre-computed per-group medians (for scoring). Computed from df when training. + global_medians: Pre-computed global medians, used for groups absent from training. """ is_training = frequency_maps is None if frequency_maps is None: frequency_maps = {} if onehot_categories is None: onehot_categories = {} + baseline_by = list(baseline_by or []) + if baseline_medians is None: + baseline_medians = {} + if global_medians is None: + global_medians = {} transformed_df = df engineered_features: list[str] = [] @@ -804,10 +950,27 @@ def apply_feature_engineering( transformed_df = _process_numeric_columns(transformed_df, numeric_cols, engineered_features) + # Must stay last: engineered_feature_names is positional, so features may only be appended. + transformed_df = _process_baseline_relative_features( + transformed_df, + numeric_cols, + baseline_by, + is_training, + baseline_medians, + global_medians, + engineered_features, + ) + # Select engineered features + preserve any extra columns not in column_infos - # (e.g., __dqx_row_id__ for joining results back) + # (e.g., __dqx_row_id__ for joining results back). Group columns are the comparison basis, + # not features, so they are excluded here — they must not reach the sklearn pipeline or the + # inferred MLflow signature. feature_col_names = [c.name for c in column_infos] - extra_cols = [c for c in transformed_df.columns if c not in feature_col_names and c not in engineered_features] + extra_cols = [ + c + for c in transformed_df.columns + if c not in feature_col_names and c not in engineered_features and c not in baseline_by + ] result_df = transformed_df.select(*engineered_features, *extra_cols) # Use only the features that actually exist in the result DataFrame @@ -829,6 +992,9 @@ def apply_feature_engineering( onehot_categories=onehot_categories, engineered_feature_names=actual_engineered_features, categorical_cardinality_threshold=categorical_cardinality_threshold, + baseline_by=baseline_by, + baseline_medians=baseline_medians, + global_medians=global_medians, ) return result_df, metadata @@ -863,4 +1029,7 @@ def apply_feature_engineering_from_metadata( categorical_cardinality_threshold=feature_metadata.categorical_cardinality_threshold, frequency_maps=feature_metadata.categorical_frequency_maps, onehot_categories=feature_metadata.onehot_categories, + baseline_by=feature_metadata.baseline_by, + baseline_medians=feature_metadata.baseline_medians, + global_medians=feature_metadata.global_medians, ) diff --git a/src/databricks/labs/dqx/anomaly/types.py b/src/databricks/labs/dqx/anomaly/types.py index 24fad3388..9c9f545f3 100644 --- a/src/databricks/labs/dqx/anomaly/types.py +++ b/src/databricks/labs/dqx/anomaly/types.py @@ -70,7 +70,14 @@ class EnsembleTrainingResult: @dataclass(frozen=True) class AnomalyTrainingContext: - """Context containing all inputs needed for training.""" + """Context containing all inputs needed for training. + + ``baseline_by`` and ``segment_by`` are mutually exclusive, and ``_resolve_grouping`` guarantees + it: ``segment_by`` selects the legacy path that trains one model per group and dispatches + ``train()``, while ``baseline_by`` selects a single pooled model fed each metric's deviation + from its own group's baseline. Both being set would append relative features inside each + segment while leaving the model config hash unchanged. + """ spark: SparkSession df: DataFrame @@ -83,6 +90,7 @@ class AnomalyTrainingContext: expected_anomaly_rate: float exclude_columns: list[str] | None auto_discovery_used: bool + baseline_by: list[str] | None = None @dataclass(frozen=True) diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index f648a3c8e..0e71d16a7 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -210,6 +210,16 @@ class AnomalyParams: confidence scores are not available for segmented models. algorithm_config: Isolation Forest parameters (contamination, num_trees, seed). feature_engineering: Feature engineering parameters (temporal features, scaling, etc.). + max_segment_models: Ceiling on how many per-segment models one training run will attempt + (default 50). Guards the legacy *segment_by* path, which is the only one that trains a + model per group: cost is linear in the segment count and segmented training does not + ensemble, so 90 segments measures roughly 88 minutes. Raise this only if you are + prepared to wait. Irrelevant to *baseline_by*, which trains a single model. + baseline_by: Columns identifying the group a row belongs to, so a metric is judged against + its own group's baseline rather than against the whole table. Each numeric metric gains + its deviation from that baseline as an extra feature, on one pooled model, so cost does + not grow with the group count. Normally set by passing *baseline_by* to + ``AnomalyEngine.train()``, which populates this. """ sample_fraction: float = 0.3 @@ -218,6 +228,10 @@ class AnomalyParams: ensemble_size: int | None = 3 # Default 3-model ensemble for robustness, tie-breaking, and confidence scores algorithm_config: IsolationForestConfig = field(default_factory=IsolationForestConfig) feature_engineering: FeatureEngineeringConfig = field(default_factory=FeatureEngineeringConfig) + # Kept in sync with anomaly.group_config.MAX_SEGMENT_MODELS by a unit test; not imported from + # there because that package requires the 'anomaly' extras and this module must not. + max_segment_models: int = 50 + baseline_by: list[str] | None = None @dataclass @@ -225,9 +239,12 @@ class AnomalyConfig: """Configuration for row anomaly detection.""" columns: list[str] | None = None # Auto-discovered if omitted - segment_by: list[str] | None = None # Auto-discovered if omitted (when columns also omitted) + segment_by: list[str] | None = None # Legacy: one model per segment. Prefer baseline_by. model_name: str | None = None # Optional in workflows; defaults to dqx_anomaly_ registry_table: str | None = None + # Preferred over segment_by: declares the basis each metric is judged against, on one pooled + # model. Optional, so installed run-config YAML written before it existed still loads. + baseline_by: list[str] | None = None @dataclass diff --git a/tests/unit/test_anomaly_configs.py b/tests/unit/test_anomaly_configs.py index 2a0a9854e..8e06714ed 100644 --- a/tests/unit/test_anomaly_configs.py +++ b/tests/unit/test_anomaly_configs.py @@ -1,5 +1,6 @@ """Unit tests for anomaly detection configuration classes.""" +from databricks.labs.dqx.anomaly.group_config import MAX_SEGMENT_MODELS from databricks.labs.dqx.config import ( AnomalyConfig, AnomalyParams, @@ -85,9 +86,20 @@ def test_anomaly_params_defaults(): assert params.sample_fraction == 0.3 assert params.max_rows == 1_000_000 assert params.train_ratio == 0.8 + assert params.max_segment_models == 50 assert isinstance(params.algorithm_config, IsolationForestConfig) +def test_max_group_models_default_matches_group_config(): + """The default must track the shared constant. + + ``config`` cannot import it, because the anomaly package requires the 'anomaly' extras + and ``config`` must stay importable without them, so the value is duplicated as a + literal. This test is the thing that stops the two drifting apart. + """ + assert AnomalyParams().max_segment_models == MAX_SEGMENT_MODELS + + def test_anomaly_params_custom_sample_fraction(): """Test AnomalyParams with custom sample fraction values.""" # Valid sample fractions diff --git a/tests/unit/test_anomaly_group_calibration_points.py b/tests/unit/test_anomaly_group_calibration_points.py new file mode 100644 index 000000000..191c650e2 --- /dev/null +++ b/tests/unit/test_anomaly_group_calibration_points.py @@ -0,0 +1,66 @@ +"""Unit tests for the SHAP gate under per-group score calibration. + +`permissive_quantile_points` collapses many per-group calibrations into the single set of bounds +the scorers use to decide which rows are worth a SHAP call. Pure dictionary arithmetic, so it is +testable without Spark. +""" + +from databricks.labs.dqx.anomaly.scoring_utils import permissive_quantile_points + +GLOBAL_POINTS = [(0.0, 0.10), (0.5, 0.50), (0.95, 0.90), (1.0, 1.00)] + + +def test_falls_back_to_global_when_no_group_calibration(): + """An ungrouped model must gate exactly as it did before.""" + assert permissive_quantile_points({}, GLOBAL_POINTS) == GLOBAL_POINTS + + +def test_takes_the_minimum_bound_across_groups(): + """The gate must admit a row that *any* group would consider anomalous. + + A per-group maximum, or the global bound, would silently drop contributions from anomalous + rows in the groups whose scores run low. + """ + group_points = { + "quiet": [(0.0, 0.01), (0.5, 0.05), (0.95, 0.20), (1.0, 0.30)], + "busy": [(0.0, 0.40), (0.5, 0.60), (0.95, 0.95), (1.0, 1.20)], + } + + result = dict(permissive_quantile_points(group_points, GLOBAL_POINTS)) + + assert result[0.95] == 0.20 + assert result[0.5] == 0.05 + assert result[1.0] == 0.30 + + +def test_gate_is_never_stricter_than_the_strictest_group(): + """Restates the safety property directly: the bound never exceeds any group's own bound.""" + group_points = { + "a": [(0.95, 0.70)], + "b": [(0.95, 0.30)], + "c": [(0.95, 0.55)], + } + + result = dict(permissive_quantile_points(group_points, GLOBAL_POINTS)) + + assert result[0.95] <= min(0.70, 0.30, 0.55) + + +def test_percentiles_only_the_global_calibration_has_are_kept(): + """A partial group calibration must not shrink the set of interpolation points.""" + group_points = {"a": [(0.95, 0.30)]} + + result = dict(permissive_quantile_points(group_points, GLOBAL_POINTS)) + + assert set(result) == {0.0, 0.5, 0.95, 1.0} + assert result[0.0] == 0.10 + assert result[0.95] == 0.30 + + +def test_result_is_ordered_by_percentile(): + """The interpolation walks these in order, so ordering is part of the contract.""" + group_points = {"a": [(1.0, 0.9), (0.0, 0.1), (0.5, 0.4)]} + + result = permissive_quantile_points(group_points, GLOBAL_POINTS) + + assert [p for p, _ in result] == sorted(p for p, _ in result) diff --git a/tests/unit/test_anomaly_transformers.py b/tests/unit/test_anomaly_transformers.py index 885a62272..9591fcf21 100644 --- a/tests/unit/test_anomaly_transformers.py +++ b/tests/unit/test_anomaly_transformers.py @@ -1,5 +1,6 @@ """Unit tests for feature engineering data structures and metadata.""" +import dataclasses import json from pyspark.sql import types as T @@ -343,3 +344,96 @@ def test_spark_feature_metadata_preserves_order(): restored = SparkFeatureMetadata.from_json(metadata.to_json()) assert restored.column_infos[0]["name"] == "z_col" assert restored.engineered_feature_names[0] == "z_col_scaled" + + +# ============================================================================ +# Group conditioning metadata (databrickslabs/dqx#1484) +# ============================================================================ + +# A feature_metadata payload exactly as DQX wrote it before group conditioning existed. +# Held verbatim rather than generated: the point is to prove that a model trained by the +# previous release still deserializes and still produces an identical feature list. +PRE_GROUPING_FEATURE_METADATA_JSON = ( + '{"column_infos": [{"name": "amount", "category": "numeric", "cardinality": null, "null_count": 0}, ' + '{"name": "region", "category": "categorical", "cardinality": 3, "null_count": 0}], ' + '"categorical_frequency_maps": {}, ' + '"onehot_categories": {"region": ["APAC", "EU", "US"]}, ' + '"engineered_feature_names": ["region_APAC", "region_EU", "region_US", "amount"], ' + '"categorical_cardinality_threshold": 20}' +) + + +def test_pre_grouping_metadata_deserializes_with_empty_grouping(): + """A model trained before grouping existed must load, with grouping simply absent.""" + restored = SparkFeatureMetadata.from_json(PRE_GROUPING_FEATURE_METADATA_JSON) + + assert not restored.baseline_by + assert not restored.baseline_medians + assert not restored.global_medians + + +def test_pre_grouping_metadata_keeps_its_feature_list_unchanged(): + """The feature list is positional, so it must survive byte-for-byte. + + An empty ``baseline_by`` makes the relative transform a no-op, which is what guarantees an + already-trained model is handed its features in exactly the order it was fitted on. + """ + restored = SparkFeatureMetadata.from_json(PRE_GROUPING_FEATURE_METADATA_JSON) + + assert restored.engineered_feature_names == ["region_APAC", "region_EU", "region_US", "amount"] + + +def test_from_json_ignores_unknown_keys(): + """A record written by a newer DQX must not break an older reader. + + ``from_json`` used to do ``cls(**data)``, so an unrecognised key raised TypeError and the + model could not be loaded at all. + """ + payload = json.loads(PRE_GROUPING_FEATURE_METADATA_JSON) + payload["some_field_from_the_future"] = {"a": 1} + + restored = SparkFeatureMetadata.from_json(json.dumps(payload)) + + assert restored.engineered_feature_names == ["region_APAC", "region_EU", "region_US", "amount"] + assert not hasattr(restored, "some_field_from_the_future") + + +def test_to_json_persists_every_dataclass_field(): + """``to_json`` is the single writer of features.feature_metadata. + + It previously named its keys literally, so a field added to the dataclass was dropped at + persistence and the model scored with a different feature set than it trained on. Iterating + the fields is what makes that failure impossible; this test pins it. + """ + metadata = SparkFeatureMetadata( + column_infos=[{"name": "amount", "category": "numeric"}], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["amount", "amount_rel_baseline"], + baseline_by=["country"], + baseline_medians={"amount": {"DE": 100.0}}, + global_medians={"amount": 90.0}, + ) + + payload = json.loads(metadata.to_json()) + + assert set(payload) == {f.name for f in dataclasses.fields(SparkFeatureMetadata)} + + +def test_group_metadata_survives_a_json_roundtrip(): + """Baselines are looked up by group key at scoring time, so they must round-trip exactly.""" + metadata = SparkFeatureMetadata( + column_infos=[{"name": "amount", "category": "numeric"}], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["amount", "amount_rel_baseline"], + baseline_by=["country", "product"], + baseline_medians={"amount": {"DE\x1fcasino": 3284.0, "IT\x1flive": 657.0}}, + global_medians={"amount": 1200.5}, + ) + + restored = SparkFeatureMetadata.from_json(metadata.to_json()) + + assert restored.baseline_by == ["country", "product"] + assert restored.baseline_medians == {"amount": {"DE\x1fcasino": 3284.0, "IT\x1flive": 657.0}} + assert restored.global_medians == {"amount": 1200.5} From 839f637845e695a90f81069772c3a73e307b2337 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:01:11 +0100 Subject: [PATCH 007/107] Make the anomaly benchmarks reach the published report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Predates this feature — `git log -- tests/perf/test_anomaly_benchmark.py` last touches it in #1129/#990 — and is separable from it. It lands here rather than as its own PR because the anomaly benchmark is what this branch changes the behaviour of. `generate_md_report.py` has always contained a fully written "Anomaly Benchmarks" section with ROC-AUC, precision, recall, F1 and precision@N columns. It has never rendered. The cause was one missing line: neither benchmark carried `@pytest.mark.benchmark(group="anomaly_synthetic")`, so pytest-benchmark recorded `group: null` and the report, which filters on exact equality with `anomaly_synthetic`, found nothing. `baseline.json` held 213 benchmarks and zero anomaly entries. Also fixed, all found while making the section actually appear: * The two benchmarks would have listed **twice** once the marker worked — once in the main results table, which is built from every benchmark, and once in their own section. * Five rounds of a Databricks-backed train-and-register cycle ran per nightly, twice over since the job invokes pytest again for the comparison step. Converted to `benchmark.pedantic(rounds=1)`, which takes its round count explicitly; verified against `--benchmark-min-rounds=5` and confirmed to record one round. * A module-global `_TRAINED_MODEL` with `needs_training` fallbacks made the score benchmark order-dependent and able to train up to three times. Each test now trains its own model as unmeasured setup. * The score benchmark read `_dq_info`, which does not exist on the frame `has_no_row_anomalies` returns — that column is assembled a layer up by `DQEngine`. So the test failed and recorded none of the quality numbers this section exists to publish. It now reads the struct column the check actually returns. Quality is published as *indicative and first-observed only*, and labelled as such. The nightly's baseline merge keeps the existing entry on conflict, so once a benchmark exists its `extra_info` is frozen at first observation and never refreshed — publishing quality that way publishes a fossil. Quality regressions are gated by assertions in `tests/integration_anomaly/` instead. The anomaly module is marked `pytest.mark.anomaly` and excluded from the timing gate. `--benchmark-compare-fail=mean:25%` is global and cannot be scoped per test, and these benchmarks are dominated by MLflow and Unity Catalog control-plane latency, so they would trip it on variance alone. They still produce a baseline; only the comparison skips them. --- .github/workflows/nightly.yml | 7 +- tests/perf/generate_md_report.py | 29 +++- tests/perf/test_anomaly_benchmark.py | 239 ++++++++++++++++----------- 3 files changed, 173 insertions(+), 102 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8e98bd02a..16c0b5c47 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -626,7 +626,12 @@ jobs: # The run fails if performance degrades by more than 25%. # Tests are run sequentially to reduce variability. # Do at least 5 rounds to get more stable results. - UV_FROZEN=1 uv run --all-extras pytest tests/perf -v -n 1 \ + # Anomaly benchmarks are deselected here: --benchmark-compare-fail applies to the whole + # invocation and cannot be scoped per test, and those benchmarks are dominated by MLflow + # and Unity Catalog control-plane latency, so they would trip a 25% mean gate on variance + # alone. They still produce a baseline in the step above; only the timing comparison skips + # them. Their detection quality is gated by assertions in tests/integration_anomaly/. + UV_FROZEN=1 uv run --all-extras pytest tests/perf -v -n 1 -m "not anomaly" \ --benchmark-storage=$BENCHMARKS_DIR \ --benchmark-compare=baseline \ --benchmark-compare-fail=mean:25% \ diff --git a/tests/perf/generate_md_report.py b/tests/perf/generate_md_report.py index 5ffef1cb5..da25aa734 100644 --- a/tests/perf/generate_md_report.py +++ b/tests/perf/generate_md_report.py @@ -21,6 +21,12 @@ data = json.loads(baseline_path.read_text()) +# Anomaly benchmarks get their own section below, with detection-quality columns the check +# benchmarks have no use for. Split them out here so they are not also listed in the main table. +ANOMALY_GROUP = "anomaly_synthetic" +check_benchmarks = [b for b in data["benchmarks"] if b.get("group") != ANOMALY_GROUP] +anomaly_benchmarks = [b for b in data["benchmarks"] if b.get("group") == ANOMALY_GROUP] + lines = [] lines.append("---\n") lines.append("title: Benchmarks\n") @@ -48,7 +54,7 @@ "|------|----------|------------|---------|---------|------------|---------|--------|--------|--------|--------------|-----------------|-------|" ) -for bench in data["benchmarks"]: +for bench in check_benchmarks: stats = bench["stats"] lines.append( f"| {bench['name']} " @@ -66,10 +72,27 @@ f"| {stats['ops']:.2f} |" ) -# Optional anomaly benchmark section (extra_info like roc_auc). -anomaly_benchmarks = [b for b in data["benchmarks"] if b.get("group") == "anomaly_synthetic"] +# Anomaly benchmark section: timings plus indicative detection quality carried in extra_info. if anomaly_benchmarks: lines.append("\n## Anomaly Benchmarks\n") + provenance: dict = next((b.get("extra_info", {}) for b in anomaly_benchmarks if b.get("extra_info")), {}) + if provenance.get("dataset"): + lines.append( + f"* Measured on {provenance['dataset']} " + f"({provenance.get('n_train_rows', 'n/a')} train / {provenance.get('n_test_rows', 'n/a')} test rows, " + f"{provenance.get('n_features', 'n/a')} features, " + f"{provenance.get('anomaly_frac', 'n/a')} anomaly fraction, seed {provenance.get('seed', 'n/a')})." + ) + lines.append( + "* Quality columns are **indicative and first-observed only**: the nightly baseline merge keeps " + "existing entries on conflict, so `extra_info` is not refreshed once a benchmark has been " + "recorded. Quality regressions are caught by assertions in " + "`tests/integration_anomaly/test_anomaly_quality.py`, not by this table." + ) + lines.append( + "* These are synthetic distributions chosen to be moderately hard, not a general claim about " + "detection quality on your own data.\n" + ) lines.append( "| Test | Mean (s) | Median (s) | Min (s) | Max (s) | Stddev (s) | Rounds | Ops/s | ROC-AUC | Precision | Recall | F1 | Precision@N |" ) diff --git a/tests/perf/test_anomaly_benchmark.py b/tests/perf/test_anomaly_benchmark.py index d8882081c..257b0ba7f 100644 --- a/tests/perf/test_anomaly_benchmark.py +++ b/tests/perf/test_anomaly_benchmark.py @@ -1,4 +1,30 @@ -from pyspark.sql import SparkSession +"""Performance benchmarks for row anomaly detection. + +Two things make these different from the check benchmarks in ``test_apply_checks.py``, and both +shape how they are written: + +**They are Databricks-backed.** Each one trains an Isolation Forest and registers it in MLflow under +Unity Catalog, so a single round costs minutes of control-plane latency rather than milliseconds of +Spark. They therefore use ``benchmark.pedantic(rounds=1)`` rather than the fixture's default +calibration — ``--benchmark-min-rounds=5`` in the nightly would otherwise mean five full +train-and-register cycles per test, twice over, since the benchmark job runs ``pytest tests/perf`` +again for the comparison step. + +**They carry quality metrics, not just timings.** ROC-AUC and friends ride along in +``benchmark.extra_info`` so the published report can show them. Treat those as *indicative and +first-observed only*: the nightly merges baselines keep-old-on-conflict, so once a benchmark exists in +``baseline.json`` its ``extra_info`` is never refreshed. Real quality-regression detection lives in +``tests/integration_anomaly/test_anomaly_quality.py``, which asserts rather than reports. + +The module is marked ``anomaly`` so the nightly can deselect it from the timing-comparison gate +(``--benchmark-compare-fail=mean:25%``), which is global and would flake on control-plane variance. +""" + +from typing import cast + +import pandas as pd +import pytest +from pyspark.sql import DataFrame, SparkSession from pyspark.sql import functions as F import mlflow @@ -7,14 +33,22 @@ from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRegistry -from databricks.labs.dqx.errors import InvalidParameterError from tests.constants import TEST_CATALOG from tests.integration_anomaly.synthetic_generators import ( generate_heavy_tail_data, generate_overlapping_gaussian_data, ) -_TRAINED_MODEL: dict[str, str] = {} +pytestmark = pytest.mark.anomaly + +BENCHMARK_GROUP = "anomaly_synthetic" + +# Fixed so the published quality numbers mean something across runs, and recorded in extra_info so +# the report can state what they were measured on. +SEED = 42 +N_SAMPLES = 3000 +N_FEATURES = 16 +ANOMALY_FRAC = 0.03 def _cleanup_anomaly_mlflow(model_name: str, registry_table: str, spark: SparkSession) -> None: @@ -37,29 +71,22 @@ def _cleanup_anomaly_mlflow(model_name: str, registry_table: str, spark: SparkSe pass -def _prepare_synthetic_data( - spark, - *, - seed: int = 42, - n_samples: int = 3000, - n_features: int = 16, - anomaly_frac: float = 0.03, -): - # Blend overlapping and heavy-tail scenarios for less optimistic quality estimates. +def _prepare_synthetic_data(spark) -> tuple[list[str], DataFrame, DataFrame]: + """Blend overlapping and heavy-tail scenarios for less optimistic quality estimates.""" overlap_cols, overlap_train, overlap_test = generate_overlapping_gaussian_data( spark, - seed=seed, - n_samples=n_samples, - n_features=n_features, - anomaly_frac=anomaly_frac, + seed=SEED, + n_samples=N_SAMPLES, + n_features=N_FEATURES, + anomaly_frac=ANOMALY_FRAC, anomaly_shift=2.0, ) _heavy_cols, heavy_train, heavy_test = generate_heavy_tail_data( spark, - seed=seed + 7, - n_samples=n_samples, - n_features=n_features, - anomaly_frac=anomaly_frac, + seed=SEED + 7, + n_samples=N_SAMPLES, + n_features=N_FEATURES, + anomaly_frac=ANOMALY_FRAC, ) train_df = overlap_train.unionByName(heavy_train) @@ -67,12 +94,40 @@ def _prepare_synthetic_data( return overlap_cols, train_df, test_df -def test_benchmark_anomaly_arrhythmia_train(benchmark, spark, ws, make_schema, make_random): - feature_cols, train_df, _ = _prepare_synthetic_data(spark) +def _record_provenance(benchmark, n_train: int, n_test: int) -> None: + """Attach what the numbers were measured on. + + A published table of quality metrics with no stated dataset invites being read as a general + claim about DQX's detection quality, which it is not — this is blended synthetic data. + """ + benchmark.extra_info["dataset"] = "synthetic: overlapping-gaussian + heavy-tail blend" + benchmark.extra_info["seed"] = SEED + benchmark.extra_info["n_features"] = N_FEATURES + benchmark.extra_info["anomaly_frac"] = ANOMALY_FRAC + benchmark.extra_info["n_train_rows"] = n_train + benchmark.extra_info["n_test_rows"] = n_test + +def _new_model_names(make_schema, make_random) -> tuple[str, str]: schema = make_schema(catalog_name=TEST_CATALOG).name - model_name = f"{TEST_CATALOG}.{schema}.bench_model_{make_random(6).lower()}" - registry_table = f"{TEST_CATALOG}.{schema}.bench_registry_{make_random(6).lower()}" + suffix = make_random(6).lower() + return ( + f"{TEST_CATALOG}.{schema}.bench_model_{suffix}", + f"{TEST_CATALOG}.{schema}.bench_registry_{suffix}", + ) + + +@pytest.mark.benchmark(group=BENCHMARK_GROUP) +def test_benchmark_anomaly_train(benchmark, request, spark, ws, make_schema, make_random): + """Time a full train-and-register cycle. + + One round: the cost is dominated by MLflow and Unity Catalog control-plane latency, so repeating + it buys variance rather than precision, and each repeat would register another model version + under the same name. + """ + feature_cols, train_df, test_df = _prepare_synthetic_data(spark) + model_name, registry_table = _new_model_names(make_schema, make_random) + request.addfinalizer(lambda: _cleanup_anomaly_mlflow(model_name, registry_table, spark)) engine = AnomalyEngine(workspace_client=ws, spark=spark) @@ -84,101 +139,89 @@ def run_train(): columns=feature_cols, ) - benchmark(run_train) - _TRAINED_MODEL["model_name"] = model_name - _TRAINED_MODEL["registry_table"] = registry_table + benchmark.pedantic(run_train, rounds=1, iterations=1, warmup_rounds=0) + _record_provenance(benchmark, train_df.count(), test_df.count()) -def test_benchmark_anomaly_arrhythmia_score(benchmark, request, spark, ws, make_schema, make_random): - feature_cols, train_df, test_df = _prepare_synthetic_data(spark) - - def _cleanup(): - _cleanup_anomaly_mlflow( - _TRAINED_MODEL.get("model_name"), - _TRAINED_MODEL.get("registry_table"), - spark, - ) +@pytest.mark.benchmark(group=BENCHMARK_GROUP) +def test_benchmark_anomaly_score(benchmark, request, spark, ws, make_schema, make_random): + """Time scoring, and record the detection quality achieved on the same data. - request.addfinalizer(_cleanup) + Trains its own model as unmeasured setup rather than sharing one with the train benchmark. The + previous shared-module-global arrangement made this test order-dependent and able to retrain up + to three times; one extra training run per nightly is a cheap price for removing that. + """ + feature_cols, train_df, test_df = _prepare_synthetic_data(spark) + model_name, registry_table = _new_model_names(make_schema, make_random) + request.addfinalizer(lambda: _cleanup_anomaly_mlflow(model_name, registry_table, spark)) engine = AnomalyEngine(workspace_client=ws, spark=spark) - model_name = _TRAINED_MODEL.get("model_name") - registry_table = _TRAINED_MODEL.get("registry_table") - needs_training = not model_name or not registry_table - if not needs_training: - registry_client = AnomalyModelRegistry(spark) - record = registry_client.get_active_model(registry_table, model_name) - needs_training = record is None - - if needs_training: - schema = make_schema(catalog_name=TEST_CATALOG).name - model_name = f"{TEST_CATALOG}.{schema}.bench_model_{make_random(6).lower()}" - registry_table = f"{TEST_CATALOG}.{schema}.bench_registry_{make_random(6).lower()}" - engine.train( - df=train_df, - model_name=model_name, - registry_table=registry_table, - columns=feature_cols, - ) - _TRAINED_MODEL["model_name"] = model_name - _TRAINED_MODEL["registry_table"] = registry_table + engine.train( + df=train_df, + model_name=model_name, + registry_table=registry_table, + columns=feature_cols, + ) - try: - _, apply_fn, _ = has_no_row_anomalies( - model_name=model_name, - registry_table=registry_table, - enable_contributions=False, - enable_ai_explanation=False, - ) - except InvalidParameterError: - schema = make_schema(catalog_name=TEST_CATALOG).name - model_name = f"{TEST_CATALOG}.{schema}.bench_model_{make_random(6).lower()}" - registry_table = f"{TEST_CATALOG}.{schema}.bench_registry_{make_random(6).lower()}" - engine.train( - df=train_df, - model_name=model_name, - registry_table=registry_table, - columns=feature_cols, - ) - _TRAINED_MODEL["model_name"] = model_name - _TRAINED_MODEL["registry_table"] = registry_table - _, apply_fn, _ = has_no_row_anomalies( - model_name=model_name, - registry_table=registry_table, - enable_contributions=False, - enable_ai_explanation=False, - ) + # The third element is the name of the struct column the check writes. `_dq_info` is assembled a + # layer up by DQEngine from that column, so reading `_dq_info` here resolves against nothing. + _, apply_fn, info_col = has_no_row_anomalies( + model_name=model_name, + registry_table=registry_table, + enable_contributions=False, + enable_ai_explanation=False, + ) def run_score(): scored_df = apply_fn(test_df) + anomaly = F.col(info_col).getField("anomaly") return scored_df.select( F.col("is_anomaly").cast("double").alias("label"), - F.col("_dq_info").getItem(0).getField("anomaly").getField("score").alias("score"), - F.col("_dq_info").getItem(0).getField("anomaly").getField("is_anomaly").cast("double").alias("pred"), + anomaly.getField("score").alias("score"), + anomaly.getField("is_anomaly").cast("double").alias("pred"), ) - scored = benchmark(run_score) - metrics = scored.select("label", "score", "pred").toPandas() + scored = benchmark.pedantic(run_score, rounds=1, iterations=1, warmup_rounds=0) + + _record_provenance(benchmark, train_df.count(), test_df.count()) + for name, value in _detection_quality(scored).items(): + benchmark.extra_info[name] = value + + +def _detection_quality(scored: DataFrame) -> dict[str, float]: + """Compute indicative detection quality from a scored frame. + + Deliberately unadjusted: no point-adjustment of any kind. Kim et al. (AAAI 2022) showed that + point-adjusted F1 lets random scores beat published state of the art, so it is not a metric worth + reporting even as an indicator. + """ + metrics = cast(pd.DataFrame, scored.select("label", "score", "pred").toPandas()) + if metrics["label"].nunique() > 1: from sklearn.metrics import roc_auc_score # type: ignore[import-untyped] - roc_auc = roc_auc_score(metrics["label"], metrics["score"]) + roc_auc = float(roc_auc_score(metrics["label"], metrics["score"])) else: roc_auc = float("nan") - tp = float(((metrics["label"] == 1.0) & (metrics["pred"] == 1.0)).sum()) - fp = float(((metrics["label"] == 0.0) & (metrics["pred"] == 1.0)).sum()) - fn = float(((metrics["label"] == 1.0) & (metrics["pred"] == 0.0)).sum()) - precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 - recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 - f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0 + + true_pos = float(((metrics["label"] == 1.0) & (metrics["pred"] == 1.0)).sum()) + false_pos = float(((metrics["label"] == 0.0) & (metrics["pred"] == 1.0)).sum()) + false_neg = float(((metrics["label"] == 1.0) & (metrics["pred"] == 0.0)).sum()) + precision = true_pos / (true_pos + false_pos) if (true_pos + false_pos) > 0 else 0.0 + recall = true_pos / (true_pos + false_neg) if (true_pos + false_neg) > 0 else 0.0 + f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0 + n_anomalies = int(metrics["label"].sum()) if n_anomalies > 0: top_n = metrics.sort_values("score", ascending=False).head(n_anomalies) precision_at_n = float(top_n["label"].mean()) else: precision_at_n = 0.0 - benchmark.extra_info["roc_auc"] = roc_auc - benchmark.extra_info["precision"] = precision - benchmark.extra_info["recall"] = recall - benchmark.extra_info["f1_score"] = f1 - benchmark.extra_info["precision_at_n"] = precision_at_n + + return { + "roc_auc": roc_auc, + "precision": precision, + "recall": recall, + "f1_score": f1_score, + "precision_at_n": precision_at_n, + } From 1d4abc744cff23c4acfef577b79a816bfd0b4764 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:01:13 +0100 Subject: [PATCH 008/107] Test offline that the relative feature separates a contextual anomaly Reimplements the transform in numpy and fits sklearn's IsolationForest on raw columns and on raw-plus-relative columns, so the claim behind #1484 is checked in the unit suite in under five seconds with no Spark session, no workspace and no MLflow. Measured: PR-AUC 0.0028 to 0.6962 on a contextual collapse, against a 0.0026 base rate -- so the pooled model is at chance, not merely worse. The reverse case is asserted not to regress: an anomaly already extreme against every group stays at 1.0000 either way. This guards the *mechanism*, not the pipeline, and the docstring says so. The pipeline is covered by tests/integration_anomaly/, which needs a real session. --- ...t_anomaly_relative_feature_separability.py | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 tests/unit/test_anomaly_relative_feature_separability.py diff --git a/tests/unit/test_anomaly_relative_feature_separability.py b/tests/unit/test_anomaly_relative_feature_separability.py new file mode 100644 index 000000000..4609c6108 --- /dev/null +++ b/tests/unit/test_anomaly_relative_feature_separability.py @@ -0,0 +1,179 @@ +"""Does the group-relative feature actually make a contextual anomaly separable? + +This guards the **mechanism**, not the pipeline. It reimplements the transform in numpy and fits +scikit-learn directly, so it runs offline in the unit suite — no Spark session, no workspace, no +MLflow. The pipeline that carries the same transform through Spark is covered by +``tests/integration_anomaly/``. + +Two scenarios, because a feature that helps one case and silently hurts the other is not an +improvement: + +* **contextual** — a group's metric collapses to a value that is entirely ordinary *globally*. This + is the case from databrickslabs/dqx#1484 that a pooled model scored at the 45th percentile. The + relative feature must make it separable. +* **global magnitude** — the anomaly is extreme against every group. A pooled model already sees + this, so the relative feature must not make it materially worse. Adding features to an Isolation + Forest is not free: it picks split dimensions at random, so extra columns dilute the informative + ones. +""" + +import numpy as np +import pytest +from sklearn.ensemble import IsolationForest +from sklearn.metrics import average_precision_score + +SEED = 20260825 +N_GROUPS = 12 +N_PER_GROUP = 160 +NOISE_FEATURES = 2 + +# Detection floors. Deliberately loose: the point is the *direction and size* of the gap, not a +# precise number that would make this test a tripwire for scikit-learn's RNG. +MIN_CONTEXTUAL_GAIN = 0.15 +MAX_GLOBAL_REGRESSION = 0.05 + + +def _signed_log1p(values: np.ndarray) -> np.ndarray: + """``signum(x) * log1p(|x|)`` — mirrors ``anomaly/transformers.py::_signed_log1p``. + + Defined over all reals, unlike plain ``log1p``, which is NaN for ``x <= -1``. + """ + return np.sign(values) * np.log1p(np.abs(values)) + + +def _relative_to_group_median(metric: np.ndarray, groups: np.ndarray) -> np.ndarray: + """Deviation of each value from its own group's median, in signed-log space. + + The training-time half of the transform. Scoring uses persisted medians instead, but the + arithmetic being checked here is identical. + """ + relative = np.empty_like(metric, dtype=float) + for group in np.unique(groups): + mask = groups == group + median = float(np.median(metric[mask])) + relative[mask] = _signed_log1p(metric[mask]) - _signed_log1p(np.full(mask.sum(), median)) + return relative + + +def _make_grouped_data(*, contextual: bool, seed: int = SEED) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Build a grouped metric with a planted anomaly. + + Group levels span an order of magnitude, which is what makes one group's normal look like + another group's anomaly — the heterogeneity the relative feature exists to remove. + + Returns ``(metric, groups, labels)``. + """ + rng = np.random.default_rng(seed) + levels = np.geomspace(200.0, 4000.0, N_GROUPS) + + metric_parts, group_parts, label_parts = [], [], [] + for index, level in enumerate(levels): + values = rng.normal(level, level * 0.05, N_PER_GROUP) + labels = np.zeros(N_PER_GROUP) + + # Plant the anomaly in a mid-level group, so a contextual collapse lands squarely inside + # the range other groups occupy normally. + if index == N_GROUPS // 2: + if contextual: + # 80% collapse. Ordinary globally: 0.2 * mid-level sits among the low groups. + values[:5] = level * 0.2 + else: + # Extreme against every group, so pooling can already see it. + values[:5] = levels[-1] * 4.0 + labels[:5] = 1.0 + + metric_parts.append(values) + group_parts.append(np.full(N_PER_GROUP, index)) + label_parts.append(labels) + + return np.concatenate(metric_parts), np.concatenate(group_parts), np.concatenate(label_parts) + + +def _average_precision(features: np.ndarray, labels: np.ndarray) -> float: + """PR-AUC of an Isolation Forest on *features*. + + Scores are negated ``score_samples`` so that higher means more anomalous, matching + ``anomaly/core.py``. PR-AUC rather than ROC-AUC because the positive class is ~0.3%. + """ + forest = IsolationForest(n_estimators=200, random_state=42, n_jobs=1) + forest.fit(features) + scores = -forest.score_samples(features) + return float(average_precision_score(labels, scores)) + + +def _pooled_and_relative(*, contextual: bool) -> tuple[float, float]: + """PR-AUC without and with the relative feature, on identical data and noise.""" + metric, groups, labels = _make_grouped_data(contextual=contextual) + rng = np.random.default_rng(SEED + 1) + noise = rng.normal(0.0, 1.0, (metric.size, NOISE_FEATURES)) + + pooled = np.column_stack([metric, noise]) + relative = np.column_stack([metric, noise, _relative_to_group_median(metric, groups)]) + + return _average_precision(pooled, labels), _average_precision(relative, labels) + + +def test_relative_feature_makes_a_contextual_anomaly_separable(): + """The claim the feature exists for: conditioning finds what pooling cannot.""" + pooled, relative = _pooled_and_relative(contextual=True) + + assert relative > pooled + MIN_CONTEXTUAL_GAIN, ( + f"relative feature should separate a contextual anomaly that pooling misses " + f"(pooled PR-AUC {pooled:.3f}, relative {relative:.3f})" + ) + + +def test_pooling_alone_misses_the_contextual_anomaly(): + """Pins the negative half. + + If pooling ever starts catching this, the fixture has stopped isolating a purely contextual + anomaly and the positive result above no longer means what it claims. + """ + pooled, _ = _pooled_and_relative(contextual=True) + + assert pooled < 0.5, f"fixture no longer isolates a contextual anomaly: pooled PR-AUC {pooled:.3f}" + + +def test_relative_feature_does_not_hurt_a_global_anomaly(): + """The cost side. An extra Isolation Forest dimension dilutes split selection, so a feature + that is useless for a given anomaly must still not meaningfully degrade detection.""" + pooled, relative = _pooled_and_relative(contextual=False) + + assert relative > pooled - MAX_GLOBAL_REGRESSION, ( + f"relative feature degraded a globally-extreme anomaly too far " + f"(pooled PR-AUC {pooled:.3f}, relative {relative:.3f})" + ) + + +# ============================================================================ +# The transform itself +# ============================================================================ + + +def test_signed_log1p_matches_log1p_for_non_negative_input(): + values = np.array([0.0, 0.5, 1.0, 100.0, 3284.0]) + np.testing.assert_allclose(_signed_log1p(values), np.log1p(values)) + + +@pytest.mark.parametrize("value", [-0.5, -1.0, -2.0, -500.0, -1e6]) +def test_signed_log1p_is_finite_where_plain_log1p_is_not(value): + """``log1p(x)`` is NaN for ``x <= -1``; the signed form must survive signed metrics.""" + result = _signed_log1p(np.array([value])) + + assert np.isfinite(result).all(), f"signed_log1p({value}) was not finite" + assert result[0] < 0.0, "a negative input must map to a negative output" + + +def test_signed_log1p_is_monotone_over_the_whole_real_line(): + """Monotonicity is what lets the transform preserve Isolation Forest's axis-aligned splits.""" + values = np.linspace(-1000.0, 1000.0, 2001) + + assert np.all(np.diff(_signed_log1p(values)) > 0) + + +def test_relative_deviation_is_zero_at_the_group_median(): + """Two groups an order of magnitude apart both centre on zero — the entire point.""" + metric = np.array([100.0, 100.0, 100.0, 10.0, 10.0, 10.0]) + groups = np.array([0, 0, 0, 1, 1, 1]) + + np.testing.assert_allclose(_relative_to_group_median(metric, groups), np.zeros(6), atol=1e-12) From 1548c64a23b9c69711cd20e79e8ca3d87827f5e3 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:01:28 +0100 Subject: [PATCH 009/107] Verify baseline conditioning through Spark, MLflow and Unity Catalog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No Spark path in this feature had ever been executed before these tests existed, which was by far its largest outstanding risk: the baseline-key agreement, the relative transform, the broadcast joins, per-baseline severity and unseen-baseline marking were verified only by inspection and by unit tests that never start a session. Running them found real defects that inspection had not: * Training any grouped model raised `UNRESOLVED_COLUMN`. Feature engineering drops the raw group columns so they cannot reach the sklearn pipeline or the inferred MLflow signature, but training-time severity calibration runs downstream of that, on the scored engineered frame, and rebuilt the key from columns that were no longer there. Fixed by computing the key once and reading it thereafter, via `with_baseline_key`. * The Python and Spark halves of the baseline key disagreed on booleans. * A discovered grouping trained one conditioned model where the test still expected one model per segment — the intended behaviour change, asserted the old way. `test_anomaly_quality.py` measures detection quality rather than shape: it trains a conditioned and an unconditioned model on identical data and asserts conditioning wins by a margin, beats a random and a max-abs-z baseline, and does not concentrate false positives in one group. Metric discipline lives in `quality_metrics.py`, which computes no point-adjusted F1 and says why. Two measurement flaws in those tests, both fixed here and both worth recording because they would have made the assertions meaningless rather than wrong: * Average precision over a single planted positive is a coin flip, so the fixture now plants an incident across several groups and days. * A per-group false-positive rate over four rows can only be 0, 0.25, 0.5, 0.75 or 1, so the maximum across ninety groups reached 1.0 by chance and measured the fixture rather than the model. The fixture now emits enough control days for the rate to exist, the metric ignores groups too small to express one, and the test asserts the statistic was actually measured instead of silently NaN. The severity assertion also matched on country alone, where one country spans fifteen groups and only one collapsed, so `first()` returned an arbitrary sibling — passing at 95.5 on one run and failing at 82.3 on the next with identical data. It now matches the whole key, built with the production function so the assertion depends on Python/Spark agreement too. --- tests/integration_anomaly/conftest.py | 14 +- tests/integration_anomaly/quality_metrics.py | 132 ++++++++++ .../synthetic_generators.py | 150 +++++++++++ .../test_anomaly_apply_checks.py | 2 + .../test_anomaly_autodiscovery.py | 16 +- .../test_anomaly_group_relative_features.py | 232 ++++++++++++++++++ .../test_anomaly_groups.py | 177 +++++++++++++ .../test_anomaly_quality.py | 144 +++++++++++ 8 files changed, 861 insertions(+), 6 deletions(-) create mode 100644 tests/integration_anomaly/quality_metrics.py create mode 100644 tests/integration_anomaly/test_anomaly_group_relative_features.py create mode 100644 tests/integration_anomaly/test_anomaly_groups.py create mode 100644 tests/integration_anomaly/test_anomaly_quality.py diff --git a/tests/integration_anomaly/conftest.py b/tests/integration_anomaly/conftest.py index e0673827e..334cc8e27 100644 --- a/tests/integration_anomaly/conftest.py +++ b/tests/integration_anomaly/conftest.py @@ -140,6 +140,7 @@ def train_model_with_params( params: AnomalyParams, segment_by: list[str] | None = None, expected_anomaly_rate: float = 0.02, + baseline_by: list[str] | None = None, ) -> str: """Train a model with internal params (test-only).""" return engine.train( @@ -148,6 +149,7 @@ def train_model_with_params( model_name=model_name, registry_table=registry_table, segment_by=segment_by, + baseline_by=baseline_by, params=params, expected_anomaly_rate=expected_anomaly_rate, ) @@ -800,6 +802,8 @@ def _train( segment_by: list[str] | None = None, catalog: str = TEST_CATALOG, schema: str | None = None, + baseline_by: list[str] | None = None, + train_schema: str | None = None, ): """ Train a quick test model. @@ -811,6 +815,9 @@ def _train( train_data (list[tuple] | None): Custom training data tuples (overrides train_size) params (AnomalyParams | None): Internal training params (test-only) segment_by (list[str] | None): Segment columns for segmented models + baseline_by (list[str] | None): Group columns for group-conditioned models + train_schema (str | None): Explicit DDL for train_data (needed when group columns + are not doubles) catalog (str): Catalog name schema (str | None): Schema name @@ -835,8 +842,9 @@ def _train( if train_data is None: train_data = [(100.0 + i * 0.5, 2.0) for i in range(train_size)] - # Infer schema from columns - schema_str = ", ".join(f"{col} double" for col in columns) + # Infer schema from columns unless the caller gave explicit DDL (group columns are + # typically strings, which cannot be inferred from the feature column list) + schema_str = train_schema or ", ".join(f"{col} double" for col in columns) train_df = session.createDataFrame(train_data, schema_str) # Create engine with shared ws client @@ -849,6 +857,7 @@ def _train( model_name=model_name, registry_table=registry_table, segment_by=segment_by, + baseline_by=baseline_by, ) else: full_model_name = train_model_with_params( @@ -859,6 +868,7 @@ def _train( columns=columns, params=params, segment_by=segment_by, + baseline_by=baseline_by, ) return full_model_name, registry_table, columns diff --git a/tests/integration_anomaly/quality_metrics.py b/tests/integration_anomaly/quality_metrics.py new file mode 100644 index 000000000..fae763843 --- /dev/null +++ b/tests/integration_anomaly/quality_metrics.py @@ -0,0 +1,132 @@ +"""Detection-quality metrics for anomaly integration tests. + +Deliberately **no point-adjustment of any kind.** Kim et al., "Towards a Rigorous Evaluation of +Time-series Anomaly Detection" (AAAI 2022) showed that under the point-adjust protocol — crediting an +entire labelled anomaly segment when any single point inside it is detected — a *random* anomaly score +achieves state-of-the-art F1. Any number produced that way is uninterpretable, so none is computed +here. If you are tempted to add one, read that paper first. + +What is here instead: + +* **PR-AUC** as the primary metric (``average_precision_score``), because these datasets are heavily + imbalanced and ROC-AUC flatters a detector that ranks the majority class well. +* **Per-group and macro-averaged** metrics, not just aggregate. An aggregate average is dominated by + the largest groups, which is exactly how a catastrophic failure in one group stays invisible — the + SMD result that motivated this work had one entity at a 56% false-alarm rate while the aggregate + looked merely mediocre. +* **Worst-group false-positive rate**, which is the statistic that exposed that failure. +* **Trivial baselines**, so a result can be compared against doing almost nothing. Kim et al.'s other + recommendation: report improvement over a baseline, not an absolute number. +""" + +import numpy as np +import pandas as pd +from sklearn.metrics import average_precision_score, roc_auc_score + + +def pr_auc(labels: pd.Series, scores: pd.Series) -> float: + """Area under the precision-recall curve. NaN when only one class is present.""" + if labels.nunique() < 2: + return float("nan") + return float(average_precision_score(labels, scores)) + + +def roc_auc(labels: pd.Series, scores: pd.Series) -> float: + """Area under the ROC curve. Reported alongside PR-AUC, never instead of it.""" + if labels.nunique() < 2: + return float("nan") + return float(roc_auc_score(labels, scores)) + + +def false_positive_rate(labels: pd.Series, flagged: pd.Series) -> float: + """Share of genuinely normal rows that were flagged.""" + normal = labels == 0 + if not normal.any(): + return float("nan") + return float(flagged[normal].mean()) + + +def per_group_metrics( + frame: pd.DataFrame, + *, + group_col: str, + label_col: str = "label", + score_col: str = "score", + flag_col: str = "flagged", +) -> pd.DataFrame: + """One row of metrics per group. + + Groups with a single class present yield NaN PR-AUC rather than a misleading 0 or 1; callers + should drop those before averaging rather than have this function guess. + """ + rows = [] + for group, chunk in frame.groupby(group_col): + rows.append( + { + group_col: group, + "n": len(chunk), + "n_normal": int((chunk[label_col] == 0).sum()), + "n_anomalies": int(chunk[label_col].sum()), + "pr_auc": pr_auc(chunk[label_col], chunk[score_col]), + "false_positive_rate": false_positive_rate(chunk[label_col], chunk[flag_col]), + } + ) + return pd.DataFrame(rows) + + +def macro_average(group_metrics: pd.DataFrame, column: str) -> float: + """Unweighted mean across groups, ignoring groups where the metric is undefined. + + Unweighted on purpose: weighting by group size reproduces the aggregate metric and re-hides the + small-group failures this exists to surface. + """ + values = group_metrics[column].dropna() + if values.empty: + return float("nan") + return float(values.mean()) + + +def worst_group_false_positive_rate(group_metrics: pd.DataFrame, *, min_normal_rows: int = 10) -> float: + """Highest per-group false-positive rate, over groups large enough to have one. + + The single most useful number for judging per-group models: their failure mode is one group's + threshold collapsing while the rest look fine. + + *min_normal_rows* is not a convenience. A rate over four normal rows can only take the values + 0, 0.25, 0.5, 0.75 or 1 — so on a small-group fixture the maximum across many groups is + approximately guaranteed to hit 1.0 by chance, and an assertion on it measures the fixture + rather than the model. The failure this statistic exists to catch was 56% over 28,392 rows; + groups that cannot express such a rate are excluded rather than allowed to fabricate one. + + Returns NaN when no group is large enough, which callers must treat as "not measured" rather + than as a pass. + """ + eligible = group_metrics + if "n_normal" in group_metrics.columns: + eligible = group_metrics[group_metrics["n_normal"] >= min_normal_rows] + values = eligible["false_positive_rate"].dropna() + if values.empty: + return float("nan") + return float(values.max()) + + +def trivial_baselines(frame: pd.DataFrame, metric_cols: list[str], *, seed: int = 42) -> dict[str, float]: + """PR-AUC of scores that required no model, as a floor to beat. + + ``random`` is the sanity check Kim et al. recommend. ``max_abs_z`` is the cheapest defensible + detector — the largest absolute z-score across metrics — and is a genuinely competitive baseline + on data whose anomalies are simply extreme, which makes it the honest thing to compare against. + """ + rng = np.random.default_rng(seed) + labels = frame["label"] + + values = frame[metric_cols].to_numpy(dtype=float) + means = np.nanmean(values, axis=0) + stds = np.nanstd(values, axis=0) + stds[stds == 0] = 1.0 + max_abs_z = np.nanmax(np.abs((values - means) / stds), axis=1) + + return { + "random": pr_auc(labels, pd.Series(rng.random(len(frame)), index=frame.index)), + "max_abs_z": pr_auc(labels, pd.Series(max_abs_z, index=frame.index)), + } diff --git a/tests/integration_anomaly/synthetic_generators.py b/tests/integration_anomaly/synthetic_generators.py index e9fa76fcb..0099e79b2 100644 --- a/tests/integration_anomaly/synthetic_generators.py +++ b/tests/integration_anomaly/synthetic_generators.py @@ -1,9 +1,13 @@ """Reusable synthetic data generators for row anomaly detection tests.""" +from dataclasses import dataclass + import numpy as np from pyspark.sql import DataFrame import pyspark.sql.functions as F +from databricks.labs.dqx.anomaly.segment_utils import build_baseline_key + def _to_train_test_frames( spark, @@ -102,6 +106,152 @@ def generate_segment_conditional_data( return ["amount", "quantity"], train_df, test_df +@dataclass +class _GroupWorld: + """The per-group baseline levels a contextual incident is generated against. + + Holds what every simulated day needs, so the generator itself stays a short sequence of + "build the world, emit control days, emit incident days" rather than carrying a dozen + loop variables alongside its Spark plumbing. + """ + + rng: np.random.Generator + groups: list[tuple[str, str, str]] + levels: dict[tuple[str, str, str], float] + incident_groups: list[tuple[str, str, str]] + collapse_factor: float + + @classmethod + def build( + cls, + *, + seed: int, + n_countries: int, + n_event_types: int, + n_products: int, + collapse_factor: float, + n_incident_groups: int, + ) -> "_GroupWorld": + rng = np.random.default_rng(seed) + groups = [ + (f"C{c}", f"E{e}", f"P{p}") + for c in range(n_countries) + for e in range(n_event_types) + for p in range(n_products) + ] + # Each group gets its own baseline level, spanning an order of magnitude. This + # heterogeneity is the point: it is what makes one group's normal look like another + # group's anomaly. + levels = {group: float(rng.uniform(200.0, 4000.0)) for group in groups} + # Spread the affected groups across the level range rather than taking a contiguous + # slice, so the result does not depend on whether high- or low-volume groups happen to + # be easier to flag. + stride = max(1, len(groups) // max(1, n_incident_groups)) + incident_groups = [groups[i * stride] for i in range(n_incident_groups)] + return cls( + rng=rng, + groups=groups, + levels=levels, + incident_groups=incident_groups, + collapse_factor=collapse_factor, + ) + + def day_rows(self, day: int, *, collapse: bool) -> list[tuple]: + """Emit one row per group for *day*, collapsing the incident groups when asked.""" + # Seasonality shared by every group, so it cannot be what distinguishes the incident. + seasonal = 1.0 + 0.15 * np.sin(2 * np.pi * (day % 7) / 7.0) + rows = [] + lost_volume = 0.0 + for group in self.groups: + level = self.levels[group] * seasonal + value = float(self.rng.normal(level, level * 0.05)) + label = 0.0 + if collapse and group in self.incident_groups: + collapsed = value * self.collapse_factor + lost_volume += value - collapsed + value = collapsed + label = 1.0 + rows.append((day, *group, value, label)) + if not lost_volume: + return rows + # Redistribute the lost volume so the daily total is unchanged. Without this the + # incident would be detectable from the total alone and would prove nothing. + share = lost_volume / (len(self.groups) - len(self.incident_groups)) + return [ + (d, c, e, p, v + share, lab) if (c, e, p) not in self.incident_groups else (d, c, e, p, v, lab) + for d, c, e, p, v, lab in rows + ] + + +def generate_group_conditional_data( + spark, + *, + seed: int = 42, + n_days: int = 120, + n_countries: int = 6, + n_event_types: int = 5, + n_products: int = 3, + collapse_factor: float = 0.2, + n_incident_days: int = 1, + n_incident_groups: int = 1, + n_control_days: int = 1, +) -> tuple[list[str], DataFrame, DataFrame, str]: + """Generate the *contextual* anomaly that motivates group conditioning. + + One group's daily volume collapses to ``collapse_factor`` of its usual level, while the + **daily total across all groups is held flat** by redistributing the lost volume over the + other groups. So the collapse is invisible in any aggregate: only a comparison against that + group's own history reveals it. + + The collapsed value is deliberately *ordinary* in the global distribution — it sits inside the + range other groups occupy normally — which is what makes a pooled model miss it. See + databrickslabs/dqx#1484 for the measured behaviour this reproduces. + + Both frames carry an ``is_anomaly`` label column (all zero in training), so the same fixture + serves a severity assertion and a PR-AUC measurement. Raise *n_incident_days* and + *n_incident_groups* for the latter: average precision over a single positive row is a coin + flip, not a measurement. + + Raise *n_control_days* for anything computed **per group**. With one control day each group has + a handful of test rows, and a per-group rate over a handful of rows can only land on a few + coarse values — so the maximum across many groups reaches 1.0 by chance and measures the + fixture rather than the model. + + Returns ``(feature_columns, train_df, test_df, incident_group_key)`` where *incident_group_key* + is the baseline key of the first affected group. + """ + world = _GroupWorld.build( + seed=seed, + n_countries=n_countries, + n_event_types=n_event_types, + n_products=n_products, + collapse_factor=collapse_factor, + n_incident_groups=n_incident_groups, + ) + incident_groups = world.incident_groups + + train_rows: list[tuple] = [] + for day in range(n_days): + train_rows.extend(world.day_rows(day, collapse=False)) + + test_rows: list[tuple] = [] + for offset in range(n_control_days): # control days, no incident + test_rows.extend(world.day_rows(n_days + offset, collapse=False)) + # Incident days follow every control day, so the two never land on the same day number and the + # latest day in the frame is always an incident day (which is what the severity assertion in + # test_anomaly_groups.py reads off). + for offset in range(n_incident_days): + test_rows.extend(world.day_rows(n_days + n_control_days + offset, collapse=True)) + + schema = "day int, country string, event_type string, product string, event_count double, is_anomaly double" + train_df = spark.createDataFrame(train_rows, schema) + test_df = spark.createDataFrame(test_rows, schema) + + first = incident_groups[0] + incident_key = build_baseline_key({"country": first[0], "event_type": first[1], "product": first[2]}) + return ["event_count"], train_df, test_df, incident_key + + def inject_missingness_spike( df: DataFrame, *, diff --git a/tests/integration_anomaly/test_anomaly_apply_checks.py b/tests/integration_anomaly/test_anomaly_apply_checks.py index f11909456..654f2214f 100644 --- a/tests/integration_anomaly/test_anomaly_apply_checks.py +++ b/tests/integration_anomaly/test_anomaly_apply_checks.py @@ -275,6 +275,8 @@ def test_apply_anomaly_check_info_column_structure(ws, spark: SparkSession, shar "segment", "contributions", "confidence_std", + "is_new_baseline", + "new_baseline_key", ] for field in expected_fields: diff --git a/tests/integration_anomaly/test_anomaly_autodiscovery.py b/tests/integration_anomaly/test_anomaly_autodiscovery.py index 547bc7011..af20a65c3 100644 --- a/tests/integration_anomaly/test_anomaly_autodiscovery.py +++ b/tests/integration_anomaly/test_anomaly_autodiscovery.py @@ -79,7 +79,14 @@ def test_auto_discover_excludes_high_cardinality(spark: SparkSession): def test_zero_config_training(spark: SparkSession, make_schema, make_random, anomaly_engine): - """Test zero-configuration training with auto-discovery.""" + """Zero-config training discovers the metrics *and* a grouping, and conditions on it. + + A discovered grouping produces **one** model that judges each metric against its own group's + baseline, not one model per group. It used to produce one per group, which is the configuration + that measured worst on the Server Machine Dataset — worse than pooling, with one entity emitting + 15,963 false positives on 28,392 normal rows — while also being the only configuration whose + cost grows with group count. See databrickslabs/dqx#1484. + """ # Create unique schema for test isolation schema = make_schema(catalog_name=TEST_CATALOG) suffix = make_random(8).lower() @@ -106,12 +113,13 @@ def test_zero_config_training(spark: SparkSession, make_schema, make_random, ano # Verify models were created assert model_uri is not None - # Check registry for segment models registry = spark.table(registry_table) models = registry.filter("identity.status = 'active'").collect() - # Should create 2 segment models (US and EU) - assert len(models) == 2 + # One conditioned model, not one per region. + assert len(models) == 1 + assert models[0].segmentation.is_global_model is True + assert models[0].segmentation.segment_by is None # Verify auto-discovered columns (amount and discount) for model in models: diff --git a/tests/integration_anomaly/test_anomaly_group_relative_features.py b/tests/integration_anomaly/test_anomaly_group_relative_features.py new file mode 100644 index 000000000..dc1387724 --- /dev/null +++ b/tests/integration_anomaly/test_anomaly_group_relative_features.py @@ -0,0 +1,232 @@ +"""Integration tests for group-relative features (databrickslabs/dqx#1484). + +The Python/Spark group-key parity test here is the most important one in this file: training +persists baselines under keys built in Python and scoring looks them up with keys built in Spark. +A disagreement raises nothing — every lookup misses and silently falls back to the global +baseline, producing a model that trains and scores cleanly while conditioning on nothing at all. +""" + +import datetime +import math + +import pytest +from pyspark.sql import Row, SparkSession +from pyspark.sql import functions as F +from pyspark.sql import types as T + +from databricks.labs.dqx.anomaly.segment_utils import ( + BASELINE_KEY_NULL, + build_baseline_key, + baseline_key_column, +) +from databricks.labs.dqx.anomaly.transformers import ( + ColumnTypeInfo, + apply_feature_engineering, +) + + +def _first(df) -> Row: + """`.first()` narrowed to a Row. An empty frame is a test bug, not a valid outcome.""" + row = df.first() + assert row is not None, "expected at least one row" + return row + + +# ============================================================================ +# The parity contract +# ============================================================================ + + +@pytest.mark.parametrize( + "rows", + [ + [("DE", "casino"), ("IT", "live"), ("AT", "sports")], + [("with space", "x"), ("with-dash", "y"), ("with_underscore", "z")], + [("Ünïcödé", "a"), ("日本", "b")], + [("", "empty-left"), ("empty-right", "")], + ], +) +def test_python_and_spark_group_keys_agree(spark: SparkSession, rows): + """The two halves of the key must produce byte-identical strings.""" + df = spark.createDataFrame(rows, "country string, product string") + + actual = [ + row["key"] + for row in df.withColumn("key", baseline_key_column(["country", "product"])) + .orderBy("country", "product") + .select("key") + .collect() + ] + expected = sorted(build_baseline_key({"country": c, "product": p}) for c, p in rows) + + assert actual == expected + + +def test_python_and_spark_agree_on_null_group_values(spark: SparkSession): + """A null dimension must land in the same bucket on both sides, not propagate.""" + df = spark.createDataFrame([(None, "casino")], "country string, product string") + + spark_key = _first(df.withColumn("key", baseline_key_column(["country", "product"])))["key"] + + assert spark_key == build_baseline_key({"country": None, "product": "casino"}) + assert BASELINE_KEY_NULL in spark_key + + +def test_python_and_spark_agree_on_integral_group_values(spark: SparkSession): + """Integral group columns are permitted, so their rendering must agree.""" + df = spark.createDataFrame([(42, -7)], "store_id int, offset int") + + spark_key = _first(df.withColumn("key", baseline_key_column(["store_id", "offset"])))["key"] + + assert spark_key == build_baseline_key({"store_id": 42, "offset": -7}) + + +@pytest.mark.parametrize("flag", [True, False]) +def test_python_and_spark_agree_on_boolean_group_values(spark: SparkSession, flag): + """Regression: Spark casts booleans lowercase, Python's str() capitalises them. + + This is the disagreement that motivated ``_as_spark_string``. Left unhandled it produced + ``True\\x1f42`` at training and ``true\\x1f42`` at scoring — every baseline lookup missing, the + global baseline used instead, and no error raised anywhere. + """ + df = spark.createDataFrame([(flag, 42)], "is_vip boolean, store_id int") + + spark_key = _first(df.withColumn("key", baseline_key_column(["is_vip", "store_id"])))["key"] + + assert spark_key == build_baseline_key({"is_vip": flag, "store_id": 42}) + assert str(flag).lower() in spark_key + + +def test_python_and_spark_agree_on_date_group_values(spark: SparkSession): + """Dates are a permitted group type, so their rendering must agree too.""" + day = datetime.date(2026, 8, 25) + df = spark.createDataFrame([(day, "DE")], "day date, country string") + + spark_key = _first(df.withColumn("key", baseline_key_column(["day", "country"])))["key"] + + assert spark_key == build_baseline_key({"day": day, "country": "DE"}) + + +def test_group_key_is_independent_of_column_order(spark: SparkSession): + """Sorting by column name means the declared order cannot change the key.""" + df = spark.createDataFrame([("DE", "casino")], "country string, product string") + + forward = _first(df.withColumn("key", baseline_key_column(["country", "product"])))["key"] + reversed_order = _first(df.withColumn("key", baseline_key_column(["product", "country"])))["key"] + + assert forward == reversed_order + + +# ============================================================================ +# The transform itself +# ============================================================================ + + +def _numeric_info(name: str) -> ColumnTypeInfo: + return ColumnTypeInfo(name=name, spark_type=T.DoubleType(), category="numeric", null_count=0) + + +def test_relative_feature_is_appended_last(spark: SparkSession): + """Feature order is positional, so the new feature must land at the tail.""" + df = spark.createDataFrame( + [("DE", 100.0), ("DE", 110.0), ("IT", 20.0), ("IT", 22.0)], + "country string, amount double", + ) + + _, metadata = apply_feature_engineering(df, [_numeric_info("amount")], baseline_by=["country"]) + + assert metadata.engineered_feature_names[-1] == "amount_rel_baseline" + + +def test_feature_prefix_matches_an_ungrouped_model(spark: SparkSession): + """Everything before the appended tail must be identical to the ungrouped feature list. + + This is what makes an already-trained model safe: its features occupy the same positions. + """ + df = spark.createDataFrame( + [("DE", 100.0), ("DE", 110.0), ("IT", 20.0), ("IT", 22.0)], + "country string, amount double", + ) + + _, ungrouped = apply_feature_engineering(df.select("amount"), [_numeric_info("amount")]) + _, grouped = apply_feature_engineering(df, [_numeric_info("amount")], baseline_by=["country"]) + + prefix_len = len(ungrouped.engineered_feature_names) + assert grouped.engineered_feature_names[:prefix_len] == ungrouped.engineered_feature_names + + +def test_no_group_by_leaves_the_feature_list_untouched(spark: SparkSession): + """An empty grouping must make the transform a no-op, not merely a cheap one.""" + df = spark.createDataFrame([(100.0,), (110.0,)], "amount double") + + _, metadata = apply_feature_engineering(df, [_numeric_info("amount")], baseline_by=[]) + + assert not [f for f in metadata.engineered_feature_names if f.endswith("_rel_baseline")] + assert not metadata.baseline_medians + + +def test_group_columns_do_not_become_features(spark: SparkSession): + """A group column is the comparison basis, not a metric; it must not reach the model.""" + df = spark.createDataFrame( + [("DE", 100.0), ("DE", 110.0), ("IT", 20.0), ("IT", 22.0)], + "country string, amount double", + ) + + engineered, metadata = apply_feature_engineering(df, [_numeric_info("amount")], baseline_by=["country"]) + + assert "country" not in metadata.engineered_feature_names + assert "country" not in engineered.columns + + +def test_relative_value_is_the_signed_log_ratio_to_the_group_median(spark: SparkSession): + """Hand-computed value, so the formula is pinned rather than merely self-consistent.""" + rows = [("DE", 100.0), ("DE", 100.0), ("DE", 100.0), ("IT", 10.0), ("IT", 10.0), ("IT", 10.0)] + df = spark.createDataFrame(rows, "country string, amount double") + + engineered, _ = apply_feature_engineering(df, [_numeric_info("amount")], baseline_by=["country"]) + values = { + round(row["amount"], 4): round(row["amount_rel_baseline"], 4) + for row in engineered.select("amount", "amount_rel_baseline").collect() + } + + # Every row sits exactly on its own group's median, so the deviation is zero for both groups + # despite their levels differing by an order of magnitude. That is the whole point. + assert values[100.0] == pytest.approx(0.0, abs=1e-6) + assert values[10.0] == pytest.approx(0.0, abs=1e-6) + + # And a value at twice its group median gives signed_log(200) - signed_log(100) + df2 = spark.createDataFrame(rows + [("DE", 200.0)], "country string, amount double") + engineered2, _ = apply_feature_engineering(df2, [_numeric_info("amount")], baseline_by=["country"]) + row = _first(engineered2.filter(F.col("amount") == 200.0)) + assert row["amount_rel_baseline"] == pytest.approx(math.log1p(200.0) - math.log1p(100.0), abs=1e-6) + + +def test_negative_metrics_stay_finite(spark: SparkSession): + """Plain log1p is NaN for x <= -1; the signed form must survive signed metrics.""" + df = spark.createDataFrame( + [("DE", -500.0), ("DE", -400.0), ("DE", -450.0), ("IT", -2.0), ("IT", -3.0)], + "country string, profit double", + ) + + engineered, _ = apply_feature_engineering(df, [_numeric_info("profit")], baseline_by=["country"]) + values = [row["profit_rel_baseline"] for row in engineered.select("profit_rel_baseline").collect()] + + assert all(v is not None and math.isfinite(v) for v in values), values + + +def test_unseen_group_falls_back_to_the_global_baseline(spark: SparkSession): + """A group absent at training must resolve to a defined value, not null the feature.""" + train = spark.createDataFrame([("DE", 100.0), ("DE", 110.0)], "country string, amount double") + _, metadata = apply_feature_engineering(train, [_numeric_info("amount")], baseline_by=["country"]) + + score = spark.createDataFrame([("BRAND_NEW", 105.0)], "country string, amount double") + engineered, _ = apply_feature_engineering( + score, + [_numeric_info("amount")], + baseline_by=metadata.baseline_by, + baseline_medians=metadata.baseline_medians, + global_medians=metadata.global_medians, + ) + + value = _first(engineered)["amount_rel_baseline"] + assert value is not None diff --git a/tests/integration_anomaly/test_anomaly_groups.py b/tests/integration_anomaly/test_anomaly_groups.py new file mode 100644 index 000000000..8b877373e --- /dev/null +++ b/tests/integration_anomaly/test_anomaly_groups.py @@ -0,0 +1,177 @@ +"""The regression test that justifies databrickslabs/dqx#1484. + +A group whose volume collapses 80% while the daily total is held exactly flat. The collapsed value +is ordinary in the global distribution, so a model that compares against the whole table cannot see +it — only a comparison against that group's own baseline can. Measured on the real harness: 45.1 +without conditioning (the 45th percentile, missed) versus 95.5 with it. + +Deliberately one model, not ninety: `baseline_by` trains a single pooled model whatever the group +count, which is the scaling property being tested alongside the detection claim. + +The pooled counterpart of these numbers is asserted offline, without Spark, in +``tests/unit/test_anomaly_relative_feature_separability.py`` — there a pooled Isolation Forest scored +PR-AUC 0.0028 against a 0.0026 base rate, i.e. chance. +""" + +import pytest +from pyspark.sql import SparkSession +from pyspark.sql import functions as F + +from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRegistry +from databricks.labs.dqx.anomaly.segment_utils import baseline_key_column +from databricks.labs.dqx.config import AnomalyParams +from databricks.labs.dqx.errors import InvalidParameterError + +from tests.integration_anomaly.synthetic_generators import generate_group_conditional_data + +BASELINE_COLUMNS = ["country", "event_type", "product"] +TRAIN_SCHEMA = "day int, country string, event_type string, product string, event_count double, is_anomaly double" + +pytestmark = pytest.mark.slow + + +def _severity_of_incident_group(result_df, incident_key: str): + """Severity for the incident group on the incident day (the latest day present). + + Matches the **whole** baseline key. An earlier version filtered on the country alone and took + ``first()``, but country ``C0`` spans fifteen groups here and only one of them collapsed, so it + returned an arbitrary sibling — passing at severity 95.5 on one run and failing at 82.3 on the + next with identical data. + + Building the key with the production :func:`baseline_key_column` rather than by hand also makes + this assertion depend on Python/Spark key agreement, which is the invariant most expensive to + get wrong. + """ + latest_day = result_df.agg(F.max("day")).first()[0] + return ( + result_df.filter(F.col("day") == latest_day) + .filter(baseline_key_column(BASELINE_COLUMNS) == F.lit(incident_key)) + .select( + F.element_at(F.col("_dq_info"), 1).getField("anomaly").getField("severity_percentile").alias("severity") + ) + .first() + ) + + +def test_baseline_conditioning_catches_a_contextual_collapse(spark: SparkSession, quick_model_factory, anomaly_scorer): + """The core claim: conditioning finds what comparing against the whole table cannot.""" + columns, train_df, test_df, incident_key = generate_group_conditional_data(spark) + train_rows = [tuple(r) for r in train_df.collect()] + + model, registry, _ = quick_model_factory( + spark, + columns=columns, + train_data=train_rows, + train_schema=TRAIN_SCHEMA, + params=AnomalyParams(sample_fraction=1.0), + baseline_by=BASELINE_COLUMNS, + ) + + result = anomaly_scorer(test_df, model, registry, extract_score=False) + incident = _severity_of_incident_group(result, incident_key) + + assert incident is not None + assert incident["severity"] is not None + # A range rather than an exact value: severity interpolates an approximate quantile grid. + assert incident["severity"] >= 90.0, f"expected the collapse to be flagged, got {incident['severity']}" + + +def test_baseline_conditioning_trains_one_model_regardless_of_group_count(spark: SparkSession, quick_model_factory): + """The scaling property. Ninety groups must not mean ninety models.""" + columns, train_df, _, _ = generate_group_conditional_data(spark) + train_rows = [tuple(r) for r in train_df.collect()] + + _, registry, _ = quick_model_factory( + spark, + columns=columns, + train_data=train_rows, + train_schema=TRAIN_SCHEMA, + params=AnomalyParams(sample_fraction=1.0), + baseline_by=BASELINE_COLUMNS, + ) + + segmented = spark.table(registry).filter(F.col("identity.model_name").contains("__seg_")).count() + assert segmented == 0, "baseline_by must not register per-segment models" + + +# ============================================================================ +# API surface +# ============================================================================ + + +def test_baseline_by_and_segment_by_together_are_rejected(spark: SparkSession, quick_model_factory): + """They are alternative mechanisms; accepting both would leave the precedence undefined.""" + with pytest.raises(InvalidParameterError, match="not both"): + quick_model_factory( + spark, + columns=["amount"], + train_data=[(float(i), "DE") for i in range(60)], + train_schema="amount double, country string", + baseline_by=["country"], + segment_by=["country"], + ) + + +def test_float_baseline_columns_are_rejected(spark: SparkSession, quick_model_factory): + """Spark and Python format floats differently, which would break the baseline key silently.""" + with pytest.raises(InvalidParameterError, match="unsupported types"): + quick_model_factory( + spark, + columns=["amount"], + train_data=[(float(i), float(i % 3)) for i in range(60)], + train_schema="amount double, ratio double", + baseline_by=["ratio"], + ) + + +def test_baseline_column_cannot_also_be_a_feature(spark: SparkSession, quick_model_factory): + """A column cannot be both the basis of comparison and the thing compared.""" + with pytest.raises(InvalidParameterError, match="both as features and as baseline_by"): + quick_model_factory( + spark, + columns=["amount", "country"], + train_data=[(float(i), "DE") for i in range(60)], + train_schema="amount double, country string", + baseline_by=["country"], + ) + + +def test_segmentation_above_the_model_ceiling_names_its_escapes(spark: SparkSession, quick_model_factory): + """The error has to say how to proceed, not just that you cannot.""" + with pytest.raises(InvalidParameterError, match="max_segment_models"): + quick_model_factory( + spark, + columns=["amount"], + train_data=[(float(i), f"G{i % 60}") for i in range(600)], + train_schema="amount double, grp string", + params=AnomalyParams(sample_fraction=1.0, max_segment_models=50), + segment_by=["grp"], + ) + + +def test_segment_by_does_not_gain_baseline_relative_features(spark: SparkSession, quick_model_factory): + """Regression for the leak where the legacy path silently changed its own feature list. + + ``_resolve_grouping`` used to set both ``baseline_by`` and the effective ``segment_by`` for the + legacy path, so relative features were computed *inside* each segment — where the baseline key + is constant, because the frame has already been filtered — while ``compute_config_hash`` is + built from ``segment_by`` alone and did not change. A model whose feature list moved but whose + config hash did not is a silent train/score hazard. + """ + model, registry, _ = quick_model_factory( + spark, + columns=["amount"], + train_data=[(100.0 + i, f"G{i % 3}") for i in range(90)], + train_schema="amount double, grp string", + params=AnomalyParams(sample_fraction=1.0), + segment_by=["grp"], + ) + + records = AnomalyModelRegistry(spark).get_all_segment_models(registry, model) + assert records, "expected at least one segment model" + for record in records: + assert record.features.feature_metadata is not None + assert "_rel_baseline" not in record.features.feature_metadata, ( + "a segment_by model must not carry baseline-relative features: its config hash is " + "computed from segment_by alone and would not reflect the changed feature list" + ) diff --git a/tests/integration_anomaly/test_anomaly_quality.py b/tests/integration_anomaly/test_anomaly_quality.py new file mode 100644 index 000000000..9d651ff5e --- /dev/null +++ b/tests/integration_anomaly/test_anomaly_quality.py @@ -0,0 +1,144 @@ +"""Detection quality of baseline conditioning, end to end through Spark and MLflow. + +This is the assertion that encodes the design claim, so that a future change which reverses it fails +a test rather than quietly shipping. It complements, rather than duplicates, +``tests/unit/test_anomaly_relative_feature_separability.py``: that one measures the *mechanism* in +numpy in under five seconds, this one measures the *pipeline* — Spark feature engineering, persisted +baselines, the scoring UDF, and severity calibration all included. + +Deliberately **one test with several assertions** rather than four tests. Every property below is +read off the same pair of trained models, and each model is a real Isolation Forest registered in +Unity Catalog: splitting them would retrain that pair four times over. pytester's ``spark`` fixture +is function-scoped, so a module-scoped fixture cannot hold the pair across tests. Each assertion +carries its own message, so a failure still says which property broke. + +Metric discipline lives in ``quality_metrics.py``; in particular there is no point-adjusted F1 +anywhere, for the reason documented there. +""" + +import numpy as np +import pytest +from pyspark.sql import SparkSession +from pyspark.sql import functions as F + +from databricks.labs.dqx.config import AnomalyParams + +from tests.integration_anomaly.quality_metrics import ( + macro_average, + per_group_metrics, + pr_auc, + trivial_baselines, + worst_group_false_positive_rate, +) +from tests.integration_anomaly.synthetic_generators import generate_group_conditional_data + +BASELINE_COLUMNS = ["country", "event_type", "product"] +TRAIN_SCHEMA = "day int, country string, event_type string, product string, event_count double, is_anomaly double" + +# Enough positives for average precision to mean something: 6 groups affected across 3 days. +INCIDENT_DAYS = 3 +INCIDENT_GROUPS = 6 + +# Enough normal rows per group for a per-group false-positive *rate* to exist. With one control day +# each of the 90 groups has a handful of test rows, so the worst rate across them reaches 1.0 by +# chance — which is exactly what the first run of this test reported. +CONTROL_DAYS = 25 +MIN_NORMAL_ROWS_PER_GROUP = 10 + +# The gap conditioning must clear. Loose on purpose — the claim is the direction and rough size, not +# a number tight enough to make this a tripwire for Isolation Forest's RNG. The offline measurement +# of the same mechanism moved PR-AUC from 0.0028 to 0.6962, so 0.10 is a wide margin. +MIN_PR_AUC_GAIN = 0.10 + +# Above this, a single group is absorbing so many false alarms that the model is unusable for it. +# The per-group configuration this replaces reached 0.56 on the Server Machine Dataset. +MAX_WORST_GROUP_FPR = 0.5 + +pytestmark = pytest.mark.slow + + +def _scored_frame(anomaly_scorer, test_df, model, registry): + """Score, then collect the columns the metrics need into pandas.""" + result = anomaly_scorer(test_df, model, registry, extract_score=False) + anomaly = F.element_at(F.col("_dq_info"), 1).getField("anomaly") + return ( + result.select( + F.col("country"), + F.col("event_type"), + F.col("product"), + F.col("event_count"), + F.col("is_anomaly").alias("label"), + anomaly.getField("score").alias("score"), + anomaly.getField("is_anomaly").cast("double").alias("flagged"), + F.concat_ws("|", F.col("country"), F.col("event_type"), F.col("product")).alias("grp"), + ) + .toPandas() + .dropna(subset=["score"]) + ) + + +def test_baseline_conditioning_detection_quality(spark: SparkSession, quick_model_factory, anomaly_scorer): + """Conditioning beats whole-table comparison, beats doing almost nothing, and stays fair. + + Trains two models on identical data — one conditioned on ``baseline_by``, one not — and compares + them on the same scored rows. + """ + columns, train_df, test_df, _ = generate_group_conditional_data( + spark, + n_incident_days=INCIDENT_DAYS, + n_incident_groups=INCIDENT_GROUPS, + n_control_days=CONTROL_DAYS, + ) + train_rows = [tuple(r) for r in train_df.collect()] + + frames = {} + for name, baseline_by in (("conditioned", BASELINE_COLUMNS), ("pooled", [])): + model, registry, _ = quick_model_factory( + spark, + columns=columns, + train_data=train_rows, + train_schema=TRAIN_SCHEMA, + params=AnomalyParams(sample_fraction=1.0), + baseline_by=baseline_by, + ) + frames[name] = _scored_frame(anomaly_scorer, test_df, model, registry) + + conditioned, pooled = frames["conditioned"], frames["pooled"] + conditioned_pr_auc = pr_auc(conditioned["label"], conditioned["score"]) + pooled_pr_auc = pr_auc(pooled["label"], pooled["score"]) + + # 1. The design claim: a contextual collapse is invisible to a whole-table comparison. + assert conditioned_pr_auc > pooled_pr_auc + MIN_PR_AUC_GAIN, ( + f"baseline conditioning should beat whole-table comparison on a contextual anomaly " + f"(pooled PR-AUC {pooled_pr_auc:.4f}, conditioned {conditioned_pr_auc:.4f})" + ) + + # 2. Kim et al.'s recommendation: report improvement over doing almost nothing. max_abs_z is a + # genuine competitor on data whose anomalies are simply extreme, which is what makes beating + # it evidence that the model contributes something a one-liner would not. + baselines = trivial_baselines(conditioned, ["event_count"]) + assert ( + conditioned_pr_auc > baselines["random"] + ), f"conditioned {conditioned_pr_auc:.4f} did not beat random {baselines['random']:.4f}" + assert conditioned_pr_auc > baselines["max_abs_z"], ( + f"conditioned {conditioned_pr_auc:.4f} did not beat a max-abs-z baseline " + f"{baselines['max_abs_z']:.4f}; the fixture's anomalies may be globally extreme rather " + "than contextual" + ) + + # 3. Fairness across groups. One pooled model shares a single calibration, so no group should + # carry a wildly disproportionate share of the false positives. Asserted on the worst group + # rather than the average, because the average is what hid this failure on SMD. + group_metrics = per_group_metrics(conditioned, group_col="grp") + worst = worst_group_false_positive_rate(group_metrics, min_normal_rows=MIN_NORMAL_ROWS_PER_GROUP) + assert worst < MAX_WORST_GROUP_FPR, f"worst per-group false-positive rate was {worst:.3f}" + + # 4. Assertion 3 must have actually been measured. A NaN worst-rate means every group was too + # small to have one, which would make the assertion above pass vacuously. + assert np.isfinite(worst), ( + f"no group had at least {MIN_NORMAL_ROWS_PER_GROUP} normal rows, so the worst-group " + "false-positive rate was never measured" + ) + assert len(group_metrics) > 1, "expected more than one group to evaluate" + assert group_metrics["n_anomalies"].sum() > 1, "fixture must plant more than one positive" + macro_average(group_metrics, "pr_auc") # must not raise on this shape From 4fd087edda2cf06047a41dbd609de12b506e6fda Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:01:30 +0100 Subject: [PATCH 010/107] Measure whether conditioning actually detects better, and publish it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `benchmarks/anomaly_conditioning/` compares three ways of relating a model to a group — pooled, baseline-relative, and one model per group — across 1,395 configurations: a synthetic two-factor sweep plus the Server Machine Dataset and NSL-KDD, 15 seeds each, paired by seed and tested with Wilcoxon signed-rank. Run manually rather than nightly: it downloads third-party data and yields a correlation rather than a pass/fail. Datasets are fetched at run time and cached, never vendored, which is what keeps the licensing position simple. SMAP and MSL are excluded — "(c) Original Authors", no permissive licence. ADBench is unusable here despite being the obvious choice: it ships pre-processed numeric matrices, so the categorical identity this experiment measures is already gone. The results, in `docs/dqx/docs/reference/anomaly_detection_quality.mdx`: * Contextual anomalies: baseline-relative beats pooled by a median **+0.0734** PR-AUC. * Globally extreme anomalies: **+0.0000**, and the test does not reject (p = 0.20). Not "costs little" — costs nothing. At low heterogeneity every group median approaches the global median, so the relative feature degenerates into a monotone transform of the raw metric: a near-duplicate of an informative column rather than noise. * Real datasets: **-0.0010** at p = 0.003. Negligible in size, real in sign, and reported rather than buried — SMD and NSL-KDD anomalies are largely globally extreme or sequence-dependent, so the extra feature dilutes slightly without adding signal. * Baseline-relative beats one-model-per-group in **every** mechanism, on real data included, while training one model instead of twelve. That asymmetry — about +0.07 where conditioning applies, about -0.001 where it does not — is the actual argument for auto-discovery enabling it rather than requiring opt-in. **On the heterogeneity gate.** The decision rules were fixed before the first run: no gate if the worst delta below eta-squared 0.10 stayed above -0.01, gate needed if any such cell reached -0.02. The pre-registered rule fired. It is a minimum over single cells, which makes it maximally sensitive to estimator variance, and it fired on one seed of `nslkdd/protocol_type` whose other seeds were +0.1003 and +0.0989. Rather than rewrite the criterion to get the preferred answer, the rule is left in place and reported, with the per-grouping median published beside it and the disagreement explained in the code. Raising the seed count fivefold moved no grouping's median below zero. No grouping is systematically harmed at low heterogeneity, so the gate had nothing to gate on. Eta-squared is not useless: it correlates with the size of the benefit (Spearman rho +0.597, bootstrap CI +0.523 to +0.668, still +0.276 after controlling for anomaly mechanism). But it predicts how much you gain, never whether you lose, and a gate needs to identify harm. It cost a full Spark aggregation per training run to answer a question that never changed the decision. `scipy` joins the existing mypy `ignore_missing_imports` list, alongside pandas, sklearn, shap and mlflow — the harness needs Wilcoxon and Spearman, and hand-rolling statistical tests invites subtler errors than the missing stubs do. --- benchmarks/anomaly_conditioning/README.md | 116 +++++ .../anomaly_conditioning/conditioning.py | 135 +++++ .../anomaly_conditioning/datasets/__init__.py | 10 + .../anomaly_conditioning/datasets/real.py | 126 +++++ .../datasets/synthetic.py | 104 ++++ benchmarks/anomaly_conditioning/metrics.py | 143 ++++++ .../anomaly_conditioning/run_experiment.py | 484 ++++++++++++++++++ .../reference/anomaly_detection_quality.mdx | 167 ++++++ pyproject.toml | 2 +- 9 files changed, 1286 insertions(+), 1 deletion(-) create mode 100644 benchmarks/anomaly_conditioning/README.md create mode 100644 benchmarks/anomaly_conditioning/conditioning.py create mode 100644 benchmarks/anomaly_conditioning/datasets/__init__.py create mode 100644 benchmarks/anomaly_conditioning/datasets/real.py create mode 100644 benchmarks/anomaly_conditioning/datasets/synthetic.py create mode 100644 benchmarks/anomaly_conditioning/metrics.py create mode 100644 benchmarks/anomaly_conditioning/run_experiment.py create mode 100644 docs/dqx/docs/reference/anomaly_detection_quality.mdx diff --git a/benchmarks/anomaly_conditioning/README.md b/benchmarks/anomaly_conditioning/README.md new file mode 100644 index 000000000..32625c63b --- /dev/null +++ b/benchmarks/anomaly_conditioning/README.md @@ -0,0 +1,116 @@ +# Anomaly conditioning experiment + +Measures whether conditioning row anomaly detection on a group actually detects better, and which +mechanism to use. This is the evidence behind [#1484](https://github.com/databrickslabs/dqx/issues/1484) +and behind the decision to remove the heterogeneity gate rather than keep it. + +Not part of `make test`, `make integration`, or the nightly. It is a manual harness: it downloads +third-party datasets, takes minutes to hours, and produces a correlation rather than a pass/fail. +The assertions that guard the same claims in CI live in +`tests/unit/test_anomaly_relative_feature_separability.py` (mechanism, numpy, under five seconds) +and `tests/integration_anomaly/test_anomaly_quality.py` (the real DQX pipeline through Spark and +MLflow). + +## Running it + +```shell +# Synthetic sweep only: no network, no Databricks workspace, a few minutes. +uv run python benchmarks/anomaly_conditioning/run_experiment.py --seeds 5 + +# Add the real datasets. Downloads ~250 MB on first run, cached in ~/.cache/dqx-benchmarks. +uv run python benchmarks/anomaly_conditioning/run_experiment.py --seeds 5 \ + --datasets synthetic smd nslkdd +``` + +Results are written to `results/YYYY-MM-DD-.md` and `.json`, stamped with the DQX commit they +came from. The markdown is meant to be readable on its own; the JSON holds every cell for reanalysis. + +## What it compares + +Three configurations, each fitted and scored on the same rows: + +| config | what it is | in DQX | +|---|---|---| +| `pooled` | one model over the raw metrics | DQX before #1484 | +| `relative` | one model, raw metrics **plus** each metric's deviation from its own group's baseline | `baseline_by` | +| `per_group` | one model per group | the legacy `segment_by` | + +The comparison is **paired by seed** and tested with Wilcoxon signed-rank. The seed drives both the +data draw and the forest and dominates the between-cell variance, so unpaired means would mostly +measure the seed. + +## Why the metrics are what they are + +**PR-AUC is primary.** These datasets are heavily imbalanced and ROC-AUC flatters a detector that +merely ranks the majority class well. + +**No point-adjusted F1, anywhere.** Under the point-adjust protocol — crediting a whole labelled +anomaly segment when any single point inside it is detected — a *random* score achieves +state-of-the-art F1 (Kim et al., *Towards a Rigorous Evaluation of Time-series Anomaly Detection*, +AAAI 2022). Numbers produced that way are uninterpretable, so none are computed. If you are about to +add one, read that paper first. + +**Worst-group false-positive rate, not the average.** A per-group model's failure mode is one +group's calibration collapsing while the rest look fine, and an average over groups is exactly what +hides it. + +**Trivial baselines are reported.** Kim et al.'s other recommendation: state the improvement over +doing almost nothing, not an absolute number. + +## Reading the numbers honestly + +DQX scores **rows independently**. It is not a sequence model. PR-AUC around 0.15 on SMD against +published sequence-model results above 0.80 is a *different task*, not a worse implementation — +those models consume a window of history per prediction, and DQX deliberately does not. Any table +lifted out of here needs that caveat attached, or it reads as a failure. + +The harness also reimplements the relative transform in numpy rather than calling DQX. That is +deliberate: it makes a sweep of a few thousand fits possible without a Spark session, at the cost of +measuring the *mechanism* rather than DQX's implementation of it. Pipeline fidelity is a separate +question, asserted in `tests/integration_anomaly/test_anomaly_quality.py`. + +## Datasets, licences, citations + +Everything is downloaded at run time and cached. **Nothing is vendored into this repository**, which +is what keeps the licensing position simple — DQX redistributes none of it. + +| dataset | licence | grouping candidates | citation | +|---|---|---|---| +| Server Machine Dataset (SMD) | MIT, via [`NetManAIOps/OmniAnomaly`](https://github.com/NetManAIOps/OmniAnomaly) | server entity (28), machine family (3) | Su et al., *Robust Anomaly Detection for Multivariate Time Series through Stochastic Recurrent Neural Networks*, KDD 2019 | +| NSL-KDD | redistributable with citation | `service`, `protocol_type`, `flag` | Tavallaee et al., *A detailed analysis of the KDD CUP 99 data set*, CISDA 2009 | + +**SMAP and MSL are deliberately excluded.** Their data files carry "© Original Authors" with no +permissive licence, so they cannot be used even under download-at-runtime. + +**ADBench is not used** despite being the obvious benchmark suite: it ships pre-processed `.npz` +numeric matrices, so categorical column identity is gone and there is no group left to condition on. +Only raw datasets retain what this experiment measures. + +Two adjustments are made to the real data, both documented at their definitions in `datasets/real.py`: +SMD is capped at 4,000 rows per entity so a run takes minutes rather than hours, and NSL-KDD's +attacks are downsampled from ~46% to 2% so the task is anomaly detection rather than classification. +Every configuration sees identical rows, so neither affects the comparison. + +## The heterogeneity gate question + +DQX briefly had `MIN_GROUP_HETEROGENEITY = 0.10`: conditioning was skipped when eta-squared — the +share of variance explained by the grouping — fell below it, on the theory that a grouping which +explains little contributes noise. + +The counter-argument, and the reason it was removed rather than kept pending: at low heterogeneity +every group median approaches the global median, so `signed_log(x) - signed_log(median)` becomes a +monotone transform of `x` — a near-duplicate of an informative column, not noise. Redundancy, not +misdirection. + +That is a falsifiable prediction, and the decision rules are fixed in `run_experiment.py` **before** +any run, so a reader can check the conclusion against the criterion rather than against a narrative +written afterwards: + +- **No gate needed** if the worst delta below eta-squared 0.10 stays above −0.01. +- **Gate needed** if any such cell reaches −0.02, in which case refit the threshold from the sweep + rather than reinstating 0.10 by inheritance. + +The harness also reports Spearman ρ(eta-squared, delta) with a bootstrap CI, and regresses +`delta ~ eta_squared + is_contextual`, because the gate's premise requires eta-squared to carry +signal *after* controlling for which mechanism produced the anomaly. A correlation that disappears +under that control was the mechanism all along. diff --git a/benchmarks/anomaly_conditioning/conditioning.py b/benchmarks/anomaly_conditioning/conditioning.py new file mode 100644 index 000000000..505fa7316 --- /dev/null +++ b/benchmarks/anomaly_conditioning/conditioning.py @@ -0,0 +1,135 @@ +"""The three ways to condition an anomaly model on a group, reimplemented offline. + +Mirrors ``databricks.labs.dqx.anomaly.transformers`` closely enough to measure the *mechanism*, +while running in numpy and sklearn so a sweep of a few thousand fits needs no Spark session, no +Databricks workspace, and no MLflow registry. Pipeline equivalence is a separate question and is +asserted in ``tests/integration_anomaly/test_anomaly_quality.py``; this harness is about which +mechanism detects better, not about whether DQX implements it faithfully. + +The three configurations: + +``pooled`` + One model over the raw metrics. No notion of a group at all. This is DQX before #1484. + +``relative`` + One model over the raw metrics *plus* each metric's deviation from its own group's baseline. + This is what ``baseline_by`` does. + +``per_group`` + One model per group, each trained only on that group's rows. This is what the legacy + ``segment_by`` does, and what auto-discovery used to select. +""" + +import time +from dataclasses import dataclass + +import numpy as np +from sklearn.ensemble import IsolationForest + + +def signed_log1p(values: np.ndarray) -> np.ndarray: + """``signum(x) * log1p(|x|)``, matching ``transformers._signed_log1p``. + + Defined for negative input, unlike a bare log, and symmetric about zero so halving and + doubling move the same distance in opposite directions. + """ + return np.sign(values) * np.log1p(np.abs(values)) + + +def relative_to_group_baseline(values: np.ndarray, groups: np.ndarray) -> np.ndarray: + """Each column's deviation from its own group's median, in signed-log space. + + The medians come from the data being transformed, which is correct here because every + configuration is fitted and scored on the same split; DQX instead persists the training + medians and reuses them at scoring, falling back to a global median for unseen groups. + """ + out = np.zeros_like(values, dtype=float) + for group in np.unique(groups): + mask = groups == group + medians = np.median(values[mask], axis=0) + out[mask] = signed_log1p(values[mask]) - signed_log1p(medians) + return out + + +def eta_squared(values: np.ndarray, groups: np.ndarray) -> float: + """Share of total variance that lies *between* groups: ``SS_between / SS_total``. + + Removed from DQX itself — it gated whether conditioning was applied at all, and cost a full + Spark aggregation to compute a number whose predictive value had never been measured. It lives + here because measuring that value is precisely this harness's job. Averaged over columns so a + multi-metric dataset gets one number. + """ + per_column = [] + for column in range(values.shape[1]): + series = values[:, column] + grand_mean = series.mean() + ss_total = float(((series - grand_mean) ** 2).sum()) + if ss_total == 0: + continue + ss_between = 0.0 + for group in np.unique(groups): + member = series[groups == group] + ss_between += len(member) * (member.mean() - grand_mean) ** 2 + per_column.append(float(ss_between) / ss_total) + return float(np.mean(per_column)) if per_column else 0.0 + + +@dataclass +class FitResult: + """Scores for every row, plus what it cost to produce them.""" + + scores: np.ndarray + n_models: int + seconds: float + + +def _forest(seed: int) -> IsolationForest: + """One estimator configuration for every cell, so comparisons are not confounded by tuning.""" + return IsolationForest(n_estimators=100, contamination="auto", random_state=seed, n_jobs=-1) + + +def fit_pooled(values: np.ndarray, groups: np.ndarray, seed: int) -> FitResult: + """One model over raw metrics. *groups* is accepted and ignored, to keep one call signature.""" + del groups + started = time.perf_counter() + model = _forest(seed).fit(values) + scores = -model.score_samples(values) + return FitResult(scores=scores, n_models=1, seconds=time.perf_counter() - started) + + +def fit_relative(values: np.ndarray, groups: np.ndarray, seed: int) -> FitResult: + """One model over raw metrics plus baseline-relative ones.""" + started = time.perf_counter() + features = np.hstack([values, relative_to_group_baseline(values, groups)]) + model = _forest(seed).fit(features) + scores = -model.score_samples(features) + return FitResult(scores=scores, n_models=1, seconds=time.perf_counter() - started) + + +def fit_per_group(values: np.ndarray, groups: np.ndarray, seed: int) -> FitResult: + """One model per group. + + A group too small to fit falls back to a score of zero rather than being dropped, so every + configuration returns a score for every row and the metrics stay comparable. Note what this + costs even when it works: each model calibrates its own contamination on its own group, so a + group whose rows are all normal still has its most-unusual few percent scored as extreme. That + is the mechanism behind the 56% false-alarm entity in the SMD result. + """ + started = time.perf_counter() + scores = np.zeros(len(values), dtype=float) + n_models = 0 + for group in np.unique(groups): + mask = groups == group + if mask.sum() < 10: + continue + model = _forest(seed).fit(values[mask]) + scores[mask] = -model.score_samples(values[mask]) + n_models += 1 + return FitResult(scores=scores, n_models=n_models, seconds=time.perf_counter() - started) + + +CONFIGS = { + "pooled": fit_pooled, + "relative": fit_relative, + "per_group": fit_per_group, +} diff --git a/benchmarks/anomaly_conditioning/datasets/__init__.py b/benchmarks/anomaly_conditioning/datasets/__init__.py new file mode 100644 index 000000000..4c2aa56ea --- /dev/null +++ b/benchmarks/anomaly_conditioning/datasets/__init__.py @@ -0,0 +1,10 @@ +"""Dataset loaders for the conditioning harness. + +Every real dataset is downloaded at run time and cached under ``~/.cache/dqx-benchmarks``. Nothing +is vendored into this repository, which is what keeps the licensing position simple: DQX +redistributes none of it. Citations and licences are recorded in the harness README and reproduced +in any published results table. + +SMAP and MSL are deliberately absent. Their data files carry "(c) Original Authors" with no +permissive licence, so they cannot be used even under download-at-runtime. +""" diff --git a/benchmarks/anomaly_conditioning/datasets/real.py b/benchmarks/anomaly_conditioning/datasets/real.py new file mode 100644 index 000000000..ee2e320ad --- /dev/null +++ b/benchmarks/anomaly_conditioning/datasets/real.py @@ -0,0 +1,126 @@ +"""Real datasets with genuine categorical groupings, downloaded at run time. + +Both are used because they fail differently. SMD's grouping (server entity) explains a great deal of +the variance and is the case conditioning is meant for. NSL-KDD's groupings (network service and +protocol) are the awkward case: hundreds of services, many tiny, which is where per-group models +become both expensive and badly calibrated. + +ADBench is deliberately not used despite being the obvious choice: it ships pre-processed ``.npz`` +numeric matrices, so categorical column identity is gone and there is no group left to condition +on. Only raw datasets retain what this experiment measures. + +Licences, for the results page: + +* **SMD** (Server Machine Dataset) — MIT, from ``NetManAIOps/OmniAnomaly``. Cite Su et al., KDD + 2019, "Robust Anomaly Detection for Multivariate Time Series through Stochastic Recurrent Neural + Networks". +* **NSL-KDD** — redistributable with citation. Cite Tavallaee et al., CISDA 2009, "A detailed + analysis of the KDD CUP 99 data set". + +Nothing is committed to this repository; files are cached under ``~/.cache/dqx-benchmarks``. +""" + +import pathlib +import urllib.request +from collections.abc import Iterator + +import numpy as np + +CACHE = pathlib.Path.home() / ".cache" / "dqx-benchmarks" + +SMD_BASE = "https://raw.githubusercontent.com/NetManAIOps/OmniAnomaly/master/ServerMachineDataset" +# 28 entities: verified against the repository, where machine-3-12 is a 404. +SMD_ENTITIES = ( + [f"machine-1-{i}" for i in range(1, 9)] + + [f"machine-2-{i}" for i in range(1, 10)] + + [f"machine-3-{i}" for i in range(1, 12)] +) +NSLKDD_URL = "https://raw.githubusercontent.com/defcom17/NSL_KDD/master/KDDTrain%2B.txt" + +# Rows per SMD entity. The full test set is ~708k rows over 38 features, and the sweep fits every +# configuration five times; capping keeps a run to minutes without changing the comparison, since +# every configuration sees exactly the same rows. +SMD_MAX_ROWS_PER_ENTITY = 4000 + +# NSL-KDD is ~46% attacks, which is a classification problem rather than an anomaly-detection one. +# Attacks are downsampled to this rate so the task matches what DQX actually does, and so PR-AUC +# means what it means everywhere else in this harness. +NSLKDD_ANOMALY_RATE = 0.02 + + +def _fetch(url: str, name: str) -> pathlib.Path: + """Download *url* into the cache once, and return the local path.""" + CACHE.mkdir(parents=True, exist_ok=True) + path = CACHE / name + if path.exists() and path.stat().st_size > 0: + return path + path.parent.mkdir(parents=True, exist_ok=True) + with urllib.request.urlopen(url, timeout=120) as resp: # noqa: S310 - fixed https literals above + path.write_bytes(resp.read()) + return path + + +def _load_smd() -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Concatenate every entity's labelled test split. + + SMD ships train (unlabelled) and test (labelled) per entity. Only the test split carries labels, + so that is what is measured; a configuration is fitted and scored on it, exactly as in the + synthetic sweep, which keeps the two comparable. + """ + values_list, labels_list, entity_list = [], [], [] + for entity in SMD_ENTITIES: + test = np.loadtxt(_fetch(f"{SMD_BASE}/test/{entity}.txt", f"smd/test-{entity}.txt"), delimiter=",") + labels = np.loadtxt(_fetch(f"{SMD_BASE}/test_label/{entity}.txt", f"smd/label-{entity}.txt"), delimiter=",") + take = min(len(test), len(labels), SMD_MAX_ROWS_PER_ENTITY) + values_list.append(test[:take]) + labels_list.append(labels[:take]) + entity_list.append(np.full(take, entity)) + return np.vstack(values_list), np.concatenate(labels_list), np.concatenate(entity_list) + + +def _load_nslkdd() -> tuple[np.ndarray, np.ndarray, dict[str, np.ndarray]]: + """Return ``(numeric values, labels, {grouping name: group labels})``. + + Columns 1-3 are ``protocol_type``, ``service`` and ``flag``; column 41 is the attack name, where + ``normal`` is the only non-attack value. Everything else numeric becomes a feature. + """ + path = _fetch(NSLKDD_URL, "nslkdd/KDDTrain+.txt") + rows = [line.split(",") for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + categorical_idx = {1: "protocol_type", 2: "service", 3: "flag"} + label_idx = 41 + numeric_idx = [i for i in range(len(rows[0])) if i not in categorical_idx and i not in (label_idx, label_idx + 1)] + + values = np.array([[float(row[i]) for i in numeric_idx] for row in rows]) + labels = np.array([0.0 if row[label_idx] == "normal" else 1.0 for row in rows]) + groupings = {name: np.array([row[idx] for row in rows]) for idx, name in categorical_idx.items()} + + # Downsample attacks to a realistic anomaly rate. Seeded so the frame is identical across + # configurations and seeds -- the forest seed varies, the data does not. + rng = np.random.default_rng(0) + normal_idx = np.flatnonzero(labels == 0) + attack_idx = np.flatnonzero(labels == 1) + n_keep = max(1, int(len(normal_idx) * NSLKDD_ANOMALY_RATE / (1 - NSLKDD_ANOMALY_RATE))) + keep = np.sort(np.concatenate([normal_idx, rng.choice(attack_idx, min(n_keep, len(attack_idx)), replace=False)])) + + return values[keep], labels[keep], {name: groups[keep] for name, groups in groupings.items()} + + +def load_groupings(name: str) -> Iterator[tuple[str, np.ndarray, np.ndarray, np.ndarray]]: + """Yield ``(grouping label, values, labels, groups)`` for each candidate grouping of *name*.""" + if name == "smd": + values, labels, entities = _load_smd() + yield "entity", values, labels, entities + # The machine family prefix: a coarser grouping over the same data, which is the kind of + # choice a user actually faces. Included so the sweep has a within-dataset contrast between + # a fine and a coarse grouping rather than one point per dataset. + yield "machine_family", values, labels, np.array([e.rsplit("-", 1)[0] for e in entities]) + return + + if name == "nslkdd": + values, labels, groupings = _load_nslkdd() + for grouping_name, groups in groupings.items(): + yield grouping_name, values, labels, groups + return + + raise ValueError(f"unknown dataset {name!r}") diff --git a/benchmarks/anomaly_conditioning/datasets/synthetic.py b/benchmarks/anomaly_conditioning/datasets/synthetic.py new file mode 100644 index 000000000..b72e87c5f --- /dev/null +++ b/benchmarks/anomaly_conditioning/datasets/synthetic.py @@ -0,0 +1,104 @@ +"""The two-factor synthetic sweep that decides whether a heterogeneity gate is needed. + +Real datasets contribute roughly ten grouping candidates in total, which is far too few to +establish a correlation between heterogeneity and the benefit of conditioning. Synthetic data has +no licensing constraints and lets both factors be set directly, so the sweep carries the breadth +and the real datasets check that its conclusion survives contact with real data. + +**Factor 1 — level spread.** How far apart the per-group baseline levels sit. At spread 0 every +group has the same level and eta-squared is ~0; at the top of the range levels differ by an order +of magnitude and eta-squared approaches 0.9. This moves the gate's input continuously across its +whole range, including the region below the 0.10 threshold that was removed. + +**Factor 2 — anomaly mechanism.** What actually makes a row anomalous: + +``global`` + The value is extreme for the table as a whole. A pooled model should find these, and the + baseline-relative feature should be redundant rather than harmful — the prediction that + justified removing the gate rather than keeping it pending. + +``contextual`` + The value is ordinary for the table but wrong for its own group. Only a comparison against the + group's own baseline can see these. This is #1484. + +The two mechanisms are the confound that matters: without splitting on it, any correlation between +eta-squared and the benefit of conditioning could be entirely an artefact of contextual anomalies +being more common at high spread. +""" + +import numpy as np + +MECHANISMS = ("global", "contextual") + + +def generate( + *, + seed: int, + level_spread: float, + mechanism: str, + n_groups: int = 12, + n_rows_per_group: int = 400, + anomaly_rate: float = 0.02, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Return ``(values, labels, groups)`` for one cell of the sweep. + + Args: + seed: Controls the data draw. Paired with the forest seed by the caller. + level_spread: 0.0 gives every group the same baseline level; 1.0 spans an order of + magnitude. This is the axis eta-squared tracks. + mechanism: ``"global"`` or ``"contextual"`` — see the module docstring. + n_groups: Number of groups. + n_rows_per_group: Rows per group. + anomaly_rate: Share of rows labelled anomalous. + """ + if mechanism not in MECHANISMS: + raise ValueError(f"mechanism must be one of {MECHANISMS}, got {mechanism!r}") + + rng = np.random.default_rng(seed) + base_level = 1000.0 + # Levels are spread multiplicatively and symmetrically about base_level, so the *mean* level is + # roughly constant across the sweep. Otherwise raising the spread would also raise the overall + # scale, and the two effects would be inseparable. + factors = np.linspace(1.0 - 0.9 * level_spread, 1.0 + 0.9 * level_spread, n_groups) + levels = base_level * np.clip(factors, 0.05, None) + + values_list, labels_list, groups_list = [], [], [] + for index in range(n_groups): + level = levels[index] + rows = rng.normal(level, level * 0.08, size=(n_rows_per_group, 2)) + rows[:, 1] = rng.normal(level * 0.5, level * 0.05, size=n_rows_per_group) + labels = np.zeros(n_rows_per_group) + + n_anomalies = max(1, int(n_rows_per_group * anomaly_rate)) + picks = rng.choice(n_rows_per_group, n_anomalies, replace=False) + if mechanism == "global": + # Extreme against the whole table: far above the largest group's normal range. + rows[picks, 0] = base_level * (1.0 + 0.9 * level_spread) * rng.uniform(4.0, 6.0, n_anomalies) + else: + # Ordinary globally, wrong for this group: collapse to a fifth of the group's own + # level, which lands inside the range other groups occupy normally. + rows[picks, 0] = level * 0.2 + labels[picks] = 1.0 + + values_list.append(rows) + labels_list.append(labels) + groups_list.append(np.full(n_rows_per_group, f"g{index:02d}")) + + return ( + np.vstack(values_list), + np.concatenate(labels_list), + np.concatenate(groups_list), + ) + + +def sweep_points(n_spreads: int = 9) -> list[tuple[float, str]]: + """The (level_spread, mechanism) grid. + + Spreads are dense at the low end because that is where the removed gate would have fired, and + where the "monotone duplicate rather than noise" prediction has to hold for its removal to be + defensible. + """ + spreads = sorted( + {round(v, 3) for v in np.concatenate([np.linspace(0.0, 0.2, 5), np.linspace(0.2, 1.0, n_spreads)])} + ) + return [(spread, mechanism) for spread in spreads for mechanism in MECHANISMS] diff --git a/benchmarks/anomaly_conditioning/metrics.py b/benchmarks/anomaly_conditioning/metrics.py new file mode 100644 index 000000000..7f19f8157 --- /dev/null +++ b/benchmarks/anomaly_conditioning/metrics.py @@ -0,0 +1,143 @@ +"""Detection-quality metrics and the paired statistics used to compare configurations. + +Deliberately **no point-adjustment**, for the reason given at length in +``tests/integration_anomaly/quality_metrics.py``: under the point-adjust protocol a random score +reaches state-of-the-art F1 (Kim et al., AAAI 2022), so any number produced that way is +uninterpretable. This module and that one are kept separate rather than shared because the test +tree is not importable from a top-level script directory, and because the paired statistics below +have no place in a test. +""" + +from dataclasses import asdict, dataclass, field + +import numpy as np +from scipy import stats +from sklearn.metrics import average_precision_score, roc_auc_score + + +@dataclass +class CellMetrics: + """Every number recorded for one (dataset, grouping, config, seed) cell.""" + + pr_auc: float + roc_auc: float + precision_at_n: float + macro_pr_auc: float + worst_group_fpr: float + n_models: int + seconds: float + eta_squared: float = 0.0 + warnings: list[str] = field(default_factory=list) + + def as_dict(self) -> dict: + return asdict(self) + + +def pr_auc(labels: np.ndarray, scores: np.ndarray) -> float: + """Average precision. NaN when a split holds only one class.""" + if len(np.unique(labels)) < 2: + return float("nan") + return float(average_precision_score(labels, scores)) + + +def roc_auc(labels: np.ndarray, scores: np.ndarray) -> float: + """Reported alongside PR-AUC, never instead of it: these datasets are heavily imbalanced.""" + if len(np.unique(labels)) < 2: + return float("nan") + return float(roc_auc_score(labels, scores)) + + +def precision_at_n(labels: np.ndarray, scores: np.ndarray) -> float: + """Precision over the top-*n* scored rows, where *n* is the true number of anomalies. + + The operational metric: an analyst reviews a fixed-size queue, so what matters is how much of + that queue is worth reviewing. + """ + n_anomalies = int(labels.sum()) + if n_anomalies == 0: + return float("nan") + top = np.argsort(scores)[::-1][:n_anomalies] + return float(labels[top].sum() / n_anomalies) + + +def per_group_pr_auc(labels: np.ndarray, scores: np.ndarray, groups: np.ndarray) -> dict[str, float]: + """PR-AUC within each group, skipping groups that hold only one class.""" + out = {} + for group in np.unique(groups): + mask = groups == group + if len(np.unique(labels[mask])) < 2: + continue + out[str(group)] = pr_auc(labels[mask], scores[mask]) + return out + + +def worst_group_false_positive_rate( + labels: np.ndarray, scores: np.ndarray, groups: np.ndarray, *, quantile: float = 0.95 +) -> float: + """Highest per-group false-positive rate at a global score threshold. + + The threshold is the *global* score quantile, which is the point: a per-group model's scores + are calibrated within its own group, so a group of entirely normal rows still emits a full + complement of extreme scores. Averaging over groups hides that; taking the worst group does + not, and it is the statistic that exposed the SMD failure. + """ + if len(np.unique(labels)) < 2: + return float("nan") + threshold = float(np.quantile(scores, quantile)) + rates = [] + for group in np.unique(groups): + mask = (groups == group) & (labels == 0) + if mask.sum() == 0: + continue + rates.append(float((scores[mask] > threshold).mean())) + return max(rates) if rates else float("nan") + + +def macro_average(values: dict[str, float]) -> float: + """Unweighted mean across groups. + + Unweighted on purpose: weighting by group size reproduces the aggregate metric and re-hides + the small-group failures this exists to surface. + """ + finite = [v for v in values.values() if np.isfinite(v)] + return float(np.mean(finite)) if finite else float("nan") + + +def wilcoxon_paired(deltas: list[float]) -> tuple[float, float]: + """Wilcoxon signed-rank on per-seed differences: returns ``(statistic, p_value)``. + + Paired by seed rather than comparing unpaired means, because the seed controls both the data + draw and the forest, and is by far the largest source of variance between cells. + """ + finite = [d for d in deltas if np.isfinite(d)] + if len(finite) < 2 or all(d == 0 for d in finite): + return float("nan"), float("nan") + result = stats.wilcoxon(finite) + return float(result.statistic), float(result.pvalue) + + +def spearman_with_bootstrap_ci( + xs: list[float], ys: list[float], *, n_boot: int = 2000, seed: int = 0 +) -> tuple[float, float, float]: + """Spearman ρ plus a percentile bootstrap 95% CI: ``(rho, lo, hi)``. + + A point correlation over a few dozen configurations is not evidence on its own; the interval is + what says whether the sign is even determined. + """ + x_arr, y_arr = np.asarray(xs, dtype=float), np.asarray(ys, dtype=float) + keep = np.isfinite(x_arr) & np.isfinite(y_arr) + x_arr, y_arr = x_arr[keep], y_arr[keep] + if len(x_arr) < 4: + return float("nan"), float("nan"), float("nan") + + rho = float(stats.spearmanr(x_arr, y_arr).statistic) + rng = np.random.default_rng(seed) + boots = [] + for _ in range(n_boot): + idx = rng.integers(0, len(x_arr), len(x_arr)) + if len(np.unique(x_arr[idx])) < 3: + continue + boots.append(stats.spearmanr(x_arr[idx], y_arr[idx]).statistic) + if not boots: + return rho, float("nan"), float("nan") + return rho, float(np.percentile(boots, 2.5)), float(np.percentile(boots, 97.5)) diff --git a/benchmarks/anomaly_conditioning/run_experiment.py b/benchmarks/anomaly_conditioning/run_experiment.py new file mode 100644 index 000000000..962da4d9c --- /dev/null +++ b/benchmarks/anomaly_conditioning/run_experiment.py @@ -0,0 +1,484 @@ +"""Run the conditioning experiment and write a dated results page. + + python benchmarks/anomaly_conditioning/run_experiment.py --seeds 5 + python benchmarks/anomaly_conditioning/run_experiment.py --seeds 5 --datasets synthetic smd nslkdd + +Synthetic runs need no network and no Databricks workspace. Real datasets are downloaded at run +time and cached; see ``datasets/__init__.py`` for why none of them are vendored. + +The decision rules below were fixed *before* the first run, and are printed alongside the result so +a reader can check the conclusion against the criterion rather than against a narrative written +afterwards. +""" + +import argparse +import datetime as dt +import json +import pathlib +import subprocess +import sys +from dataclasses import dataclass + +import numpy as np + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from conditioning import CONFIGS, eta_squared # noqa: E402 +from datasets import synthetic # noqa: E402 +from metrics import ( # noqa: E402 + CellMetrics, + macro_average, + per_group_pr_auc, + pr_auc, + precision_at_n, + roc_auc, + spearman_with_bootstrap_ci, + wilcoxon_paired, + worst_group_false_positive_rate, +) + +RESULTS_DIR = pathlib.Path(__file__).resolve().parent / "results" + +# Decision rules, fixed in advance. See the plan for #1484. +# +# The gate being tested is the removed MIN_GROUP_HETEROGENEITY = 0.10: conditioning used to be +# skipped when eta-squared fell below it, on the theory that a grouping which explains little +# variance contributes noise. The counter-argument is that at low heterogeneity every group median +# approaches the global median, so the relative feature degenerates into a monotone transform of +# the raw metric -- a near-duplicate of an informative column, not noise. That is a prediction, and +# NO_GATE_FLOOR is where it gets tested. +LOW_ETA = 0.10 +NO_GATE_FLOOR = -0.01 # relative never materially worse than pooled below LOW_ETA +GATE_NEEDED_DELTA = -0.02 # a real cost, concentrated below LOW_ETA + + +@dataclass +class Cell: + """One measured configuration.""" + + dataset: str + grouping: str + config: str + seed: int + mechanism: str + level_spread: float + metrics: CellMetrics + + def as_dict(self) -> dict: + record = { + "dataset": self.dataset, + "grouping": self.grouping, + "config": self.config, + "seed": self.seed, + "mechanism": self.mechanism, + "level_spread": self.level_spread, + } + record.update(self.metrics.as_dict()) + return record + + +def measure(values: np.ndarray, labels: np.ndarray, groups: np.ndarray, config: str, seed: int) -> CellMetrics: + """Fit one configuration and compute every metric for it.""" + result = CONFIGS[config](values, groups, seed) + scores = result.scores + return CellMetrics( + pr_auc=pr_auc(labels, scores), + roc_auc=roc_auc(labels, scores), + precision_at_n=precision_at_n(labels, scores), + macro_pr_auc=macro_average(per_group_pr_auc(labels, scores, groups)), + worst_group_fpr=worst_group_false_positive_rate(labels, scores, groups), + n_models=result.n_models, + seconds=result.seconds, + eta_squared=eta_squared(values, groups), + ) + + +def run_synthetic(seeds: int) -> list[Cell]: + """The two-factor sweep: spread x mechanism x config x seed.""" + cells: list[Cell] = [] + points = synthetic.sweep_points() + total = len(points) * len(CONFIGS) * seeds + done = 0 + for level_spread, mechanism in points: + for seed in range(seeds): + values, labels, groups = synthetic.generate(seed=seed, level_spread=level_spread, mechanism=mechanism) + for config in CONFIGS: + cells.append( + Cell( + dataset="synthetic", + grouping=f"spread={level_spread:.3f}", + config=config, + seed=seed, + mechanism=mechanism, + level_spread=level_spread, + metrics=measure(values, labels, groups, config, seed), + ) + ) + done += 1 + print(f"\r {done}/{total} cells", end="", flush=True) + print() + return cells + + +def run_real(real_module, name: str, seeds: int) -> list[Cell]: + """Every candidate grouping of a real dataset, across configs and seeds. + + *mechanism* is recorded as ``"real"``: which mechanism produced a real anomaly is not knowable, + which is exactly why the synthetic sweep carries the mechanism contrast and these datasets + check that its conclusion survives. + """ + cells: list[Cell] = [] + for grouping, values, labels, groups in real_module.load_groupings(name): + print(f" {grouping}: {values.shape[0]} rows, {len(np.unique(groups))} groups") + for seed in range(seeds): + for config in CONFIGS: + cells.append( + Cell( + dataset=name, + grouping=grouping, + config=config, + seed=seed, + mechanism="real", + level_spread=float("nan"), + metrics=measure(values, labels, groups, config, seed), + ) + ) + print(f" done ({seeds} seeds x {len(CONFIGS)} configs)") + return cells + + +def paired_deltas(cells: list[Cell], left: str, right: str, metric: str = "pr_auc") -> list[dict]: + """``metric(left) - metric(right)`` for every (dataset, grouping, seed) the two share. + + Paired rather than averaged: the seed drives both the data draw and the forest, and dominates + the variance between cells. + """ + # The key must identify a cell uniquely. Mechanism belongs in it: the synthetic sweep runs both + # mechanisms at the same spread and seed, so a key without it collapses each pair of cells onto + # one entry and silently discards half the grid — which is what it did, until the absent + # "global" row in the summary table gave it away. + index: dict[tuple[str, str, str, int], dict[str, Cell]] = {} + for cell in cells: + index.setdefault((cell.dataset, cell.grouping, cell.mechanism, cell.seed), {})[cell.config] = cell + + out = [] + for (dataset, grouping, _mechanism, seed), by_config in sorted(index.items()): + if left not in by_config or right not in by_config: + continue + left_cell, right_cell = by_config[left], by_config[right] + left_value = getattr(left_cell.metrics, metric) + right_value = getattr(right_cell.metrics, metric) + out.append( + { + "dataset": dataset, + "grouping": grouping, + "seed": seed, + "mechanism": left_cell.mechanism, + "eta_squared": left_cell.metrics.eta_squared, + "delta": left_value - right_value, + "left": left_value, + "right": right_value, + } + ) + return out + + +def regress_delta_on_eta(deltas: list[dict]) -> dict[str, float]: + """Least squares of ``delta ~ 1 + eta_squared + is_contextual``. + + The gate's premise needs eta-squared to carry signal *after* controlling for the anomaly + mechanism. If the eta coefficient collapses once the mechanism dummy is present, the apparent + correlation was the mechanism all along. + + Synthetic rows only. On real data the mechanism behind each anomaly is unknown, so folding those + rows in would silently code them as "global" and bias the very coefficient being tested. + """ + rows = [ + d + for d in deltas + if np.isfinite(d["delta"]) and np.isfinite(d["eta_squared"]) and d["mechanism"] in synthetic.MECHANISMS + ] + if len(rows) < 4: + return {} + design = np.column_stack( + [ + np.ones(len(rows)), + np.array([r["eta_squared"] for r in rows]), + np.array([1.0 if r["mechanism"] == "contextual" else 0.0 for r in rows]), + ] + ) + target = np.array([r["delta"] for r in rows]) + coeffs, *_ = np.linalg.lstsq(design, target, rcond=None) + residual = target - design @ coeffs + ss_res = float((residual**2).sum()) + ss_tot = float(((target - target.mean()) ** 2).sum()) + return { + "intercept": float(coeffs[0]), + "eta_squared": float(coeffs[1]), + "is_contextual": float(coeffs[2]), + "r_squared": 1.0 - ss_res / ss_tot if ss_tot else float("nan"), + "n": float(len(rows)), + } + + +def decide(deltas: list[dict]) -> tuple[str, str, dict[str, float]]: + """Apply the pre-registered rules, and a variance-robust companion. + + Returns ``(pre_registered_verdict, robust_verdict, evidence)``. + + The pre-registered rule is a **minimum over single cells**, which makes it maximally sensitive + to the variance of the estimator it is applied to — and Isolation Forest on a 3-group, + 68k-row dataset is high variance. It fired on the first run against real data, on one seed of + ``nslkdd/protocol_type`` whose other seeds were **+0.1003** and **+0.0989**. + + That is a mis-specification, not a finding: the rule's stated intent was that conditioning is + never *materially worse* on homogeneous data, which is a claim about the distribution rather + than about the unluckiest single draw. So the per-(dataset, grouping) **median** delta is + reported beside it, and both verdicts are published. The pre-registered rule is deliberately + left in place rather than replaced, so that a reader can see the criterion that was set in + advance, the answer it gave, and why a second statistic was added. + """ + low = [d for d in deltas if np.isfinite(d["delta"]) and d["eta_squared"] < LOW_ETA] + if not low: + return "inconclusive: nothing fell below the eta-squared threshold", "inconclusive", {} + + worst_cell = min(d["delta"] for d in low) + harmful = [d for d in low if d["delta"] <= GATE_NEEDED_DELTA] + + # Median per (dataset, grouping): the distributional form of the same question. + by_grouping: dict[tuple[str, str], list[float]] = {} + for d in low: + by_grouping.setdefault((str(d["dataset"]), str(d["grouping"])), []).append(float(d["delta"])) + medians = {key: float(np.median(values)) for key, values in by_grouping.items()} + worst_median = min(medians.values()) + harmful_groupings = [key for key, value in medians.items() if value <= GATE_NEEDED_DELTA] + + evidence = { + "n_low_eta_cells": float(len(low)), + "n_low_eta_groupings": float(len(medians)), + "worst_delta_single_cell": worst_cell, + "n_harmful_cells": float(len(harmful)), + "worst_median_delta_per_grouping": worst_median, + "n_harmful_groupings": float(len(harmful_groupings)), + "median_delta_below_threshold": float(np.median([d["delta"] for d in low])), + } + + def verdict(worst: float, any_harmful: bool) -> str: + if worst > NO_GATE_FLOOR: + return "no gate needed" + if any_harmful: + return "gate needed; refit the threshold from this sweep" + return "borderline: below the floor but above the harm threshold" + + return verdict(worst_cell, bool(harmful)), verdict(worst_median, bool(harmful_groupings)), evidence + + +def git_sha() -> str: + """Short SHA of the tree these numbers came from. Results without it are unreproducible.""" + try: + return subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=True, + cwd=pathlib.Path(__file__).resolve().parent, + ).stdout.strip() + except (subprocess.CalledProcessError, OSError): + return "unknown" + + +def summarise(deltas: list[dict], label: str) -> list[str]: + """Median, IQR and a paired test for one comparison, split by anomaly mechanism.""" + lines = [f"### {label}", ""] + lines.append("| mechanism | n | median delta | IQR | Wilcoxon p |") + lines.append("|---|---|---|---|---|") + present = sorted({str(d["mechanism"]) for d in deltas}) + for mechanism in [*present, "*all*"]: + subset = deltas if mechanism == "*all*" else [d for d in deltas if d["mechanism"] == mechanism] + values = [d["delta"] for d in subset if np.isfinite(d["delta"])] + if not values: + continue + q25, q75 = np.percentile(values, [25, 75]) + _, p_value = wilcoxon_paired(values) + p_text = "n/a" if not np.isfinite(p_value) else f"{p_value:.2e}" + lines.append( + f"| {mechanism} | {len(values)} | {np.median(values):+.4f} | " f"{q25:+.4f} to {q75:+.4f} | {p_text} |" + ) + lines.append("") + return lines + + +def low_eta_table(deltas: list[dict]) -> list[str]: + """Every low-eta grouping, seed by seed. + + The table the two verdicts have to be read against: a grouping whose seeds straddle zero is + telling you about variance, and one whose seeds agree is telling you about the mechanism. + """ + low = [d for d in deltas if np.isfinite(d["delta"]) and d["eta_squared"] < LOW_ETA] + if not low: + return [] + + by_grouping: dict[tuple[str, str], list[dict]] = {} + for d in low: + by_grouping.setdefault((str(d["dataset"]), str(d["grouping"])), []).append(d) + + lines = [ + f"### Every grouping below eta-squared {LOW_ETA}, seed by seed", + "", + "| dataset | grouping | eta-squared | median delta | min | max | seeds agree? |", + "|---|---|---|---|---|---|---|", + ] + for (dataset, grouping), group in sorted(by_grouping.items()): + values = [g["delta"] for g in group] + agree = "yes, all negative" if max(values) < 0 else ("yes, none negative" if min(values) >= 0 else "**no**") + lines.append( + f"| {dataset} | {grouping} | {group[0]['eta_squared']:.4f} | {np.median(values):+.4f} | " + f"{min(values):+.4f} | {max(values):+.4f} | {agree} |" + ) + lines.append("") + return lines + + +def write_report(cells: list[Cell], seeds: int, datasets: list[str]) -> pathlib.Path: + """Write the dated markdown and JSON results, and return the markdown path.""" + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + stamp = dt.date.today().isoformat() + sha = git_sha() + stem = f"{stamp}-{sha}" + + rel_vs_pooled = paired_deltas(cells, "relative", "pooled") + rel_vs_per_group = paired_deltas(cells, "relative", "per_group") + pre_registered_verdict, robust_verdict, evidence = decide(rel_vs_pooled) + rho, lo, hi = spearman_with_bootstrap_ci( + [d["eta_squared"] for d in rel_vs_pooled], [d["delta"] for d in rel_vs_pooled] + ) + regression = regress_delta_on_eta(rel_vs_pooled) + + lines = [ + f"# Anomaly conditioning experiment — {stamp}", + "", + f"DQX `{sha}` · datasets: {', '.join(datasets)} · seeds per cell: {seeds} · " f"cells: {len(cells)}", + "", + "PR-AUC is the primary metric. No point-adjusted F1 appears anywhere; see `metrics.py`.", + "DQX scores rows independently, so figures on time-series data are not comparable with", + "published sequence-model results — that is a different task, not a worse implementation.", + "", + "## Does removing the heterogeneity gate cost anything?", + "", + f"Rules fixed before running: **no gate** if the worst delta below eta-squared " + f"{LOW_ETA} exceeds {NO_GATE_FLOOR:+.2f}; **gate needed** if any such cell reaches " + f"{GATE_NEEDED_DELTA:+.2f}.", + "", + f"- **Pre-registered rule (worst single cell): {pre_registered_verdict}**", + f"- **Variance-robust companion (worst per-grouping median): {robust_verdict}**", + "", + "The pre-registered rule takes a minimum over individual cells, so it is maximally " + "sensitive to estimator variance. It is reported unchanged, alongside the median form of " + "the same question, so the criterion set in advance and the answer it gave are both " + "visible. Where the two disagree, the per-grouping deltas below show why.", + "", + ] + if evidence: + lines.append("| statistic | value |") + lines.append("|---|---|") + for key, value in evidence.items(): + lines.append(f"| {key} | {value:+.4f} |" if "delta" in key else f"| {key} | {value:.0f} |") + lines.append("") + + lines += [ + "### Does eta-squared predict the benefit at all?", + "", + f"Spearman rho(eta-squared, delta) = **{rho:+.3f}** (bootstrap 95% CI {lo:+.3f} to {hi:+.3f}).", + "", + ] + if regression: + lines += [ + "Least squares `delta ~ 1 + eta_squared + is_contextual`, which asks whether " + "eta-squared still carries signal once the anomaly mechanism is controlled for:", + "", + "| term | coefficient |", + "|---|---|", + f"| intercept | {regression['intercept']:+.4f} |", + f"| eta_squared | {regression['eta_squared']:+.4f} |", + f"| is_contextual | {regression['is_contextual']:+.4f} |", + f"| R-squared | {regression['r_squared']:.3f} (n={regression['n']:.0f}) |", + "", + ] + + lines += low_eta_table(rel_vs_pooled) + lines += ["## Paired comparisons", ""] + lines += summarise(rel_vs_pooled, "baseline-relative minus pooled (PR-AUC)") + lines += summarise(rel_vs_per_group, "baseline-relative minus per-group (PR-AUC)") + + lines += ["## Cost", "", "| config | median models | median seconds |", "|---|---|---|"] + for config in CONFIGS: + subset = [c for c in cells if c.config == config] + if subset: + lines.append( + f"| {config} | {np.median([c.metrics.n_models for c in subset]):.0f} | " + f"{np.median([c.metrics.seconds for c in subset]):.2f} |" + ) + lines.append("") + + md_path = RESULTS_DIR / f"{stem}.md" + md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + (RESULTS_DIR / f"{stem}.json").write_text( + json.dumps( + { + "generated": stamp, + "git_sha": sha, + "seeds": seeds, + "datasets": datasets, + "verdict_pre_registered": pre_registered_verdict, + "verdict_robust": robust_verdict, + "evidence": evidence, + "spearman": {"rho": rho, "ci_low": lo, "ci_high": hi}, + "regression": regression, + "cells": [c.as_dict() for c in cells], + }, + indent=2, + default=str, + ), + encoding="utf-8", + ) + return md_path + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seeds", type=int, default=5, help="seeds per cell (paired across configs)") + parser.add_argument( + "--datasets", + nargs="+", + default=["synthetic"], + choices=["synthetic", "smd", "nslkdd"], + help="synthetic needs no network; the others download at run time", + ) + args = parser.parse_args() + + cells: list[Cell] = [] + if "synthetic" in args.datasets: + print("synthetic sweep:") + cells += run_synthetic(args.seeds) + + real_names = [n for n in ("smd", "nslkdd") if n in args.datasets] + if real_names: + # Imported lazily: it downloads data, so a synthetic-only run stays offline. + from datasets import real # noqa: PLC0415 + + for name in real_names: + print(f"{name}:") + cells += run_real(real, name, args.seeds) + + if not cells: + print("no cells measured", file=sys.stderr) + return 1 + + path = write_report(cells, args.seeds, args.datasets) + print(f"\nwrote {path}") + print(path.read_text(encoding="utf-8")) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx new file mode 100644 index 000000000..20050b3cb --- /dev/null +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -0,0 +1,167 @@ +--- + +title: Anomaly detection quality + +sidebar_position: 509 + +--- + +# Anomaly Detection Quality + +How well does row anomaly detection actually detect? This page reports measured detection quality +rather than timing — [Benchmarks](/docs/reference/benchmarks) covers performance. + +The numbers come from `benchmarks/anomaly_conditioning/` in the DQX repository, which is run manually +rather than nightly: it downloads third-party datasets and produces a correlation rather than a +pass/fail. Its README documents how to reproduce everything below. + +## Read this first + +**DQX scores rows independently.** It is not a sequence model, and it does not consume a window of +history to make a prediction. Published results on time-series anomaly benchmarks routinely exceed +0.80 PR-AUC using models that do. A DQX figure of 0.15 on the same data is **a different task, not a +worse implementation**. Comparing the two directly is a category error. + +**PR-AUC is the primary metric.** These datasets are heavily imbalanced, and ROC-AUC flatters a +detector that merely ranks the majority class well. + +**No point-adjusted F1 appears anywhere.** Under the point-adjust protocol — crediting an entire +labelled anomaly segment when any single point inside it is detected — a *random* anomaly score +achieves state-of-the-art F1 ([Kim et al., AAAI 2022](https://arxiv.org/abs/2109.05257)). Numbers +produced that way are uninterpretable, so DQX does not compute them. + +## What was compared + +Three ways of relating a model to a group, each fitted and scored on identical rows: + +| configuration | what it does | DQX equivalent | +|---|---|---| +| pooled | one model over the raw metrics; no notion of a group | no `baseline_by` | +| relative | one model over the raw metrics **plus** each metric's deviation from its own group's baseline | `baseline_by` | +| per-group | one model per group, trained only on that group's rows | the legacy `segment_by` | + +Comparisons are **paired by seed** and tested with Wilcoxon signed-rank. The seed drives both the +data draw and the forest, and dominates the variance between configurations, so unpaired means would +largely measure the seed. + +## Results + +1,395 cells: a synthetic two-factor sweep plus the Server Machine Dataset (28 entities) and NSL-KDD, +15 seeds each. + +### `baseline_by` versus comparing against the whole table + +| anomaly mechanism | n | median ΔPR-AUC | IQR | Wilcoxon p | +|---|---|---|---|---| +| contextual (ordinary globally, wrong for its group) | 195 | **+0.0734** | +0.0068 to +0.2649 | 5.3e-32 | +| global (extreme for the whole table) | 195 | +0.0000 | −0.0000 to +0.0000 | 2.0e-01 | +| real datasets (mechanism unknown) | 75 | −0.0010 | −0.1622 to +0.0110 | 3.1e-03 | + +Read these three rows together, because they are the whole argument: + +- **When anomalies are contextual, conditioning is transformative.** This is the case + [#1484](https://github.com/databrickslabs/dqx/issues/1484) exists for: a group whose volume + collapses while the daily total stays flat is invisible to any whole-table comparison. Measured + offline on such a collapse, a pooled model scored PR-AUC 0.0028 against a 0.0026 base rate — that + is chance — while conditioning reached 0.6962. +- **When anomalies are globally extreme, conditioning costs nothing.** Not "costs little" — + the median difference is zero and the test does not reject (p = 0.20). At low heterogeneity every + group median approaches the global median, so the relative feature degenerates into a monotone + transform of the raw metric: a near-duplicate of an informative column rather than noise. +- **On real datasets, conditioning is marginally worse, and detectably so.** Median −0.0010 at + p = 0.003. The magnitude is negligible; the sign is real. SMD and NSL-KDD anomalies are largely + globally extreme or sequence-dependent rather than contextual with respect to a categorical group, + so the extra feature dilutes slightly without adding signal. + +The practical reading: `baseline_by` buys roughly +0.07 PR-AUC where it applies and costs roughly +0.001 where it does not. That asymmetry is why auto-discovery enables it rather than requiring you to +opt in. + +### `baseline_by` versus one model per group + +| anomaly mechanism | n | median ΔPR-AUC | IQR | Wilcoxon p | +|---|---|---|---|---| +| contextual | 195 | **+0.0274** | +0.0128 to +0.0413 | 1.2e-33 | +| global | 195 | +0.0006 | +0.0000 to +0.0017 | 3.4e-26 | +| real datasets | 75 | **+0.0123** | −0.0043 to +0.2785 | 3.2e-05 | + +One conditioned model beats one model per group in **every** mechanism, including on real data, while +training a single model instead of one per group: + +| configuration | median models trained | median fit seconds | +|---|---|---| +| pooled | 1 | 0.09 | +| relative | 1 | 0.09 | +| per-group | 12 | 0.79 | + +Per-group models also fail in a particular way that averages hide. On the Server Machine Dataset one +entity produced 15,963 false positives across 28,392 normal rows — a 56% false-alarm rate — while the +aggregate metric looked merely mediocre. Each per-group model calibrates its own contamination on its +own rows, so a group containing nothing unusual still has its most-unusual few percent scored as +extreme. This is why DQX reports the **worst** group's false-positive rate rather than the average, +and why `segment_by` is retained for compatibility rather than recommended. + +## Why there is no heterogeneity threshold + +DQX briefly gated conditioning on eta-squared — the share of variance the grouping explains — skipping +it below 0.10, on the theory that a grouping which explains little contributes noise. + +The decision rules were fixed before the experiment ran: **no gate** if the worst delta below +eta-squared 0.10 stayed above −0.01; **gate needed** if any such cell reached −0.02. + +The two forms of that criterion disagreed, and both are published: + +| criterion | verdict | +|---|---| +| pre-registered (worst single cell) | gate needed — 3 harmful cells of 90 | +| variance-robust (worst per-grouping median) | **no gate needed** — 0 harmful groupings | + +The seed-by-seed breakdown shows why: + +| dataset | grouping | eta-squared | median Δ | min | max | seeds agree? | +|---|---|---|---|---|---|---| +| NSL-KDD | `protocol_type` | 0.0509 | +0.0285 | −0.0744 | +0.1003 | no | +| SMD | machine family | 0.0669 | +0.0033 | −0.0047 | +0.0125 | no | +| synthetic | spread 0.000 | 0.0008 | +0.0000 | −0.0019 | +0.0283 | no | +| synthetic | spread 0.050 | 0.0593 | +0.0000 | −0.0019 | +0.0169 | no | + +Every low-heterogeneity grouping straddles zero. The pre-registered rule takes a minimum over +individual cells, which makes it maximally sensitive to estimator variance, and it fired on single +unlucky seeds of groupings whose medians are positive. Raising the seed count fivefold moved no +grouping's median below zero. **No grouping is systematically harmed at low heterogeneity**, so the +gate has nothing to gate on and was removed. + +Eta-squared is not useless — it correlates with the *size* of the benefit (Spearman ρ = +0.597, +bootstrap 95% CI +0.523 to +0.668, and a coefficient of +0.276 in +`Δ ~ eta_squared + is_contextual`, so it carries signal even after controlling for the anomaly +mechanism). But it predicts **how much you gain, never whether you lose**, and a gate needs to +identify harm. Computing it cost a full Spark aggregation per training run to answer a question that +never changes the decision. + +## Datasets + +Downloaded at run time and cached; DQX redistributes none of them. + +| dataset | licence | groupings tested | citation | +|---|---|---|---| +| Server Machine Dataset | MIT, via [`NetManAIOps/OmniAnomaly`](https://github.com/NetManAIOps/OmniAnomaly) | server entity (28), machine family (3) | Su et al., *Robust Anomaly Detection for Multivariate Time Series through Stochastic Recurrent Neural Networks*, KDD 2019 | +| NSL-KDD | redistributable with citation | `service` (65), `flag` (11), `protocol_type` (3) | Tavallaee et al., *A detailed analysis of the KDD CUP 99 data set*, CISDA 2009 | +| synthetic | n/a | 13 heterogeneity levels × 2 anomaly mechanisms | `benchmarks/anomaly_conditioning/datasets/synthetic.py` | + +SMAP and MSL are excluded: their data files carry "© Original Authors" with no permissive licence. +ADBench is not used because it ships pre-processed numeric matrices, so categorical column identity — +the grouping this experiment measures — is gone. + +SMD is capped at 4,000 rows per entity so a run takes minutes rather than hours, and NSL-KDD's attacks +are downsampled from roughly 46% to 2% so the task is anomaly detection rather than classification. +Every configuration sees identical rows, so neither affects the comparison. + +## Caveats + +- The harness reimplements the baseline-relative transform in numpy rather than calling DQX, so a + sweep of a few thousand fits needs no Spark session. It therefore measures the **mechanism**, not + DQX's implementation of it. Pipeline fidelity is asserted separately in + `tests/integration_anomaly/test_anomaly_quality.py`, which runs the real Spark and MLflow path. +- Each configuration is fitted and scored on the same rows. This measures separability, not + generalisation to unseen data. +- Results are indicative. Always benchmark against your own data and environment. diff --git a/pyproject.toml b/pyproject.toml index 4a8d831bb..5b9695f51 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -207,7 +207,7 @@ exclude = ['venv', '.venv', 'demos/*', 'tests/e2e/*', 'app/*', 'mcp-server/*'] [[tool.mypy.overrides]] # External packages without PEP 561 type stubs (unavoidable - these are third-party optional dependencies) -module = ["google", "google.*", "pandas", "sklearn.*", "shap", "cloudpickle", "mlflow", "mlflow.*", "urllib3.*", "aiohttp.*", "openpyxl", "openpyxl.*", "websockets", "websockets.*"] +module = ["google", "google.*", "pandas", "sklearn.*", "scipy", "scipy.*", "shap", "cloudpickle", "mlflow", "mlflow.*", "urllib3.*", "aiohttp.*", "openpyxl", "openpyxl.*", "websockets", "websockets.*"] ignore_missing_imports = true From d38bf958eb3010aa0b01a55d60edc901b3f7894d Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:01:32 +0100 Subject: [PATCH 011/107] Document baseline conditioning, and record it as a breaking change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guide gains a "Baseline conditioning" section built around the one mechanism there is: what `baseline_by` compares against, that it costs one model however many groups you have, that baseline columns are the basis of comparison rather than metrics to be compared, and what happens to a group that was never trained on. The measured numbers are quoted with a link to `anomaly_detection_quality.mdx` for the full sweep and, more importantly, for what they do not mean — DQX scores rows independently, so its figures on time-series benchmarks are a different task, not a worse implementation. `segment_by` is documented as legacy with the reason rather than a bare recommendation. On the Server Machine Dataset per-entity models were the worst of three configurations, and one entity produced 15,963 false positives across 28,392 normal rows. Each per-group model calibrates its own contamination on its own rows, so a group containing nothing unusual still has its most-unusual few percent scored as extreme. Four breaking changes, of which the second is the one existing users will notice: * Above the segment ceiling, training now raises instead of warning. * **An auto-discovered grouping now trains one conditioned model instead of N per-group models**, so those runs produce different scores. * A row whose group was never seen at training returns a null score and is reported via `is_new_baseline` rather than being scored 0.0. * The `_dq_info` struct is wider, so appending to an existing results table needs `mergeSchema`. --- CHANGELOG.md | 5 + demos/dqx_row_anomaly_detection_demo.py | 7 +- .../guide/row_anomaly_detection/index.mdx | 118 +++++++++++++++++- docs/dqx/docs/reference/quality_checks.mdx | 2 + 4 files changed, 128 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0461d983a..8cc2df999 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.16.0 +* Added baseline conditioning to row anomaly detection ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). Anomaly detection could not detect a **contextual** anomaly — a value that is unremarkable across the table but wrong for its own group. On the measurements in the issue, one group's volume dropping 80% behind a flat daily total scored 45.1, the 45th percentile, so no threshold recovered it. `AnomalyEngine.train()` now takes `baseline_by`: each numeric metric gains its deviation from that metric's own baseline within the row's group, as a signed log-ratio, on a **single** pooled model — so the cost does not grow with the group count, and the same collapse scores above 95. Measured offline in the unit suite, a contextual collapse goes from PR-AUC 0.0028 (chance) to 0.6962, while an anomaly that was already globally extreme is unchanged at 1.0000, so conditioning costs nothing measurable when there is nothing to gain. Baseline columns must be string, integral, boolean or date; floating-point and decimal types are rejected because Spark and Python format them differently, which would silently break the key lookup that matches persisted baselines to rows. Grouping auto-discovery is no longer coupled to column discovery, so passing explicit `columns` no longer silently gives up conditioning, and a discovered grouping is used as `baseline_by` rather than turning into one model per group. Across a wider sweep — 1,395 configurations over synthetic data, the Server Machine Dataset and NSL-KDD — conditioning is worth a median +0.0734 PR-AUC where anomalies are contextual and −0.0010 where they are not, and beats one-model-per-group in every case measured; see [Anomaly detection quality](https://databrickslabs.github.io/dqx/docs/reference/anomaly_detection_quality) for the full results, the licences, and what these numbers do not mean. * Added a pluggable actions and alerting subsystem ([#1289](https://github.com/databrickslabs/dqx/issues/1289)). DQX now supports extensible *actions* that run when checked data violates an optional condition evaluated against the summary metrics produced by `DQMetricsObserver`. The built-in `DQAlert` action can send notifications to Slack, Microsoft Teams, a generic HTTPS webhook, or the log, so pipelines can react to data quality regressions without custom plumbing. You can create your own custom actions as well, and custom alerting is possible via the callback destination, which invokes an in-process Python callable for each alert. * Added an MCP (Model Context Protocol) server for DQX ([#1252](https://github.com/databrickslabs/dqx/issues/1252)). The server exposes DQX's data quality capabilities as tools that any MCP-compatible AI agent (Claude, Genie Code, Cursor, Mosaic AI) can discover and orchestrate. It runs as a Databricks App with on-behalf-of (OBO) authentication, so all data access is governed by the calling user's Unity Catalog permissions. * Added support for summary metrics in Lakeflow Declarative Pipelines (LDP/DLT) ([#1301](https://github.com/databrickslabs/dqx/issues/1301)). A new `DQEngine.compute_summary_metrics(...)` produces the same row counts, per-check breakdown, and custom observer metrics as a lazy aggregation over the results DataFrame, so metrics can be computed inside Spark Declarative Pipelines where the observer- and streaming-listener-based paths cannot be used. @@ -48,6 +49,10 @@ BREAKING CHANGES! +* Row anomaly detection now **errors** instead of warning when segmentation would produce more than `AnomalyParams.max_segment_models` segments (default 50). One model is trained per segment and segmented training does not ensemble, so cost is linear in the segment count: 90 segments measures roughly 88 minutes, and a 90-segment run was cancelled after 70 minutes without finishing. Auto-discovery could reach tens of thousands of segments behind a single log line. Runs that previously segmented into 51 or more segments will now fail; pass a higher `max_segment_models` to keep the old behaviour, or use `baseline_by`, which trains one model regardless of the group count. ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) +* `segment_by` is now legacy, and an auto-discovered grouping is used as `baseline_by` instead of training one model per group. Runs that relied on auto-segmentation will train a single conditioned model rather than N models, and will produce different scores. The evidence: on the Server Machine Dataset per-group models were the worst of three configurations (PR-AUC 0.1416 against 0.1499 for plain pooling and 0.1536 with baseline conditioning), and one entity produced 15,963 false positives out of 28,392 normal rows. `segment_by` still works if you pass it explicitly; a `DeprecationWarning` follows one release later. Passing both `segment_by` and `baseline_by` raises. ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) +* Rows whose group was absent from training now return a **null** score and severity instead of a number, and are not flagged as violations. Previously they were scored 0.0 — the most normal-looking value in the table — because one-hot encoding emits all zeros for an unseen category, which resembles the majority on every axis; frequency encoding has the same defect with the opposite sign, coalescing the miss to a frequency below anything seen in training. Neither is a signal a caller can act on. The new `_dq_info[].anomaly.is_new_baseline` and `.new_baseline_key` fields report the fact. To fail on unrecognised group values, use `foreign_key` or `is_in_list` on the baseline column, which is the check built for that question. ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) +* The anomaly struct inside `_dq_info` gains `is_new_baseline` (boolean) and `new_baseline_key` (string). Existing named-field queries such as `_dq_info[0].anomaly.score` keep working, but the struct is wider, so **appending to a Delta table that already holds `_dq_info` requires `mergeSchema`** (`.option("mergeSchema", "true")` or `spark.databricks.delta.schema.autoMerge.enabled`). ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) * `is_in_list`, `is_not_in_list`, and `is_not_null_and_is_in_list` now resolve their `allowed` / `forbidden` string values as **column expressions** (consistent with the comparison checks), not string literals. A bare string is interpreted as a column reference, a numeric string (e.g. `"3"`) is parsed as a number, and an ISO-date string (e.g. `"2024-01-01"`) as a date. To match a string literal, single-quote the value (e.g. `'value'`) or wrap it in `F.lit("value")`. Existing checks that relied on bare strings being treated as literals must quote them. ([#1419](https://github.com/databrickslabs/dqx/issues/1419)) * `user_metadata` saved through the **Delta** table storage backend is now JSON-encoded at rest to preserve non-string types through the `MAP` column. Save→load via DQX is transparent (you get the original typed value back), but the stored representation changes: direct SQL/dashboard consumers now read JSON-encoded values (decode with `from_json`), existing tables are not migrated, and legacy string values that look like JSON atoms (`"true"`, `"1"`, `"null"`) read back as typed values (`True` / `1` / `None`) — re-save affected rule sets after upgrading to normalize. The File/Volume (YAML/JSON) and Lakebase (JSONB) backends are unaffected. ([#1319](https://github.com/databrickslabs/dqx/issues/1319)) diff --git a/demos/dqx_row_anomaly_detection_demo.py b/demos/dqx_row_anomaly_detection_demo.py index 6576ca56f..1087e2e3a 100644 --- a/demos/dqx_row_anomaly_detection_demo.py +++ b/demos/dqx_row_anomaly_detection_demo.py @@ -699,7 +699,10 @@ def inject_anomalies_and_dq_issues( # MAGIC # MAGIC **Training options (`AnomalyEngine.train` / `AnomalyParams`):** # MAGIC - `columns` (list[str]): explicit feature list (disables auto‑discovery) -# MAGIC - `segment_by` (list[str]): explicit segmentation columns +# MAGIC - `baseline_by` (list[str]): columns identifying a row's group, so each metric is judged +# MAGIC against its own group's baseline rather than the whole table — catches values that are +# MAGIC ordinary globally but wrong in context. One model, whatever the group count. +# MAGIC - `segment_by` (list[str]): legacy, trains one model per group; prefer `baseline_by` # MAGIC - `sample_fraction`, `max_rows`: training sample controls # MAGIC - `ensemble_size`: number of models in the ensemble # MAGIC - `expected_anomaly_rate`: expected anomaly rate for calibration @@ -828,7 +831,7 @@ def inject_anomalies_and_dq_issues( # MAGIC ``` # MAGIC # MAGIC **Optional next steps:** -# MAGIC - Add segmentation (`segment_by` option for training), drift detection, and scheduled scoring. +# MAGIC - Add group conditioning (`group_by` for training), drift detection, and scheduled scoring. # MAGIC - Automate retraining and alerting. # COMMAND ---------- diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 74e270497..5d4eb0359 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -289,6 +289,118 @@ Scores are normalized into `severity_percentile` (0–100). The anomaly threshol The threshold controls how many rows are flagged as anomalies (percentile cutoff; default 95). Lower values (for example 90) flag more rows; higher values (for example 98) flag fewer (only extreme anomalies). Start with the default (95). If you get too many alerts, raise the threshold; if you are missing issues you care about, lower it. +## Baseline conditioning + +Some values are only wrong *in context*. If one country's daily order volume drops 80% while the +overall total holds steady — because other countries absorbed the difference — the collapsed number +still sits comfortably inside the range other countries occupy normally. A model that compares every +row against the whole table cannot see it: on the measurements behind +[#1484](https://github.com/databrickslabs/dqx/issues/1484) such a collapse scored **45.1**, the 45th +percentile. No threshold recovers that without flagging half the table. + +Pass `baseline_by` to give DQX the right basis for comparison: + +```python +anomaly_engine.train( + df, + model_name="catalog.schema.orders_model", + registry_table="catalog.schema.dqx_anomaly_models", + columns=["order_count"], + baseline_by=["country", "product"], +) +``` + +The same collapse then scores above 95. + +### How it works, and what it costs + +For each numeric metric, DQX adds one feature: that metric's deviation from its own group's +baseline, as a signed log-ratio. + +``` +signed_log(x) = signum(x) * log1p(abs(x)) +_rel_baseline = signed_log(value) - signed_log(group_median(metric)) +``` + +The raw metric is kept alongside, so a globally absurd value stays detectable even where it is +ordinary for its group. The log-ratio form is stable when a baseline is near zero and symmetric for +halving versus doubling; the *signed* form matters because plain `log1p` is NaN for values at or +below −1, which would silently produce NaN features on any signed metric such as profit or balance. + +There is **one model**, however many groups you have. Baselines are computed once at training and +persisted with the model, then broadcast-joined at scoring time. So the cost does not grow with the +group count — which is the whole point, and the difference from `segment_by`. + +Measured offline (see `tests/unit/test_anomaly_relative_feature_separability.py`, which runs in the +unit suite): + +| scenario | without conditioning | with `baseline_by` | +|---|---|---| +| contextual collapse (ordinary globally) | PR-AUC 0.0028 — chance | PR-AUC 0.6962 | +| anomaly extreme against every group | PR-AUC 1.0000 | PR-AUC 1.0000 | + +The second row is why conditioning is on by default when a grouping is available: it costs nothing +measurable when the anomaly was already visible. + +Across a wider sweep — 1,395 configurations over synthetic data, the Server Machine Dataset and +NSL-KDD — conditioning is worth about **+0.07 PR-AUC** where anomalies are contextual, and about +**−0.001** where they are not. That asymmetry, not a hunch, is why a discovered grouping is used +rather than ignored. See [Anomaly detection quality](/docs/reference/anomaly_detection_quality) for +the full results, the datasets, and what these numbers do *not* mean. + +### Baseline columns are not features + +A baseline column is the basis of comparison, not a metric being compared, so it never becomes a +model feature. Passing the same column as both `columns` and `baseline_by` is an error. If you let +DQX auto-discover `columns`, it drops your declared baseline columns from the feature list for you. + +Baseline columns must be string, integral, boolean or date. Float, double and decimal are +**rejected**: Spark and Python format floating-point values differently, and DQX builds each row's +baseline key in both — Python when saving baselines, Spark when looking them up. A mismatch would +not raise; it would silently miss every lookup and quietly condition on nothing. Bucket the value or +cast it to a string first. + +:::note +That failure mode is not hypothetical. Booleans hit it during development — Spark renders `true`, +Python renders `True` — and it was caught only by a test that compares the two implementations +against a live session. Both halves of the key are now pinned by that test. +::: + +### Groups that appear after training + +A row whose group was never seen during training gets a **null** score and severity, plus +`is_new_baseline = true` and the unrecognised key in `new_baseline_key`: + +```python +result.filter(F.col("_dq_info")[0].anomaly.is_new_baseline).select("_dq_info") +``` + +It is not flagged as a violation. Neither categorical encoder can represent an unseen value honestly +— one-hot makes it look maximally normal, frequency encoding maximally extreme — so DQX cannot judge +the row, and "could not judge" is a different claim from "is anomalous". Previously such rows were +scored 0.0 and silently passed, which is the most normal-looking value in the table. + +If an unrecognised group value is itself something you want to fail on, that is a membership question +rather than an anomaly one: use `foreign_key` or `is_in_list` on the baseline column against your set +of known values. + +### segment_by is legacy + +`segment_by` trains one model per group. It still works, but prefer `baseline_by`, because +per-group models lost on every axis that was measured: + +* **Detection.** On the Server Machine Dataset they were the worst of three configurations — + PR-AUC 0.1416 against 0.1499 for plain pooling and 0.1536 with baseline conditioning. +* **Reliability.** One entity produced 15,963 false positives out of 28,392 normal rows, a 56% + false-alarm rate. Each per-group model calibrates its severity threshold on its own small sample, + so each is independently fragile. +* **Cost.** Linear in the group count, and segmented training does not ensemble. 90 groups measured + roughly 88 minutes; a 90-group run was cancelled after 70 minutes without finishing. Runs above + `AnomalyParams.max_segment_models` (default 50) now raise rather than warn. + +Passing both `baseline_by` and `segment_by` is an error. A `DeprecationWarning` on `segment_by` +follows one release later. + ## How it works under the hood For full parameter and schema details, see [Row Anomaly Detection in Quality Checks](/docs/reference/quality_checks#row-anomaly-detection). @@ -299,7 +411,7 @@ For full parameter and schema details, see [Row Anomaly Detection in Quality Che 2. **Smart sampling and training**: DQX samples your data (default 30%, capped at 1M rows), trains an ensemble of Isolation Forest models, and captures baseline statistics for drift detection. 3. **Model registry**: Models and metadata live in MLflow and a Delta table; segmented models use deterministic names (for example `__seg_region=US_tier=gold`). 4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. SHAP contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. -5. **Auto-discovery of columns and segments**: When you call `train()` without `columns` or `segment_by`, DQX automatically discovers both. It selects numeric columns with enough variance as features and may **auto-segment** when it finds suitable segment columns: categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per segment (for example region, product category). If your auto-trained model is segmented, that is expected. To force a single global model, pass `segment_by=[]` (or omit segment columns from the data used for discovery) or set `columns` and `segment_by` explicitly. +5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping — categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group (for example region, product category) — and this happens whether or not you passed `columns`, since what to measure and what to compare it against are independent questions. A discovered grouping is used as `baseline_by` (see [Baseline conditioning](#baseline-conditioning)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. ### Why Isolation Forest? @@ -330,6 +442,8 @@ Each element is a **struct** with a shared “wide” schema. Currently, the onl | `segment` | map<string, string> | Segment key-value pairs for segmented models; `null` for global models. | | `contributions` | map<string, double> | Per-feature contribution percentages (0–100). On by default (`enable_contributions=True`); populated only for anomalous rows — `null` for non-anomalous rows or if you set it `False`. | | `confidence_std` | double | Ensemble score standard deviation. Present when `enable_confidence_std=True`; `null` otherwise. | +| `is_new_baseline` | boolean | `true` when the row's group was absent from training, in which case `score` and `severity_percentile` are `null` and the row is **not** flagged. See [Groups that appear after training](#groups-that-appear-after-training). | +| `new_baseline_key` | string | The unrecognised group key, for rows where `is_new_baseline` is `true`; `null` otherwise. | | `ai_explanation` | struct | LLM-generated explanation for the row's `(segment, pattern)` group. On by default (`enable_ai_explanation=True`); `null` for non-anomalous rows, when disabled, or when no serving endpoint is reachable. See [AI explanations](#ai-explanations) below. | The nested `ai_explanation` struct (populated when AI explanations are on — the default): @@ -423,7 +537,7 @@ See the **Training data requirements** tip under Quick start. In short: 1,000+ r
Q: Why is my auto-trained model segmented? -When you train without specifying `columns` or `segment_by`, DQX auto-discovers both. If your data has columns that look like good segment dimensions (for example region, category) — low cardinality (2–50 distinct values), low null rate, and enough rows per segment — DQX will train **one model per segment**. That is intentional: segmented models often fit better when behavior differs by segment. To get a single global model instead, pass `segment_by=[]` or provide explicit `columns` (and no segment columns) when calling `train()`. +When you train without specifying a grouping, DQX looks for one: columns that look like good dimensions (for example region, category) — low cardinality (2–50 distinct values), low null rate, and enough rows per group. That is intentional, because behaviour often does differ by group, and the user least likely to pass `baseline_by` explicitly is the one most likely to miss a contextual anomaly. A discovered grouping is used as `baseline_by`: each metric gains its deviation from that group's baseline, on a **single** model, so the discovery cannot turn into an hours-long run however many groups it finds. The grouping it chose is logged. To compare against the whole table instead, pass `baseline_by=[]`.
diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 4507f90ae..88ca2ea56 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -3513,6 +3513,8 @@ Pass an `AnomalyParams` object to the `params` argument to customize training be | `max_rows` | int | 1,000,000 | Maximum rows to use for training. Caps memory usage for very large datasets. | | `train_ratio` | float | 0.8 | Train/validation split ratio (80% train, 20% validation). | | `ensemble_size` | int or None | 3 | Number of models in ensemble. Set to None for single model. More models = more robust but slower training. Applies to a single global model only: **segmented training always trains one model per segment and ignores this setting**, so `confidence_std` is unavailable for segmented models. | +| `baseline_by` | list[str] or None | None | Columns identifying the group a row belongs to, so each metric is judged against its own group's baseline rather than against the whole table. Adds one feature per metric — its signed log-ratio to that group's median — on a **single** model, so cost does not grow with the group count. Normally set by passing `baseline_by` to `train()`. See [Baseline conditioning](/docs/guide/row_anomaly_detection#baseline-conditioning). | +| `max_segment_models` | int | 50 | Ceiling on per-segment models one run will attempt, guarding the legacy `segment_by` path. Cost there is linear in the segment count and segmented training does not ensemble, so 90 segments measures roughly 88 minutes. Exceeding this raises rather than warns. Irrelevant to `baseline_by`. | #### IsolationForestConfig (Algorithm Parameters) From d480802a5f383c69bdb97530e3250695ca1d207d Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 14:13:03 +0100 Subject: [PATCH 012/107] Record the first run of the conditioning experiment 1,395 cells: the synthetic two-factor sweep plus the Server Machine Dataset and NSL-KDD, 15 seeds each, stamped with the commit they were produced from so the numbers are reproducible rather than merely asserted. Committed as its own change because the results are an observation, not code. Re-running the harness at the same seeds reproduces this file, which is how the determinism claim in the README is checked; the figures quoted in docs/dqx/docs/reference/anomaly_detection_quality.mdx are read directly from here. --- .../results/2026-08-25-55ebd5ca.json | 23750 ++++++++++++++++ .../results/2026-08-25-55ebd5ca.md | 77 + 2 files changed, 23827 insertions(+) create mode 100644 benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.json create mode 100644 benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.md diff --git a/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.json b/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.json new file mode 100644 index 000000000..d45798352 --- /dev/null +++ b/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.json @@ -0,0 +1,23750 @@ +{ + "generated": "2026-08-25", + "git_sha": "55ebd5ca", + "seeds": 15, + "datasets": [ + "synthetic", + "smd", + "nslkdd" + ], + "verdict_pre_registered": "gate needed; refit the threshold from this sweep", + "verdict_robust": "no gate needed", + "evidence": { + "n_low_eta_cells": 90.0, + "n_low_eta_groupings": 4.0, + "worst_delta_single_cell": -0.07436193395506441, + "n_harmful_cells": 3.0, + "worst_median_delta_per_grouping": 0.0, + "n_harmful_groupings": 0.0, + "median_delta_below_threshold": 1.6653345369377348e-16 + }, + "spearman": { + "rho": 0.5973468499523505, + "ci_low": 0.5227401597548323, + "ci_high": 0.6681276119704511 + }, + "regression": { + "intercept": -0.09975022217842222, + "eta_squared": 0.2760137159852486, + "is_contextual": 0.08302997541505142, + "r_squared": 0.5605703092383967, + "n": 390.0 + }, + "cells": [ + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.09368937500039465, + "eta_squared": 0.0008371029862412371, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07720820900067338, + "eta_squared": 0.0008371029862412371, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7040970829984872, + "eta_squared": 0.0008371029862412371, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.0718564589988091, + "eta_squared": 0.0012807601285772677, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9995636400137604, + "roc_auc": 0.9999911422902495, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07067412499964121, + "eta_squared": 0.0012807601285772677, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9962594841774015, + "roc_auc": 0.999926923894558, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7128532499991707, + "eta_squared": 0.0012807601285772677, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07275233299878892, + "eta_squared": 0.00097235471194274, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9980505965884341, + "roc_auc": 0.9999623547335601, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07040591600161861, + "eta_squared": 0.00097235471194274, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9979498197998601, + "roc_auc": 0.9999557114512473, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7100916669987782, + "eta_squared": 0.00097235471194274, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.06776700000045821, + "eta_squared": 0.00043327407293021155, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07179583299875958, + "eta_squared": 0.00043327407293021155, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7085450409977057, + "eta_squared": 0.00043327407293021155, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.0739716669995687, + "eta_squared": 0.0011831306529618937, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07306791700102622, + "eta_squared": 0.0011831306529618937, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9988727861917877, + "roc_auc": 0.9999778557256236, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7117822090003756, + "eta_squared": 0.0011831306529618937, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07129791599800228, + "eta_squared": 0.001278796513683895, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.0722847079996427, + "eta_squared": 0.001278796513683895, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9985712607506163, + "roc_auc": 0.9999712124433106, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6925027090001095, + "eta_squared": 0.001278796513683895, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.0716216250002617, + "eta_squared": 0.0013720012875030802, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.08743787500134204, + "eta_squared": 0.0013720012875030802, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9946789809149659, + "roc_auc": 0.999913637329932, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.710910709000018, + "eta_squared": 0.0013720012875030802, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.06848508300026879, + "eta_squared": 0.0017954572306766912, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07606524999937392, + "eta_squared": 0.0017954572306766912, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9973931657464519, + "roc_auc": 0.9999468537414967, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7286392080022779, + "eta_squared": 0.0017954572306766912, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9986808847878315, + "roc_auc": 0.9999734268707483, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9963831018518517, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.08059941700048512, + "eta_squared": 0.00084717659914683, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07574087500324822, + "eta_squared": 0.00084717659914683, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9950291456983933, + "roc_auc": 0.9999180661848074, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7295693329979258, + "eta_squared": 0.00084717659914683, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07634137499917415, + "eta_squared": 0.0012853540416852677, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.0719384589974652, + "eta_squared": 0.0012853540416852677, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.731033625001146, + "eta_squared": 0.0012853540416852677, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07527325000046403, + "eta_squared": 0.0012397194238675945, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07464133299799869, + "eta_squared": 0.0012397194238675945, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9980517045519615, + "roc_auc": 0.9999601403061226, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.699046958998224, + "eta_squared": 0.0012397194238675945, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07438695900054881, + "eta_squared": 0.0003383620246718858, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.08109362500181305, + "eta_squared": 0.0003383620246718858, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9948596494505617, + "roc_auc": 0.9999158517573696, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7164103750001232, + "eta_squared": 0.0003383620246718858, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 1, + "seconds": 0.07404937500177766, + "eta_squared": 0.0010503666837600957, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07101924999733455, + "eta_squared": 0.0010503666837600957, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7057310419986607, + "eta_squared": 0.0010503666837600957, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.08904483299920685, + "eta_squared": 0.002415635561632408, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9993384082076204, + "roc_auc": 0.9999867134353743, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07063283300158218, + "eta_squared": 0.002415635561632408, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.999572638333684, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7176964999998745, + "eta_squared": 0.002415635561632408, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 1, + "seconds": 0.07436258300003828, + "eta_squared": 0.0009561098242806195, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07921012500082725, + "eta_squared": 0.0009561098242806195, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.0, + "pr_auc": 0.9985638435893296, + "roc_auc": 0.9999712124433107, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9948950066137566, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7040092079987517, + "eta_squared": 0.0009561098242806195, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07160279199888464, + "eta_squared": 0.0007681373849954036, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07975612500013085, + "eta_squared": 0.0007681373849954036, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.6898896250022517, + "eta_squared": 0.0007681373849954036, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07450095899912412, + "eta_squared": 0.0023345860714670385, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07206262500039884, + "eta_squared": 0.0023345860714670385, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9523907508279281, + "roc_auc": 0.9992980265022675, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6941066250001313, + "eta_squared": 0.0023345860714670385, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07249937500091619, + "eta_squared": 0.0013075922239467576, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.998136673741906, + "roc_auc": 0.9999645691609977, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.0823573329980718, + "eta_squared": 0.0013075922239467576, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7552631670005212, + "eta_squared": 0.0013075922239467576, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9944819614427851, + "roc_auc": 0.9999092084750566, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07165954100128147, + "eta_squared": 0.0010029218229787487, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725625, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.0729617920005694, + "eta_squared": 0.0010029218229787487, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7429075420004665, + "eta_squared": 0.0010029218229787487, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07104595800046809, + "eta_squared": 0.0017217081640612903, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07243112500145799, + "eta_squared": 0.0017217081640612903, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7270605000012438, + "eta_squared": 0.0017217081640612903, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9970752459494735, + "roc_auc": 0.9999468537414967, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07321583299926715, + "eta_squared": 0.0012261319468726715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9991129213154376, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07625950000146986, + "eta_squared": 0.0012261319468726715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7202732090008794, + "eta_squared": 0.0012261319468726715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9964798833343682, + "roc_auc": 0.9999313527494331, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9976851851851851, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07321233300172025, + "eta_squared": 0.0012204410626207138, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9972031494854697, + "roc_auc": 0.9999490681689343, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07305512500170153, + "eta_squared": 0.0012204410626207138, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7081770830009191, + "eta_squared": 0.0012204410626207138, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.992820838204789, + "roc_auc": 0.9998804209183673, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9896288029100528, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07393937500091852, + "eta_squared": 0.0019679556564418787, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9974816945212338, + "roc_auc": 0.9999534970238095, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.06988745899798232, + "eta_squared": 0.0019679556564418787, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7241048750001937, + "eta_squared": 0.0019679556564418787, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07015695900190622, + "eta_squared": 0.001183606140824988, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.08318279200102552, + "eta_squared": 0.001183606140824988, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7309281660018314, + "eta_squared": 0.001183606140824988, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07772495899916976, + "eta_squared": 0.0010477149599897315, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07547470799909206, + "eta_squared": 0.0010477149599897315, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7729518750020361, + "eta_squared": 0.0010477149599897315, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.997490523278894, + "roc_auc": 0.9999534970238095, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07022299999880488, + "eta_squared": 0.0015314052300759431, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9996744556165971, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07097679100115784, + "eta_squared": 0.0015314052300759431, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7067097500003001, + "eta_squared": 0.0015314052300759431, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9713348315763177, + "roc_auc": 0.9997231965702948, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9783895502645503, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07170770800075843, + "eta_squared": 0.0008561578874620307, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9996744556165974, + "roc_auc": 0.9999933567176872, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.0818978329989477, + "eta_squared": 0.0008561578874620307, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.961741641669088, + "roc_auc": 0.9994884672619048, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7422807080001803, + "eta_squared": 0.0008561578874620307, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.08475429100144538, + "eta_squared": 0.0009042790866601086, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.0698327079990122, + "eta_squared": 0.0009042790866601086, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7110866250004619, + "eta_squared": 0.0009042790866601086, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9929780154537662, + "roc_auc": 0.9998937074829932, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9896288029100528, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07018608399812365, + "eta_squared": 0.0023824674825981534, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9995636400137602, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.08228349999990314, + "eta_squared": 0.0023824674825981534, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7147403329981898, + "eta_squared": 0.0023824674825981534, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07568112499939161, + "eta_squared": 0.0017212676917411547, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.9997841047394044, + "roc_auc": 0.9999955711451246, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07336058399960166, + "eta_squared": 0.0017212676917411547, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.000", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.0, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7232982089990401, + "eta_squared": 0.0017212676917411547, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06377551020408163, + "n_models": 1, + "seconds": 0.07654600000023493, + "eta_squared": 0.03955842865464311, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07850200000029872, + "eta_squared": 0.03955842865464311, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7702025409998896, + "eta_squared": 0.03955842865464311, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07867550000082701, + "eta_squared": 0.035766956730376886, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.08078166600171244, + "eta_squared": 0.035766956730376886, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9968128111867416, + "roc_auc": 0.999937996031746, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7648674589981965, + "eta_squared": 0.035766956730376886, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.08140275000187103, + "eta_squared": 0.0395464466000243, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.999019577035818, + "roc_auc": 0.9999800701530612, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9976851851851851, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07521029200142948, + "eta_squared": 0.0395464466000243, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9979533443255417, + "roc_auc": 0.9999557114512471, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7182910410010663, + "eta_squared": 0.0395464466000243, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07188695800141431, + "eta_squared": 0.038621152227959823, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07432549999793991, + "eta_squared": 0.038621152227959823, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7132319590018597, + "eta_squared": 0.038621152227959823, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07170508299896028, + "eta_squared": 0.0391862568727475, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.07496162499955972, + "eta_squared": 0.0391862568727475, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9989911574039089, + "roc_auc": 0.9999800701530611, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7146162909994018, + "eta_squared": 0.0391862568727475, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07139183300023433, + "eta_squared": 0.03294941366121079, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07158858299953863, + "eta_squared": 0.03294941366121079, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9983257405764419, + "roc_auc": 0.9999667835884354, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7419408749992726, + "eta_squared": 0.03294941366121079, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9991081986024108, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.07530224999936763, + "eta_squared": 0.03415590740792779, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.061224489795918366, + "n_models": 1, + "seconds": 0.07935620799980825, + "eta_squared": 0.03415590740792779, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9963227694148249, + "roc_auc": 0.9999357816043084, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7186046250026266, + "eta_squared": 0.03415590740792779, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.07252712499757763, + "eta_squared": 0.04191038320105364, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07778999999936786, + "eta_squared": 0.04191038320105364, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9982416315100598, + "roc_auc": 0.9999645691609977, + "precision_at_n": 0.96875, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7112914999997884, + "eta_squared": 0.04191038320105364, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.07115170800170745, + "eta_squared": 0.04475252345121021, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07341995900060283, + "eta_squared": 0.04475252345121021, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9960141105034992, + "roc_auc": 0.9999313527494331, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6964845830007107, + "eta_squared": 0.04475252345121021, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06377551020408163, + "n_models": 1, + "seconds": 0.07056295799702639, + "eta_squared": 0.03474882738233468, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07717137500003446, + "eta_squared": 0.03474882738233468, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6975382499986154, + "eta_squared": 0.03474882738233468, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.06700420800189022, + "eta_squared": 0.04213584867477234, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07658679099768051, + "eta_squared": 0.04213584867477234, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9984977607821051, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7315555829991354, + "eta_squared": 0.04213584867477234, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07062958299866295, + "eta_squared": 0.03654307103775154, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.06992241599800764, + "eta_squared": 0.03654307103775154, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9953729381748984, + "roc_auc": 0.9999224950396824, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7172980830000597, + "eta_squared": 0.03654307103775154, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.06928816700019524, + "eta_squared": 0.03988094020033876, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.0711717080012022, + "eta_squared": 0.03988094020033876, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7151609159991494, + "eta_squared": 0.03988094020033876, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9984216555199152, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.06899037500261329, + "eta_squared": 0.038582085800609046, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9988727861917879, + "roc_auc": 0.9999778557256236, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07521916699988651, + "eta_squared": 0.038582085800609046, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9995726383336838, + "roc_auc": 0.9999911422902493, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7040710830005992, + "eta_squared": 0.038582085800609046, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07072458299808204, + "eta_squared": 0.036627617689678975, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07034900000144262, + "eta_squared": 0.036627617689678975, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.05, + "pr_auc": 0.999244808841958, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7096689579993836, + "eta_squared": 0.036627617689678975, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.06995812500099419, + "eta_squared": 0.059293082499552834, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.07958750000034343, + "eta_squared": 0.059293082499552834, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7175909170000523, + "eta_squared": 0.059293082499552834, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9875576233231009, + "roc_auc": 0.9998494189342404, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.09570874999917578, + "eta_squared": 0.05670231707537825, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.10129179100113106, + "eta_squared": 0.05670231707537825, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9523907508279281, + "roc_auc": 0.9992980265022675, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7102379169991764, + "eta_squared": 0.05670231707537825, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9960506815613993, + "roc_auc": 0.9999247094671202, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9950810185185185, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07414150000113295, + "eta_squared": 0.05921841304093976, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.07848229100272874, + "eta_squared": 0.05921841304093976, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7420237920014188, + "eta_squared": 0.05921841304093976, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9994596097759277, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07248145799894701, + "eta_squared": 0.058057615078346766, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07383162499900209, + "eta_squared": 0.058057615078346766, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7287121660010598, + "eta_squared": 0.058057615078346766, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9992239393431516, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.07586733299831394, + "eta_squared": 0.05544497562800828, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07230329199956032, + "eta_squared": 0.05544497562800828, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6943149999970046, + "eta_squared": 0.05544497562800828, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9986319303600137, + "roc_auc": 0.9999734268707483, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.06787462500142283, + "eta_squared": 0.051197551161426436, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.061224489795918366, + "n_models": 1, + "seconds": 0.07406333299877588, + "eta_squared": 0.051197551161426436, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.6935085420009273, + "eta_squared": 0.051197551161426436, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9958579726891336, + "roc_auc": 0.9999291383219955, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.0663265306122449, + "n_models": 1, + "seconds": 0.07717508299901965, + "eta_squared": 0.050492980765247095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.07736512500196113, + "eta_squared": 0.050492980765247095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7311062079970725, + "eta_squared": 0.050492980765247095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9996744556165972, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07627204099844676, + "eta_squared": 0.05910198864179911, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9992239393431517, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.07063824999931967, + "eta_squared": 0.05910198864179911, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.695684500002244, + "eta_squared": 0.05910198864179911, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.06994925000253716, + "eta_squared": 0.06438107327632228, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07746212500205729, + "eta_squared": 0.06438107327632228, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6991803749988321, + "eta_squared": 0.06438107327632228, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.07582712500152411, + "eta_squared": 0.054805584022504586, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 1, + "seconds": 0.07222833300329512, + "eta_squared": 0.054805584022504586, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7066742080023687, + "eta_squared": 0.054805584022504586, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9937045293446132, + "roc_auc": 0.9998848497732427, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.061224489795918366, + "n_models": 1, + "seconds": 0.0901148339980864, + "eta_squared": 0.06303098656943196, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.08605449999959092, + "eta_squared": 0.06303098656943196, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7075918749978882, + "eta_squared": 0.06303098656943196, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9983853734038979, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.07811904200207209, + "eta_squared": 0.05142686270297512, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9964737355983997, + "roc_auc": 0.999937996031746, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07093691699992632, + "eta_squared": 0.05142686270297512, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9619580084145981, + "roc_auc": 0.99949289611678, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7219730830001936, + "eta_squared": 0.05142686270297512, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07191874999989523, + "eta_squared": 0.05895449280709131, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07913745800033212, + "eta_squared": 0.05895449280709131, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.712019375001546, + "eta_squared": 0.05895449280709131, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.983066282557806, + "roc_auc": 0.9997386975623582, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.07142857142857142, + "n_models": 1, + "seconds": 0.07618983300199034, + "eta_squared": 0.052277832661798654, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07268458399994415, + "eta_squared": 0.052277832661798654, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7438016249980137, + "eta_squared": 0.052277832661798654, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.9997841047394044, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.07378295799935586, + "eta_squared": 0.05617290915117752, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07609516699812957, + "eta_squared": 0.05617290915117752, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.050", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.05, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7129640410021238, + "eta_squared": 0.05617290915117752, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08673469387755102, + "n_models": 1, + "seconds": 0.07423887499680859, + "eta_squared": 0.12687309144122869, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07873454200307606, + "eta_squared": 0.12687309144122869, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.6979090000022552, + "eta_squared": 0.12687309144122869, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.0663265306122449, + "n_models": 1, + "seconds": 0.07222316700062947, + "eta_squared": 0.12119391519465993, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.07347983400177327, + "eta_squared": 0.12119391519465993, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9968373110910685, + "roc_auc": 0.999937996031746, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6928048749978188, + "eta_squared": 0.12119391519465993, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9902144031663376, + "roc_auc": 0.9998339179421768, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9922401094276094, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.07798179199744482, + "eta_squared": 0.12540942706236655, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9997841047394043, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.0712847499999043, + "eta_squared": 0.12540942706236655, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.998047536533169, + "roc_auc": 0.9999579258786848, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7157716250003432, + "eta_squared": 0.12540942706236655, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.07014016600078321, + "eta_squared": 0.12527474321356308, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07505729100012104, + "eta_squared": 0.12527474321356308, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7035883330026991, + "eta_squared": 0.12527474321356308, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.07179441699918243, + "eta_squared": 0.12512147981398453, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07887549999941257, + "eta_squared": 0.12512147981398453, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9986319303600136, + "roc_auc": 0.9999734268707483, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7104391660031979, + "eta_squared": 0.12512147981398453, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06377551020408163, + "n_models": 1, + "seconds": 0.07062662500175065, + "eta_squared": 0.1151837730978297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.07220429199878708, + "eta_squared": 0.1151837730978297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9990030018845484, + "roc_auc": 0.9999800701530611, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7412907910002104, + "eta_squared": 0.1151837730978297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9950557391870531, + "roc_auc": 0.9999158517573695, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07358395899791503, + "eta_squared": 0.11684378654258522, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.07783791700057918, + "eta_squared": 0.11684378654258522, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9966225451222093, + "roc_auc": 0.9999402104591838, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7159545840004284, + "eta_squared": 0.11684378654258522, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9992401664395811, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.07142070799818612, + "eta_squared": 0.12961477677268454, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07209687499926076, + "eta_squared": 0.12961477677268454, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9985539550199853, + "roc_auc": 0.9999712124433107, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.7082592910010135, + "eta_squared": 0.12961477677268454, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.07899929099949077, + "eta_squared": 0.13340934644786664, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.07169262499883189, + "eta_squared": 0.13340934644786664, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9955346576239892, + "roc_auc": 0.9999247094671201, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7164824999999837, + "eta_squared": 0.13340934644786664, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.07427245800136006, + "eta_squared": 0.11917540465192976, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07583237500148243, + "eta_squared": 0.11917540465192976, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7166787919995841, + "eta_squared": 0.11917540465192976, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.06855162499778089, + "eta_squared": 0.12876131345125413, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.07894258400119725, + "eta_squared": 0.12876131345125413, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9986062434248825, + "roc_auc": 0.9999712124433106, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7314287919980416, + "eta_squared": 0.12876131345125413, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07099874999767053, + "eta_squared": 0.1235703476453848, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07508233299813583, + "eta_squared": 0.1235703476453848, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9943210112718865, + "roc_auc": 0.9999092084750567, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7117179999986547, + "eta_squared": 0.1235703476453848, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.07244870900103706, + "eta_squared": 0.12854427437882665, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 1, + "seconds": 0.07468970800255192, + "eta_squared": 0.12854427437882665, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7214789999998175, + "eta_squared": 0.12854427437882665, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9981218997092937, + "roc_auc": 0.99996235473356, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.08673469387755102, + "n_models": 1, + "seconds": 0.07330120799815631, + "eta_squared": 0.1246847039613907, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9980007756782858, + "roc_auc": 0.9999601403061226, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.061224489795918366, + "n_models": 1, + "seconds": 0.08027399999991758, + "eta_squared": 0.1246847039613907, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9995726383336838, + "roc_auc": 0.9999911422902493, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7322338340018177, + "eta_squared": 0.1246847039613907, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06377551020408163, + "n_models": 1, + "seconds": 0.06885337499988964, + "eta_squared": 0.12175002598109345, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 1, + "seconds": 0.06756683399726171, + "eta_squared": 0.12175002598109345, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.1, + "pr_auc": 0.9992448088419582, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7063922909983376, + "eta_squared": 0.12175002598109345, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.07291979100045864, + "eta_squared": 0.19451621854380768, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07728233300076681, + "eta_squared": 0.19451621854380768, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7823825830018905, + "eta_squared": 0.19451621854380768, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9821201179067449, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9861565806878306, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.08046337500127265, + "eta_squared": 0.1898993994842026, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.07804666700030793, + "eta_squared": 0.1898993994842026, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.999289168792517, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7099072080018232, + "eta_squared": 0.1898993994842026, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9812650841708556, + "roc_auc": 0.9997874149659864, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9797908399470899, + "worst_group_fpr": 0.07142857142857142, + "n_models": 1, + "seconds": 0.0738493329990888, + "eta_squared": 0.1936752703132149, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.072544791000837, + "eta_squared": 0.1936752703132149, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7283174579970364, + "eta_squared": 0.1936752703132149, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08673469387755102, + "n_models": 1, + "seconds": 0.07802712500051712, + "eta_squared": 0.1934640118312397, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 1, + "seconds": 0.07160441700034426, + "eta_squared": 0.1934640118312397, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7246251670003403, + "eta_squared": 0.1934640118312397, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07575679200090235, + "eta_squared": 0.18861888416601494, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06377551020408163, + "n_models": 1, + "seconds": 0.07119949999832897, + "eta_squared": 0.18861888416601494, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6969671250008105, + "eta_squared": 0.18861888416601494, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9985428054545235, + "roc_auc": 0.9999712124433107, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.0693282909996924, + "eta_squared": 0.1805813085726683, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.06892870800220408, + "eta_squared": 0.1805813085726683, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.6985914169999887, + "eta_squared": 0.1805813085726683, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9928499229185896, + "roc_auc": 0.9998693487811791, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9918568121693121, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07400295900151832, + "eta_squared": 0.17980556561098363, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725622, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.08185012499961886, + "eta_squared": 0.17980556561098363, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7189363329998741, + "eta_squared": 0.17980556561098363, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.07125499999892781, + "eta_squared": 0.1935769586972524, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.0792962500017893, + "eta_squared": 0.1935769586972524, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.741542916999606, + "eta_squared": 0.1935769586972524, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.999233211489758, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.08608595800251351, + "eta_squared": 0.20068556264939091, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.08244766599818831, + "eta_squared": 0.20068556264939091, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7124477079996723, + "eta_squared": 0.20068556264939091, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.07079854100084049, + "eta_squared": 0.1874671200172165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.061224489795918366, + "n_models": 1, + "seconds": 0.0786639580001065, + "eta_squared": 0.1874671200172165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7268245829982334, + "eta_squared": 0.1874671200172165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9911302410778361, + "roc_auc": 0.9998405612244897, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.06961750000118627, + "eta_squared": 0.19900625679340145, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06377551020408163, + "n_models": 1, + "seconds": 0.07949883399851387, + "eta_squared": 0.19900625679340145, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.703285749998031, + "eta_squared": 0.19900625679340145, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9906410394177838, + "roc_auc": 0.9998604910714286, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.07260295900050551, + "eta_squared": 0.1831366419552775, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9953721770912125, + "roc_auc": 0.9999224950396826, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07940979200066067, + "eta_squared": 0.1831366419552775, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.961741641669088, + "roc_auc": 0.9994884672619048, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7247961670000223, + "eta_squared": 0.1831366419552775, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9991106879479051, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07304641699738568, + "eta_squared": 0.1962296761690674, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.07132141700276406, + "eta_squared": 0.1962296761690674, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6932837919994199, + "eta_squared": 0.1962296761690674, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9726214262302605, + "roc_auc": 0.9996700503117913, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.991075562169312, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.06829887500134646, + "eta_squared": 0.18278781206386457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.07537133300138521, + "eta_squared": 0.18278781206386457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.6942515420014388, + "eta_squared": 0.18278781206386457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.06851670900141471, + "eta_squared": 0.18987545660900912, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07073066699740593, + "eta_squared": 0.18987545660900912, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.100", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.1, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7188157089985907, + "eta_squared": 0.18987545660900912, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.06851633399855928, + "eta_squared": 0.21842060879328182, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.0663265306122449, + "n_models": 1, + "seconds": 0.07292166599654593, + "eta_squared": 0.21842060879328182, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.6916868750013236, + "eta_squared": 0.21842060879328182, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.07799687499937136, + "eta_squared": 0.2129441367707082, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.07319541599645163, + "eta_squared": 0.2129441367707082, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9968326319040454, + "roc_auc": 0.999937996031746, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7092204580003454, + "eta_squared": 0.2129441367707082, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9993556244447951, + "roc_auc": 0.9999867134353742, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.07785770800182945, + "eta_squared": 0.21607807398685436, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.06997070800207439, + "eta_squared": 0.21607807398685436, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9985636101934156, + "roc_auc": 0.9999689980158731, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7070650839996233, + "eta_squared": 0.21607807398685436, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.0689469579992874, + "eta_squared": 0.2167824088169093, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.0663265306122449, + "n_models": 1, + "seconds": 0.0699157919989375, + "eta_squared": 0.2167824088169093, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7044350830001349, + "eta_squared": 0.2167824088169093, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.06935874999908265, + "eta_squared": 0.2158502792872289, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.06596420899950317, + "eta_squared": 0.2158502792872289, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9993384082076204, + "roc_auc": 0.9999867134353742, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.709873374999006, + "eta_squared": 0.2158502792872289, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, + "n_models": 1, + "seconds": 0.07122862499818439, + "eta_squared": 0.20579994667793172, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 1, + "seconds": 0.07308333300170489, + "eta_squared": 0.20579994667793172, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9988859606860464, + "roc_auc": 0.9999778557256235, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7061194160014566, + "eta_squared": 0.20579994667793172, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.07734245900064707, + "eta_squared": 0.20697832970174135, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.07056250000096043, + "eta_squared": 0.20697832970174135, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9969139346631588, + "roc_auc": 0.999944639314059, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7153266250024899, + "eta_squared": 0.20697832970174135, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.06616024999675574, + "eta_squared": 0.2209510705543195, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.07109645799937425, + "eta_squared": 0.2209510705543195, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9987072535258925, + "roc_auc": 0.9999734268707484, + "precision_at_n": 0.96875, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.6969909580002422, + "eta_squared": 0.2209510705543195, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.07319562500197208, + "eta_squared": 0.223914333840144, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.0793693749983504, + "eta_squared": 0.223914333840144, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9956960258104144, + "roc_auc": 0.9999269238945577, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7002469580002071, + "eta_squared": 0.223914333840144, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.0686213749977469, + "eta_squared": 0.21066498526646607, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.0663265306122449, + "n_models": 1, + "seconds": 0.07141987499926472, + "eta_squared": 0.21066498526646607, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.682583000001614, + "eta_squared": 0.21066498526646607, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07027670800016494, + "eta_squared": 0.21894049322652165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07142857142857142, + "n_models": 1, + "seconds": 0.07136733299921616, + "eta_squared": 0.21894049322652165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9990329939892202, + "roc_auc": 0.9999800701530611, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7137086670009012, + "eta_squared": 0.21894049322652165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9993651772660819, + "roc_auc": 0.9999867134353742, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07226612500016927, + "eta_squared": 0.2159597863844126, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07840720899912412, + "eta_squared": 0.2159597863844126, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9939428770383998, + "roc_auc": 0.9999047796201814, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.6842563329992117, + "eta_squared": 0.2159597863844126, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08673469387755102, + "n_models": 1, + "seconds": 0.07295425000120304, + "eta_squared": 0.22105758156006608, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07012737499826471, + "eta_squared": 0.22105758156006608, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7247925829979067, + "eta_squared": 0.22105758156006608, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9987684430207726, + "roc_auc": 0.9999734268707483, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.07108045800123364, + "eta_squared": 0.21621224561046687, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.07759091600019019, + "eta_squared": 0.21621224561046687, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9996789080215416, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7220111249989714, + "eta_squared": 0.21621224561046687, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.07145308300096076, + "eta_squared": 0.21276020362000367, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.0916170830023475, + "eta_squared": 0.21276020362000367, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.15, + "pr_auc": 0.9990243190307221, + "roc_auc": 0.9999800701530612, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7191717089990561, + "eta_squared": 0.21276020362000367, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.07261966599980951, + "eta_squared": 0.3454157417517599, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.0727134999979171, + "eta_squared": 0.3454157417517599, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7135545409983024, + "eta_squared": 0.3454157417517599, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9801921610275169, + "roc_auc": 0.9997541985544217, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.988702876984127, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.07139412499964237, + "eta_squared": 0.34102334572506343, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.07210879099875456, + "eta_squared": 0.34102334572506343, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.9992891687925171, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7238602500001434, + "eta_squared": 0.34102334572506343, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9997874149659862, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.08066958299968974, + "eta_squared": 0.34453696882915624, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.07343699999910314, + "eta_squared": 0.34453696882915624, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7385112909978488, + "eta_squared": 0.34453696882915624, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.07873783299874049, + "eta_squared": 0.3456480927214205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, + "n_models": 1, + "seconds": 0.0889895410000463, + "eta_squared": 0.3456480927214205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7152433749979537, + "eta_squared": 0.3456480927214205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9993395503859831, + "roc_auc": 0.9999867134353743, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.07209704100023373, + "eta_squared": 0.34002683577528714, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07142857142857142, + "n_models": 1, + "seconds": 0.07163737500013667, + "eta_squared": 0.34002683577528714, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7301543749999837, + "eta_squared": 0.34002683577528714, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.971795730603734, + "roc_auc": 0.9995593289399093, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.1377551020408163, + "n_models": 1, + "seconds": 0.07648383299965644, + "eta_squared": 0.3305219088439165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9998926116838486, + "roc_auc": 0.9999977855725625, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07291724999959115, + "eta_squared": 0.3305219088439165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7337241659988649, + "eta_squared": 0.3305219088439165, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9997874149659864, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12244897959183673, + "n_models": 1, + "seconds": 0.08109112499732873, + "eta_squared": 0.32973445787453604, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, + "n_models": 1, + "seconds": 0.075488000002224, + "eta_squared": 0.32973445787453604, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7605138750004699, + "eta_squared": 0.32973445787453604, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9764156875457186, + "roc_auc": 0.9996545493197279, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9951264880952381, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.07583337500182097, + "eta_squared": 0.3446151723429949, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.07259120800154051, + "eta_squared": 0.3446151723429949, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7225472089994582, + "eta_squared": 0.3446151723429949, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9992239393431516, + "roc_auc": 0.9999844990079366, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.07207162499980768, + "eta_squared": 0.3507338333119392, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06377551020408163, + "n_models": 1, + "seconds": 0.07437016599942581, + "eta_squared": 0.3507338333119392, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6980373340011283, + "eta_squared": 0.3507338333119392, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9777801087320123, + "roc_auc": 0.9997918438208617, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9861565806878306, + "worst_group_fpr": 0.14030612244897958, + "n_models": 1, + "seconds": 0.07867308300046716, + "eta_squared": 0.33856980964700606, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.07843920899904333, + "eta_squared": 0.33856980964700606, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7272615830006544, + "eta_squared": 0.33856980964700606, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9991094704786827, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.07898945800116053, + "eta_squared": 0.3494306148914419, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06377551020408163, + "n_models": 1, + "seconds": 0.07821712499935529, + "eta_squared": 0.3494306148914419, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7282064579994767, + "eta_squared": 0.3494306148914419, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9936280415960002, + "roc_auc": 0.9998914930555556, + "precision_at_n": 0.96875, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.07279424999796902, + "eta_squared": 0.33458322121718237, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9991081986024111, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.07553291600197554, + "eta_squared": 0.33458322121718237, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9619580084145981, + "roc_auc": 0.99949289611678, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7148141659999965, + "eta_squared": 0.33458322121718237, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.07439137499750359, + "eta_squared": 0.3490755282832429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.08225108300030115, + "eta_squared": 0.3490755282832429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7257115419997717, + "eta_squared": 0.3490755282832429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9238049983388068, + "roc_auc": 0.9992094494047619, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9592280693843195, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.06664079099937226, + "eta_squared": 0.3333216444268115, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, + "n_models": 1, + "seconds": 0.07816883299892652, + "eta_squared": 0.3333216444268115, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7100955829992017, + "eta_squared": 0.3333216444268115, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9864046213824539, + "roc_auc": 0.9998361323696145, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.07676120799806085, + "eta_squared": 0.34143017232311446, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.0663265306122449, + "n_models": 1, + "seconds": 0.07141820800097776, + "eta_squared": 0.34143017232311446, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.150", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7144282080007542, + "eta_squared": 0.34143017232311446, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.07433545900130412, + "eta_squared": 0.2935663417355178, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07142857142857142, + "n_models": 1, + "seconds": 0.07685870799832628, + "eta_squared": 0.2935663417355178, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7047439999987546, + "eta_squared": 0.2935663417355178, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.06890383399877464, + "eta_squared": 0.288953232932902, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.07123420800053282, + "eta_squared": 0.288953232932902, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9972700593819733, + "roc_auc": 0.9999468537414966, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7110342499981925, + "eta_squared": 0.288953232932902, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9995636400137603, + "roc_auc": 0.9999911422902493, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.07179974999962724, + "eta_squared": 0.29120655697879977, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07142857142857142, + "n_models": 1, + "seconds": 0.07621354099683231, + "eta_squared": 0.29120655697879977, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9984301094341246, + "roc_auc": 0.9999667835884354, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7028574159994605, + "eta_squared": 0.29120655697879977, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.07623379199867486, + "eta_squared": 0.2923445749968756, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.08033066699863411, + "eta_squared": 0.2923445749968756, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6950934169981338, + "eta_squared": 0.2923445749968756, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.06914320799842244, + "eta_squared": 0.29078029700670166, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11989795918367346, + "n_models": 1, + "seconds": 0.07281108300230699, + "eta_squared": 0.29078029700670166, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9993384082076205, + "roc_auc": 0.9999867134353742, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7129051669980981, + "eta_squared": 0.29078029700670166, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.06847087500136695, + "eta_squared": 0.282343864590447, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.0663265306122449, + "n_models": 1, + "seconds": 0.07125195800108486, + "eta_squared": 0.282343864590447, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9985281258723928, + "roc_auc": 0.9999712124433106, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7077696249980363, + "eta_squared": 0.282343864590447, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9991081986024108, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11989795918367346, + "n_models": 1, + "seconds": 0.06953916599741206, + "eta_squared": 0.2827224241416107, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07077208299961057, + "eta_squared": 0.2827224241416107, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9973362833817405, + "roc_auc": 0.9999512825963719, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7117570829977922, + "eta_squared": 0.2827224241416107, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.07288400000106776, + "eta_squared": 0.2957817245569126, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.06842374999905587, + "eta_squared": 0.2957817245569126, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9986862958989656, + "roc_auc": 0.9999734268707483, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.718693249997159, + "eta_squared": 0.2957817245569126, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12755102040816327, + "n_models": 1, + "seconds": 0.07095466699684039, + "eta_squared": 0.2975501660008078, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05612244897959184, + "n_models": 1, + "seconds": 0.0700917499998468, + "eta_squared": 0.2975501660008078, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9958562822206711, + "roc_auc": 0.9999291383219955, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.703653917000338, + "eta_squared": 0.2975501660008078, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.07769012500284589, + "eta_squared": 0.28690079185598694, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.06761479199849418, + "eta_squared": 0.28690079185598694, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7115977500034205, + "eta_squared": 0.28690079185598694, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.10467395900195697, + "eta_squared": 0.2932065449666956, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07399174999954994, + "eta_squared": 0.2932065449666956, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9988204781635737, + "roc_auc": 0.9999756412981858, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7070473329986271, + "eta_squared": 0.2932065449666956, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.06950833299924852, + "eta_squared": 0.2919163375746864, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07058487500034971, + "eta_squared": 0.2919163375746864, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9950307939051444, + "roc_auc": 0.9999180661848073, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7124103749993083, + "eta_squared": 0.2919163375746864, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.07182687499880558, + "eta_squared": 0.2964580999866939, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.061224489795918366, + "n_models": 1, + "seconds": 0.07359462499880465, + "eta_squared": 0.2964580999866939, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.6911854169993603, + "eta_squared": 0.2964580999866939, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.07328529200094636, + "eta_squared": 0.29175866652881005, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9998926116838486, + "roc_auc": 0.9999977855725625, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.06887755102040816, + "n_models": 1, + "seconds": 0.07719345800069277, + "eta_squared": 0.29175866652881005, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9995726383336836, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7007327499995881, + "eta_squared": 0.29175866652881005, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.06992137500128592, + "eta_squared": 0.28829596838598986, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, + "n_models": 1, + "seconds": 0.0699008750016219, + "eta_squared": 0.28829596838598986, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.2, + "pr_auc": 0.9992448088419582, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.7195295839992468, + "eta_squared": 0.28829596838598986, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9826354837074835, + "roc_auc": 0.9996766935941044, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9892361111111111, + "worst_group_fpr": 0.16071428571428573, + "n_models": 1, + "seconds": 0.07226437500139582, + "eta_squared": 0.4778476892757978, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.07525420900128665, + "eta_squared": 0.4778476892757978, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7041030830005184, + "eta_squared": 0.4778476892757978, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9686266973751816, + "roc_auc": 0.9996611926020408, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.981712962962963, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.07087304200103972, + "eta_squared": 0.4744098790454379, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.06929333399966708, + "eta_squared": 0.4744098790454379, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.9992891687925171, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7102770840028825, + "eta_squared": 0.4744098790454379, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9419522669342344, + "roc_auc": 0.9991651608560091, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9726416523291522, + "worst_group_fpr": 0.1377551020408163, + "n_models": 1, + "seconds": 0.07590425000307732, + "eta_squared": 0.4772875504667477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08673469387755102, + "n_models": 1, + "seconds": 0.07962629200119409, + "eta_squared": 0.4772875504667477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7056990419987414, + "eta_squared": 0.4772875504667477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9816622918044869, + "roc_auc": 0.9997364831349207, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9950810185185185, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.07474441699741874, + "eta_squared": 0.47920852546400317, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.07266374999744585, + "eta_squared": 0.47920852546400317, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7114176249997399, + "eta_squared": 0.47920852546400317, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9736044464091245, + "roc_auc": 0.999696623441043, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9818617724867723, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.07567225000093458, + "eta_squared": 0.47375497337598166, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07178654200106394, + "eta_squared": 0.47375497337598166, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7036598749982659, + "eta_squared": 0.47375497337598166, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9172572822230664, + "roc_auc": 0.9992648100907029, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.96766587000962, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.07218012499652104, + "eta_squared": 0.4645660409992461, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.07338595900000655, + "eta_squared": 0.4645660409992461, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.6958567920009955, + "eta_squared": 0.4645660409992461, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9976369764612152, + "roc_auc": 0.9999557114512472, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14030612244897958, + "n_models": 1, + "seconds": 0.07794729200031725, + "eta_squared": 0.46373875898462746, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07319979200110538, + "eta_squared": 0.46373875898462746, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7167679579979449, + "eta_squared": 0.46373875898462746, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9861848812887417, + "roc_auc": 0.9998007015306123, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9929976851851853, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.06588654200095334, + "eta_squared": 0.47745063229238155, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.07609766699897591, + "eta_squared": 0.47745063229238155, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7205418749981618, + "eta_squared": 0.47745063229238155, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9895686047987848, + "roc_auc": 0.999820631377551, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.0699456670008658, + "eta_squared": 0.48200866700409517, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.07046799999807263, + "eta_squared": 0.48200866700409517, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6927875830006087, + "eta_squared": 0.48200866700409517, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9800971167255851, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9941137566137566, + "worst_group_fpr": 0.1683673469387755, + "n_models": 1, + "seconds": 0.07665895799800637, + "eta_squared": 0.4722715287711093, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.07793999999921652, + "eta_squared": 0.4722715287711093, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7258374170014577, + "eta_squared": 0.4722715287711093, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9659399603165795, + "roc_auc": 0.9997121244331066, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9888186177248678, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.07170687499819905, + "eta_squared": 0.4809671284581783, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07701929199902224, + "eta_squared": 0.4809671284581783, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7040943329993752, + "eta_squared": 0.4809671284581783, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9251687155968192, + "roc_auc": 0.9991939484126984, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9708276966089465, + "worst_group_fpr": 0.1377551020408163, + "n_models": 1, + "seconds": 0.07568179200097802, + "eta_squared": 0.469061169841381, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07196716600083164, + "eta_squared": 0.469061169841381, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9619580084145981, + "roc_auc": 0.99949289611678, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.6845039580002776, + "eta_squared": 0.469061169841381, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9993384082076205, + "roc_auc": 0.9999867134353742, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15051020408163265, + "n_models": 1, + "seconds": 0.07087045799926273, + "eta_squared": 0.4822963153336835, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.0727752499988128, + "eta_squared": 0.4822963153336835, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7130109160025313, + "eta_squared": 0.4822963153336835, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9398813761507946, + "roc_auc": 0.9993223852040816, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.988420664983165, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.07730920900212368, + "eta_squared": 0.4673396279875538, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.07188620799934142, + "eta_squared": 0.4673396279875538, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.705412957999215, + "eta_squared": 0.4673396279875538, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9820778190927275, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.15051020408163265, + "n_models": 1, + "seconds": 0.06871708399921772, + "eta_squared": 0.4750062627828297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07639395799924387, + "eta_squared": 0.4750062627828297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.200", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.2, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7285070000034466, + "eta_squared": 0.4750062627828297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1836734693877551, + "n_models": 1, + "seconds": 0.0747485419997247, + "eta_squared": 0.3927094336787097, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.07151725000221631, + "eta_squared": 0.3927094336787097, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.6960212080011843, + "eta_squared": 0.3927094336787097, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.07316204100061441, + "eta_squared": 0.38945558020781024, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.06780370799970115, + "eta_squared": 0.38945558020781024, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.997984202808983, + "roc_auc": 0.9999601403061225, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7270031250009197, + "eta_squared": 0.38945558020781024, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9997841047394044, + "roc_auc": 0.9999955711451246, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.06545379100134596, + "eta_squared": 0.391417879351872, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07216316600170103, + "eta_squared": 0.391417879351872, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9980440120074874, + "roc_auc": 0.9999579258786848, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7017973340007302, + "eta_squared": 0.391417879351872, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.20408163265306123, + "n_models": 1, + "seconds": 0.06790637499943841, + "eta_squared": 0.3925992743462009, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07433179200234008, + "eta_squared": 0.3925992743462009, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7195739169983426, + "eta_squared": 0.3925992743462009, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.07371741600218229, + "eta_squared": 0.39013809068873967, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.06969729200136499, + "eta_squared": 0.39013809068873967, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9994516328453016, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7018840829987312, + "eta_squared": 0.39013809068873967, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.06848408400037442, + "eta_squared": 0.385337437465984, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.06798533400069573, + "eta_squared": 0.385337437465984, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9984204639542932, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.710395709000295, + "eta_squared": 0.385337437465984, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16581632653061223, + "n_models": 1, + "seconds": 0.07513691599888261, + "eta_squared": 0.38448012991671376, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.06959566600198741, + "eta_squared": 0.38448012991671376, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9982598713958658, + "roc_auc": 0.9999667835884354, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7011397080023016, + "eta_squared": 0.38448012991671376, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17346938775510204, + "n_models": 1, + "seconds": 0.06881287499709288, + "eta_squared": 0.3945500497199951, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.06850195799779613, + "eta_squared": 0.3945500497199951, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9982623386901864, + "roc_auc": 0.9999645691609979, + "precision_at_n": 0.96875, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05102040816326531, + "n_models": 12, + "seconds": 0.7089162079973903, + "eta_squared": 0.3945500497199951, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17857142857142858, + "n_models": 1, + "seconds": 0.07024066699887044, + "eta_squared": 0.39452480740459633, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07150866699885228, + "eta_squared": 0.39452480740459633, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9960141105034992, + "roc_auc": 0.9999313527494331, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7044262499985052, + "eta_squared": 0.39452480740459633, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16581632653061223, + "n_models": 1, + "seconds": 0.0690706669993233, + "eta_squared": 0.3881032222309504, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.07449799999812967, + "eta_squared": 0.3881032222309504, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6935719579996658, + "eta_squared": 0.3881032222309504, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.0743379579980683, + "eta_squared": 0.39197938133375715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07260508299805224, + "eta_squared": 0.39197938133375715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9990474418934239, + "roc_auc": 0.9999800701530612, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.6825684169998567, + "eta_squared": 0.39197938133375715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11989795918367346, + "n_models": 1, + "seconds": 0.0703274160005094, + "eta_squared": 0.39171866046305087, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.06744299999991199, + "eta_squared": 0.39171866046305087, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9961709131539347, + "roc_auc": 0.9999335671768708, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7208609170011187, + "eta_squared": 0.39171866046305087, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.07486599999901955, + "eta_squared": 0.39513960651559954, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07409995800117031, + "eta_squared": 0.39513960651559954, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7254400829988299, + "eta_squared": 0.39513960651559954, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.06718825000280049, + "eta_squared": 0.39167748869152313, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.06884524999986752, + "eta_squared": 0.39167748869152313, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9996789080215419, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7007277500015334, + "eta_squared": 0.39167748869152313, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.07341037500009406, + "eta_squared": 0.38870468849861617, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.06719725000220933, + "eta_squared": 0.38870468849861617, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.3, + "pr_auc": 0.9996789080215419, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.716530290999799, + "eta_squared": 0.38870468849861617, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9263657652290938, + "roc_auc": 0.9989968643707483, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9875038156288157, + "worst_group_fpr": 0.20918367346938777, + "n_models": 1, + "seconds": 0.07047358400086523, + "eta_squared": 0.6640971283653139, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.07679395900049713, + "eta_squared": 0.6640971283653139, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7105053749983199, + "eta_squared": 0.6640971283653139, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9265826092568507, + "roc_auc": 0.9990920847505669, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9703647336459836, + "worst_group_fpr": 0.18622448979591838, + "n_models": 1, + "seconds": 0.06875366700114682, + "eta_squared": 0.6621686277077374, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.07715662500049802, + "eta_squared": 0.6621686277077374, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.999289168792517, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6922855829980108, + "eta_squared": 0.6621686277077374, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9363247964140696, + "roc_auc": 0.9990677260487528, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9933903769841269, + "worst_group_fpr": 0.1760204081632653, + "n_models": 1, + "seconds": 0.06781591699837008, + "eta_squared": 0.6640597584313623, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.06943708300241269, + "eta_squared": 0.6640597584313623, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6979062919999706, + "eta_squared": 0.6640597584313623, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9676978194854132, + "roc_auc": 0.9994176055839001, + "precision_at_n": 0.8854166666666666, + "macro_pr_auc": 0.9979166666666667, + "worst_group_fpr": 0.18112244897959184, + "n_models": 1, + "seconds": 0.07692316700195079, + "eta_squared": 0.6662381870514966, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.07189474999904633, + "eta_squared": 0.6662381870514966, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7306125830000383, + "eta_squared": 0.6662381870514966, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9907521318855751, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.07344716700026765, + "eta_squared": 0.6620305481663469, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.06924670800071908, + "eta_squared": 0.6620305481663469, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7220302500027174, + "eta_squared": 0.6620305481663469, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9048015569661748, + "roc_auc": 0.9989968643707483, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9823273659211158, + "worst_group_fpr": 0.1989795918367347, + "n_models": 1, + "seconds": 0.0702958750007383, + "eta_squared": 0.6550141848809792, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.07197683300182689, + "eta_squared": 0.6550141848809792, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7107337910019851, + "eta_squared": 0.6550141848809792, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9884679414612956, + "roc_auc": 0.9998117736678005, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18112244897959184, + "n_models": 1, + "seconds": 0.06940833300177474, + "eta_squared": 0.6541683708984072, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.07447241599948029, + "eta_squared": 0.6541683708984072, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7033861659983813, + "eta_squared": 0.6541683708984072, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9679302253727047, + "roc_auc": 0.9995681866496597, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9916087962962963, + "worst_group_fpr": 0.18622448979591838, + "n_models": 1, + "seconds": 0.06832433399904403, + "eta_squared": 0.6641774551533273, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.06596158300089883, + "eta_squared": 0.6641774551533273, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.6846688330006145, + "eta_squared": 0.6641774551533273, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9711104008701593, + "roc_auc": 0.9996124751984127, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.18112244897959184, + "n_models": 1, + "seconds": 0.07261400000061258, + "eta_squared": 0.666609652326715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.0710116249974817, + "eta_squared": 0.666609652326715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7163267500000075, + "eta_squared": 0.666609652326715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9659613202417214, + "roc_auc": 0.9997143388605442, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9842179232804232, + "worst_group_fpr": 0.1913265306122449, + "n_models": 1, + "seconds": 0.07235275000130059, + "eta_squared": 0.6607993707927986, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12755102040816327, + "n_models": 1, + "seconds": 0.07504649999827961, + "eta_squared": 0.6607993707927986, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7094101250004314, + "eta_squared": 0.6607993707927986, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.941703041760084, + "roc_auc": 0.9992714533730158, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9908966901154401, + "worst_group_fpr": 0.1760204081632653, + "n_models": 1, + "seconds": 0.07642904200110934, + "eta_squared": 0.6655080804629401, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.07010483300109627, + "eta_squared": 0.6655080804629401, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7144434169968008, + "eta_squared": 0.6655080804629401, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9367880988062811, + "roc_auc": 0.999209449404762, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9861137415824915, + "worst_group_fpr": 0.16071428571428573, + "n_models": 1, + "seconds": 0.0789701249996142, + "eta_squared": 0.6590680406076661, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08673469387755102, + "n_models": 1, + "seconds": 0.07323458300015773, + "eta_squared": 0.6590680406076661, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9619580084145981, + "roc_auc": 0.99949289611678, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7030437920002441, + "eta_squared": 0.6590680406076661, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9796856156394346, + "roc_auc": 0.999734268707483, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9976851851851851, + "worst_group_fpr": 0.18622448979591838, + "n_models": 1, + "seconds": 0.06972791699809022, + "eta_squared": 0.6680621080435837, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.07147479199920781, + "eta_squared": 0.6680621080435837, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7273979999990843, + "eta_squared": 0.6680621080435837, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9004911739406816, + "roc_auc": 0.9989105017006803, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9697574705387204, + "worst_group_fpr": 0.17346938775510204, + "n_models": 1, + "seconds": 0.0703660420003871, + "eta_squared": 0.6571877895615115, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.07190329200238921, + "eta_squared": 0.6571877895615115, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7175949580014276, + "eta_squared": 0.6571877895615115, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9726997232911961, + "roc_auc": 0.9996944090136054, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.990549467893218, + "worst_group_fpr": 0.18877551020408162, + "n_models": 1, + "seconds": 0.07027662499967846, + "eta_squared": 0.6627275317190392, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.07170183299967903, + "eta_squared": 0.6627275317190392, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.300", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.3, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7343082910010708, + "eta_squared": 0.6627275317190392, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1989795918367347, + "n_models": 1, + "seconds": 0.06916733300022315, + "eta_squared": 0.44964427142161606, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.07196804200066254, + "eta_squared": 0.44964427142161606, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7126680000001215, + "eta_squared": 0.44964427142161606, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15051020408163265, + "n_models": 1, + "seconds": 0.07681637499990757, + "eta_squared": 0.4469631140734328, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12244897959183673, + "n_models": 1, + "seconds": 0.0704585000021325, + "eta_squared": 0.4469631140734328, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9982327768083502, + "roc_auc": 0.9999645691609977, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7689973329979694, + "eta_squared": 0.4469631140734328, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1760204081632653, + "n_models": 1, + "seconds": 0.06915779200062389, + "eta_squared": 0.449549676187598, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07148883300033049, + "eta_squared": 0.449549676187598, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.99935217360804, + "roc_auc": 0.999986713435374, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7156301670001994, + "eta_squared": 0.449549676187598, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.23214285714285715, + "n_models": 1, + "seconds": 0.07223383399832528, + "eta_squared": 0.45043297969341634, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.07250633300282061, + "eta_squared": 0.45043297969341634, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7102636250019714, + "eta_squared": 0.45043297969341634, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17091836734693877, + "n_models": 1, + "seconds": 0.07457300000169198, + "eta_squared": 0.4473170350696837, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.07632920900141471, + "eta_squared": 0.4473170350696837, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9995636400137603, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7027896670006157, + "eta_squared": 0.4473170350696837, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18112244897959184, + "n_models": 1, + "seconds": 0.07392875000005006, + "eta_squared": 0.4449952275639174, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07942145800188882, + "eta_squared": 0.4449952275639174, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9990030018845484, + "roc_auc": 0.9999800701530612, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7168002500002331, + "eta_squared": 0.4449952275639174, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17857142857142858, + "n_models": 1, + "seconds": 0.07417616699967766, + "eta_squared": 0.4435643816373198, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.07187375000285101, + "eta_squared": 0.4435643816373198, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9981328388755407, + "roc_auc": 0.9999645691609979, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8298017499982961, + "eta_squared": 0.4435643816373198, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18622448979591838, + "n_models": 1, + "seconds": 0.07912500000020373, + "eta_squared": 0.4514009747359702, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.07275858399952995, + "eta_squared": 0.4514009747359702, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9985821949802475, + "roc_auc": 0.9999712124433106, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7548948750009004, + "eta_squared": 0.4514009747359702, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1913265306122449, + "n_models": 1, + "seconds": 0.07519174999833922, + "eta_squared": 0.4503463925177883, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07878029199855519, + "eta_squared": 0.4503463925177883, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9955908290925521, + "roc_auc": 0.9999247094671203, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9931588955026456, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7416448750009295, + "eta_squared": 0.4503463925177883, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18877551020408162, + "n_models": 1, + "seconds": 0.06985316700229305, + "eta_squared": 0.4460711711615254, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.0776381669966213, + "eta_squared": 0.4460711711615254, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7081212079974648, + "eta_squared": 0.4460711711615254, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.07157841700245626, + "eta_squared": 0.44924129916852445, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07996595800068462, + "eta_squared": 0.44924129916852445, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9991450011576791, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.721172416000627, + "eta_squared": 0.44924129916852445, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15051020408163265, + "n_models": 1, + "seconds": 0.06692212500274763, + "eta_squared": 0.44859304750127427, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.07617491600103676, + "eta_squared": 0.44859304750127427, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.996324417621576, + "roc_auc": 0.9999357816043084, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7081812920005177, + "eta_squared": 0.44859304750127427, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.19642857142857142, + "n_models": 1, + "seconds": 0.06883758299954934, + "eta_squared": 0.4514629140297388, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07397959183673469, + "n_models": 1, + "seconds": 0.06850795800346532, + "eta_squared": 0.4514629140297388, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7134700000024168, + "eta_squared": 0.4514629140297388, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.07810383400283172, + "eta_squared": 0.4491049550478841, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07709904199873563, + "eta_squared": 0.4491049550478841, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9996789080215419, + "roc_auc": 0.9999933567176872, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7326332919983543, + "eta_squared": 0.4491049550478841, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18877551020408162, + "n_models": 1, + "seconds": 0.07244779199754703, + "eta_squared": 0.44655634205910016, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07034012499934761, + "eta_squared": 0.44655634205910016, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.4, + "pr_auc": 0.9996744556165973, + "roc_auc": 0.999993356717687, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.7143351250015257, + "eta_squared": 0.44655634205910016, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.919494070332322, + "roc_auc": 0.9988728564342403, + "precision_at_n": 0.8854166666666666, + "macro_pr_auc": 0.9884958791208791, + "worst_group_fpr": 0.25255102040816324, + "n_models": 1, + "seconds": 0.07150508300037473, + "eta_squared": 0.7723857444189095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.08206308399894624, + "eta_squared": 0.7723857444189095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7402828750018671, + "eta_squared": 0.7723857444189095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9222452035785629, + "roc_auc": 0.999171804138322, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9704387626262626, + "worst_group_fpr": 0.19387755102040816, + "n_models": 1, + "seconds": 0.08093129199914983, + "eta_squared": 0.7711687637922273, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12755102040816327, + "n_models": 1, + "seconds": 0.07064316700052586, + "eta_squared": 0.7711687637922273, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9523907508279281, + "roc_auc": 0.9992980265022675, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7176182500006689, + "eta_squared": 0.7711687637922273, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9502640920205745, + "roc_auc": 0.9993002409297052, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9971590909090909, + "worst_group_fpr": 0.22193877551020408, + "n_models": 1, + "seconds": 0.07786358299927088, + "eta_squared": 0.772529544751337, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.07384237499718438, + "eta_squared": 0.772529544751337, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7333828750015527, + "eta_squared": 0.772529544751337, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.8310530351619684, + "roc_auc": 0.9972474666950113, + "precision_at_n": 0.7708333333333334, + "macro_pr_auc": 0.9805530118030119, + "worst_group_fpr": 0.24489795918367346, + "n_models": 1, + "seconds": 0.0734578750016226, + "eta_squared": 0.7743402728249429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12244897959183673, + "n_models": 1, + "seconds": 0.07175804199869162, + "eta_squared": 0.7743402728249429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7108539579967328, + "eta_squared": 0.7743402728249429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9504117563479513, + "roc_auc": 0.9995128259637187, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9752645502645502, + "worst_group_fpr": 0.20918367346938777, + "n_models": 1, + "seconds": 0.06905545900008292, + "eta_squared": 0.7712908836823555, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.07561912500023027, + "eta_squared": 0.7712908836823555, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.741537582998717, + "eta_squared": 0.7712908836823555, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9157899293207066, + "roc_auc": 0.9991784474206349, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.992205710955711, + "worst_group_fpr": 0.23979591836734693, + "n_models": 1, + "seconds": 0.07177312500061817, + "eta_squared": 0.7661755359039057, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.07254974999887054, + "eta_squared": 0.7661755359039057, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7203230000013718, + "eta_squared": 0.7661755359039057, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9714622579844718, + "roc_auc": 0.9994818239795918, + "precision_at_n": 0.90625, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18877551020408162, + "n_models": 1, + "seconds": 0.07678883300104644, + "eta_squared": 0.765390043535227, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.06896087499990244, + "eta_squared": 0.765390043535227, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.72620470900074, + "eta_squared": 0.765390043535227, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.8365449171531172, + "roc_auc": 0.997674851190476, + "precision_at_n": 0.8020833333333334, + "macro_pr_auc": 0.9779265873015873, + "worst_group_fpr": 0.21683673469387754, + "n_models": 1, + "seconds": 0.06944787499742233, + "eta_squared": 0.7725358121772077, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.0715800420002779, + "eta_squared": 0.7725358121772077, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7381474160029029, + "eta_squared": 0.7725358121772077, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9533739644744637, + "roc_auc": 0.9993644593253969, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9951264880952381, + "worst_group_fpr": 0.2193877551020408, + "n_models": 1, + "seconds": 0.07220670799870277, + "eta_squared": 0.7740110430937635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.06808333399749245, + "eta_squared": 0.7740110430937635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7301062500009721, + "eta_squared": 0.7740110430937635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9609648437754886, + "roc_auc": 0.9993556016156463, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9879133597883598, + "worst_group_fpr": 0.21428571428571427, + "n_models": 1, + "seconds": 0.07667612499790266, + "eta_squared": 0.7703322802500099, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.08043412499682745, + "eta_squared": 0.7703322802500099, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7481633329989563, + "eta_squared": 0.7703322802500099, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.923013720180947, + "roc_auc": 0.999109800170068, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9822616041366041, + "worst_group_fpr": 0.19642857142857142, + "n_models": 1, + "seconds": 0.06946004199926392, + "eta_squared": 0.7727219503559253, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07302362500195159, + "eta_squared": 0.7727219503559253, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7149331670007086, + "eta_squared": 0.7727219503559253, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.8567167741821362, + "roc_auc": 0.998764349489796, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9580439814814815, + "worst_group_fpr": 0.1913265306122449, + "n_models": 1, + "seconds": 0.07800854200104368, + "eta_squared": 0.769516047151419, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.07188725000014529, + "eta_squared": 0.769516047151419, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9619580084145981, + "roc_auc": 0.99949289611678, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7189485419985431, + "eta_squared": 0.769516047151419, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9995692588987349, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.20918367346938777, + "n_models": 1, + "seconds": 0.07393587499973364, + "eta_squared": 0.7753234694608296, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.06857591600055457, + "eta_squared": 0.7753234694608296, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7238538749988948, + "eta_squared": 0.7753234694608296, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9323072531442229, + "roc_auc": 0.9991297300170068, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9873594576719578, + "worst_group_fpr": 0.20153061224489796, + "n_models": 1, + "seconds": 0.07103625000308966, + "eta_squared": 0.7678020127711631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.07522237499870243, + "eta_squared": 0.7678020127711631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9329070122690577, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7089806250005495, + "eta_squared": 0.7678020127711631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.8998203771520226, + "roc_auc": 0.9989857922335601, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9762887286324786, + "worst_group_fpr": 0.20153061224489796, + "n_models": 1, + "seconds": 0.07112204199802363, + "eta_squared": 0.7715725614681234, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.07155020799837075, + "eta_squared": 0.7715725614681234, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.400", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.4, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7092746669986809, + "eta_squared": 0.7715725614681234, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.21683673469387754, + "n_models": 1, + "seconds": 0.07030904099883628, + "eta_squared": 0.4857308726173069, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.07049504200040246, + "eta_squared": 0.4857308726173069, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.6890660829994886, + "eta_squared": 0.4857308726173069, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1760204081632653, + "n_models": 1, + "seconds": 0.06879945900072926, + "eta_squared": 0.4832253415278028, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.06889812500230619, + "eta_squared": 0.4832253415278028, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9979007406391652, + "roc_auc": 0.9999579258786848, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7033612500017625, + "eta_squared": 0.4832253415278028, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.23979591836734693, + "n_models": 1, + "seconds": 0.0700648749989341, + "eta_squared": 0.48660199721043473, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.07226320799964014, + "eta_squared": 0.48660199721043473, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9992448088419579, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7190990830022201, + "eta_squared": 0.48660199721043473, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.25255102040816324, + "n_models": 1, + "seconds": 0.07167383399792016, + "eta_squared": 0.4871724603545716, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.07642195800144691, + "eta_squared": 0.4871724603545716, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7078079580023768, + "eta_squared": 0.4871724603545716, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16326530612244897, + "n_models": 1, + "seconds": 0.07651633400018909, + "eta_squared": 0.4835220144166391, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14540816326530612, + "n_models": 1, + "seconds": 0.0694544580001093, + "eta_squared": 0.4835220144166391, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9995636400137602, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.6862827090008068, + "eta_squared": 0.4835220144166391, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9998926116838486, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1989795918367347, + "n_models": 1, + "seconds": 0.07201791599800345, + "eta_squared": 0.4827570725669257, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.06927183299922035, + "eta_squared": 0.4827570725669257, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9994516328453016, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.727608708999469, + "eta_squared": 0.4827570725669257, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.19642857142857142, + "n_models": 1, + "seconds": 0.07113816700075404, + "eta_squared": 0.4811183421127033, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.07138120899981004, + "eta_squared": 0.4811183421127033, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9980042380524952, + "roc_auc": 0.9999623547335601, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7304766249981185, + "eta_squared": 0.4811183421127033, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18622448979591838, + "n_models": 1, + "seconds": 0.06869233299948974, + "eta_squared": 0.48752906081921704, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.07400212500215275, + "eta_squared": 0.48752906081921704, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9983640883002555, + "roc_auc": 0.9999667835884354, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.7849107920010283, + "eta_squared": 0.48752906081921704, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2423469387755102, + "n_models": 1, + "seconds": 0.08595991699985461, + "eta_squared": 0.4858291404456496, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.08572004199959338, + "eta_squared": 0.4858291404456496, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9973362833817403, + "roc_auc": 0.9999512825963719, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7693029170004593, + "eta_squared": 0.4858291404456496, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18877551020408162, + "n_models": 1, + "seconds": 0.08046441700207652, + "eta_squared": 0.4825689121407673, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11989795918367346, + "n_models": 1, + "seconds": 0.07818054200106417, + "eta_squared": 0.4825689121407673, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7497013749998587, + "eta_squared": 0.4825689121407673, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16581632653061223, + "n_models": 1, + "seconds": 0.07646537500113482, + "eta_squared": 0.48575387175214657, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07978204200117034, + "eta_squared": 0.48575387175214657, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7509155000007013, + "eta_squared": 0.48575387175214657, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18622448979591838, + "n_models": 1, + "seconds": 0.08028016699972795, + "eta_squared": 0.4844186055700644, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, + "n_models": 1, + "seconds": 0.0883259160000307, + "eta_squared": 0.4844186055700644, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9939481572022735, + "roc_auc": 0.9999047796201813, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9945643187830688, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 1.1186415419979312, + "eta_squared": 0.4844186055700644, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.21173469387755103, + "n_models": 1, + "seconds": 0.143989625001268, + "eta_squared": 0.48710566059576565, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.08993958299834048, + "eta_squared": 0.48710566059576565, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7706514999990759, + "eta_squared": 0.48710566059576565, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.07688920899818186, + "eta_squared": 0.4855240340488236, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.08556545800092863, + "eta_squared": 0.4855240340488236, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9996789080215418, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7525589169999876, + "eta_squared": 0.4855240340488236, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.20408163265306123, + "n_models": 1, + "seconds": 0.08668029100226704, + "eta_squared": 0.48320943890358836, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.08095079199847532, + "eta_squared": 0.48320943890358836, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.5, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7630562089980231, + "eta_squared": 0.48320943890358836, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8205571978459951, + "roc_auc": 0.9977169253117913, + "precision_at_n": 0.8020833333333334, + "macro_pr_auc": 0.9679937394781145, + "worst_group_fpr": 0.25, + "n_models": 1, + "seconds": 0.07825237499855575, + "eta_squared": 0.8366350521930952, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.08690137499797856, + "eta_squared": 0.8366350521930952, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7718835000014224, + "eta_squared": 0.8366350521930952, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8921185258048079, + "roc_auc": 0.9989392892573696, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9646262591575092, + "worst_group_fpr": 0.21683673469387754, + "n_models": 1, + "seconds": 0.08617970899649663, + "eta_squared": 0.8357517463861228, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9997841047394042, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15051020408163265, + "n_models": 1, + "seconds": 0.07478454199736007, + "eta_squared": 0.8357517463861228, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.999289168792517, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7834026669988816, + "eta_squared": 0.8357517463861228, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9112206394512736, + "roc_auc": 0.9989791489512472, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9868371212121212, + "worst_group_fpr": 0.24744897959183673, + "n_models": 1, + "seconds": 0.08341195799948764, + "eta_squared": 0.8368167743920167, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.08615370800180244, + "eta_squared": 0.8368167743920167, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7578562080016127, + "eta_squared": 0.8368167743920167, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8788587192526313, + "roc_auc": 0.9977080676020409, + "precision_at_n": 0.7916666666666666, + "macro_pr_auc": 0.9920460037647537, + "worst_group_fpr": 0.2780612244897959, + "n_models": 1, + "seconds": 0.07655620900186477, + "eta_squared": 0.8382354298920958, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.08366674999706447, + "eta_squared": 0.8382354298920958, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7506139580000308, + "eta_squared": 0.8382354298920958, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9772469572383853, + "roc_auc": 0.999796272675737, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.1913265306122449, + "n_models": 1, + "seconds": 0.07961420900028315, + "eta_squared": 0.8359843649672052, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.08680995800023084, + "eta_squared": 0.8359843649672052, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7827762499982782, + "eta_squared": 0.8359843649672052, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8981325648851586, + "roc_auc": 0.9987931370464852, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9958570075757577, + "worst_group_fpr": 0.2729591836734694, + "n_models": 1, + "seconds": 0.07629529100086074, + "eta_squared": 0.8321590306687349, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.08843254100065678, + "eta_squared": 0.8321590306687349, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7516903329997149, + "eta_squared": 0.8321590306687349, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9138970246678486, + "roc_auc": 0.9986912733843537, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9971590909090909, + "worst_group_fpr": 0.20918367346938777, + "n_models": 1, + "seconds": 0.0761759160013753, + "eta_squared": 0.8314517992187488, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.0872115419988404, + "eta_squared": 0.8314517992187488, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7406817920018511, + "eta_squared": 0.8314517992187488, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.7751847692274307, + "roc_auc": 0.9966407135770975, + "precision_at_n": 0.7395833333333334, + "macro_pr_auc": 0.9787990944240944, + "worst_group_fpr": 0.2423469387755102, + "n_models": 1, + "seconds": 0.08818870799950673, + "eta_squared": 0.8367353195642477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12244897959183673, + "n_models": 1, + "seconds": 0.07610624999870197, + "eta_squared": 0.8367353195642477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7761402079995605, + "eta_squared": 0.8367353195642477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9286330254336417, + "roc_auc": 0.9988285678854876, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9966145833333333, + "worst_group_fpr": 0.22448979591836735, + "n_models": 1, + "seconds": 0.0843877919978695, + "eta_squared": 0.83775825552584, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.07827283399819862, + "eta_squared": 0.83775825552584, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7463412920005794, + "eta_squared": 0.83775825552584, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9293409565936561, + "roc_auc": 0.999171804138322, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9826007326007327, + "worst_group_fpr": 0.2423469387755102, + "n_models": 1, + "seconds": 0.08487620899904869, + "eta_squared": 0.8352327138393587, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.08720804199765553, + "eta_squared": 0.8352327138393587, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7998721670010127, + "eta_squared": 0.8352327138393587, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8742404129640526, + "roc_auc": 0.998782064909297, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9756076388888889, + "worst_group_fpr": 0.20918367346938777, + "n_models": 1, + "seconds": 0.0876146249975136, + "eta_squared": 0.8363958210386342, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.087231375000556, + "eta_squared": 0.8363958210386342, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7806689999997616, + "eta_squared": 0.8363958210386342, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8110908446160179, + "roc_auc": 0.9978320755385488, + "precision_at_n": 0.8125, + "macro_pr_auc": 0.9634441368816368, + "worst_group_fpr": 0.22959183673469388, + "n_models": 1, + "seconds": 0.07668337500217604, + "eta_squared": 0.8349061810858989, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.07496341700243647, + "eta_squared": 0.8349061810858989, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.961741641669088, + "roc_auc": 0.9994884672619048, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7732104579990846, + "eta_squared": 0.8349061810858989, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9929563180122893, + "roc_auc": 0.999875992063492, + "precision_at_n": 0.96875, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2423469387755102, + "n_models": 1, + "seconds": 0.07441083300000173, + "eta_squared": 0.838772846460863, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12244897959183673, + "n_models": 1, + "seconds": 0.07725229200150352, + "eta_squared": 0.838772846460863, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7636144169991894, + "eta_squared": 0.838772846460863, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8345176660614, + "roc_auc": 0.9977545705782312, + "precision_at_n": 0.7916666666666666, + "macro_pr_auc": 0.9732875631313131, + "worst_group_fpr": 0.23214285714285715, + "n_models": 1, + "seconds": 0.08674941700155614, + "eta_squared": 0.8333960800737457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.07854441699964809, + "eta_squared": 0.8333960800737457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7687328330030141, + "eta_squared": 0.8333960800737457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.8730680702206932, + "roc_auc": 0.9985761231575963, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9735504079254079, + "worst_group_fpr": 0.23469387755102042, + "n_models": 1, + "seconds": 0.08224779199736076, + "eta_squared": 0.8360376953619889, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.0859369579993654, + "eta_squared": 0.8360376953619889, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.500", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.5, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7832443750012317, + "eta_squared": 0.8360376953619889, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.25, + "n_models": 1, + "seconds": 0.08491629099808051, + "eta_squared": 0.5109379001151101, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11989795918367346, + "n_models": 1, + "seconds": 0.08680275000006077, + "eta_squared": 0.5109379001151101, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7681977909996931, + "eta_squared": 0.5109379001151101, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1913265306122449, + "n_models": 1, + "seconds": 0.0772125830008008, + "eta_squared": 0.5084467520162148, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.08866916699844296, + "eta_squared": 0.5084467520162148, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9980151809221449, + "roc_auc": 0.9999601403061225, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7772392080005375, + "eta_squared": 0.5084467520162148, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.24489795918367346, + "n_models": 1, + "seconds": 0.08360429200183717, + "eta_squared": 0.512559151132954, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.08503254199968069, + "eta_squared": 0.512559151132954, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9993599176792338, + "roc_auc": 0.9999867134353743, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.776030165998236, + "eta_squared": 0.512559151132954, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.26785714285714285, + "n_models": 1, + "seconds": 0.09037379099754617, + "eta_squared": 0.5128629688830891, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12755102040816327, + "n_models": 1, + "seconds": 0.07780999999886262, + "eta_squared": 0.5128629688830891, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7654231659980724, + "eta_squared": 0.5128629688830891, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18622448979591838, + "n_models": 1, + "seconds": 0.08365862499704235, + "eta_squared": 0.508762898773635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.08557658300196636, + "eta_squared": 0.508762898773635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9994516328453017, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7562834170021233, + "eta_squared": 0.508762898773635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.23469387755102042, + "n_models": 1, + "seconds": 0.07370516599985422, + "eta_squared": 0.5090176444765433, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.08651979199930793, + "eta_squared": 0.5090176444765433, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9994516328453016, + "roc_auc": 0.9999889278628119, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.763468124998326, + "eta_squared": 0.5090176444765433, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.22959183673469388, + "n_models": 1, + "seconds": 0.08445399999982328, + "eta_squared": 0.5073389623798141, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.07331941600205027, + "eta_squared": 0.5073389623798141, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9983853734038979, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7771152919995075, + "eta_squared": 0.5073389623798141, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2372448979591837, + "n_models": 1, + "seconds": 0.08142979200056288, + "eta_squared": 0.5128210303210285, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.08359362500050338, + "eta_squared": 0.5128210303210285, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9986895597463293, + "roc_auc": 0.9999734268707484, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7754584160029481, + "eta_squared": 0.5128210303210285, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2372448979591837, + "n_models": 1, + "seconds": 0.07261362500139512, + "eta_squared": 0.5106686466317674, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.07893954100291012, + "eta_squared": 0.5106686466317674, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9976086261705304, + "roc_auc": 0.9999557114512472, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7453919169965957, + "eta_squared": 0.5106686466317674, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.22193877551020408, + "n_models": 1, + "seconds": 0.07460391700078617, + "eta_squared": 0.5078990358901121, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.08502054199925624, + "eta_squared": 0.5078990358901121, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7404959579980641, + "eta_squared": 0.5078990358901121, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1836734693877551, + "n_models": 1, + "seconds": 0.07493704200169304, + "eta_squared": 0.5113451805013186, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.078347250000661, + "eta_squared": 0.5113451805013186, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9994695668020408, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7691615000003367, + "eta_squared": 0.5113451805013186, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9996744556165973, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.22448979591836735, + "n_models": 1, + "seconds": 0.07297562500025379, + "eta_squared": 0.5093438660938511, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08163265306122448, + "n_models": 1, + "seconds": 0.07727287499801605, + "eta_squared": 0.5093438660938511, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9931567828976611, + "roc_auc": 0.9998959219104309, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7367227909999201, + "eta_squared": 0.5093438660938511, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2193877551020408, + "n_models": 1, + "seconds": 0.07862629200099036, + "eta_squared": 0.5120210648371795, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, + "n_models": 1, + "seconds": 0.07654770799854305, + "eta_squared": 0.5120210648371795, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7411699999975099, + "eta_squared": 0.5120210648371795, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9996843434343433, + "roc_auc": 0.999993356717687, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.08959908299948438, + "eta_squared": 0.5109849469679029, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, + "n_models": 1, + "seconds": 0.08650095900156884, + "eta_squared": 0.5109849469679029, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9998926116838486, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7631417499978852, + "eta_squared": 0.5109849469679029, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2372448979591837, + "n_models": 1, + "seconds": 0.08152608300224529, + "eta_squared": 0.5087750149271244, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.08800775000054273, + "eta_squared": 0.5087750149271244, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.6, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7683398330009368, + "eta_squared": 0.5087750149271244, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.7885567270879117, + "roc_auc": 0.9967957234977325, + "precision_at_n": 0.75, + "macro_pr_auc": 0.978984302054155, + "worst_group_fpr": 0.28061224489795916, + "n_models": 1, + "seconds": 0.0855373749982391, + "eta_squared": 0.8766830602256049, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16581632653061223, + "n_models": 1, + "seconds": 0.08905991600113339, + "eta_squared": 0.8766830602256049, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7801802079993649, + "eta_squared": 0.8766830602256049, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9263263993422348, + "roc_auc": 0.9991031568877551, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9799768518518519, + "worst_group_fpr": 0.22193877551020408, + "n_models": 1, + "seconds": 0.0870615000021644, + "eta_squared": 0.8759756368485012, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9983880098609269, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16581632653061223, + "n_models": 1, + "seconds": 0.08714395900096861, + "eta_squared": 0.8759756368485012, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.999289168792517, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7651897499999905, + "eta_squared": 0.8759756368485012, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.87515182259879, + "roc_auc": 0.9986536281179138, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9836749188311688, + "worst_group_fpr": 0.28061224489795916, + "n_models": 1, + "seconds": 0.07952075000139303, + "eta_squared": 0.8768603441135048, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14030612244897958, + "n_models": 1, + "seconds": 0.08885616699990351, + "eta_squared": 0.8768603441135048, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7935098329980974, + "eta_squared": 0.8768603441135048, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.5919385078950008, + "roc_auc": 0.9934165072278911, + "precision_at_n": 0.625, + "macro_pr_auc": 0.9361432189170192, + "worst_group_fpr": 0.288265306122449, + "n_models": 1, + "seconds": 0.08872162500119884, + "eta_squared": 0.8779733389869431, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1377551020408163, + "n_models": 1, + "seconds": 0.07963954099977855, + "eta_squared": 0.8779733389869431, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7658361249996233, + "eta_squared": 0.8779733389869431, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9737943240947925, + "roc_auc": 0.9997852005385487, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9896288029100528, + "worst_group_fpr": 0.22448979591836735, + "n_models": 1, + "seconds": 0.08150162500169245, + "eta_squared": 0.8762481102453108, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9964766874443567, + "roc_auc": 0.9999379960317459, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.086150874998566, + "eta_squared": 0.8762481102453108, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7922221669978171, + "eta_squared": 0.8762481102453108, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.7774689882969182, + "roc_auc": 0.9966982886904763, + "precision_at_n": 0.75, + "macro_pr_auc": 0.9810831529581531, + "worst_group_fpr": 0.2780612244897959, + "n_models": 1, + "seconds": 0.08492241700150771, + "eta_squared": 0.8732821441751656, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.08977879200028838, + "eta_squared": 0.8732821441751656, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7473138329987705, + "eta_squared": 0.8732821441751656, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.8118769820654939, + "roc_auc": 0.9974999114229025, + "precision_at_n": 0.8333333333333334, + "macro_pr_auc": 0.9899305555555555, + "worst_group_fpr": 0.2653061224489796, + "n_models": 1, + "seconds": 0.08858470899940585, + "eta_squared": 0.8726482673910156, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.0864155839990417, + "eta_squared": 0.8726482673910156, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7839988749983604, + "eta_squared": 0.8726482673910156, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.6566644460928512, + "roc_auc": 0.9937996031746031, + "precision_at_n": 0.6041666666666666, + "macro_pr_auc": 0.9704588779956427, + "worst_group_fpr": 0.24489795918367346, + "n_models": 1, + "seconds": 0.08813191700028256, + "eta_squared": 0.8767223767771402, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12755102040816327, + "n_models": 1, + "seconds": 0.08840037499976461, + "eta_squared": 0.8767223767771402, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7705785830003151, + "eta_squared": 0.8767223767771402, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9132700475497844, + "roc_auc": 0.9985141191893425, + "precision_at_n": 0.8333333333333334, + "macro_pr_auc": 0.9951388888888889, + "worst_group_fpr": 0.2576530612244898, + "n_models": 1, + "seconds": 0.08638079199954518, + "eta_squared": 0.8774993715157073, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14540816326530612, + "n_models": 1, + "seconds": 0.08703520799826947, + "eta_squared": 0.8774993715157073, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7560972079991188, + "eta_squared": 0.8774993715157073, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.8440371582166274, + "roc_auc": 0.9977567850056689, + "precision_at_n": 0.8333333333333334, + "macro_pr_auc": 0.9803240740740741, + "worst_group_fpr": 0.2653061224489796, + "n_models": 1, + "seconds": 0.07792279199929908, + "eta_squared": 0.8756459899827929, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.08649587500258349, + "eta_squared": 0.8756459899827929, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7773372499977995, + "eta_squared": 0.8756459899827929, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.7648853117072381, + "roc_auc": 0.9973183283730158, + "precision_at_n": 0.8125, + "macro_pr_auc": 0.96149172008547, + "worst_group_fpr": 0.24744897959183673, + "n_models": 1, + "seconds": 0.07740291699883528, + "eta_squared": 0.8761496699086859, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.08015466599681531, + "eta_squared": 0.8761496699086859, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7577294590009842, + "eta_squared": 0.8761496699086859, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.7016601981681742, + "roc_auc": 0.9958634495464853, + "precision_at_n": 0.7395833333333334, + "macro_pr_auc": 0.9485367063492064, + "worst_group_fpr": 0.2780612244897959, + "n_models": 1, + "seconds": 0.08493816700138268, + "eta_squared": 0.8755759250473935, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.08781279100003303, + "eta_squared": 0.8755759250473935, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9619580084145981, + "roc_auc": 0.99949289611678, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7508680829996592, + "eta_squared": 0.8755759250473935, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9406836090385928, + "roc_auc": 0.9991275155895691, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.25, + "n_models": 1, + "seconds": 0.08873591600058717, + "eta_squared": 0.8782757121696583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.08785258399802842, + "eta_squared": 0.8782757121696583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7660697499995877, + "eta_squared": 0.8782757121696583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.8008599649452659, + "roc_auc": 0.9969285891439909, + "precision_at_n": 0.75, + "macro_pr_auc": 0.9716713263588264, + "worst_group_fpr": 0.24489795918367346, + "n_models": 1, + "seconds": 0.07465600000068662, + "eta_squared": 0.8742451489593618, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9944359305718173, + "roc_auc": 0.9998937074829932, + "precision_at_n": 0.96875, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.07024633399851155, + "eta_squared": 0.8742451489593618, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.8164289589985856, + "eta_squared": 0.8742451489593618, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.8042180619734371, + "roc_auc": 0.9975663442460317, + "precision_at_n": 0.8229166666666666, + "macro_pr_auc": 0.9657957782957783, + "worst_group_fpr": 0.25510204081632654, + "n_models": 1, + "seconds": 0.08552541699827998, + "eta_squared": 0.8761879838657488, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.08105375000013737, + "eta_squared": 0.8761879838657488, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.600", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.6, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7777734590017644, + "eta_squared": 0.8761879838657488, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.22959183673469388, + "n_models": 1, + "seconds": 0.08777550000013434, + "eta_squared": 0.5298890398187, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.08882937500311527, + "eta_squared": 0.5298890398187, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7712032080016797, + "eta_squared": 0.5298890398187, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.19387755102040816, + "n_models": 1, + "seconds": 0.08165000000008149, + "eta_squared": 0.52735131605007, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.07766783300030511, + "eta_squared": 0.52735131605007, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9984607720576867, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7554894579989195, + "eta_squared": 0.52735131605007, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.26785714285714285, + "n_models": 1, + "seconds": 0.07812595799987321, + "eta_squared": 0.5320991054125361, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.08511112500127638, + "eta_squared": 0.5320991054125361, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9993685567010306, + "roc_auc": 0.9999867134353742, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7884545829983836, + "eta_squared": 0.5320991054125361, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.29081632653061223, + "n_models": 1, + "seconds": 0.0858608330017887, + "eta_squared": 0.5321841551665233, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14030612244897958, + "n_models": 1, + "seconds": 0.08419258399953833, + "eta_squared": 0.5321841551665233, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7621752499981085, + "eta_squared": 0.5321841551665233, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1836734693877551, + "n_models": 1, + "seconds": 0.08351683300134027, + "eta_squared": 0.5277037636692434, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.08679024999946705, + "eta_squared": 0.5277037636692434, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9995636400137602, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7911076249984035, + "eta_squared": 0.5277037636692434, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.25255102040816324, + "n_models": 1, + "seconds": 0.08264058299755561, + "eta_squared": 0.5286644856867957, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.07730950000041048, + "eta_squared": 0.5286644856867957, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9994516328453017, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8215333750013087, + "eta_squared": 0.5286644856867957, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.24489795918367346, + "n_models": 1, + "seconds": 0.08576008300224203, + "eta_squared": 0.5270205162013888, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.08757841600163374, + "eta_squared": 0.5270205162013888, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9981328388755404, + "roc_auc": 0.9999645691609977, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7834615419997135, + "eta_squared": 0.5270205162013888, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2372448979591837, + "n_models": 1, + "seconds": 0.07499008299782872, + "eta_squared": 0.5318672897267391, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.08617700000104378, + "eta_squared": 0.5318672897267391, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.999243364595796, + "roc_auc": 0.9999844990079364, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.77146250000078, + "eta_squared": 0.5318672897267391, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2576530612244898, + "n_models": 1, + "seconds": 0.0849276250010007, + "eta_squared": 0.5293730129243481, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.07352420799725223, + "eta_squared": 0.5293730129243481, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9967692587372328, + "roc_auc": 0.9999424248866213, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7896272920006595, + "eta_squared": 0.5293730129243481, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.22959183673469388, + "n_models": 1, + "seconds": 0.08258941600070102, + "eta_squared": 0.5268459802946263, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.08991237499867566, + "eta_squared": 0.5268459802946263, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7764590839979064, + "eta_squared": 0.5268459802946263, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.21428571428571427, + "n_models": 1, + "seconds": 0.08458404100019834, + "eta_squared": 0.5306182183453104, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.09078204199977336, + "eta_squared": 0.5306182183453104, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.794608000000153, + "eta_squared": 0.5306182183453104, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2627551020408163, + "n_models": 1, + "seconds": 0.08935416700114729, + "eta_squared": 0.5280422190863733, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.09058883300167508, + "eta_squared": 0.5280422190863733, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9935541423625902, + "roc_auc": 0.9999003507653061, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9945023148148149, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7893917919973319, + "eta_squared": 0.5280422190863733, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2423469387755102, + "n_models": 1, + "seconds": 0.07543029099906562, + "eta_squared": 0.5307835609735325, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.07623579099890776, + "eta_squared": 0.5307835609735325, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7468186670012074, + "eta_squared": 0.5307835609735325, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17346938775510204, + "n_models": 1, + "seconds": 0.08429450000039651, + "eta_squared": 0.5301452136480338, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.08776687499994296, + "eta_squared": 0.5301452136480338, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9996789080215417, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7628876249982568, + "eta_squared": 0.5301452136480338, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2193877551020408, + "n_models": 1, + "seconds": 0.08443741699738894, + "eta_squared": 0.5279659930517174, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, + "n_models": 1, + "seconds": 0.08273758400173392, + "eta_squared": 0.5279659930517174, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.7746504579990869, + "eta_squared": 0.5279659930517174, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.6472915324390636, + "roc_auc": 0.9940609056122449, + "precision_at_n": 0.6875, + "macro_pr_auc": 0.9523411195286196, + "worst_group_fpr": 0.29081632653061223, + "n_models": 1, + "seconds": 0.07696279200172285, + "eta_squared": 0.9029294791735885, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17091836734693877, + "n_models": 1, + "seconds": 0.09040825000192854, + "eta_squared": 0.9029294791735885, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.8112011669982166, + "eta_squared": 0.9029294791735885, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.7335148768545713, + "roc_auc": 0.996383839994331, + "precision_at_n": 0.7604166666666666, + "macro_pr_auc": 0.9474263583638584, + "worst_group_fpr": 0.25255102040816324, + "n_models": 1, + "seconds": 0.08776487499926588, + "eta_squared": 0.9023260653166297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9987543844046409, + "roc_auc": 0.999975641298186, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.08751541599849588, + "eta_squared": 0.9023260653166297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.999289168792517, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7803810410005099, + "eta_squared": 0.9023260653166297, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.7566046046890987, + "roc_auc": 0.996273118622449, + "precision_at_n": 0.71875, + "macro_pr_auc": 0.9692839318698493, + "worst_group_fpr": 0.3010204081632653, + "n_models": 1, + "seconds": 0.08632062499964377, + "eta_squared": 0.9030920988447193, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.08881295900209807, + "eta_squared": 0.9030920988447193, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7908264580000832, + "eta_squared": 0.9030920988447193, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.5444770662299433, + "roc_auc": 0.9901413690476191, + "precision_at_n": 0.4479166666666667, + "macro_pr_auc": 0.9579326923076922, + "worst_group_fpr": 0.3112244897959184, + "n_models": 1, + "seconds": 0.08835808400181122, + "eta_squared": 0.9039807536694324, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16581632653061223, + "n_models": 1, + "seconds": 0.08825104200150236, + "eta_squared": 0.9039807536694324, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7938991660012107, + "eta_squared": 0.9039807536694324, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.907544427474411, + "roc_auc": 0.9989636479591837, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9743055555555555, + "worst_group_fpr": 0.2423469387755102, + "n_models": 1, + "seconds": 0.07891408299838076, + "eta_squared": 0.9026069122444567, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9995636400137604, + "roc_auc": 0.9999911422902495, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.08629674999974668, + "eta_squared": 0.9026069122444567, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7687257500001579, + "eta_squared": 0.9026069122444567, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.7374702609384092, + "roc_auc": 0.9964635593820861, + "precision_at_n": 0.7604166666666666, + "macro_pr_auc": 0.9770084422657952, + "worst_group_fpr": 0.29336734693877553, + "n_models": 1, + "seconds": 0.08752916600133176, + "eta_squared": 0.9002286409429918, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.0856532090001565, + "eta_squared": 0.9002286409429918, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7807869169992045, + "eta_squared": 0.9002286409429918, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.832233935178333, + "roc_auc": 0.9975131979875284, + "precision_at_n": 0.8020833333333334, + "macro_pr_auc": 0.9936342592592592, + "worst_group_fpr": 0.23979591836734693, + "n_models": 1, + "seconds": 0.08214566700189607, + "eta_squared": 0.8996587213625616, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.08768945899646496, + "eta_squared": 0.8996587213625616, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8176077089992759, + "eta_squared": 0.8996587213625616, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.6626887062555417, + "roc_auc": 0.9928097541099774, + "precision_at_n": 0.6041666666666666, + "macro_pr_auc": 0.9801338281601439, + "worst_group_fpr": 0.29081632653061223, + "n_models": 1, + "seconds": 0.08764775000236114, + "eta_squared": 0.9029208108231173, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1377551020408163, + "n_models": 1, + "seconds": 0.0888431670027785, + "eta_squared": 0.9029208108231173, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.8138497500003723, + "eta_squared": 0.9029208108231173, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.7680113801219064, + "roc_auc": 0.9959099525226758, + "precision_at_n": 0.7395833333333334, + "macro_pr_auc": 0.9742989417989417, + "worst_group_fpr": 0.288265306122449, + "n_models": 1, + "seconds": 0.08371254200028488, + "eta_squared": 0.9035454354972019, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9982626909014108, + "roc_auc": 0.9999667835884353, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.07779562499854364, + "eta_squared": 0.9035454354972019, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7602968749997672, + "eta_squared": 0.9035454354972019, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.7166776861324206, + "roc_auc": 0.9954714958900227, + "precision_at_n": 0.6875, + "macro_pr_auc": 0.9653441930955519, + "worst_group_fpr": 0.2857142857142857, + "n_models": 1, + "seconds": 0.08736404099909123, + "eta_squared": 0.9021131136700368, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.08585641700119595, + "eta_squared": 0.9021131136700368, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7692449580026732, + "eta_squared": 0.9021131136700368, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.6628779134839293, + "roc_auc": 0.9960029584750567, + "precision_at_n": 0.78125, + "macro_pr_auc": 0.9226900314723591, + "worst_group_fpr": 0.2729591836734694, + "n_models": 1, + "seconds": 0.07446300000083284, + "eta_squared": 0.9022494388744451, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10459183673469388, + "n_models": 1, + "seconds": 0.08461487499880604, + "eta_squared": 0.9022494388744451, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7702019580028718, + "eta_squared": 0.9022494388744451, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.750630516683807, + "roc_auc": 0.9969418757086168, + "precision_at_n": 0.7708333333333334, + "macro_pr_auc": 0.9536864755614755, + "worst_group_fpr": 0.29336734693877553, + "n_models": 1, + "seconds": 0.08416654100074084, + "eta_squared": 0.9021789744937543, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15051020408163265, + "n_models": 1, + "seconds": 0.07893762499952572, + "eta_squared": 0.9021789744937543, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.961741641669088, + "roc_auc": 0.9994884672619048, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7485258340020664, + "eta_squared": 0.9021789744937543, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.7616804829455618, + "roc_auc": 0.9962930484693877, + "precision_at_n": 0.75, + "macro_pr_auc": 0.9817997685185186, + "worst_group_fpr": 0.29336734693877553, + "n_models": 1, + "seconds": 0.08561912500226754, + "eta_squared": 0.9041553072370441, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11989795918367346, + "n_models": 1, + "seconds": 0.08452362500247546, + "eta_squared": 0.9041553072370441, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7471194579993607, + "eta_squared": 0.9041553072370441, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.7058328327915732, + "roc_auc": 0.995469281462585, + "precision_at_n": 0.6979166666666666, + "macro_pr_auc": 0.9647858796296296, + "worst_group_fpr": 0.2627551020408163, + "n_models": 1, + "seconds": 0.08218399999896064, + "eta_squared": 0.9009953154147747, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9994516328453016, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.08120787500229198, + "eta_squared": 0.9009953154147747, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7353911250029341, + "eta_squared": 0.9009953154147747, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.7180977512797182, + "roc_auc": 0.9964657738095237, + "precision_at_n": 0.78125, + "macro_pr_auc": 0.9498210139318886, + "worst_group_fpr": 0.2857142857142857, + "n_models": 1, + "seconds": 0.07534395799666527, + "eta_squared": 0.9024948788179256, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15051020408163265, + "n_models": 1, + "seconds": 0.08118570800070302, + "eta_squared": 0.9024948788179256, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.700", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.7, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7686429999994289, + "eta_squared": 0.9024948788179256, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2576530612244898, + "n_models": 1, + "seconds": 0.08530229200187023, + "eta_squared": 0.5448994609628225, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12244897959183673, + "n_models": 1, + "seconds": 0.08992466700146906, + "eta_squared": 0.5448994609628225, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7847364580011345, + "eta_squared": 0.5448994609628225, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.20153061224489796, + "n_models": 1, + "seconds": 0.08345112499955576, + "eta_squared": 0.5422953822643336, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.08259204199930537, + "eta_squared": 0.5422953822643336, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.999130139958115, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7781143330030318, + "eta_squared": 0.5422953822643336, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.28316326530612246, + "n_models": 1, + "seconds": 0.08957220799857168, + "eta_squared": 0.5475800579356785, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.07663624999986496, + "eta_squared": 0.5475800579356785, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9996789080215418, + "roc_auc": 0.999993356717687, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03316326530612245, + "n_models": 12, + "seconds": 0.7863824170017324, + "eta_squared": 0.5475800579356785, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.29846938775510207, + "n_models": 1, + "seconds": 0.08090100000117673, + "eta_squared": 0.5474858301661086, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.07967862499936018, + "eta_squared": 0.5474858301661086, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7925279999981285, + "eta_squared": 0.5474858301661086, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2066326530612245, + "n_models": 1, + "seconds": 0.08738858300057473, + "eta_squared": 0.5426833011841099, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.08409604099870194, + "eta_squared": 0.5426833011841099, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9995636400137603, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7880617500013614, + "eta_squared": 0.5426833011841099, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2653061224489796, + "n_models": 1, + "seconds": 0.08894629199858173, + "eta_squared": 0.5441570579629037, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.08500491599988891, + "eta_squared": 0.5441570579629037, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9994516328453017, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7914371249971737, + "eta_squared": 0.5441570579629037, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2729591836734694, + "n_models": 1, + "seconds": 0.08759733300030348, + "eta_squared": 0.5425795873833327, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.12755102040816327, + "n_models": 1, + "seconds": 0.09105041599832475, + "eta_squared": 0.5425795873833327, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9976086261705306, + "roc_auc": 0.9999557114512472, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7902605419985775, + "eta_squared": 0.5425795873833327, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.24744897959183673, + "n_models": 1, + "seconds": 0.08753587500177673, + "eta_squared": 0.5469699481347092, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14540816326530612, + "n_models": 1, + "seconds": 0.09185233400057768, + "eta_squared": 0.5469699481347092, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9992433645957961, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7912287920007657, + "eta_squared": 0.5469699481347092, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2780612244897959, + "n_models": 1, + "seconds": 0.08330637499966542, + "eta_squared": 0.544205346444814, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.08133950000046752, + "eta_squared": 0.544205346444814, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9966244734203707, + "roc_auc": 0.9999402104591837, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7714259999993374, + "eta_squared": 0.544205346444814, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.25510204081632654, + "n_models": 1, + "seconds": 0.07723812499898486, + "eta_squared": 0.5417981360356992, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.08279041700006928, + "eta_squared": 0.5417981360356992, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8046925830021792, + "eta_squared": 0.5417981360356992, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.21428571428571427, + "n_models": 1, + "seconds": 0.0878153330013447, + "eta_squared": 0.5458940752362631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.09355462500025169, + "eta_squared": 0.5458940752362631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.8644236660002207, + "eta_squared": 0.5458940752362631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9996744556165973, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.2653061224489796, + "n_models": 1, + "seconds": 0.09201187500002561, + "eta_squared": 0.5428380555165477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, + "n_models": 1, + "seconds": 0.08699370799877215, + "eta_squared": 0.5428380555165477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.994855534587282, + "roc_auc": 0.9999158517573696, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7936065409994626, + "eta_squared": 0.5428380555165477, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2653061224489796, + "n_models": 1, + "seconds": 0.08785570799955167, + "eta_squared": 0.5456713636299851, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.08340708299874677, + "eta_squared": 0.5456713636299851, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.762391000000207, + "eta_squared": 0.5456713636299851, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.20408163265306123, + "n_models": 1, + "seconds": 0.08767304199864157, + "eta_squared": 0.5453344397432399, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.09091595800055075, + "eta_squared": 0.5453344397432399, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9997874149659862, + "roc_auc": 0.9999955711451246, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7814873749994149, + "eta_squared": 0.5453344397432399, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.25255102040816324, + "n_models": 1, + "seconds": 0.0868818340022699, + "eta_squared": 0.5431459755082583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08673469387755102, + "n_models": 1, + "seconds": 0.07824070900096558, + "eta_squared": 0.5431459755082583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.8067619579996972, + "eta_squared": 0.5431459755082583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.681798304119326, + "roc_auc": 0.9949909651360545, + "precision_at_n": 0.6875, + "macro_pr_auc": 0.9718088624338624, + "worst_group_fpr": 0.29336734693877553, + "n_models": 1, + "seconds": 0.08836204200270004, + "eta_squared": 0.9209097965562205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18112244897959184, + "n_models": 1, + "seconds": 0.07975916700161179, + "eta_squared": 0.9209097965562205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.8325424169997859, + "eta_squared": 0.9209097965562205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.6975289453069624, + "roc_auc": 0.9948160253684807, + "precision_at_n": 0.6354166666666666, + "macro_pr_auc": 0.951994825708061, + "worst_group_fpr": 0.29336734693877553, + "n_models": 1, + "seconds": 0.08175404099893058, + "eta_squared": 0.920373719914181, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9958578127329641, + "roc_auc": 0.9999291383219955, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1377551020408163, + "n_models": 1, + "seconds": 0.0871164169984695, + "eta_squared": 0.920373719914181, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9523907508279281, + "roc_auc": 0.9992980265022675, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8049216250001336, + "eta_squared": 0.920373719914181, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.6518896249722297, + "roc_auc": 0.9932592828798186, + "precision_at_n": 0.5729166666666666, + "macro_pr_auc": 0.9735127005347594, + "worst_group_fpr": 0.32142857142857145, + "n_models": 1, + "seconds": 0.08367216699843993, + "eta_squared": 0.9210567957325315, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16071428571428573, + "n_models": 1, + "seconds": 0.08688112500021816, + "eta_squared": 0.9210567957325315, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8075408750009956, + "eta_squared": 0.9210567957325315, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.4683855431996348, + "roc_auc": 0.9850481859410432, + "precision_at_n": 0.3645833333333333, + "macro_pr_auc": 0.9690972222222222, + "worst_group_fpr": 0.3086734693877551, + "n_models": 1, + "seconds": 0.08949387499887962, + "eta_squared": 0.9217812642726251, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.08769679200122482, + "eta_squared": 0.9217812642726251, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7730962090026878, + "eta_squared": 0.9217812642726251, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.920340014165772, + "roc_auc": 0.9986669146825397, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9895367364117363, + "worst_group_fpr": 0.25255102040816324, + "n_models": 1, + "seconds": 0.08265704099903814, + "eta_squared": 0.920649352214294, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9998926116838486, + "roc_auc": 0.9999977855725625, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.09078566700191004, + "eta_squared": 0.920649352214294, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7948213329982536, + "eta_squared": 0.920649352214294, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.7376304330422577, + "roc_auc": 0.9951326884920635, + "precision_at_n": 0.6666666666666666, + "macro_pr_auc": 0.9928685897435897, + "worst_group_fpr": 0.29591836734693877, + "n_models": 1, + "seconds": 0.08608991599976434, + "eta_squared": 0.9186868472692276, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1683673469387755, + "n_models": 1, + "seconds": 0.08270133300175075, + "eta_squared": 0.9186868472692276, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8256208329985384, + "eta_squared": 0.9186868472692276, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.7419668543274931, + "roc_auc": 0.9959210246598639, + "precision_at_n": 0.7395833333333334, + "macro_pr_auc": 0.9858743686868686, + "worst_group_fpr": 0.2576530612244898, + "n_models": 1, + "seconds": 0.08848787500028266, + "eta_squared": 0.9181718730647732, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.08740104199750931, + "eta_squared": 0.9181718730647732, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8212779999994382, + "eta_squared": 0.9181718730647732, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.5073131114931568, + "roc_auc": 0.9889832234977325, + "precision_at_n": 0.4791666666666667, + "macro_pr_auc": 0.9532295380625122, + "worst_group_fpr": 0.3010204081632653, + "n_models": 1, + "seconds": 0.08298383299916168, + "eta_squared": 0.9208674523939366, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.0848398329981137, + "eta_squared": 0.9208674523939366, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.8311338749990682, + "eta_squared": 0.9208674523939366, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.6895028537478385, + "roc_auc": 0.9936556653911566, + "precision_at_n": 0.6041666666666666, + "macro_pr_auc": 0.9805205098343684, + "worst_group_fpr": 0.29846938775510207, + "n_models": 1, + "seconds": 0.0786492500010354, + "eta_squared": 0.9213876858018002, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9963283583334289, + "roc_auc": 0.9999357816043083, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.08939033299975563, + "eta_squared": 0.9213876858018002, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8267360840000038, + "eta_squared": 0.9213876858018002, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.6339613178668613, + "roc_auc": 0.9939656852324263, + "precision_at_n": 0.6458333333333334, + "macro_pr_auc": 0.9464195526695526, + "worst_group_fpr": 0.3086734693877551, + "n_models": 1, + "seconds": 0.07994841700201505, + "eta_squared": 0.9202352791959443, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.08697612499963725, + "eta_squared": 0.9202352791959443, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.8108292079996318, + "eta_squared": 0.9202352791959443, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.7125730237172959, + "roc_auc": 0.9963461947278911, + "precision_at_n": 0.7604166666666666, + "macro_pr_auc": 0.952378684088243, + "worst_group_fpr": 0.28316326530612246, + "n_models": 1, + "seconds": 0.0884319590004452, + "eta_squared": 0.9201601632705698, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.08454666699981317, + "eta_squared": 0.9201601632705698, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7721837089993642, + "eta_squared": 0.9201601632705698, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.7410573923465887, + "roc_auc": 0.9964613449546484, + "precision_at_n": 0.75, + "macro_pr_auc": 0.9539373394636552, + "worst_group_fpr": 0.30357142857142855, + "n_models": 1, + "seconds": 0.08881941599975107, + "eta_squared": 0.9203742833505759, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.08157641700017848, + "eta_squared": 0.9203742833505759, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9619580084145981, + "roc_auc": 0.99949289611678, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7682856249994074, + "eta_squared": 0.9203742833505759, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.7621316150026999, + "roc_auc": 0.9956663655045351, + "precision_at_n": 0.6875, + "macro_pr_auc": 0.9945549242424243, + "worst_group_fpr": 0.29591836734693877, + "n_models": 1, + "seconds": 0.08588337499895715, + "eta_squared": 0.9218843396708194, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.12737383299827343, + "eta_squared": 0.9218843396708194, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7875508329998411, + "eta_squared": 0.9218843396708194, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.5876624542741195, + "roc_auc": 0.9925993835034013, + "precision_at_n": 0.6354166666666666, + "macro_pr_auc": 0.9481195371667185, + "worst_group_fpr": 0.2755102040816326, + "n_models": 1, + "seconds": 0.08751795900025172, + "eta_squared": 0.9193097916370743, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9757123371101586, + "roc_auc": 0.9996014030612245, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9956018518518519, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.08832674999939627, + "eta_squared": 0.9193097916370743, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9329070122690577, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.8594744999973045, + "eta_squared": 0.9193097916370743, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.7111010445901043, + "roc_auc": 0.9958878082482994, + "precision_at_n": 0.7291666666666666, + "macro_pr_auc": 0.9571602009102009, + "worst_group_fpr": 0.31887755102040816, + "n_models": 1, + "seconds": 0.07846191600037855, + "eta_squared": 0.920517468315107, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17091836734693877, + "n_models": 1, + "seconds": 0.08422133400017628, + "eta_squared": 0.920517468315107, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.800", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.8, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.780792792000284, + "eta_squared": 0.920517468315107, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.288265306122449, + "n_models": 1, + "seconds": 0.07455820799805224, + "eta_squared": 0.5572328878968438, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08673469387755102, + "n_models": 1, + "seconds": 0.07642225000017788, + "eta_squared": 0.5572328878968438, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8349476659968786, + "eta_squared": 0.5572328878968438, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.21683673469387754, + "n_models": 1, + "seconds": 0.08829708299890626, + "eta_squared": 0.5545595572982635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.0792739580028865, + "eta_squared": 0.5545595572982635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9988020366397061, + "roc_auc": 0.9999767485119048, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.800436333000107, + "eta_squared": 0.5545595572982635, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.30612244897959184, + "n_models": 1, + "seconds": 0.08130583399906754, + "eta_squared": 0.5602958775475205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.08753462499953457, + "eta_squared": 0.5602958775475205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.999575836489899, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.7678894999990007, + "eta_squared": 0.5602958775475205, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.3163265306122449, + "n_models": 1, + "seconds": 0.08538550000230316, + "eta_squared": 0.5600535097729571, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, + "n_models": 1, + "seconds": 0.0778062499994121, + "eta_squared": 0.5600535097729571, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7888367919986194, + "eta_squared": 0.5600535097729571, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.20918367346938777, + "n_models": 1, + "seconds": 0.0808924160010065, + "eta_squared": 0.5549776244660766, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.07597116699980688, + "eta_squared": 0.5549776244660766, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9995636400137602, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7883488749976095, + "eta_squared": 0.5549776244660766, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2780612244897959, + "n_models": 1, + "seconds": 0.08512200000041048, + "eta_squared": 0.5568389917399191, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.0915095830023347, + "eta_squared": 0.5568389917399191, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9995636400137603, + "roc_auc": 0.9999911422902492, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8239208750019316, + "eta_squared": 0.5568389917399191, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.29081632653061223, + "n_models": 1, + "seconds": 0.08209349999742699, + "eta_squared": 0.55533969332111, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.08643829199718311, + "eta_squared": 0.55533969332111, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9983853734038977, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8057116249983665, + "eta_squared": 0.55533969332111, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.28316326530612246, + "n_models": 1, + "seconds": 0.08546391700292588, + "eta_squared": 0.5593879759814088, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1326530612244898, + "n_models": 1, + "seconds": 0.08844525000313297, + "eta_squared": 0.5593879759814088, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9992332114897579, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.7703120000005583, + "eta_squared": 0.5593879759814088, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.28316326530612246, + "n_models": 1, + "seconds": 0.07417500000155997, + "eta_squared": 0.5564035218173411, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, + "n_models": 1, + "seconds": 0.07529554099892266, + "eta_squared": 0.5564035218173411, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9960159379888793, + "roc_auc": 0.9999313527494331, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7456938750001427, + "eta_squared": 0.5564035218173411, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.24744897959183673, + "n_models": 1, + "seconds": 0.0773404580031638, + "eta_squared": 0.5540528723307074, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11224489795918367, + "n_models": 1, + "seconds": 0.09114179100288311, + "eta_squared": 0.5540528723307074, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.8689664580015233, + "eta_squared": 0.5540528723307074, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.22193877551020408, + "n_models": 1, + "seconds": 0.07785095800136332, + "eta_squared": 0.5584465959181422, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.0865530420014693, + "eta_squared": 0.5584465959181422, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.76247237500138, + "eta_squared": 0.5584465959181422, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.288265306122449, + "n_models": 1, + "seconds": 0.08170925000013085, + "eta_squared": 0.5549925397074991, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07653061224489796, + "n_models": 1, + "seconds": 0.08589354200012167, + "eta_squared": 0.5549925397074991, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9953696119468665, + "roc_auc": 0.9999224950396826, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8043951660001767, + "eta_squared": 0.5549925397074991, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2755102040816326, + "n_models": 1, + "seconds": 0.08595641700230772, + "eta_squared": 0.5579244171259582, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11989795918367346, + "n_models": 1, + "seconds": 0.08596374999979162, + "eta_squared": 0.5579244171259582, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7921546249999665, + "eta_squared": 0.5579244171259582, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.22193877551020408, + "n_models": 1, + "seconds": 0.0836649579978257, + "eta_squared": 0.5578235062169735, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.08290887499970268, + "eta_squared": 0.5578235062169735, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9996789080215419, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8014245000013034, + "eta_squared": 0.5578235062169735, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.26785714285714285, + "n_models": 1, + "seconds": 0.0813014999985171, + "eta_squared": 0.5556051803235251, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.07846112500192248, + "eta_squared": 0.5556051803235251, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.8116107500027283, + "eta_squared": 0.5556051803235251, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.48370182641704884, + "roc_auc": 0.9887352076247166, + "precision_at_n": 0.4791666666666667, + "macro_pr_auc": 0.94273530765998, + "worst_group_fpr": 0.3163265306122449, + "n_models": 1, + "seconds": 0.08873683300043922, + "eta_squared": 0.9337002346939631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17346938775510204, + "n_models": 1, + "seconds": 0.08215395899969735, + "eta_squared": 0.9337002346939631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7766321250019246, + "eta_squared": 0.9337002346939631, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.48841516890928915, + "roc_auc": 0.9884827628968254, + "precision_at_n": 0.4583333333333333, + "macro_pr_auc": 0.948426906046769, + "worst_group_fpr": 0.30612244897959184, + "n_models": 1, + "seconds": 0.07183579200136592, + "eta_squared": 0.9332108003153095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9903355234771398, + "roc_auc": 0.9998693487811792, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.08315720799873816, + "eta_squared": 0.9332108003153095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.999289168792517, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7925684170004388, + "eta_squared": 0.9332108003153095, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.49233872316581706, + "roc_auc": 0.9881218112244898, + "precision_at_n": 0.4270833333333333, + "macro_pr_auc": 0.9649555077597841, + "worst_group_fpr": 0.32653061224489793, + "n_models": 1, + "seconds": 0.08811762500045006, + "eta_squared": 0.9338330621664808, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14030612244897958, + "n_models": 1, + "seconds": 0.08562370800063945, + "eta_squared": 0.9338330621664808, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7701699589997588, + "eta_squared": 0.9338330621664808, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.5443471300698407, + "roc_auc": 0.9854932858560091, + "precision_at_n": 0.3541666666666667, + "macro_pr_auc": 0.9873006333943835, + "worst_group_fpr": 0.30612244897959184, + "n_models": 1, + "seconds": 0.08923404199958895, + "eta_squared": 0.9344356642098697, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1836734693877551, + "n_models": 1, + "seconds": 0.08842350000122678, + "eta_squared": 0.9344356642098697, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8114923339999223, + "eta_squared": 0.9344356642098697, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.7228507251411649, + "roc_auc": 0.9961579683956916, + "precision_at_n": 0.7291666666666666, + "macro_pr_auc": 0.9599738666834255, + "worst_group_fpr": 0.28316326530612246, + "n_models": 1, + "seconds": 0.08098579099896597, + "eta_squared": 0.9334757370341583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9997841047394042, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14540816326530612, + "n_models": 1, + "seconds": 0.09482916599881719, + "eta_squared": 0.9334757370341583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8320669169988832, + "eta_squared": 0.9334757370341583, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.4341574536013676, + "roc_auc": 0.985728015164399, + "precision_at_n": 0.3958333333333333, + "macro_pr_auc": 0.9419825605680869, + "worst_group_fpr": 0.32142857142857145, + "n_models": 1, + "seconds": 0.0860680420009885, + "eta_squared": 0.9318170650470089, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1760204081632653, + "n_models": 1, + "seconds": 0.08812975000182632, + "eta_squared": 0.9318170650470089, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7867327090025356, + "eta_squared": 0.9318170650470089, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.8306151753552133, + "roc_auc": 0.9970902423469388, + "precision_at_n": 0.7916666666666666, + "macro_pr_auc": 0.9971590909090909, + "worst_group_fpr": 0.29846938775510207, + "n_models": 1, + "seconds": 0.08717691700076102, + "eta_squared": 0.9313493667097374, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.08346687500306871, + "eta_squared": 0.9313493667097374, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7639452910007094, + "eta_squared": 0.9313493667097374, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.49641269539946786, + "roc_auc": 0.9876434948979592, + "precision_at_n": 0.4270833333333333, + "macro_pr_auc": 0.9692120927318296, + "worst_group_fpr": 0.3112244897959184, + "n_models": 1, + "seconds": 0.07500079100282164, + "eta_squared": 0.9336353454255104, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13520408163265307, + "n_models": 1, + "seconds": 0.08246458299981896, + "eta_squared": 0.9336353454255104, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.759892708996631, + "eta_squared": 0.9336353454255104, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.5603500230997622, + "roc_auc": 0.9891692354024944, + "precision_at_n": 0.46875, + "macro_pr_auc": 0.972537088350666, + "worst_group_fpr": 0.30612244897959184, + "n_models": 1, + "seconds": 0.08461950000128127, + "eta_squared": 0.9340786626302715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9728929481135735, + "roc_auc": 0.9997165532879818, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9864045965608467, + "worst_group_fpr": 0.1556122448979592, + "n_models": 1, + "seconds": 0.08807795800021267, + "eta_squared": 0.9340786626302715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7622362080001039, + "eta_squared": 0.9340786626302715, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.5749684361771564, + "roc_auc": 0.9910359977324262, + "precision_at_n": 0.4583333333333333, + "macro_pr_auc": 0.9552121489621489, + "worst_group_fpr": 0.3112244897959184, + "n_models": 1, + "seconds": 0.07600716700108023, + "eta_squared": 0.9331214740684672, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.0798523750017921, + "eta_squared": 0.9331214740684672, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7766599579990725, + "eta_squared": 0.9331214740684672, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.6433556438094067, + "roc_auc": 0.9947562358276645, + "precision_at_n": 0.6666666666666666, + "macro_pr_auc": 0.9443264634670885, + "worst_group_fpr": 0.3137755102040816, + "n_models": 1, + "seconds": 0.08223033399917767, + "eta_squared": 0.9329217798181637, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.08641679100037436, + "eta_squared": 0.9329217798181637, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7963710830008495, + "eta_squared": 0.9329217798181637, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.487662332276554, + "roc_auc": 0.9900085034013606, + "precision_at_n": 0.4583333333333333, + "macro_pr_auc": 0.9343976263898139, + "worst_group_fpr": 0.31887755102040816, + "n_models": 1, + "seconds": 0.0876374580002448, + "eta_squared": 0.9333000674744998, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.998801470355826, + "roc_auc": 0.999975641298186, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15816326530612246, + "n_models": 1, + "seconds": 0.08732816700285184, + "eta_squared": 0.9333000674744998, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9619580084145981, + "roc_auc": 0.99949289611678, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.830850875001488, + "eta_squared": 0.9333000674744998, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.6257991263089661, + "roc_auc": 0.9913903061224489, + "precision_at_n": 0.5416666666666666, + "macro_pr_auc": 0.9787367724867725, + "worst_group_fpr": 0.32142857142857145, + "n_models": 1, + "seconds": 0.09474454199880711, + "eta_squared": 0.9344982331342713, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.08025383299900568, + "eta_squared": 0.9344982331342713, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8052421659995161, + "eta_squared": 0.9344982331342713, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.4725756220492768, + "roc_auc": 0.9875062003968254, + "precision_at_n": 0.4166666666666667, + "macro_pr_auc": 0.9477053668689698, + "worst_group_fpr": 0.30612244897959184, + "n_models": 1, + "seconds": 0.08894675000192365, + "eta_squared": 0.9323325522428791, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9824324760482137, + "roc_auc": 0.9996877657312926, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, + "n_models": 1, + "seconds": 0.08182137500261888, + "eta_squared": 0.9323325522428791, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.8128278749973106, + "eta_squared": 0.9323325522428791, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.5740868812626767, + "roc_auc": 0.9920081313775511, + "precision_at_n": 0.5729166666666666, + "macro_pr_auc": 0.9453199226297052, + "worst_group_fpr": 0.32142857142857145, + "n_models": 1, + "seconds": 0.08808708299693535, + "eta_squared": 0.9333406233039432, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.0876066249984433, + "eta_squared": 0.9333406233039432, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=0.900", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 0.9, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7643466250010533, + "eta_squared": 0.9333406233039432, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 0, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2755102040816326, + "n_models": 1, + "seconds": 0.08390070799941896, + "eta_squared": 0.5676363348862092, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 0, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, + "n_models": 1, + "seconds": 0.08287154100253247, + "eta_squared": 0.5676363348862092, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 0, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7580239579983754, + "eta_squared": 0.5676363348862092, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 1, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2372448979591837, + "n_models": 1, + "seconds": 0.0869074170004751, + "eta_squared": 0.5648975634450134, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 1, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.08407229200020083, + "eta_squared": 0.5648975634450134, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 1, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9988859606860465, + "roc_auc": 0.9999778557256236, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8368146669999987, + "eta_squared": 0.5648975634450134, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 2, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.30612244897959184, + "n_models": 1, + "seconds": 0.08855483300067135, + "eta_squared": 0.5710146129420057, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 2, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.09004162500059465, + "eta_squared": 0.5710146129420057, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 2, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9996789080215417, + "roc_auc": 0.9999933567176872, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.784261583998159, + "eta_squared": 0.5710146129420057, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 3, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.3112244897959184, + "n_models": 1, + "seconds": 0.07325204199878499, + "eta_squared": 0.5706485648142726, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 3, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.07290899999861722, + "eta_squared": 0.5706485648142726, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 3, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8076408750021074, + "eta_squared": 0.5706485648142726, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 4, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.21173469387755103, + "n_models": 1, + "seconds": 0.07311187500090455, + "eta_squared": 0.56534003587956, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 4, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14030612244897958, + "n_models": 1, + "seconds": 0.0710214589998941, + "eta_squared": 0.56534003587956, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 4, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9996744556165972, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7913264160015387, + "eta_squared": 0.56534003587956, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 5, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.25510204081632654, + "n_models": 1, + "seconds": 0.08004762499695062, + "eta_squared": 0.5675034019831222, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 5, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07908163265306123, + "n_models": 1, + "seconds": 0.09018975000071805, + "eta_squared": 0.5675034019831222, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 5, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9995636400137603, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.8272125000003143, + "eta_squared": 0.5675034019831222, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 6, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2627551020408163, + "n_models": 1, + "seconds": 0.08454420799898799, + "eta_squared": 0.5660843709494305, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 6, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14540816326530612, + "n_models": 1, + "seconds": 0.1031810829990718, + "eta_squared": 0.5660843709494305, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 6, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9981328388755406, + "roc_auc": 0.9999645691609979, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7952350419982395, + "eta_squared": 0.5660843709494305, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 7, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.28316326530612246, + "n_models": 1, + "seconds": 0.08404750000045169, + "eta_squared": 0.5698671092657079, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 7, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.125, + "n_models": 1, + "seconds": 0.08244604100036668, + "eta_squared": 0.5698671092657079, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 7, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9993464361274391, + "roc_auc": 0.9999867134353743, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.8000821249988803, + "eta_squared": 0.5698671092657079, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 8, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.29081632653061223, + "n_models": 1, + "seconds": 0.08897100000103819, + "eta_squared": 0.5667004409467457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 8, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09693877551020408, + "n_models": 1, + "seconds": 0.08988291700006812, + "eta_squared": 0.5667004409467457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 8, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9961695831403155, + "roc_auc": 0.9999335671768708, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7982099159999052, + "eta_squared": 0.5667004409467457, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 9, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2576530612244898, + "n_models": 1, + "seconds": 0.08204345800186275, + "eta_squared": 0.5643728023336351, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 9, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11479591836734694, + "n_models": 1, + "seconds": 0.08827945799930603, + "eta_squared": 0.5643728023336351, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 9, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7822977089999767, + "eta_squared": 0.5643728023336351, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 10, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.22959183673469388, + "n_models": 1, + "seconds": 0.08758908300296753, + "eta_squared": 0.5690320175635628, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 10, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, + "n_models": 1, + "seconds": 0.08838724999804981, + "eta_squared": 0.5690320175635628, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 10, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9996843434343433, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, + "n_models": 12, + "seconds": 0.8200831659996766, + "eta_squared": 0.5690320175635628, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 11, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.29846938775510207, + "n_models": 1, + "seconds": 0.0864905419985007, + "eta_squared": 0.5652473317791511, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 11, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08928571428571429, + "n_models": 1, + "seconds": 0.08354408300147043, + "eta_squared": 0.5652473317791511, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 11, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9952030832506646, + "roc_auc": 0.9999202806122449, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.79343237500143, + "eta_squared": 0.5652473317791511, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 12, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.3010204081632653, + "n_models": 1, + "seconds": 0.08703429200249957, + "eta_squared": 0.5682748535558901, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 12, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11989795918367346, + "n_models": 1, + "seconds": 0.08590316700065159, + "eta_squared": 0.5682748535558901, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 12, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.82700795799974, + "eta_squared": 0.5682748535558901, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 13, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2066326530612245, + "n_models": 1, + "seconds": 0.07719537500088336, + "eta_squared": 0.5683637055956918, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 13, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.08894529099779902, + "eta_squared": 0.5683637055956918, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 13, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9996789080215418, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8343272090023675, + "eta_squared": 0.5683637055956918, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 14, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2627551020408163, + "n_models": 1, + "seconds": 0.08936945900131832, + "eta_squared": 0.5661057125507907, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 14, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09438775510204081, + "n_models": 1, + "seconds": 0.09122275000117952, + "eta_squared": 0.5661057125507907, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 14, + "mechanism": "global", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04846938775510204, + "n_models": 12, + "seconds": 0.8167644160021155, + "eta_squared": 0.5661057125507907, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.4404474898959679, + "roc_auc": 0.9855663619614513, + "precision_at_n": 0.4375, + "macro_pr_auc": 0.9488062744037009, + "worst_group_fpr": 0.3086734693877551, + "n_models": 1, + "seconds": 0.07679250000001048, + "eta_squared": 0.9430926146131842, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 0, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.18112244897959184, + "n_models": 1, + "seconds": 0.0897597080002015, + "eta_squared": 0.9430926146131842, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 0, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9867282776223474, + "roc_auc": 0.9997608418367346, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9924355158730158, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7976526249985909, + "eta_squared": 0.9430926146131842, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 1, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.5297227019254098, + "roc_auc": 0.9909983524659864, + "precision_at_n": 0.5, + "macro_pr_auc": 0.9452332937530307, + "worst_group_fpr": 0.29336734693877553, + "n_models": 1, + "seconds": 0.08685933399829082, + "eta_squared": 0.9426371564373757, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 1, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.987198828087694, + "roc_auc": 0.9998472045068028, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.14030612244897958, + "n_models": 1, + "seconds": 0.08658529100284795, + "eta_squared": 0.9426371564373757, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 1, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9505343079049093, + "roc_auc": 0.999289168792517, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9559441137566137, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7732552500019665, + "eta_squared": 0.9426371564373757, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 2, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.5556907945800128, + "roc_auc": 0.990117010345805, + "precision_at_n": 0.5520833333333334, + "macro_pr_auc": 0.970216922238981, + "worst_group_fpr": 0.32653061224489793, + "n_models": 1, + "seconds": 0.0870056250023481, + "eta_squared": 0.9432131151496429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 2, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1377551020408163, + "n_models": 1, + "seconds": 0.08347629200216033, + "eta_squared": 0.9432131151496429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 2, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9675380860028191, + "roc_auc": 0.9995161476048753, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9709118716931218, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7792606670009263, + "eta_squared": 0.9432131151496429, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 3, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.4358561409859812, + "roc_auc": 0.9812991602891157, + "precision_at_n": 0.3229166666666667, + "macro_pr_auc": 0.9661458333333334, + "worst_group_fpr": 0.336734693877551, + "n_models": 1, + "seconds": 0.08025700000143843, + "eta_squared": 0.9437235511189537, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 3, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1683673469387755, + "n_models": 1, + "seconds": 0.07899366700075916, + "eta_squared": 0.9437235511189537, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 3, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9543889606828282, + "roc_auc": 0.9994751806972789, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9662822420634921, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7909962499979883, + "eta_squared": 0.9437235511189537, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 4, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.6522575845109674, + "roc_auc": 0.9937198837868481, + "precision_at_n": 0.6145833333333334, + "macro_pr_auc": 0.9728918650793651, + "worst_group_fpr": 0.30357142857142855, + "n_models": 1, + "seconds": 0.08615758300220477, + "eta_squared": 0.9428896083609098, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 4, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9974747370154907, + "roc_auc": 0.9999534970238096, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14540816326530612, + "n_models": 1, + "seconds": 0.08721558400065987, + "eta_squared": 0.9428896083609098, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 4, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9918297236189909, + "roc_auc": 0.999882635345805, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.792246332999639, + "eta_squared": 0.9428896083609098, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 5, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.5515014660004262, + "roc_auc": 0.9890186543367347, + "precision_at_n": 0.5104166666666666, + "macro_pr_auc": 0.9818695533769063, + "worst_group_fpr": 0.31887755102040816, + "n_models": 1, + "seconds": 0.0903698340007395, + "eta_squared": 0.9414594641783969, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 5, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1760204081632653, + "n_models": 1, + "seconds": 0.0913679580007738, + "eta_squared": 0.9414594641783969, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 5, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9725508634860032, + "roc_auc": 0.9995460423752834, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9757853835978837, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.8012882500006526, + "eta_squared": 0.9414594641783969, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 6, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.7353436685064171, + "roc_auc": 0.9948448129251701, + "precision_at_n": 0.65625, + "macro_pr_auc": 0.9971590909090909, + "worst_group_fpr": 0.29081632653061223, + "n_models": 1, + "seconds": 0.07957833299951744, + "eta_squared": 0.9410327000313501, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 6, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14795918367346939, + "n_models": 1, + "seconds": 0.08582583399766008, + "eta_squared": 0.9410327000313501, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 6, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9807289352882025, + "roc_auc": 0.9996966234410432, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9846106150793651, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7827843750019383, + "eta_squared": 0.9410327000313501, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 7, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.47024340453116165, + "roc_auc": 0.9853626346371882, + "precision_at_n": 0.3854166666666667, + "macro_pr_auc": 0.9707844672688423, + "worst_group_fpr": 0.3137755102040816, + "n_models": 1, + "seconds": 0.0861924579985498, + "eta_squared": 0.9430130855086445, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 7, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.16071428571428573, + "n_models": 1, + "seconds": 0.08583804199952283, + "eta_squared": 0.9430130855086445, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 7, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.8881024848231156, + "roc_auc": 0.9986049107142858, + "precision_at_n": 0.8541666666666666, + "macro_pr_auc": 0.9196304563492065, + "worst_group_fpr": 0.03571428571428571, + "n_models": 12, + "seconds": 0.803724166999018, + "eta_squared": 0.9430130855086445, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 8, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.5989056649722575, + "roc_auc": 0.9890607284580499, + "precision_at_n": 0.5520833333333334, + "macro_pr_auc": 0.9787465428090427, + "worst_group_fpr": 0.30357142857142855, + "n_models": 1, + "seconds": 0.08554504100175109, + "eta_squared": 0.9433967168032253, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 8, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9890192795549045, + "roc_auc": 0.9998494189342404, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17091836734693877, + "n_models": 1, + "seconds": 0.08578779100207612, + "eta_squared": 0.9433967168032253, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 8, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9806220968736649, + "roc_auc": 0.999672264739229, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9769179894179892, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.782843375000084, + "eta_squared": 0.9433967168032253, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 9, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.4566709690403336, + "roc_auc": 0.9856970131802721, + "precision_at_n": 0.3645833333333333, + "macro_pr_auc": 0.9320161716421307, + "worst_group_fpr": 0.32908163265306123, + "n_models": 1, + "seconds": 0.07899349999934202, + "eta_squared": 0.9425810756331876, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 9, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, + "n_models": 1, + "seconds": 0.08821075000014389, + "eta_squared": 0.9425810756331876, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 9, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9871735075615327, + "roc_auc": 0.9997364831349206, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9900628306878306, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7780413750006119, + "eta_squared": 0.9425810756331876, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 10, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.4558993558210772, + "roc_auc": 0.9876368516156462, + "precision_at_n": 0.4166666666666667, + "macro_pr_auc": 0.9422583645929236, + "worst_group_fpr": 0.3163265306122449, + "n_models": 1, + "seconds": 0.0782590419985354, + "eta_squared": 0.9423071534590428, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 10, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10204081632653061, + "n_models": 1, + "seconds": 0.08753012500164914, + "eta_squared": 0.9423071534590428, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 10, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9697656129469174, + "roc_auc": 0.9994707518424036, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.968812003968254, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.782373624999309, + "eta_squared": 0.9423071534590428, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 11, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.5631518605076176, + "roc_auc": 0.9916826105442177, + "precision_at_n": 0.5520833333333334, + "macro_pr_auc": 0.956760796008329, + "worst_group_fpr": 0.3086734693877551, + "n_models": 1, + "seconds": 0.08744820900028571, + "eta_squared": 0.9427808191647584, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 11, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9967753175878704, + "roc_auc": 0.9999357816043084, + "precision_at_n": 0.96875, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14285714285714285, + "n_models": 1, + "seconds": 0.08697441700132913, + "eta_squared": 0.9427808191647584, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 11, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.961741641669088, + "roc_auc": 0.9994884672619048, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9683077050264549, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7827680410009634, + "eta_squared": 0.9427808191647584, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 12, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.6117859143148155, + "roc_auc": 0.9889987244897959, + "precision_at_n": 0.5, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.3239795918367347, + "n_models": 1, + "seconds": 0.07708795899816323, + "eta_squared": 0.9437633675151137, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 12, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.14030612244897958, + "n_models": 1, + "seconds": 0.08376499999940279, + "eta_squared": 0.9437633675151137, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 12, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9874493808974725, + "roc_auc": 0.9997852005385488, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9865492724867725, + "worst_group_fpr": 0.03826530612244898, + "n_models": 12, + "seconds": 0.7783098330000939, + "eta_squared": 0.9437633675151137, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 13, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.4144666289612037, + "roc_auc": 0.9853404903628118, + "precision_at_n": 0.3958333333333333, + "macro_pr_auc": 0.9277287780895475, + "worst_group_fpr": 0.29591836734693877, + "n_models": 1, + "seconds": 0.08058441599860089, + "eta_squared": 0.9418930396685032, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 13, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9489157076696662, + "roc_auc": 0.9991607320011339, + "precision_at_n": 0.8854166666666666, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1096938775510204, + "n_models": 1, + "seconds": 0.08507408299919916, + "eta_squared": 0.9418930396685032, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 13, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.932976816087396, + "roc_auc": 0.9988529265873016, + "precision_at_n": 0.8645833333333334, + "macro_pr_auc": 0.9459077380952382, + "worst_group_fpr": 0.04336734693877551, + "n_models": 12, + "seconds": 0.7501342499999737, + "eta_squared": 0.9418930396685032, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "pooled", + "seed": 14, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.5402136199887647, + "roc_auc": 0.9890385841836735, + "precision_at_n": 0.5104166666666666, + "macro_pr_auc": 0.9504573704481792, + "worst_group_fpr": 0.3239795918367347, + "n_models": 1, + "seconds": 0.08150612499957788, + "eta_squared": 0.9427598024542532, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "relative", + "seed": 14, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.17346938775510204, + "n_models": 1, + "seconds": 0.08459220800068579, + "eta_squared": 0.9427598024542532, + "warnings": [] + }, + { + "dataset": "synthetic", + "grouping": "spread=1.000", + "config": "per_group", + "seed": 14, + "mechanism": "contextual", + "level_spread": 1.0, + "pr_auc": 0.992765746796775, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9894085948773449, + "worst_group_fpr": 0.04081632653061224, + "n_models": 12, + "seconds": 0.7775512919979519, + "eta_squared": 0.9427598024542532, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06359419427859529, + "roc_auc": 0.6844236319753716, + "precision_at_n": 0.10664523043944266, + "macro_pr_auc": 0.2615858617185089, + "worst_group_fpr": 0.32496863237139273, + "n_models": 1, + "seconds": 0.28429770799994003, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07853477794777033, + "roc_auc": 0.7221384471549323, + "precision_at_n": 0.11843515541264737, + "macro_pr_auc": 0.30104520993588496, + "worst_group_fpr": 0.34002509410288584, + "n_models": 1, + "seconds": 0.3833286670014786, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08605352394216598, + "roc_auc": 0.6270455719998697, + "precision_at_n": 0.1379957127545552, + "macro_pr_auc": 0.3471501929009753, + "worst_group_fpr": 0.076, + "n_models": 28, + "seconds": 2.303080667003087, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06479623378888605, + "roc_auc": 0.6920459730827131, + "precision_at_n": 0.11387995712754555, + "macro_pr_auc": 0.24849247128209095, + "worst_group_fpr": 0.32496863237139273, + "n_models": 1, + "seconds": 0.2619278749989462, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07872446166069408, + "roc_auc": 0.7308464207214593, + "precision_at_n": 0.1270096463022508, + "macro_pr_auc": 0.2873815302517855, + "worst_group_fpr": 0.3164366373902133, + "n_models": 1, + "seconds": 0.401927415998216, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08671785900975944, + "roc_auc": 0.615626724141447, + "precision_at_n": 0.13665594855305466, + "macro_pr_auc": 0.3560687298644604, + "worst_group_fpr": 0.07425, + "n_models": 28, + "seconds": 2.247596707999037, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06107439693748362, + "roc_auc": 0.6873777113111124, + "precision_at_n": 0.09378349410503752, + "macro_pr_auc": 0.25590641932495106, + "worst_group_fpr": 0.3686323713927227, + "n_models": 1, + "seconds": 0.26288724999903934, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07195736909369158, + "roc_auc": 0.6973923893196474, + "precision_at_n": 0.1085209003215434, + "macro_pr_auc": 0.26700186532376446, + "worst_group_fpr": 0.2466750313676286, + "n_models": 1, + "seconds": 0.36878224999964004, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08055817637796389, + "roc_auc": 0.6126525362156573, + "precision_at_n": 0.13129689174705253, + "macro_pr_auc": 0.32893986825914184, + "worst_group_fpr": 0.0905, + "n_models": 28, + "seconds": 2.046026207997784, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06429968454057976, + "roc_auc": 0.6933390865927515, + "precision_at_n": 0.11548767416934619, + "macro_pr_auc": 0.26682500835664447, + "worst_group_fpr": 0.31267252195734, + "n_models": 1, + "seconds": 0.26589262500056066, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0733964924868918, + "roc_auc": 0.7080216427628617, + "precision_at_n": 0.11280814576634512, + "macro_pr_auc": 0.31077649960963766, + "worst_group_fpr": 0.451693851944793, + "n_models": 1, + "seconds": 0.37211320899950806, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07633181816626733, + "roc_auc": 0.616948213656311, + "precision_at_n": 0.1339764201500536, + "macro_pr_auc": 0.3336965825572596, + "worst_group_fpr": 0.08616780045351474, + "n_models": 28, + "seconds": 2.312892541001929, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.060607111652736634, + "roc_auc": 0.6621210623445587, + "precision_at_n": 0.09807073954983923, + "macro_pr_auc": 0.26129734756404205, + "worst_group_fpr": 0.3575909661229611, + "n_models": 1, + "seconds": 0.2839389159998973, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08220802992573126, + "roc_auc": 0.7120001477220336, + "precision_at_n": 0.12004287245444802, + "macro_pr_auc": 0.30504437523313016, + "worst_group_fpr": 0.21530740276035132, + "n_models": 1, + "seconds": 0.3897333749991958, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08187207342916611, + "roc_auc": 0.6114700595493434, + "precision_at_n": 0.13558413719185422, + "macro_pr_auc": 0.3632590087045792, + "worst_group_fpr": 0.08225, + "n_models": 28, + "seconds": 2.3178855830010434, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06604695963924091, + "roc_auc": 0.6867261113217089, + "precision_at_n": 0.11066452304394427, + "macro_pr_auc": 0.24535099567508128, + "worst_group_fpr": 0.36662484316185695, + "n_models": 1, + "seconds": 0.2720707910011697, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07463292842148948, + "roc_auc": 0.7001492955771575, + "precision_at_n": 0.12165058949624866, + "macro_pr_auc": 0.3074674122412946, + "worst_group_fpr": 0.3214554579673777, + "n_models": 1, + "seconds": 0.3984493329990073, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0785027094613833, + "roc_auc": 0.6040210953241314, + "precision_at_n": 0.1345123258306538, + "macro_pr_auc": 0.34832790946273445, + "worst_group_fpr": 0.0865, + "n_models": 28, + "seconds": 2.34852645799765, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07777070784891844, + "roc_auc": 0.7209001280059634, + "precision_at_n": 0.11441586280814577, + "macro_pr_auc": 0.26303247641541033, + "worst_group_fpr": 0.39849435382685067, + "n_models": 1, + "seconds": 0.2540542919996369, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08682769905620842, + "roc_auc": 0.7560410473715913, + "precision_at_n": 0.13585209003215434, + "macro_pr_auc": 0.30531470499064983, + "worst_group_fpr": 0.33023839397741533, + "n_models": 1, + "seconds": 0.3711304589996871, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08797418981377883, + "roc_auc": 0.6071637424990133, + "precision_at_n": 0.1412111468381565, + "macro_pr_auc": 0.36442990032102757, + "worst_group_fpr": 0.09625, + "n_models": 28, + "seconds": 2.0829790829993726, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06128428571157356, + "roc_auc": 0.673318061100494, + "precision_at_n": 0.10691318327974277, + "macro_pr_auc": 0.26284710122393756, + "worst_group_fpr": 0.3212045169385194, + "n_models": 1, + "seconds": 0.2565165000014531, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06862293675552432, + "roc_auc": 0.6830670025447154, + "precision_at_n": 0.10744908896034298, + "macro_pr_auc": 0.29573862634474224, + "worst_group_fpr": 0.2587202007528231, + "n_models": 1, + "seconds": 0.3732836250019318, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08083032345610011, + "roc_auc": 0.6079619508154728, + "precision_at_n": 0.1377277599142551, + "macro_pr_auc": 0.3255140969349702, + "worst_group_fpr": 0.105, + "n_models": 28, + "seconds": 2.2073950419980974, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06588676629961493, + "roc_auc": 0.6846132479360989, + "precision_at_n": 0.09646302250803858, + "macro_pr_auc": 0.25388308740902954, + "worst_group_fpr": 0.43212045169385194, + "n_models": 1, + "seconds": 0.2586077080013638, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07744300864021096, + "roc_auc": 0.716798334496934, + "precision_at_n": 0.1235262593783494, + "macro_pr_auc": 0.25194493578602234, + "worst_group_fpr": 0.37164366373902136, + "n_models": 1, + "seconds": 0.37577691699698335, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08518821490986139, + "roc_auc": 0.6159125210351939, + "precision_at_n": 0.13290460878885316, + "macro_pr_auc": 0.3657924509299794, + "worst_group_fpr": 0.091, + "n_models": 28, + "seconds": 2.1043218329978117, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06167005301582487, + "roc_auc": 0.6737572351820703, + "precision_at_n": 0.09217577706323687, + "macro_pr_auc": 0.255451871575027, + "worst_group_fpr": 0.3131744040150565, + "n_models": 1, + "seconds": 0.2797340830002213, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07840338643865112, + "roc_auc": 0.7177888081582002, + "precision_at_n": 0.12031082529474812, + "macro_pr_auc": 0.3012854877632323, + "worst_group_fpr": 0.1805, + "n_models": 1, + "seconds": 0.35555091700007324, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08253209519105992, + "roc_auc": 0.6271013736466189, + "precision_at_n": 0.1347802786709539, + "macro_pr_auc": 0.3538262027169542, + "worst_group_fpr": 0.07625, + "n_models": 28, + "seconds": 1.970553833001759, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07045065852496393, + "roc_auc": 0.681170735279146, + "precision_at_n": 0.12620578778135047, + "macro_pr_auc": 0.26236888207493836, + "worst_group_fpr": 0.20325, + "n_models": 1, + "seconds": 0.24093054100012523, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0729493499368938, + "roc_auc": 0.7007142256872718, + "precision_at_n": 0.11495176848874598, + "macro_pr_auc": 0.285273221705565, + "worst_group_fpr": 0.2584692597239649, + "n_models": 1, + "seconds": 0.4011748329976399, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08790933758213504, + "roc_auc": 0.614422059966236, + "precision_at_n": 0.13612004287245444, + "macro_pr_auc": 0.3685751181249375, + "worst_group_fpr": 0.0805, + "n_models": 28, + "seconds": 2.328206541998952, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06295036559112563, + "roc_auc": 0.6962409689785314, + "precision_at_n": 0.08654876741693462, + "macro_pr_auc": 0.2544358483930452, + "worst_group_fpr": 0.42383939774153073, + "n_models": 1, + "seconds": 0.28210775000115973, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07412476212074792, + "roc_auc": 0.7297252907229415, + "precision_at_n": 0.10557341907824223, + "macro_pr_auc": 0.2895992560502265, + "worst_group_fpr": 0.47176913425345046, + "n_models": 1, + "seconds": 0.3880428749980638, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08310053096983538, + "roc_auc": 0.631422064935842, + "precision_at_n": 0.1345123258306538, + "macro_pr_auc": 0.3492017698780529, + "worst_group_fpr": 0.08175, + "n_models": 28, + "seconds": 2.2246951250017446, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06728451657966107, + "roc_auc": 0.6823611724722158, + "precision_at_n": 0.10905680600214362, + "macro_pr_auc": 0.25466656116327896, + "worst_group_fpr": 0.26025, + "n_models": 1, + "seconds": 0.2537664169976779, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08107267027136326, + "roc_auc": 0.7062605361587146, + "precision_at_n": 0.1122722400857449, + "macro_pr_auc": 0.30122007270785933, + "worst_group_fpr": 0.22936010037641155, + "n_models": 1, + "seconds": 0.37384054100039066, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08132474896503511, + "roc_auc": 0.6148238135085453, + "precision_at_n": 0.1385316184351554, + "macro_pr_auc": 0.34649758548222853, + "worst_group_fpr": 0.09225, + "n_models": 28, + "seconds": 2.2186549580001156, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06327527823213003, + "roc_auc": 0.6794141206246529, + "precision_at_n": 0.10289389067524116, + "macro_pr_auc": 0.2592741196508792, + "worst_group_fpr": 0.424090338770389, + "n_models": 1, + "seconds": 0.26233091600079206, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.09033175928257643, + "roc_auc": 0.7130177302375896, + "precision_at_n": 0.11655948553054662, + "macro_pr_auc": 0.3100238438500384, + "worst_group_fpr": 0.285069008782936, + "n_models": 1, + "seconds": 0.37331779200030724, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07805975736148492, + "roc_auc": 0.607997606748622, + "precision_at_n": 0.12968917470525188, + "macro_pr_auc": 0.35955356545587785, + "worst_group_fpr": 0.0845, + "n_models": 28, + "seconds": 2.2150131250018603, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "pooled", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06603648778281995, + "roc_auc": 0.6952049459578116, + "precision_at_n": 0.12379421221864952, + "macro_pr_auc": 0.25673275019955366, + "worst_group_fpr": 0.41405269761606023, + "n_models": 1, + "seconds": 0.25487854099992546, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "relative", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07534273780011053, + "roc_auc": 0.7171714014340422, + "precision_at_n": 0.1045016077170418, + "macro_pr_auc": 0.29780842635792343, + "worst_group_fpr": 0.4398996235884567, + "n_models": 1, + "seconds": 0.36989783399985754, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "entity", + "config": "per_group", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08793068926917626, + "roc_auc": 0.6174247142308251, + "precision_at_n": 0.13344051446945338, + "macro_pr_auc": 0.35291601118538185, + "worst_group_fpr": 0.082, + "n_models": 28, + "seconds": 2.078569792000053, + "eta_squared": 0.5619488636163167, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06359419427859529, + "roc_auc": 0.6844236319753716, + "precision_at_n": 0.10664523043944266, + "macro_pr_auc": 0.15486358973175143, + "worst_group_fpr": 0.08212905995135213, + "n_models": 1, + "seconds": 0.24159191599756014, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07110381391354774, + "roc_auc": 0.6965429727771318, + "precision_at_n": 0.11039657020364416, + "macro_pr_auc": 0.16023615098459712, + "worst_group_fpr": 0.08336910382982783, + "n_models": 1, + "seconds": 0.34601829199891654, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07561607622391128, + "roc_auc": 0.716640646027398, + "precision_at_n": 0.10209003215434084, + "macro_pr_auc": 0.14956598978209557, + "worst_group_fpr": 0.051143636262452004, + "n_models": 3, + "seconds": 0.4875754579989007, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06479623378888605, + "roc_auc": 0.6920459730827131, + "precision_at_n": 0.11387995712754555, + "macro_pr_auc": 0.15193062075560407, + "worst_group_fpr": 0.07442647970620499, + "n_models": 1, + "seconds": 0.2598764169997594, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.061225502312909894, + "roc_auc": 0.6793274272337815, + "precision_at_n": 0.11629153269024652, + "macro_pr_auc": 0.15408180312971578, + "worst_group_fpr": 0.0768350264701674, + "n_models": 1, + "seconds": 0.3561963329993887, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06826979839214103, + "roc_auc": 0.674242823601835, + "precision_at_n": 0.10637727759914255, + "macro_pr_auc": 0.1646718006293512, + "worst_group_fpr": 0.047999736807474665, + "n_models": 3, + "seconds": 0.514039916000911, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06107439693748362, + "roc_auc": 0.6873777113111124, + "precision_at_n": 0.09378349410503752, + "macro_pr_auc": 0.14409721454328103, + "worst_group_fpr": 0.08689845948395097, + "n_models": 1, + "seconds": 0.2507771659984428, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0600603802641146, + "roc_auc": 0.6737304592022867, + "precision_at_n": 0.09110396570203644, + "macro_pr_auc": 0.14919945819278152, + "worst_group_fpr": 0.09300329088567749, + "n_models": 1, + "seconds": 0.3602215419996355, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06805200030217438, + "roc_auc": 0.6631391225164691, + "precision_at_n": 0.10691318327974277, + "macro_pr_auc": 0.1627696505783975, + "worst_group_fpr": 0.05061494796594134, + "n_models": 3, + "seconds": 0.49610391599708237, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06429968454057976, + "roc_auc": 0.6933390865927515, + "precision_at_n": 0.11548767416934619, + "macro_pr_auc": 0.16147972457948392, + "worst_group_fpr": 0.08160442600276625, + "n_models": 1, + "seconds": 0.26588524999897345, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0675781312701662, + "roc_auc": 0.7055571500533133, + "precision_at_n": 0.10610932475884244, + "macro_pr_auc": 0.16140909650780297, + "worst_group_fpr": 0.08019745314064959, + "n_models": 1, + "seconds": 0.3723146660013299, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07330773925395759, + "roc_auc": 0.7058960360996931, + "precision_at_n": 0.10530546623794212, + "macro_pr_auc": 0.16869099135666157, + "worst_group_fpr": 0.05222883855528967, + "n_models": 3, + "seconds": 0.521546166997723, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.060607111652736634, + "roc_auc": 0.6621210623445587, + "precision_at_n": 0.09807073954983923, + "macro_pr_auc": 0.1553474240367889, + "worst_group_fpr": 0.08241522392330805, + "n_models": 1, + "seconds": 0.2576507089979714, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.062949022238146, + "roc_auc": 0.6722181447363893, + "precision_at_n": 0.10905680600214362, + "macro_pr_auc": 0.15459923664407205, + "worst_group_fpr": 0.07335336481137025, + "n_models": 1, + "seconds": 0.37582820800162153, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06551991773583199, + "roc_auc": 0.6867615210019707, + "precision_at_n": 0.09271168274383708, + "macro_pr_auc": 0.15440383007296768, + "worst_group_fpr": 0.05334186654794368, + "n_models": 3, + "seconds": 0.5093504579999717, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06604695963924091, + "roc_auc": 0.6867261113217089, + "precision_at_n": 0.11066452304394427, + "macro_pr_auc": 0.15593379811930022, + "worst_group_fpr": 0.08127056803548434, + "n_models": 1, + "seconds": 0.2543880840021302, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07093348871493281, + "roc_auc": 0.702998017038106, + "precision_at_n": 0.1152197213290461, + "macro_pr_auc": 0.16396008509767293, + "worst_group_fpr": 0.08124672103782134, + "n_models": 1, + "seconds": 0.37792704199819127, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06589072714445193, + "roc_auc": 0.670718957158076, + "precision_at_n": 0.09753483386923902, + "macro_pr_auc": 0.16423524584103774, + "worst_group_fpr": 0.04871941622549721, + "n_models": 3, + "seconds": 0.5126857920004113, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07777070784891844, + "roc_auc": 0.7209001280059634, + "precision_at_n": 0.11441586280814577, + "macro_pr_auc": 0.16230417381678816, + "worst_group_fpr": 0.08503839366623742, + "n_models": 1, + "seconds": 0.27584125000066706, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07303214749902401, + "roc_auc": 0.7166424143458706, + "precision_at_n": 0.13156484458735263, + "macro_pr_auc": 0.1510767031270082, + "worst_group_fpr": 0.07390184575761911, + "n_models": 1, + "seconds": 0.31722187499690335, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07504078781692751, + "roc_auc": 0.6967395172299012, + "precision_at_n": 0.10209003215434084, + "macro_pr_auc": 0.15327462100832814, + "worst_group_fpr": 0.05387055484445434, + "n_models": 3, + "seconds": 0.4245285420001892, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06128428571157356, + "roc_auc": 0.673318061100494, + "precision_at_n": 0.10691318327974277, + "macro_pr_auc": 0.15311394812407336, + "worst_group_fpr": 0.08212905995135213, + "n_models": 1, + "seconds": 0.2261497090003104, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06510429288954053, + "roc_auc": 0.6997572238569124, + "precision_at_n": 0.10316184351554127, + "macro_pr_auc": 0.1570887840363783, + "worst_group_fpr": 0.08737539943721086, + "n_models": 1, + "seconds": 0.3157693749999453, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06677001333844357, + "roc_auc": 0.6868312637795196, + "precision_at_n": 0.09512325830653805, + "macro_pr_auc": 0.1595599701478352, + "worst_group_fpr": 0.05384272914463799, + "n_models": 3, + "seconds": 0.45073600000250735, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06588676629961493, + "roc_auc": 0.6846132479360989, + "precision_at_n": 0.09646302250803858, + "macro_pr_auc": 0.15902473898089842, + "worst_group_fpr": 0.0913578480469309, + "n_models": 1, + "seconds": 0.23203920800006017, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06855202796036898, + "roc_auc": 0.6967101599258813, + "precision_at_n": 0.1122722400857449, + "macro_pr_auc": 0.16224185024824136, + "worst_group_fpr": 0.08456145371297753, + "n_models": 1, + "seconds": 0.319037916000525, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06644022259586063, + "roc_auc": 0.6799007522161968, + "precision_at_n": 0.08654876741693462, + "macro_pr_auc": 0.14515283268998994, + "worst_group_fpr": 0.05754354722021259, + "n_models": 3, + "seconds": 0.4790727910003625, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06167005301582487, + "roc_auc": 0.6737572351820703, + "precision_at_n": 0.09217577706323687, + "macro_pr_auc": 0.15580273173958395, + "worst_group_fpr": 0.08131826203081031, + "n_models": 1, + "seconds": 0.23473350000131177, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.074212649947672, + "roc_auc": 0.7091004073651384, + "precision_at_n": 0.1160235798499464, + "macro_pr_auc": 0.13841713015373577, + "worst_group_fpr": 0.07895740926217389, + "n_models": 1, + "seconds": 0.3204471669996565, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06818715912624354, + "roc_auc": 0.6891144636284435, + "precision_at_n": 0.10155412647374062, + "macro_pr_auc": 0.14820412819477294, + "worst_group_fpr": 0.05259057265290222, + "n_models": 3, + "seconds": 0.4410273329995107, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07045065852496393, + "roc_auc": 0.681170735279146, + "precision_at_n": 0.12620578778135047, + "macro_pr_auc": 0.16035347406903172, + "worst_group_fpr": 0.07220870892354653, + "n_models": 1, + "seconds": 0.2355420409985527, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07841440771859345, + "roc_auc": 0.7026667450320075, + "precision_at_n": 0.10878885316184352, + "macro_pr_auc": 0.1650645677416863, + "worst_group_fpr": 0.06805933133018553, + "n_models": 1, + "seconds": 0.31727712499923655, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06670036755437919, + "roc_auc": 0.6876468533424916, + "precision_at_n": 0.09673097534833869, + "macro_pr_auc": 0.15906439767498926, + "worst_group_fpr": 0.0548444543380266, + "n_models": 3, + "seconds": 0.43318066600113525, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06295036559112563, + "roc_auc": 0.6962409689785314, + "precision_at_n": 0.08654876741693462, + "macro_pr_auc": 0.1505493358424266, + "worst_group_fpr": 0.08751848142318883, + "n_models": 1, + "seconds": 0.22850850000031642, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06500424942725971, + "roc_auc": 0.6974645104793549, + "precision_at_n": 0.10235798499464094, + "macro_pr_auc": 0.15148000604998416, + "worst_group_fpr": 0.07952973720608575, + "n_models": 1, + "seconds": 0.31415566699797637, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06274572055032215, + "roc_auc": 0.6837210625880892, + "precision_at_n": 0.07797427652733119, + "macro_pr_auc": 0.1581081348336286, + "worst_group_fpr": 0.052630323842228266, + "n_models": 3, + "seconds": 0.45697745900179143, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06728451657966107, + "roc_auc": 0.6823611724722158, + "precision_at_n": 0.10905680600214362, + "macro_pr_auc": 0.15658017565737375, + "worst_group_fpr": 0.07964897219440073, + "n_models": 1, + "seconds": 0.2216652079987398, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07708351627138926, + "roc_auc": 0.7115021389996028, + "precision_at_n": 0.1235262593783494, + "macro_pr_auc": 0.16202960823035675, + "worst_group_fpr": 0.07218486192588353, + "n_models": 1, + "seconds": 0.31133950000003097, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07047187139546018, + "roc_auc": 0.6935858047619596, + "precision_at_n": 0.10128617363344052, + "macro_pr_auc": 0.1656376575504184, + "worst_group_fpr": 0.05167644393570849, + "n_models": 3, + "seconds": 0.4432239580019086, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06327527823213003, + "roc_auc": 0.6794141206246529, + "precision_at_n": 0.10289389067524116, + "macro_pr_auc": 0.1455228603956286, + "worst_group_fpr": 0.08656460151666905, + "n_models": 1, + "seconds": 0.2286644999985583, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06949117319065048, + "roc_auc": 0.7011086089672838, + "precision_at_n": 0.12379421221864952, + "macro_pr_auc": 0.16179957982069756, + "worst_group_fpr": 0.08241522392330805, + "n_models": 1, + "seconds": 0.330637957998988, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07209625062706362, + "roc_auc": 0.7010738897850679, + "precision_at_n": 0.10021436227224008, + "macro_pr_auc": 0.1586095944590266, + "worst_group_fpr": 0.04979253112033195, + "n_models": 3, + "seconds": 0.4323672499995155, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "pooled", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06603648778281995, + "roc_auc": 0.6952049459578116, + "precision_at_n": 0.12379421221864952, + "macro_pr_auc": 0.16289907678511376, + "worst_group_fpr": 0.082653693899938, + "n_models": 1, + "seconds": 0.2240179159998661, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "relative", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06606210290769324, + "roc_auc": 0.6989305157904578, + "precision_at_n": 0.11173633440514469, + "macro_pr_auc": 0.15999751055708233, + "worst_group_fpr": 0.08212905995135213, + "n_models": 1, + "seconds": 0.3161649159992521, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "smd", + "grouping": "machine_family", + "config": "per_group", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0743450190784837, + "roc_auc": 0.7035847980702564, + "precision_at_n": 0.11843515541264737, + "macro_pr_auc": 0.15763795542242776, + "worst_group_fpr": 0.049891479770716236, + "n_models": 3, + "seconds": 0.4461614579995512, + "eta_squared": 0.06691168539386136, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6768047861689719, + "roc_auc": 0.972266055193209, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.5313209194469692, + "worst_group_fpr": 0.04235074626865672, + "n_models": 1, + "seconds": 0.20720208400234696, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7771351789296148, + "roc_auc": 0.972047529775493, + "precision_at_n": 0.7743813682678311, + "macro_pr_auc": 0.5802319576815024, + "worst_group_fpr": 0.041100746268656715, + "n_models": 1, + "seconds": 0.24668929200197454, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7620519974790155, + "roc_auc": 0.9831381972681902, + "precision_at_n": 0.8144104803493449, + "macro_pr_auc": 0.6711622930197206, + "worst_group_fpr": 0.08307865529998391, + "n_models": 3, + "seconds": 0.3629343749998952, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6857222009368626, + "roc_auc": 0.9737622464205439, + "precision_at_n": 0.7510917030567685, + "macro_pr_auc": 0.5535748214241942, + "worst_group_fpr": 0.04272388059701492, + "n_models": 1, + "seconds": 0.1986891669985198, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6113602669817982, + "roc_auc": 0.9740465348039771, + "precision_at_n": 0.764919941775837, + "macro_pr_auc": 0.5072647036952613, + "worst_group_fpr": 0.04080223880597015, + "n_models": 1, + "seconds": 0.2542918750004901, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6860051393690949, + "roc_auc": 0.9832123305571526, + "precision_at_n": 0.7445414847161572, + "macro_pr_auc": 0.6350978057997398, + "worst_group_fpr": 0.07519704037317033, + "n_models": 3, + "seconds": 0.35823345799872186, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6297422146981834, + "roc_auc": 0.9741568620407105, + "precision_at_n": 0.7299854439592431, + "macro_pr_auc": 0.545633517740742, + "worst_group_fpr": 0.04003731343283582, + "n_models": 1, + "seconds": 0.20318404100180487, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7286468197463736, + "roc_auc": 0.9748349446827006, + "precision_at_n": 0.7758369723435226, + "macro_pr_auc": 0.5879134098171662, + "worst_group_fpr": 0.037667910447761195, + "n_models": 1, + "seconds": 0.24570520799898077, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6603901413486373, + "roc_auc": 0.9830915309598965, + "precision_at_n": 0.745269286754003, + "macro_pr_auc": 0.6320887626863021, + "worst_group_fpr": 0.09449895447965256, + "n_models": 3, + "seconds": 0.3419267499994021, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6430275805941272, + "roc_auc": 0.9705622269931803, + "precision_at_n": 0.7802037845705968, + "macro_pr_auc": 0.4909138377370974, + "worst_group_fpr": 0.03792910447761194, + "n_models": 1, + "seconds": 0.20373449999897275, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6705523695466854, + "roc_auc": 0.9705192189862665, + "precision_at_n": 0.784570596797671, + "macro_pr_auc": 0.54605265099269, + "worst_group_fpr": 0.04074626865671642, + "n_models": 1, + "seconds": 0.24851304199910373, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6400187149102201, + "roc_auc": 0.9825899113752985, + "precision_at_n": 0.7758369723435226, + "macro_pr_auc": 0.586999374960694, + "worst_group_fpr": 0.10503458259610744, + "n_models": 3, + "seconds": 0.3355528749998484, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6018090042265931, + "roc_auc": 0.9710144405962212, + "precision_at_n": 0.7489082969432315, + "macro_pr_auc": 0.5032442675968255, + "worst_group_fpr": 0.04126865671641791, + "n_models": 1, + "seconds": 0.20233870800075238, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6812099737634885, + "roc_auc": 0.9714637848373232, + "precision_at_n": 0.7787481804949054, + "macro_pr_auc": 0.5465542275381834, + "worst_group_fpr": 0.03947761194029851, + "n_models": 1, + "seconds": 0.2541480420004518, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6218142259808852, + "roc_auc": 0.9809207694921918, + "precision_at_n": 0.7074235807860262, + "macro_pr_auc": 0.5716419581078703, + "worst_group_fpr": 0.0653048093935982, + "n_models": 3, + "seconds": 0.3419192910005222, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7219691731333892, + "roc_auc": 0.9724569407120224, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.549924573247098, + "worst_group_fpr": 0.039850746268656714, + "n_models": 1, + "seconds": 0.2112539170011587, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6647913663607559, + "roc_auc": 0.9717020499521438, + "precision_at_n": 0.7736535662299855, + "macro_pr_auc": 0.5191153660713054, + "worst_group_fpr": 0.03953358208955224, + "n_models": 1, + "seconds": 0.262136166998971, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7126872409664603, + "roc_auc": 0.982975745991415, + "precision_at_n": 0.7983988355167394, + "macro_pr_auc": 0.6219767122446509, + "worst_group_fpr": 0.09112111951101817, + "n_models": 3, + "seconds": 0.3417797090005479, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5477739185722166, + "roc_auc": 0.971442326765272, + "precision_at_n": 0.7430858806404658, + "macro_pr_auc": 0.4502184719290038, + "worst_group_fpr": 0.03852611940298507, + "n_models": 1, + "seconds": 0.20219741700202576, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6070284009175415, + "roc_auc": 0.971787871432959, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.49669027776810853, + "worst_group_fpr": 0.03880597014925373, + "n_models": 1, + "seconds": 0.2465083750030317, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6469253178574408, + "roc_auc": 0.9826695942588207, + "precision_at_n": 0.759825327510917, + "macro_pr_auc": 0.6120341502004562, + "worst_group_fpr": 0.07962039568923918, + "n_models": 3, + "seconds": 0.3544100420003815, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.656610287216522, + "roc_auc": 0.9702420418651903, + "precision_at_n": 0.7794759825327511, + "macro_pr_auc": 0.4361360618102787, + "worst_group_fpr": 0.03962686567164179, + "n_models": 1, + "seconds": 0.1933791250012291, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7064937151856711, + "roc_auc": 0.970018717966492, + "precision_at_n": 0.7714701601164483, + "macro_pr_auc": 0.5121206601547154, + "worst_group_fpr": 0.037052238805970146, + "n_models": 1, + "seconds": 0.25321816699943156, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6673518435943845, + "roc_auc": 0.9820557723553934, + "precision_at_n": 0.7976710334788938, + "macro_pr_auc": 0.6227622361027524, + "worst_group_fpr": 0.08862795560559755, + "n_models": 3, + "seconds": 0.3529735830015852, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6879044743725874, + "roc_auc": 0.9685583856578505, + "precision_at_n": 0.7554585152838428, + "macro_pr_auc": 0.5064574481765121, + "worst_group_fpr": 0.03992537313432836, + "n_models": 1, + "seconds": 0.1997192080016248, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6683263018119266, + "roc_auc": 0.9727062455753196, + "precision_at_n": 0.7707423580786026, + "macro_pr_auc": 0.5390720178834354, + "worst_group_fpr": 0.03583955223880597, + "n_models": 1, + "seconds": 0.2481015839985048, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7168319555925291, + "roc_auc": 0.9852743804928692, + "precision_at_n": 0.8056768558951966, + "macro_pr_auc": 0.618973540170405, + "worst_group_fpr": 0.09900273443783175, + "n_models": 3, + "seconds": 0.35266041599970777, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6407583233057124, + "roc_auc": 0.9725172513496863, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.4462010831582593, + "worst_group_fpr": 0.03604477611940299, + "n_models": 1, + "seconds": 0.20796087499911664, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.669307405730254, + "roc_auc": 0.9754224127665878, + "precision_at_n": 0.7590975254730713, + "macro_pr_auc": 0.5597129030487102, + "worst_group_fpr": 0.034384328358208954, + "n_models": 1, + "seconds": 0.24508341700129677, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6503634825175936, + "roc_auc": 0.9822675809804727, + "precision_at_n": 0.772197962154294, + "macro_pr_auc": 0.6361188963477792, + "worst_group_fpr": 0.10463245938555574, + "n_models": 3, + "seconds": 0.34629341600157204, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6353575815204143, + "roc_auc": 0.9720192630479938, + "precision_at_n": 0.7561863173216885, + "macro_pr_auc": 0.46688389334892527, + "worst_group_fpr": 0.03820170500241274, + "n_models": 1, + "seconds": 0.19593587499912246, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.681123198451289, + "roc_auc": 0.9719263897454645, + "precision_at_n": 0.7765647743813683, + "macro_pr_auc": 0.5041782349951254, + "worst_group_fpr": 0.03718283582089552, + "n_models": 1, + "seconds": 0.24913533300059498, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6647451339671704, + "roc_auc": 0.9829602590021178, + "precision_at_n": 0.7161572052401747, + "macro_pr_auc": 0.6087803813113698, + "worst_group_fpr": 0.08637606562650796, + "n_models": 3, + "seconds": 0.3410571249987697, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6043056093954045, + "roc_auc": 0.9726922932353457, + "precision_at_n": 0.7358078602620087, + "macro_pr_auc": 0.5093356935366994, + "worst_group_fpr": 0.03921641791044776, + "n_models": 1, + "seconds": 0.1936662919979426, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6560189855442788, + "roc_auc": 0.9721102072314795, + "precision_at_n": 0.7736535662299855, + "macro_pr_auc": 0.5149827041492328, + "worst_group_fpr": 0.03949626865671642, + "n_models": 1, + "seconds": 0.25070270800279104, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6954184299748126, + "roc_auc": 0.981737894605083, + "precision_at_n": 0.7765647743813683, + "macro_pr_auc": 0.6170665503805763, + "worst_group_fpr": 0.07495576644683931, + "n_models": 3, + "seconds": 0.3438656250000349, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6275629286313821, + "roc_auc": 0.9720377058583466, + "precision_at_n": 0.7328966521106259, + "macro_pr_auc": 0.50662550657358, + "worst_group_fpr": 0.03824626865671642, + "n_models": 1, + "seconds": 0.20500329100104864, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6446591525183992, + "roc_auc": 0.9754932498017221, + "precision_at_n": 0.7481804949053857, + "macro_pr_auc": 0.5595449760087942, + "worst_group_fpr": 0.03886194029850746, + "n_models": 1, + "seconds": 0.25379166700076894, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6426365135474428, + "roc_auc": 0.9842551193685908, + "precision_at_n": 0.7976710334788938, + "macro_pr_auc": 0.6091390698178812, + "worst_group_fpr": 0.10398906224867299, + "n_models": 3, + "seconds": 0.346133415998338, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7285997390796592, + "roc_auc": 0.9726331065662004, + "precision_at_n": 0.7802037845705968, + "macro_pr_auc": 0.533125894293412, + "worst_group_fpr": 0.041026119402985076, + "n_models": 1, + "seconds": 0.20132883300175308, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.720546436302856, + "roc_auc": 0.9734600934221017, + "precision_at_n": 0.7758369723435226, + "macro_pr_auc": 0.575537891034216, + "worst_group_fpr": 0.0407089552238806, + "n_models": 1, + "seconds": 0.256971082999371, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7213639938542473, + "roc_auc": 0.9827387021116191, + "precision_at_n": 0.8078602620087336, + "macro_pr_auc": 0.6256143354854139, + "worst_group_fpr": 0.0662699050989223, + "n_models": 3, + "seconds": 0.3361283749982249, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "pooled", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6233078022796769, + "roc_auc": 0.9716151747508427, + "precision_at_n": 0.740174672489083, + "macro_pr_auc": 0.5207234690335601, + "worst_group_fpr": 0.0428544776119403, + "n_models": 1, + "seconds": 0.19976808299907134, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "relative", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5663276870854721, + "roc_auc": 0.9736081276411505, + "precision_at_n": 0.7023289665211062, + "macro_pr_auc": 0.5392097197171283, + "worst_group_fpr": 0.043115671641791045, + "n_models": 1, + "seconds": 0.2500787089993537, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "protocol_type", + "config": "per_group", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6482634991740984, + "roc_auc": 0.982784898298465, + "precision_at_n": 0.7197962154294032, + "macro_pr_auc": 0.6063187120045425, + "worst_group_fpr": 0.07085410969921184, + "n_models": 3, + "seconds": 0.3386523330009368, + "eta_squared": 0.05087186473761437, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6768047861689719, + "roc_auc": 0.972266055193209, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.6996034070040531, + "worst_group_fpr": 0.7096774193548387, + "n_models": 1, + "seconds": 0.20758445799947367, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5406925996404937, + "roc_auc": 0.9614455184035687, + "precision_at_n": 0.4912663755458515, + "macro_pr_auc": 0.7856591729582578, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.274014165999688, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.18703204247733796, + "roc_auc": 0.8739845727971822, + "precision_at_n": 0.19068413391557495, + "macro_pr_auc": 0.6760946942545061, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.076055790999817, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6857222009368626, + "roc_auc": 0.9737622464205439, + "precision_at_n": 0.7510917030567685, + "macro_pr_auc": 0.7586410316256198, + "worst_group_fpr": 0.8333333333333334, + "n_models": 1, + "seconds": 0.19729187499979162, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5252317276128622, + "roc_auc": 0.9620086806682451, + "precision_at_n": 0.5356622998544396, + "macro_pr_auc": 0.7585627369212566, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.2684215420013061, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.20112796571471114, + "roc_auc": 0.8791306680624627, + "precision_at_n": 0.21033478893740903, + "macro_pr_auc": 0.6656088655658063, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.096728083000926, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6297422146981834, + "roc_auc": 0.9741568620407105, + "precision_at_n": 0.7299854439592431, + "macro_pr_auc": 0.7043019881527529, + "worst_group_fpr": 0.8387096774193549, + "n_models": 1, + "seconds": 0.19956187500065425, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5720614240791909, + "roc_auc": 0.9709951980390381, + "precision_at_n": 0.5291120815138283, + "macro_pr_auc": 0.7814869495898868, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.25566479199915193, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.2095921913874511, + "roc_auc": 0.8796752956539747, + "precision_at_n": 0.2096069868995633, + "macro_pr_auc": 0.6743987061641321, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.0936844589996326, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6430275805941272, + "roc_auc": 0.9705622269931803, + "precision_at_n": 0.7802037845705968, + "macro_pr_auc": 0.7337349843379073, + "worst_group_fpr": 0.8387096774193549, + "n_models": 1, + "seconds": 0.19409620800070115, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5500951341718753, + "roc_auc": 0.9607994526532693, + "precision_at_n": 0.5312954876273653, + "macro_pr_auc": 0.753382778831343, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.2788463329998194, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.19649999140658245, + "roc_auc": 0.8780009824349442, + "precision_at_n": 0.20232896652110627, + "macro_pr_auc": 0.6518937356853428, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.0975127090023307, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6018090042265931, + "roc_auc": 0.9710144405962212, + "precision_at_n": 0.7489082969432315, + "macro_pr_auc": 0.6721139290877173, + "worst_group_fpr": 0.8763440860215054, + "n_models": 1, + "seconds": 0.20953120899866917, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5257244009940591, + "roc_auc": 0.9591343527338729, + "precision_at_n": 0.5269286754002911, + "macro_pr_auc": 0.6976737326981719, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.2813873330014758, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.1834832859642059, + "roc_auc": 0.8778802260672464, + "precision_at_n": 0.2081513828238719, + "macro_pr_auc": 0.6563759276300162, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.078388417001406, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7219691731333892, + "roc_auc": 0.9724569407120224, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.7446667828428115, + "worst_group_fpr": 0.8333333333333334, + "n_models": 1, + "seconds": 0.19650229200124159, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5062848011346063, + "roc_auc": 0.9612991809446872, + "precision_at_n": 0.5254730713245997, + "macro_pr_auc": 0.7592604380773622, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.27197862500179326, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.20201422842494055, + "roc_auc": 0.8773170259767064, + "precision_at_n": 0.21033478893740903, + "macro_pr_auc": 0.6796184995818257, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.0553991250017134, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5477739185722166, + "roc_auc": 0.971442326765272, + "precision_at_n": 0.7430858806404658, + "macro_pr_auc": 0.716946321427026, + "worst_group_fpr": 0.8548387096774194, + "n_models": 1, + "seconds": 0.2002458339993609, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5016511517428832, + "roc_auc": 0.9635457454430479, + "precision_at_n": 0.524745269286754, + "macro_pr_auc": 0.7856706887167291, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.27726262499709264, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.20804780617622678, + "roc_auc": 0.8785029910855678, + "precision_at_n": 0.21251819505094613, + "macro_pr_auc": 0.674123994825253, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.034101875000488, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.656610287216522, + "roc_auc": 0.9702420418651903, + "precision_at_n": 0.7794759825327511, + "macro_pr_auc": 0.7494908624299447, + "worst_group_fpr": 0.8548387096774194, + "n_models": 1, + "seconds": 0.20102370800304925, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5607263597688444, + "roc_auc": 0.9665532690505476, + "precision_at_n": 0.5334788937409025, + "macro_pr_auc": 0.7897788734647209, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.270870415999525, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.20854992987888132, + "roc_auc": 0.8772972592611278, + "precision_at_n": 0.2205240174672489, + "macro_pr_auc": 0.6662087227753097, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.1973240000006626, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6879044743725874, + "roc_auc": 0.9685583856578505, + "precision_at_n": 0.7554585152838428, + "macro_pr_auc": 0.7280317213838148, + "worst_group_fpr": 0.8387096774193549, + "n_models": 1, + "seconds": 0.1988795410034072, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5277315215516899, + "roc_auc": 0.9567197063087554, + "precision_at_n": 0.5276564774381368, + "macro_pr_auc": 0.717254263513192, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.28242937499817344, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.20067006461093626, + "roc_auc": 0.8797338987240818, + "precision_at_n": 0.2066957787481805, + "macro_pr_auc": 0.6559956670259489, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.094073208001646, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6407583233057124, + "roc_auc": 0.9725172513496863, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.6836882921538495, + "worst_group_fpr": 0.8279569892473119, + "n_models": 1, + "seconds": 0.19848512500175275, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.47676605049364645, + "roc_auc": 0.9601640700075895, + "precision_at_n": 0.5058224163027657, + "macro_pr_auc": 0.7270344397438341, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.26985037499980535, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.20052733262055475, + "roc_auc": 0.8692715890738243, + "precision_at_n": 0.2183406113537118, + "macro_pr_auc": 0.6531491636743189, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.0625154159970407, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6353575815204143, + "roc_auc": 0.9720192630479938, + "precision_at_n": 0.7561863173216885, + "macro_pr_auc": 0.69812154611129, + "worst_group_fpr": 0.8494623655913979, + "n_models": 1, + "seconds": 0.20798570799888694, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.4809364976628495, + "roc_auc": 0.9634443288990397, + "precision_at_n": 0.5160116448326055, + "macro_pr_auc": 0.7000143546785108, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.26160629099831567, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.2001193602506792, + "roc_auc": 0.8723764440320632, + "precision_at_n": 0.19723435225618632, + "macro_pr_auc": 0.655558862847107, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.245599709000089, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6043056093954045, + "roc_auc": 0.9726922932353457, + "precision_at_n": 0.7358078602620087, + "macro_pr_auc": 0.7005306512787453, + "worst_group_fpr": 0.8494623655913979, + "n_models": 1, + "seconds": 0.21010816599664395, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5033086492667329, + "roc_auc": 0.962167760039465, + "precision_at_n": 0.5269286754002911, + "macro_pr_auc": 0.694393634085864, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.30407349999950384, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.20085933528134287, + "roc_auc": 0.8727757662704008, + "precision_at_n": 0.21179039301310043, + "macro_pr_auc": 0.6688128241082326, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.297948749997886, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6275629286313821, + "roc_auc": 0.9720377058583466, + "precision_at_n": 0.7328966521106259, + "macro_pr_auc": 0.7174989474930186, + "worst_group_fpr": 0.8494623655913979, + "n_models": 1, + "seconds": 0.21575704199858592, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.38342074843131196, + "roc_auc": 0.9585648411278064, + "precision_at_n": 0.462882096069869, + "macro_pr_auc": 0.7117930588430045, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.27413754100052756, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.19642812702900658, + "roc_auc": 0.8711648059692065, + "precision_at_n": 0.20087336244541484, + "macro_pr_auc": 0.659899391775854, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.0494829999988724, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7285997390796592, + "roc_auc": 0.9726331065662004, + "precision_at_n": 0.7802037845705968, + "macro_pr_auc": 0.7818843759635191, + "worst_group_fpr": 0.8494623655913979, + "n_models": 1, + "seconds": 0.2014633340004366, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5273879980108448, + "roc_auc": 0.9632161578861057, + "precision_at_n": 0.524745269286754, + "macro_pr_auc": 0.7185393777974337, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.28373254200050724, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.2014786489324183, + "roc_auc": 0.8755190654132603, + "precision_at_n": 0.21397379912663755, + "macro_pr_auc": 0.6828954458144512, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.414072291001503, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "pooled", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6233078022796769, + "roc_auc": 0.9716151747508427, + "precision_at_n": 0.740174672489083, + "macro_pr_auc": 0.7151427340837024, + "worst_group_fpr": 0.8494623655913979, + "n_models": 1, + "seconds": 0.22015479199762922, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "relative", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5207206515635024, + "roc_auc": 0.9668831105811455, + "precision_at_n": 0.5254730713245997, + "macro_pr_auc": 0.7343201490083063, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.2871878749974712, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "service", + "config": "per_group", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.19248512507512788, + "roc_auc": 0.8764153330401938, + "precision_at_n": 0.20087336244541484, + "macro_pr_auc": 0.6678446655880989, + "worst_group_fpr": 1.0, + "n_models": 50, + "seconds": 3.0972869580000406, + "eta_squared": 0.23802723873795992, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6768047861689719, + "roc_auc": 0.972266055193209, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.6962765518547999, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.23114012500082026, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.3273271540675249, + "roc_auc": 0.9664521118838899, + "precision_at_n": 0.32168850072780203, + "macro_pr_auc": 0.6363351199492512, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.263118625000061, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 0, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07968185827363976, + "roc_auc": 0.7140235023114089, + "precision_at_n": 0.1564774381368268, + "macro_pr_auc": 0.3858407985363126, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.7374150840005314, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6857222009368626, + "roc_auc": 0.9737622464205439, + "precision_at_n": 0.7510917030567685, + "macro_pr_auc": 0.6425280272669781, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.19914012500157696, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.24046309606615066, + "roc_auc": 0.9590784461074712, + "precision_at_n": 0.2059679767103348, + "macro_pr_auc": 0.5880528015383217, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.24573016599970288, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 1, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0750782235983171, + "roc_auc": 0.7167630729048562, + "precision_at_n": 0.13537117903930132, + "macro_pr_auc": 0.395465069469411, + "worst_group_fpr": 0.3983050847457627, + "n_models": 9, + "seconds": 0.6775090420014749, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6297422146981834, + "roc_auc": 0.9741568620407105, + "precision_at_n": 0.7299854439592431, + "macro_pr_auc": 0.6925223121673637, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.19627287500043167, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.46016034585866533, + "roc_auc": 0.9745935562322855, + "precision_at_n": 0.574235807860262, + "macro_pr_auc": 0.6241088073386754, + "worst_group_fpr": 0.547945205479452, + "n_models": 1, + "seconds": 0.24542595900129527, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 2, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07690179233831149, + "roc_auc": 0.7192531008724352, + "precision_at_n": 0.17248908296943233, + "macro_pr_auc": 0.40360496245498223, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.6885379580016888, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6430275805941272, + "roc_auc": 0.9705622269931803, + "precision_at_n": 0.7802037845705968, + "macro_pr_auc": 0.6836070915297823, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.20553808299882803, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.2589765025486777, + "roc_auc": 0.9598066101928685, + "precision_at_n": 0.22780203784570596, + "macro_pr_auc": 0.6008786154330514, + "worst_group_fpr": 0.576271186440678, + "n_models": 1, + "seconds": 0.2481297500016808, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 3, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0637751855247328, + "roc_auc": 0.7069178597970749, + "precision_at_n": 0.0982532751091703, + "macro_pr_auc": 0.3945021474398532, + "worst_group_fpr": 0.4011299435028249, + "n_models": 9, + "seconds": 0.6691163750001579, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6018090042265931, + "roc_auc": 0.9710144405962212, + "precision_at_n": 0.7489082969432315, + "macro_pr_auc": 0.630740432852108, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.1993304169991461, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.3234510522653249, + "roc_auc": 0.9669947833378844, + "precision_at_n": 0.3158660844250364, + "macro_pr_auc": 0.6259982222975145, + "worst_group_fpr": 0.5342465753424658, + "n_models": 1, + "seconds": 0.26843716700022924, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 4, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08640878122573586, + "roc_auc": 0.7189830836469692, + "precision_at_n": 0.1433770014556041, + "macro_pr_auc": 0.40852179948277323, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.9446155420009745, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7219691731333892, + "roc_auc": 0.9724569407120224, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.6724135119096966, + "worst_group_fpr": 0.952054794520548, + "n_models": 1, + "seconds": 0.22371366600054898, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.42668957005806235, + "roc_auc": 0.9717321701469596, + "precision_at_n": 0.5203784570596798, + "macro_pr_auc": 0.6252719546383938, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.2623475000000326, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 5, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07996626606127319, + "roc_auc": 0.7514170973465459, + "precision_at_n": 0.11499272197962154, + "macro_pr_auc": 0.4051010495650098, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.6614609999996901, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.5477739185722166, + "roc_auc": 0.971442326765272, + "precision_at_n": 0.7430858806404658, + "macro_pr_auc": 0.621669004706941, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.20584945900191087, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.20737521034393283, + "roc_auc": 0.9453792151980602, + "precision_at_n": 0.21251819505094613, + "macro_pr_auc": 0.6019847338794133, + "worst_group_fpr": 0.8698630136986302, + "n_models": 1, + "seconds": 0.25065941700086114, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 6, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08017149867796952, + "roc_auc": 0.7134330081584336, + "precision_at_n": 0.1462882096069869, + "macro_pr_auc": 0.40062922236528836, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.6690729170004488, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.656610287216522, + "roc_auc": 0.9702420418651903, + "precision_at_n": 0.7794759825327511, + "macro_pr_auc": 0.6754417961969063, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.20678316699923016, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.395278085800411, + "roc_auc": 0.9732555797850024, + "precision_at_n": 0.5094614264919942, + "macro_pr_auc": 0.5920150302290529, + "worst_group_fpr": 0.5753424657534246, + "n_models": 1, + "seconds": 0.24940754199997173, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 7, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07631598617840837, + "roc_auc": 0.7293813595138456, + "precision_at_n": 0.09243085880640466, + "macro_pr_auc": 0.39603941873799964, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.6708718749978289, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6879044743725874, + "roc_auc": 0.9685583856578505, + "precision_at_n": 0.7554585152838428, + "macro_pr_auc": 0.6771777353932684, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.1932590420001361, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.3130157461431265, + "roc_auc": 0.9644009179710267, + "precision_at_n": 0.2685589519650655, + "macro_pr_auc": 0.6222887020900015, + "worst_group_fpr": 0.8531073446327684, + "n_models": 1, + "seconds": 0.26523350000206847, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 8, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07639087290321192, + "roc_auc": 0.7220214353333034, + "precision_at_n": 0.12518195050946143, + "macro_pr_auc": 0.40693402893363, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.6679120419976243, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6407583233057124, + "roc_auc": 0.9725172513496863, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.7003670689921986, + "worst_group_fpr": 0.9383561643835616, + "n_models": 1, + "seconds": 0.19455912500052364, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.3364051211985619, + "roc_auc": 0.9655589621888561, + "precision_at_n": 0.31077147016011647, + "macro_pr_auc": 0.5965132042321021, + "worst_group_fpr": 0.6610169491525424, + "n_models": 1, + "seconds": 0.24625333299991325, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 9, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.0760674281761912, + "roc_auc": 0.7293366061135113, + "precision_at_n": 0.13901018922852984, + "macro_pr_auc": 0.4172472929121744, + "worst_group_fpr": 0.45454545454545453, + "n_models": 9, + "seconds": 0.6731629589994554, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6353575815204143, + "roc_auc": 0.9720192630479938, + "precision_at_n": 0.7561863173216885, + "macro_pr_auc": 0.650372407487826, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.20414445899950806, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.3480830797503195, + "roc_auc": 0.9703641329455036, + "precision_at_n": 0.38573508005822416, + "macro_pr_auc": 0.5814514554227593, + "worst_group_fpr": 0.6301369863013698, + "n_models": 1, + "seconds": 0.26166016700153705, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 10, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06438051583272147, + "roc_auc": 0.7163338196010209, + "precision_at_n": 0.07496360989810771, + "macro_pr_auc": 0.3859154054264402, + "worst_group_fpr": 0.45454545454545453, + "n_models": 9, + "seconds": 0.706516541999008, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6043056093954045, + "roc_auc": 0.9726922932353457, + "precision_at_n": 0.7358078602620087, + "macro_pr_auc": 0.634287625877879, + "worst_group_fpr": 0.9315068493150684, + "n_models": 1, + "seconds": 0.3409611249990121, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.38277031030813824, + "roc_auc": 0.9698135396749323, + "precision_at_n": 0.38355167394468703, + "macro_pr_auc": 0.6796146639100208, + "worst_group_fpr": 0.6027397260273972, + "n_models": 1, + "seconds": 0.2751555830000143, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 11, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.07489059141273685, + "roc_auc": 0.7470951303826177, + "precision_at_n": 0.09970887918486172, + "macro_pr_auc": 0.4024024556919111, + "worst_group_fpr": 0.423728813559322, + "n_models": 9, + "seconds": 0.6938686250032333, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6275629286313821, + "roc_auc": 0.9720377058583466, + "precision_at_n": 0.7328966521106259, + "macro_pr_auc": 0.6401234203460929, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.23530379200019524, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.26900659319698456, + "roc_auc": 0.9645175188974232, + "precision_at_n": 0.21033478893740903, + "macro_pr_auc": 0.6039773125607502, + "worst_group_fpr": 0.5, + "n_models": 1, + "seconds": 0.27582854199863505, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 12, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.08429731402335623, + "roc_auc": 0.7179036199589228, + "precision_at_n": 0.1586608442503639, + "macro_pr_auc": 0.4099720782816015, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.7225801250024233, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.7285997390796592, + "roc_auc": 0.9726331065662004, + "precision_at_n": 0.7802037845705968, + "macro_pr_auc": 0.7060445397113666, + "worst_group_fpr": 0.9315068493150684, + "n_models": 1, + "seconds": 0.22078720900026383, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.39602434244127666, + "roc_auc": 0.9715310932597532, + "precision_at_n": 0.4104803493449782, + "macro_pr_auc": 0.6008872050536501, + "worst_group_fpr": 0.9915254237288136, + "n_models": 1, + "seconds": 0.27268545900005847, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 13, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.06710228407045235, + "roc_auc": 0.6961358945809176, + "precision_at_n": 0.10116448326055313, + "macro_pr_auc": 0.3926832918461665, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.6953132090020517, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "pooled", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.6233078022796769, + "roc_auc": 0.9716151747508427, + "precision_at_n": 0.740174672489083, + "macro_pr_auc": 0.6710047505870849, + "worst_group_fpr": 0.9383561643835616, + "n_models": 1, + "seconds": 0.21069662499940023, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "relative", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.3534844492103426, + "roc_auc": 0.9712877162496516, + "precision_at_n": 0.31368267831149926, + "macro_pr_auc": 0.5719889990063187, + "worst_group_fpr": 1.0, + "n_models": 1, + "seconds": 0.2774734589984291, + "eta_squared": 0.18908304175391513, + "warnings": [] + }, + { + "dataset": "nslkdd", + "grouping": "flag", + "config": "per_group", + "seed": 14, + "mechanism": "real", + "level_spread": NaN, + "pr_auc": 0.09778967463280377, + "roc_auc": 0.7424727504099728, + "precision_at_n": 0.19359534206695778, + "macro_pr_auc": 0.407532612033249, + "worst_group_fpr": 0.5454545454545454, + "n_models": 9, + "seconds": 0.8698225000007369, + "eta_squared": 0.18908304175391513, + "warnings": [] + } + ] +} \ No newline at end of file diff --git a/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.md b/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.md new file mode 100644 index 000000000..0305678a5 --- /dev/null +++ b/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.md @@ -0,0 +1,77 @@ +# Anomaly conditioning experiment — 2026-08-25 + +DQX `55ebd5ca` · datasets: synthetic, smd, nslkdd · seeds per cell: 15 · cells: 1395 + +PR-AUC is the primary metric. No point-adjusted F1 appears anywhere; see `metrics.py`. +DQX scores rows independently, so figures on time-series data are not comparable with +published sequence-model results — that is a different task, not a worse implementation. + +## Does removing the heterogeneity gate cost anything? + +Rules fixed before running: **no gate** if the worst delta below eta-squared 0.1 exceeds -0.01; **gate needed** if any such cell reaches -0.02. + +- **Pre-registered rule (worst single cell): gate needed; refit the threshold from this sweep** +- **Variance-robust companion (worst per-grouping median): no gate needed** + +The pre-registered rule takes a minimum over individual cells, so it is maximally sensitive to estimator variance. It is reported unchanged, alongside the median form of the same question, so the criterion set in advance and the answer it gave are both visible. Where the two disagree, the per-grouping deltas below show why. + +| statistic | value | +|---|---| +| n_low_eta_cells | 90 | +| n_low_eta_groupings | 4 | +| worst_delta_single_cell | -0.0744 | +| n_harmful_cells | 3 | +| worst_median_delta_per_grouping | +0.0000 | +| n_harmful_groupings | 0 | +| median_delta_below_threshold | +0.0000 | + +### Does eta-squared predict the benefit at all? + +Spearman rho(eta-squared, delta) = **+0.597** (bootstrap 95% CI +0.523 to +0.668). + +Least squares `delta ~ 1 + eta_squared + is_contextual`, which asks whether eta-squared still carries signal once the anomaly mechanism is controlled for: + +| term | coefficient | +|---|---| +| intercept | -0.0998 | +| eta_squared | +0.2760 | +| is_contextual | +0.0830 | +| R-squared | 0.561 (n=390) | + +### Every grouping below eta-squared 0.1, seed by seed + +| dataset | grouping | eta-squared | median delta | min | max | seeds agree? | +|---|---|---|---|---|---|---| +| nslkdd | protocol_type | 0.0509 | +0.0285 | -0.0744 | +0.1003 | **no** | +| smd | machine_family | 0.0669 | +0.0033 | -0.0047 | +0.0125 | **no** | +| synthetic | spread=0.000 | 0.0008 | +0.0000 | -0.0019 | +0.0283 | **no** | +| synthetic | spread=0.050 | 0.0593 | +0.0000 | -0.0019 | +0.0169 | **no** | + +## Paired comparisons + +### baseline-relative minus pooled (PR-AUC) + +| mechanism | n | median delta | IQR | Wilcoxon p | +|---|---|---|---|---| +| contextual | 195 | +0.0734 | +0.0068 to +0.2649 | 5.28e-32 | +| global | 195 | +0.0000 | -0.0000 to +0.0000 | 2.01e-01 | +| real | 75 | -0.0010 | -0.1622 to +0.0110 | 3.05e-03 | +| *all* | 465 | +0.0000 | +0.0000 to +0.0466 | 1.07e-23 | + +### baseline-relative minus per-group (PR-AUC) + +| mechanism | n | median delta | IQR | Wilcoxon p | +|---|---|---|---|---| +| contextual | 195 | +0.0274 | +0.0128 to +0.0413 | 1.20e-33 | +| global | 195 | +0.0006 | +0.0000 to +0.0017 | 3.35e-26 | +| real | 75 | +0.0123 | -0.0043 to +0.2785 | 3.24e-05 | +| *all* | 465 | +0.0066 | +0.0004 to +0.0302 | 7.95e-55 | + +## Cost + +| config | median models | median seconds | +|---|---|---| +| pooled | 1 | 0.08 | +| relative | 1 | 0.08 | +| per_group | 12 | 0.73 | + From 9993c388bf0c845e41de15673b157b9281215a79 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 16:43:12 +0100 Subject: [PATCH 013/107] Add the plain-tabular regime to the conditioning harness Ten ADBench datasets (BSD-2-Clause, redistributing the ODDS / UCI / Kaggle collections), chosen to span the axes that actually change a detector's behaviour: 1.8k to 285k rows, 9 to 100 features, and base rates from 0.17% (the Kaggle credit-card set) to 40%. These answer a different question from the rest of the harness. SMD and NSL-KDD exist to test whether conditioning on a group helps; these have no grouping at all, so they characterise the ungrouped path -- which this branch does not change -- and serve as its regression baseline. Reported per dataset against its own random floor and its own max-abs-z baseline, never pooled into a headline: PR-AUC moves with the base rate, so 0.19 at a 0.17% base rate (a 99x lift) and 0.48 at 40% are not comparable numbers. An earlier draft of this harness rejected ADBench outright because its preprocessing discards categorical column identity. That is disqualifying for the grouping question but not for plain tabular data, so the exclusion was too broad. --- .../anomaly_conditioning/datasets/tabular.py | 99 +++++++++++++++++++ benchmarks/anomaly_conditioning/metrics.py | 18 ++++ .../anomaly_conditioning/run_experiment.py | 86 +++++++++++++++- 3 files changed, 200 insertions(+), 3 deletions(-) create mode 100644 benchmarks/anomaly_conditioning/datasets/tabular.py diff --git a/benchmarks/anomaly_conditioning/datasets/tabular.py b/benchmarks/anomaly_conditioning/datasets/tabular.py new file mode 100644 index 000000000..d24b99592 --- /dev/null +++ b/benchmarks/anomaly_conditioning/datasets/tabular.py @@ -0,0 +1,99 @@ +"""Classical tabular anomaly benchmarks, for the regime that has no grouping at all. + +These answer a different question from the rest of the harness. SMD and NSL-KDD exist to test whether +conditioning on a group helps; these exist to characterise how DQX's mechanism performs on the +bread-and-butter case the field actually benchmarks on, and to confirm that adding baseline +conditioning did not disturb it. + +Source: [ADBench](https://github.com/Minqi824/ADBench) (BSD-2-Clause), which redistributes the ODDS / +UCI / Kaggle collections as `.npz` matrices of `X` and `y`. Downloaded at run time and cached; never +vendored. + +An earlier draft of this harness dismissed ADBench outright because its pre-processing discards +categorical column identity. That is a real limitation, but only for the grouping question — a +plain-tabular regime does not need a grouping, so excluding it here was too broad a call. + +The selection spans three axes deliberately, because a single dataset tells you almost nothing about +a detector: + +* **Scale**: 1.8k rows (cardio) to 285k (fraud). +* **Dimensionality**: 9 features (shuttle) to 100 (mnist). +* **Anomaly rate**: 0.17% (fraud) to ~32% (campaign-adjacent), which matters enormously — PR-AUC is + not comparable across base rates, so each dataset is reported against its own base rate and its own + trivial baselines rather than pooled into one headline number. + +`13_fraud` is the Kaggle credit-card competition set; `10_cover` is Covertype; `32_shuttle`, +`30_satellite`, `23_mammography`, `38_thyroid` and `6_cardio` are the long-standing ODDS benchmarks +most papers report. +""" + +import io +import pathlib +import urllib.request +from collections.abc import Iterator + +import numpy as np + +CACHE = pathlib.Path.home() / ".cache" / "dqx-benchmarks" / "adbench" +BASE = "https://raw.githubusercontent.com/Minqi824/ADBench/main/adbench/datasets/Classical" + +# name -> ADBench file. Kept explicit rather than globbed so a run is reproducible and a new upstream +# dataset cannot silently change the published table. +DATASETS = { + "cardio": "6_cardio.npz", + "thyroid": "38_thyroid.npz", + "mammography": "23_mammography.npz", + "satellite": "30_satellite.npz", + "shuttle": "32_shuttle.npz", + "covertype": "10_cover.npz", + "spambase": "35_SpamBase.npz", + "campaign": "5_campaign.npz", + "mnist": "24_mnist.npz", + "fraud": "13_fraud.npz", +} + +# Cap for the largest sets so a full sweep stays in minutes. Stratified so the anomaly rate -- the +# thing PR-AUC is most sensitive to -- is preserved rather than resampled away. +MAX_ROWS = 30000 + + +def _fetch(filename: str) -> pathlib.Path: + CACHE.mkdir(parents=True, exist_ok=True) + path = CACHE / filename + if path.exists() and path.stat().st_size > 0: + return path + with urllib.request.urlopen(f"{BASE}/{filename}", timeout=300) as resp: # noqa: S310 - fixed https literal + path.write_bytes(resp.read()) + return path + + +def _stratified_cap(values: np.ndarray, labels: np.ndarray, seed: int) -> tuple[np.ndarray, np.ndarray]: + """Cap row count while preserving the anomaly rate.""" + if len(values) <= MAX_ROWS: + return values, labels + rng = np.random.default_rng(seed) + keep_fraction = MAX_ROWS / len(values) + keep = [] + for label in (0.0, 1.0): + idx = np.flatnonzero(labels == label) + n = max(1, int(round(len(idx) * keep_fraction))) + keep.append(rng.choice(idx, size=min(n, len(idx)), replace=False)) + selected = np.sort(np.concatenate(keep)) + return values[selected], labels[selected] + + +def load(name: str, *, seed: int = 0) -> tuple[np.ndarray, np.ndarray]: + """Return ``(values, labels)`` for one dataset.""" + if name not in DATASETS: + raise ValueError(f"unknown tabular dataset {name!r}; known: {sorted(DATASETS)}") + with np.load(io.BytesIO(_fetch(DATASETS[name]).read_bytes())) as data: + values = np.asarray(data["X"], dtype=float) + labels = np.asarray(data["y"], dtype=float).ravel() + return _stratified_cap(values, labels, seed) + + +def iter_datasets(names: list[str] | None = None) -> Iterator[tuple[str, np.ndarray, np.ndarray]]: + """Yield ``(name, values, labels)`` for the requested datasets, or all of them.""" + for name in names or sorted(DATASETS): + values, labels = load(name) + yield name, values, labels diff --git a/benchmarks/anomaly_conditioning/metrics.py b/benchmarks/anomaly_conditioning/metrics.py index 7f19f8157..f4b252084 100644 --- a/benchmarks/anomaly_conditioning/metrics.py +++ b/benchmarks/anomaly_conditioning/metrics.py @@ -103,6 +103,24 @@ def macro_average(values: dict[str, float]) -> float: return float(np.mean(finite)) if finite else float("nan") +def trivial_baselines(values: np.ndarray, labels: np.ndarray, *, seed: int = 42) -> dict[str, float]: + """PR-AUC of scores that required no model, as the floor a result has to clear. + + Absolute PR-AUC is uninterpretable on its own because it moves with the base rate: 0.30 is + excellent at a 0.17% anomaly rate and poor at 30%. ``random`` fixes the base rate to compare + against, and ``max_abs_z`` -- the largest absolute z-score across features -- is the cheapest + defensible detector, so beating it is what shows a model earns its cost. + """ + rng = np.random.default_rng(seed) + means, stds = np.nanmean(values, axis=0), np.nanstd(values, axis=0) + stds = np.where(stds == 0, 1.0, stds) + max_abs_z = np.nanmax(np.abs((values - means) / stds), axis=1) + return { + "random": pr_auc(labels, rng.random(len(labels))), + "max_abs_z": pr_auc(labels, max_abs_z), + } + + def wilcoxon_paired(deltas: list[float]) -> tuple[float, float]: """Wilcoxon signed-rank on per-seed differences: returns ``(statistic, p_value)``. diff --git a/benchmarks/anomaly_conditioning/run_experiment.py b/benchmarks/anomaly_conditioning/run_experiment.py index 962da4d9c..c436a8306 100644 --- a/benchmarks/anomaly_conditioning/run_experiment.py +++ b/benchmarks/anomaly_conditioning/run_experiment.py @@ -33,6 +33,7 @@ precision_at_n, roc_auc, spearman_with_bootstrap_ci, + trivial_baselines, wilcoxon_paired, worst_group_false_positive_rate, ) @@ -147,6 +148,77 @@ def run_real(real_module, name: str, seeds: int) -> list[Cell]: return cells +def run_tabular(seeds: int, names: list[str] | None = None) -> tuple[list[Cell], list[dict]]: + """The plain-tabular regime: no grouping, so only the pooled configuration is meaningful. + + Returns ``(cells, baselines)``. This does not compare configurations -- there is nothing to + condition on -- it characterises absolute quality on the benchmarks the field actually reports, + and pins each result against its own base rate and its own trivial baselines. Reported per + dataset and never pooled into one headline: PR-AUC is not comparable across base rates. + """ + from datasets import tabular # noqa: PLC0415 - downloads data, so only imported when asked for + + cells: list[Cell] = [] + baselines: list[dict] = [] + for name, values, labels in tabular.iter_datasets(names): + base_rate = float(labels.mean()) + print(f" {name}: {values.shape[0]} rows, {values.shape[1]} features, base rate {base_rate:.4%}") + groups = np.zeros(len(values), dtype=str) # one group: pooled is the only configuration + for seed in range(seeds): + cells.append( + Cell( + dataset=f"tabular:{name}", + grouping="none", + config="pooled", + seed=seed, + mechanism="tabular", + level_spread=float("nan"), + metrics=measure(values, labels, groups, "pooled", seed), + ) + ) + trivial = trivial_baselines(values, labels) + baselines.append( + { + "dataset": name, + "n_rows": int(values.shape[0]), + "n_features": int(values.shape[1]), + "base_rate": base_rate, + "dqx_pr_auc": float(np.median([c.metrics.pr_auc for c in cells if c.dataset.endswith(name)])), + **trivial, + } + ) + return cells, baselines + + +def tabular_table(baselines: list[dict]) -> list[str]: + """Per-dataset absolute quality against the floor, with the base rate alongside.""" + if not baselines: + return [] + lines = [ + "## Plain tabular benchmarks (no grouping)", + "", + "No grouping exists in these datasets, so conditioning is not applicable and only the pooled", + "configuration runs. This characterises the mechanism on the benchmarks the field reports, and", + "is the regression check that adding conditioning did not disturb the ungrouped path.", + "", + "PR-AUC is **not comparable across rows** — it moves with the base rate — so each dataset is", + "read against its own random floor. `lift` is DQX PR-AUC divided by the random floor.", + "", + "| dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random | beats max-abs-z |", + "|---|---|---|---|---|---|---|---|---|", + ] + for row in baselines: + lift = row["dqx_pr_auc"] / row["random"] if row["random"] else float("nan") + beats = "yes" if row["dqx_pr_auc"] > row["max_abs_z"] else "**no**" + lines.append( + f"| {row['dataset']} | {row['n_rows']} | {row['n_features']} | {row['base_rate']:.2%} | " + f"{row['dqx_pr_auc']:.4f} | {row['random']:.4f} | {row['max_abs_z']:.4f} | " + f"{lift:.1f}x | {beats} |" + ) + lines.append("") + return lines + + def paired_deltas(cells: list[Cell], left: str, right: str, metric: str = "pr_auc") -> list[dict]: """``metric(left) - metric(right)`` for every (dataset, grouping, seed) the two share. @@ -339,7 +411,9 @@ def low_eta_table(deltas: list[dict]) -> list[str]: return lines -def write_report(cells: list[Cell], seeds: int, datasets: list[str]) -> pathlib.Path: +def write_report( + cells: list[Cell], seeds: int, datasets: list[str], tabular_baselines: list[dict] | None = None +) -> pathlib.Path: """Write the dated markdown and JSON results, and return the markdown path.""" RESULTS_DIR.mkdir(parents=True, exist_ok=True) stamp = dt.date.today().isoformat() @@ -405,6 +479,7 @@ def write_report(cells: list[Cell], seeds: int, datasets: list[str]) -> pathlib. "", ] + lines += tabular_table(tabular_baselines or []) lines += low_eta_table(rel_vs_pooled) lines += ["## Paired comparisons", ""] lines += summarise(rel_vs_pooled, "baseline-relative minus pooled (PR-AUC)") @@ -451,12 +526,17 @@ def main() -> int: "--datasets", nargs="+", default=["synthetic"], - choices=["synthetic", "smd", "nslkdd"], + choices=["synthetic", "smd", "nslkdd", "tabular"], help="synthetic needs no network; the others download at run time", ) args = parser.parse_args() cells: list[Cell] = [] + tabular_baselines: list[dict] = [] + if "tabular" in args.datasets: + print("plain tabular benchmarks:") + tab_cells, tabular_baselines = run_tabular(args.seeds) + cells += tab_cells if "synthetic" in args.datasets: print("synthetic sweep:") cells += run_synthetic(args.seeds) @@ -474,7 +554,7 @@ def main() -> int: print("no cells measured", file=sys.stderr) return 1 - path = write_report(cells, args.seeds, args.datasets) + path = write_report(cells, args.seeds, args.datasets, tabular_baselines) print(f"\nwrote {path}") print(path.read_text(encoding="utf-8")) return 0 From fee682f94b49686fa3e833016fd9fe5d383ad7c9 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 18:06:11 +0100 Subject: [PATCH 014/107] Choose a baseline grouping on the baseline cost model, not the segmented one Auto-discovery routed a discovered grouping to `baseline_by` but kept the selection policy that only ever made sense for `segment_by`: take a single column, the lowest-cardinality candidate, requiring 100 rows per group. Both rules exist because a per-segment model has to train a forest on each group. Under baseline conditioning there is one model however many groups there are, and a group only has to yield a median. Measured on a dataset grouped by country x event_type x product -- 90 groups available: discovered baseline_by [product] -> [product, event_type, country] groups 3 -> 90 training.columns country, event_count, event_type -> event_count engineered features 13 (11 one-hot dummies) -> 2 PR-AUC, contextual 0.1302 -> 0.5703 PR-AUC, global 0.2702 -> 1.0000 The feature-list change falls out of the grouping fix rather than being a separate decision: the profiler already drops the chosen grouping from the feature columns, so recognising country and event_type as dimensions stops them being one-hot encoded as metrics. Eleven dummies alongside a single real metric were diluting the isolation signal, which is why the global scenario was scoring 0.2702 where explicit columns scored 0.7200. `select_baseline_columns` is public so its policy can be unit-tested directly rather than through a Spark session; `MIN_ROWS_PER_BASELINE_GROUP`, `MAX_BASELINE_GROUPS` and `MAX_BASELINE_COLUMN_CARDINALITY` sit beside the segmented thresholds with the reason they differ. The legacy segmented policy is untouched. Also closes a latent hole on the explicit-columns path: a discovered baseline column that the caller had named as a feature is now excluded, rather than producing the feature-and-baseline overlap that `validate_baseline_columns` rejects. --- .../labs/dqx/anomaly/group_config.py | 32 ++++- src/databricks/labs/dqx/anomaly/profiler.py | 125 +++++++++++++++--- .../labs/dqx/anomaly/training_service.py | 31 +++-- tests/unit/test_anomaly_baseline_discovery.py | 96 ++++++++++++++ 4 files changed, 249 insertions(+), 35 deletions(-) create mode 100644 tests/unit/test_anomaly_baseline_discovery.py diff --git a/src/databricks/labs/dqx/anomaly/group_config.py b/src/databricks/labs/dqx/anomaly/group_config.py index de4c43f12..0519277df 100644 --- a/src/databricks/labs/dqx/anomaly/group_config.py +++ b/src/databricks/labs/dqx/anomaly/group_config.py @@ -32,6 +32,34 @@ MIN_ROWS_TO_TRAIN_SEGMENT = 10 # Upper bound on the distinct values a column may have to be *recommended* as a grouping by -# auto-discovery. Conservative on purpose: a wide grouping is fine for baseline_by, which trains one -# model, but auto-discovery's recommendation is also what the legacy segmented path would consume. +# auto-discovery on the legacy segmented path. Conservative on purpose: each distinct value there +# becomes its own model. MAX_AUTO_GROUP_COUNT = 20 + + +# --- baseline conditioning ---------------------------------------------------------------------- +# +# These deliberately differ from the segmented thresholds above, because the cost model differs. +# A per-segment model must *train a forest* on its group, so it needs hundreds of rows and the group +# count is a direct cost. A baseline group only has to yield a *median*, and there is one model +# however many groups exist — so the constraint is statistical, not economic. +# +# Applying the segmented thresholds to baseline_by was a real defect: on a dataset whose natural +# grouping was country x event_type x product (90 groups), discovery selected the single +# lowest-cardinality column — 3 groups — and conditioning barely engaged, scoring PR-AUC 0.1302 +# against 0.1176 unconditioned. Finer grouping is what makes a baseline tight. + +# Rows per group needed for a representative median. Well below MIN_ROWS_PER_SEGMENT because a +# median is a far cheaper statistic than a fitted forest: percentile_approx over a few dozen rows is +# a usable centre, where a forest over the same rows is not a usable model. +MIN_ROWS_PER_BASELINE_GROUP = 30 + +# Ceiling on total baseline groups. Not a training cost — it bounds what gets persisted in the +# feature metadata and broadcast at scoring. Well above the 200 keys at which unseen-group marking +# switches from an isin to a broadcast join, so both strategies stay viable. +MAX_BASELINE_GROUPS = 5000 + +# Per-column distinct-value ceiling for a baseline column. Higher than MAX_AUTO_GROUP_COUNT since +# breadth is affordable here, but still below the profiler's high-cardinality warning at 50, so an +# identifier-like column is never mistaken for a dimension. +MAX_BASELINE_COLUMN_CARDINALITY = 50 diff --git a/src/databricks/labs/dqx/anomaly/profiler.py b/src/databricks/labs/dqx/anomaly/profiler.py index 7957c2b9c..66d48fe4a 100644 --- a/src/databricks/labs/dqx/anomaly/profiler.py +++ b/src/databricks/labs/dqx/anomaly/profiler.py @@ -27,7 +27,10 @@ from databricks.labs.dqx.anomaly.group_config import ( MAX_AUTO_GROUP_COUNT, + MAX_BASELINE_COLUMN_CARDINALITY, + MAX_BASELINE_GROUPS, MAX_SEGMENT_MODELS, + MIN_ROWS_PER_BASELINE_GROUP, MIN_ROWS_PER_SEGMENT, ) from databricks.labs.dqx.profiling_utils import compute_exact_distinct_counts, compute_null_and_distinct_counts @@ -48,7 +51,7 @@ class AnomalyProfile: unsupported_columns: list[str] | None = None # NEW: columns that cannot be used -def auto_discover_columns(df: DataFrame) -> AnomalyProfile: +def auto_discover_columns(df: DataFrame, *, for_baseline: bool = False) -> AnomalyProfile: """ Auto-discover columns and segments for row anomaly detection. @@ -69,12 +72,16 @@ def auto_discover_columns(df: DataFrame) -> AnomalyProfile: Args: df: DataFrame to analyze. + for_baseline: Select the grouping for baseline conditioning rather than for the legacy + segmented path. Baseline conditioning trains one model whatever the group count, so it + can afford a finer grouping and only needs enough rows per group to take a median. See + :func:`select_baseline_columns`. Returns: AnomalyProfile with recommendations and warnings. """ warnings: list[str] = [] - return _auto_discover_heuristic(df, warnings) + return _auto_discover_heuristic(df, warnings, for_baseline=for_baseline) def _compute_numeric_stats_batched(df: DataFrame, column_names: list[str]) -> dict[str, dict[str, float]]: @@ -264,6 +271,73 @@ def _calculate_total_segments( return segment_count +def _is_grouping_candidate( + distinct_count: int, + *, + null_rate: float, + is_id_column: bool, + total_count: int, + for_baseline: bool, +) -> bool: + """Whether a column may be *considered* as a grouping. + + The bar differs by destination, for the same reason the selection policy does: a segment has to + support training a forest on its own rows, a baseline group only has to yield a median. Identifier + -like names and columns that are more than 10% null are rejected either way -- a grouping keyed on + something nearly unique or frequently missing is not a peer group. + """ + max_cardinality = MAX_BASELINE_COLUMN_CARDINALITY if for_baseline else MAX_AUTO_GROUP_COUNT + min_rows = MIN_ROWS_PER_BASELINE_GROUP if for_baseline else MIN_ROWS_PER_SEGMENT + return ( + 2 <= distinct_count <= max_cardinality + and null_rate < 0.1 + and not is_id_column + and (total_count / distinct_count) >= min_rows + ) + + +def select_baseline_columns(candidates: list[tuple[str, int, float]], total_count: int) -> list[str]: + """Choose a grouping for baseline conditioning, given candidates ordered by cardinality. + + Adds columns while every resulting group still holds enough rows for a representative median and + the total group count stays within what is sensible to persist and broadcast. That is the whole + constraint: there is one model regardless of group count, so breadth costs nothing at training + time and buys a tighter baseline. + + Deliberately *not* the segmented policy of taking a single lowest-cardinality column. On a + dataset grouped by country x event_type x product, that policy selected 3 groups out of 90 and + conditioning barely engaged. Cardinality-ascending order makes the choice deterministic and + spends the row budget on the coarsest dimensions first, so the grouping degrades gracefully on + smaller tables instead of picking one arbitrary fine dimension. + """ + selected: list[str] = [] + groups = 1 + for name, distinct_count, _rows_per_group in candidates: + if distinct_count > MAX_BASELINE_COLUMN_CARDINALITY: + continue + prospective = groups * int(distinct_count) + if prospective > MAX_BASELINE_GROUPS: + continue + if total_count / prospective < MIN_ROWS_PER_BASELINE_GROUP: + continue + selected.append(name) + groups = prospective + + if selected: + logger.info( + f"Auto-detected baseline grouping {selected}: {groups} groups, " + f"~{int(total_count / groups)} rows/group (one model regardless of group count)" + ) + skipped = [c[0] for c in candidates if c[0] not in selected] + if skipped: + logger.debug( + f"Not added to the baseline grouping (would leave under " + f"{MIN_ROWS_PER_BASELINE_GROUP} rows/group, or exceed {MAX_BASELINE_GROUPS} " + f"groups): {skipped}" + ) + return selected + + def _select_segment_columns( df: DataFrame, recommended_columns: list[str], @@ -274,8 +348,14 @@ def _select_segment_columns( total_count: int, null_counts: dict[str, int], distinct_counts: dict[str, int], + for_baseline: bool = False, ) -> tuple[list[str], int]: - """Identify and validate segment columns.""" + """Identify and validate segment columns. + + *for_baseline* selects the grouping policy. See :func:`select_baseline_columns` for why the two + differ: a per-segment model has to train on its group, a baseline group only has to yield a + median. + """ recommended_segments = [] candidate_segments = [] # Track all viable candidates for user info categorical_types = (StringType, IntegerType) @@ -290,36 +370,34 @@ def _select_segment_columns( # Compute distinct count and null rate distinct_count = distinct_counts.get(col_name) - null_count = null_counts.get(col_name, 0) if distinct_count is None: distinct_row = df.select(F.countDistinct(col_name)).first() assert distinct_row is not None # to satisfy linter distinct_count = distinct_row[0] - null_rate = null_count / total_count if total_count > 0 else 1.0 - is_id_column = id_pattern.search(col_name) is not None - - # Check segment criteria: conservative for auto-discovery - # Only consider columns with 2-20 distinct values (not 50) - # Ensure at least 100 rows per segment on average - meets_segment_criteria = ( - 2 <= distinct_count <= MAX_AUTO_GROUP_COUNT # More conservative upper bound - and null_rate < 0.1 - and not is_id_column - and (total_count / distinct_count) >= MIN_ROWS_PER_SEGMENT + null_rate = null_counts.get(col_name, 0) / total_count if total_count > 0 else 1.0 + + meets_segment_criteria = _is_grouping_candidate( + distinct_count, + null_rate=null_rate, + is_id_column=id_pattern.search(col_name) is not None, + total_count=total_count, + for_baseline=for_baseline, ) - is_high_cardinality = distinct_count > 50 if meets_segment_criteria and _validate_and_add_segment_column(df, col_name, warnings): candidate_segments.append((col_name, distinct_count, total_count / distinct_count)) - elif is_high_cardinality: + elif distinct_count > 50: _check_high_cardinality_warning(field, col_name, distinct_count, warnings) - # AUTO-DISCOVERY STRATEGY: Be conservative, prefer single segment column - # Sort candidates by: lowest cardinality first (fewer segments = more reliable) + # Sort candidates by lowest cardinality first. For the segmented path that means "fewest models"; + # for the baseline path it means the cheapest granularity is added first. candidate_segments.sort(key=lambda x: x[1]) # Sort by distinct_count ascending - if candidate_segments: - # For auto-discovery, only select the FIRST (lowest cardinality) candidate + if candidate_segments and for_baseline: + recommended_segments.extend(select_baseline_columns(candidate_segments, total_count)) + elif candidate_segments: + # LEGACY SEGMENTED PATH: be conservative, take a single column. Every distinct value becomes + # its own model, so breadth here is paid for in training time. selected = candidate_segments[0] recommended_segments.append(selected[0]) @@ -440,13 +518,15 @@ def _compute_discovery_stats(df: DataFrame) -> tuple[dict[str, int], dict[str, i return null_counts, distinct_counts, numeric_stats, total_count -def _auto_discover_heuristic(df: DataFrame, warnings: list[str]) -> AnomalyProfile: +def _auto_discover_heuristic(df: DataFrame, warnings: list[str], *, for_baseline: bool = False) -> AnomalyProfile: """ Auto-discover using on-the-fly heuristics with multi-type support. Args: df: DataFrame to analyze. warnings: List to accumulate warnings. + for_baseline: Select the grouping for baseline conditioning rather than the legacy + segmented path. See :func:`select_baseline_columns`. Returns: AnomalyProfile with recommendations (max 10 columns). @@ -497,6 +577,7 @@ def _auto_discover_heuristic(df: DataFrame, warnings: list[str]) -> AnomalyProfi total_count=total_count, null_counts=null_counts, distinct_counts=distinct_counts, + for_baseline=for_baseline, ) # Remove segment columns from feature columns (they would be constant within each segment) diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index b8452e14c..cbbcc6765 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -75,8 +75,14 @@ def _perform_auto_discovery( df_filtered: DataFrame, segment_by: list[str] | None, ) -> tuple[list[str], list[str] | None]: - """Perform auto-discovery of columns and segments.""" - profile = auto_discover_columns(df_filtered) + """Perform auto-discovery of columns and segments. + + When the caller has not declared ``segment_by``, any grouping discovered here is routed to + ``baseline_by``, so discovery is asked for a baseline-shaped grouping: finer, bounded by rows + per group rather than by model count. Asking for the segmented shape was a real defect -- + it returned a single lowest-cardinality column and conditioning barely engaged. + """ + profile = auto_discover_columns(df_filtered, for_baseline=segment_by is None) discovered_columns = profile.recommended_columns discovered_segments = segment_by if segment_by is None: @@ -244,25 +250,28 @@ def _discover_columns_and_grouping( # naming your feature columns silently gave up any chance of conditioning. Those are # independent questions. Costs one extra profiling pass for callers who pass explicit # columns and no grouping. - return columns, self._discover_baseline_columns(df_filtered), None + return columns, self._discover_baseline_columns(df_filtered, columns), None return columns, declared_baseline_by, segment_by @staticmethod - def _discover_baseline_columns(df_filtered: DataFrame) -> list[str] | None: + def _discover_baseline_columns(df_filtered: DataFrame, columns: list[str]) -> list[str] | None: """Discover a baseline grouping when the caller named feature columns but no grouping. Kept separate from ``_perform_auto_discovery`` so that discovering a grouping does not require also discovering the feature columns. + + Anything the caller named as a feature is excluded. When discovery picks the columns itself + the profiler already keeps the two lists disjoint, but here the feature list came from the + caller: a column they asked to have measured must not silently become the basis it is + measured against, which ``validate_baseline_columns`` would reject anyway. """ - profile = auto_discover_columns(df_filtered) - if not profile.recommended_segments: + profile = auto_discover_columns(df_filtered, for_baseline=True) + discovered = [c for c in profile.recommended_segments if c not in set(columns)] + if not discovered: return None - logger.info( - f"Auto-detected {len(profile.recommended_segments)} baseline columns: " - f"{profile.recommended_segments} ({profile.segment_count} total groups)" - ) - return profile.recommended_segments + logger.info(f"Auto-detected {len(discovered)} baseline columns: {discovered}") + return discovered @staticmethod def _resolve_grouping( diff --git a/tests/unit/test_anomaly_baseline_discovery.py b/tests/unit/test_anomaly_baseline_discovery.py new file mode 100644 index 000000000..c78ea4ee4 --- /dev/null +++ b/tests/unit/test_anomaly_baseline_discovery.py @@ -0,0 +1,96 @@ +"""The policy that chooses a baseline grouping from candidate dimension columns. + +Pure selection logic, so it is unit-tested directly rather than through a Spark session. It exists +as a separate policy because the segmented one is wrong here: a per-segment model must train a forest +on its own rows, so it wants few groups with many rows each, while a baseline group only has to yield +a median and costs nothing extra because there is one model regardless of group count. + +Applying the segmented policy to ``baseline_by`` was a measured defect. On a dataset grouped by +country x event_type x product -- 90 groups -- it selected the single lowest-cardinality column, +giving 3 groups, and conditioning barely engaged: PR-AUC 0.1302 against 0.1176 unconditioned. With +this policy the same data yields all 90 groups. See databrickslabs/dqx#1484. +""" + +from databricks.labs.dqx.anomaly.group_config import ( + MAX_BASELINE_COLUMN_CARDINALITY, + MAX_BASELINE_GROUPS, + MIN_ROWS_PER_BASELINE_GROUP, +) +from databricks.labs.dqx.anomaly.profiler import select_baseline_columns + + +def _candidate(name: str, distinct: int, total: int) -> tuple[str, int, float]: + """Candidates arrive as (name, distinct_count, rows_per_group), ordered by cardinality.""" + return (name, distinct, total / distinct) + + +def test_combines_dimensions_while_groups_stay_estimable(): + """The case the old policy got wrong: take all three, not just the cheapest.""" + total = 10800 + candidates = [ + _candidate("product", 3, total), + _candidate("event_type", 5, total), + _candidate("country", 6, total), + ] + + selected = select_baseline_columns(candidates, total) + + assert selected == ["product", "event_type", "country"] + # 3 * 5 * 6 = 90 groups, 120 rows each -- comfortably above the median floor. + assert total / 90 >= MIN_ROWS_PER_BASELINE_GROUP + + +def test_stops_before_groups_get_too_thin_to_median(): + """A dimension that would starve every group is skipped, not accepted.""" + total = 200 + candidates = [ + _candidate("region", 2, total), # 100 rows/group -- fine + _candidate("sku", 40, total), # would give 80 groups over 200 rows -- 2.5 rows each + ] + + selected = select_baseline_columns(candidates, total) + + assert selected == ["region"] + + +def test_skips_columns_too_wide_to_be_a_dimension(): + """Above the per-column ceiling a column is an identifier, not a peer group.""" + total = 1_000_000 + candidates = [ + _candidate("region", 4, total), + _candidate("user_bucket", MAX_BASELINE_COLUMN_CARDINALITY + 1, total), + ] + + selected = select_baseline_columns(candidates, total) + + assert selected == ["region"] + + +def test_respects_the_total_group_ceiling(): + """The ceiling bounds what is persisted and broadcast, not what is affordable to train.""" + total = 100_000_000 # rows are not the binding constraint here + candidates = [ + _candidate("a", 50, total), + _candidate("b", 50, total), + _candidate("c", 50, total), # 50^3 = 125,000 groups, over the ceiling + ] + + selected = select_baseline_columns(candidates, total) + + assert selected == ["a", "b"] # 2,500 groups + assert 50 * 50 <= MAX_BASELINE_GROUPS + assert 50 * 50 * 50 > MAX_BASELINE_GROUPS + + +def test_no_candidates_means_no_grouping(): + """Returning an empty list is how the caller learns to train unconditioned.""" + assert not select_baseline_columns([], 5000) + + +def test_selection_is_deterministic_in_candidate_order(): + """Two runs over the same table must produce the same grouping, or the key changes underneath a + persisted model.""" + total = 9000 + candidates = [_candidate("a", 3, total), _candidate("b", 5, total), _candidate("c", 6, total)] + + assert select_baseline_columns(candidates, total) == select_baseline_columns(list(candidates), total) From 7d13390fa5a5e904c1770967eb28b4ff9204eeb7 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 18:43:55 +0100 Subject: [PATCH 015/107] Mark row anomaly detection as beta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page carried only `AvailableSinceVersion`, and the lifecycle reference is explicit that "a feature with no status badge is generally available" — so for three releases the docs promised semantic versioning and backward-compatibility guarantees for a feature that was Experimental and is now Beta. DQX Studio is tagged experimental, actions and the MCP server beta; this page was simply missed. The distinction is not cosmetic. It is what makes the breaking changes in the following commits legitimate: Beta states that "the API may still change in backward-incompatible ways between releases", where GA would not. --- docs/dqx/docs/guide/row_anomaly_detection/index.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 5d4eb0359..027317781 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -7,11 +7,12 @@ import Admonition from '@theme/Admonition'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import Deck, { Slide } from '@site/src/components/Deck'; -import { AvailableSinceVersion, FeatureTags } from '@site/src/components/FeatureTags'; +import { AvailableSinceVersion, FeatureLifecycleStage, FeatureTags } from '@site/src/components/FeatureTags'; # Row Anomaly Detection + From 77894a386f39aad8c3a25affa9d00738e63a4ae7 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 18:53:48 +0100 Subject: [PATCH 016/107] BREAKING: include baseline_by in the model configuration hash The hash was f(columns, segment_by). Retraining under the same model_name with a different baseline_by therefore produced an identical hash, so the single thing this hash exists to catch -- same name, different configuration -- was blind to the grouping, even though the grouping changes the feature list and the persisted baselines. baseline_by now joins the inputs, which changes the hash of every configuration including ungrouped ones, since the key is present either way. A model registered before this fails the configuration check in score_global_model and must be retrained. That break is the point rather than a side effect. Silently scoring a model whose persisted hash no longer describes its configuration is worse than refusing to, and the error now names retraining as the remedy and says which version moved the goalposts. Row anomaly detection was Experimental through 0.16.0 -- "API, behavior, and on-disk or table formats may change ... without notice or a migration path" -- so no compatibility was owed here. At scoring, baseline_by is read back from the persisted feature metadata rather than taken from the caller: it is a property of the trained model, not a scoring argument. That makes the recomputed hash match for anything this version trained, and mismatch for anything older, which is exactly the intended boundary. --- docs/dqx/docs/dev/anomaly_compatibility.mdx | 132 ++++++++++++++++++ .../labs/dqx/anomaly/model_config.py | 21 ++- .../labs/dqx/anomaly/scoring_run.py | 23 ++- .../labs/dqx/anomaly/training_service.py | 2 +- tests/unit/test_anomaly_model_registry.py | 24 ++++ 5 files changed, 191 insertions(+), 11 deletions(-) create mode 100644 docs/dqx/docs/dev/anomaly_compatibility.mdx diff --git a/docs/dqx/docs/dev/anomaly_compatibility.mdx b/docs/dqx/docs/dev/anomaly_compatibility.mdx new file mode 100644 index 000000000..838df1fe2 --- /dev/null +++ b/docs/dqx/docs/dev/anomaly_compatibility.mdx @@ -0,0 +1,132 @@ +--- + +title: Anomaly compatibility debt + +sidebar_position: 640 + +--- + +# Anomaly Detection Compatibility Debt + +Baseline conditioning ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) was added without +breaking anything: the previous grouping mechanism still works, and models trained before it still +score. That cost a set of affordances which exist **only** to keep old behaviour alive. + +This page is the removal checklist for when they go. Every site listed here carries the marker + +```python +# COMPAT(anomaly-v1): -- see docs/dev/anomaly_compatibility +``` + +so `git grep "COMPAT(anomaly-v1)"` finds the lot, and nothing has to be rediscovered by reading. + +Two things this page deliberately keeps apart, because they look identical in a diff: + +- **Compatibility debt** — exists for old callers or old models, and is removable. +- **Runtime fallbacks** — permanent behaviour for cases that will still happen after any deprecation + (a group absent at training, a group too small to calibrate). Listed at the bottom under + [Not compatibility debt](#not-compatibility-debt) so they are not deleted by mistake. + +## A. The legacy `segment_by` path + +The largest block, and the one whose removal deletes the most code. `segment_by` trains one model per +group; `baseline_by` trains one model whatever the group count. On the Server Machine Dataset the +per-segment configuration was the *worst* of three measured, with one entity emitting 15,963 false +positives across 28,392 normal rows, so nothing depends on keeping it except existing callers. + +| Site | What to remove | +|---|---| +| `config.py` · `AnomalyParams.segment_by` | the field and its docstring entry | +| `config.py` · `AnomalyParams.max_segment_models` | the field; its only consumer is this path | +| `anomaly_engine.py` · `train(segment_by=...)` | the parameter and its docs | +| `training_service.py` · `_resolve_grouping` | collapses to "use `baseline_by`"; the both-declared error and the `baseline_by = None` clearing both go | +| `training_service.py` · `_get_and_validate_segments` | delete | +| `training_service.py` · `_train_segmented` | delete, and the `if context.segment_by:` dispatch above it | +| `training_service.py` · `_perform_auto_discovery` | `for_baseline=segment_by is None` becomes unconditional `True` | +| `profiler.py` · `_select_segment_columns` | the `elif candidate_segments:` legacy branch and the `for_baseline` parameter; `select_baseline_columns` becomes the only policy | +| `profiler.py` · `_is_grouping_candidate` | drops `for_baseline`; keeps only the baseline thresholds | +| `group_config.py` | `MAX_SEGMENT_MODELS`, `MIN_ROWS_PER_SEGMENT`, `SEGMENT_COUNT_WARN_THRESHOLD`, `MIN_ROWS_TO_TRAIN_SEGMENT`, `MAX_AUTO_GROUP_COUNT` — the whole segmented half of the module | +| `segment_utils.py` | `canonicalize_segment_values`, `build_segment_name`, `build_segment_filter` | +| `scoring_orchestrator.py` · `try_segmented_scoring_fallback` | delete, and the `if config.segment_by:` branch that calls it | +| `scoring_run.py` · `score_segmented` | delete | +| `scoring_strategies.py` · `score_segmented` | delete from the protocol and its implementations | +| `scoring_config.py` · `ScoringConfig.segment_by` | delete | +| `types.py` · `AnomalyTrainingContext.segment_by` | delete | +| `model_config.py` · `SegmentationConfig` | `segment_by`, `segment_values`, `is_global_model` become meaningless — every model is global | +| `model_registry.py` · registry schema | the `segmentation` struct loses those fields. **Registry migration required** — see below | +| tests | `test_anomaly_segments.py`, the `segment_by` cases in `test_anomaly_groups.py`, `test_segment_by_does_not_gain_baseline_relative_features` | + +**Registry migration.** The `segmentation` struct is a persisted Delta schema. Dropping fields from it +is not a code-only change: existing registry tables carry them. Either keep the columns and stop +writing them, or migrate the table. Decide this before starting, because it is the only item here +that touches customer data rather than customer code. + +## B. Pre-#1484 model metadata + +Every group-conditioning field on `SparkFeatureMetadata` defaults to empty specifically so a model +trained before they existed deserializes into "no grouping", the relative transform returns +immediately, and `engineered_feature_names` is byte-identical to what it was. That is what makes old +models score unchanged. + +| Site | What to remove | +|---|---| +| `transformers.py` · `SparkFeatureMetadata` group fields | the empty defaults may become required once no old model can be loaded | +| `transformers.py` · `from_json` unknown-key tolerance | the `logger.debug` branch that ignores unknown keys. Forward-compatibility for *older DQX reading newer models*, symmetrical debt | +| `transformers.py` · OneHot category reconstruction | the "Model may be from an older version without OneHot category storage" branch — predates #1484 and is older debt still | + +Note `_process_baseline_relative_features`'s early return on `not baseline_by` is **not** in this +table. It looks like the same thing but it is the legitimate ungrouped path, which survives any +deprecation. + +## C. `compute_config_hash` excludes `baseline_by` + +The hash is `(columns, segment_by)`. A model retrained under the same name with a *different* +`baseline_by` therefore produces an identical hash, so the collision detection that exists to catch +"same name, different config" cannot see a grouping change. The persisted metadata still records the +real grouping, so this misleads rather than corrupts. + +It is left alone because a scoring-time hash mismatch **raises** +(`scoring_run.py` · `score_global_model`), so changing the formula would stop every existing model +from scoring. Fixing it correctly means adding `baseline_by` to the hash *and* accepting that break — +which is exactly the decision this page exists to inform. + +## D. `RobustScaler` residue + +Conditional on the scaler removal landing. `RobustScaler` is an affine per-feature transform and +Isolation Forest splits on per-feature thresholds, so the model is invariant to it — measured +identical to four decimal places across five ADBench datasets. New models are fitted without it; old +models still have it pickled inside their pipeline. + +| Site | What to remove | +|---|---| +| `explainability.py` · `compute_shap_values` | the `scaler.transform(...) if scaler else ...` branch | +| `explainability.py` · contribution helper | the `isinstance(model_local, Pipeline)` branch that pulls out a scaler | +| `core.py` · `fit_sklearn_model` | the single-step `Pipeline` wrapper, once nothing needs `named_steps["model"]` | + +## Not compatibility debt + +These are permanent, and deleting them would change behaviour for cases that still occur: + +- **Unseen baseline group → global median.** A group absent at training gets the global baseline, so + the row reads as ordinary rather than extreme, and `is_new_baseline` reports it. New groups appear + in production forever. +- **Missing per-group quantiles → global calibration.** A group without a full quantile set falls back + to table-wide calibration rather than being half-calibrated. +- **`_process_baseline_relative_features` early return.** The ungrouped path. +- **Drift threshold default** in `scoring_config.py`. +- **Python/Spark baseline-key agreement.** `build_baseline_key` and `baseline_key_column` must keep + matching for as long as both exist; the notes there are a contract, not debt. + +## If the break happens now instead + +Deprecating immediately rather than later removes everything in A–D in one change. What a user has to +do: + +1. **Replace `segment_by` with `baseline_by`.** Not a rename — the semantics differ. `segment_by` + partitions into N models; `baseline_by` judges each metric against its own group's baseline on one + model. Scores will differ. +2. **Retrain every model.** Persisted metadata from before #1484 would no longer be loadable, and the + config hash would change even where the configuration did not. +3. **Migrate or recreate registry tables**, per the note in section A. +4. **Expect different scores on auto-discovered groupings** even without any config change, since + discovery now selects a finer grouping routed to `baseline_by`. diff --git a/src/databricks/labs/dqx/anomaly/model_config.py b/src/databricks/labs/dqx/anomaly/model_config.py index 26ea633b5..39554eb0d 100644 --- a/src/databricks/labs/dqx/anomaly/model_config.py +++ b/src/databricks/labs/dqx/anomaly/model_config.py @@ -86,25 +86,36 @@ class AnomalyModelRecord: segmentation: SegmentationConfig -def compute_config_hash(columns: list[str], segment_by: list[str] | None) -> str: +def compute_config_hash(columns: list[str], segment_by: list[str] | None, baseline_by: list[str] | None = None) -> str: """Generate stable hash of model configuration. Args: columns: List of column names used for training segment_by: List of columns used for segmentation, or None + baseline_by: Columns the metrics are judged against, or None Returns: 16-character hex string (first 16 chars of SHA256 hash) Note: - This hash uniquely identifies a model configuration based on: - - Sorted list of columns (order-independent) - - Sorted list of segment_by columns (order-independent) - Used for collision detection when same model_name is reused with different configs. + This hash uniquely identifies a model configuration based on the sorted, order-independent + lists of feature columns, segment columns and baseline columns. It is used for collision + detection when the same model_name is reused with a different configuration. + + **Breaking change.** *baseline_by* joined the hash inputs in 0.17.0, which changes the hash of + every configuration -- including ones with no grouping, since the key is present either way. + A model registered before that therefore fails the configuration check in + :func:`~databricks.labs.dqx.anomaly.scoring_run.score_global_model` and must be retrained. + That is deliberate: without *baseline_by* in the hash, retraining under the same name with a + different grouping produced an identical hash, so the one thing this hash exists to catch -- + same name, different configuration -- was invisible for the grouping. Row anomaly detection + was Experimental through 0.16.0, which carries no backward-compatibility or on-disk format + guarantee. See https://github.com/databrickslabs/dqx/issues/1484. """ config = { "columns": sorted(columns), "segment_by": sorted(segment_by) if segment_by else None, + "baseline_by": sorted(baseline_by) if baseline_by else None, } config_str = json.dumps(config, sort_keys=True) return hashlib.sha256(config_str.encode()).hexdigest()[:16] diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index 0e6fd667d..4fb541f72 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -144,7 +144,18 @@ def score_global_model( config: ScoringConfig, ) -> DataFrame: """Score using a global (non-segmented) model.""" - expected_hash = compute_config_hash(config.columns, config.segment_by) + # baseline_by is a property of the trained model rather than something the caller supplies, so it + # is read back from the persisted metadata. That makes the recomputed hash match for any model + # trained by this version, and mismatch for one trained before baseline_by joined the hash -- + # which is the intended loud failure rather than an accident. See compute_config_hash. + # A record with no persisted feature metadata cannot have been trained with a grouping, so it + # hashes as ungrouped -- and will still mismatch, because the hash formula itself changed. + trained_baseline_by = ( + SparkFeatureMetadata.from_json(record.features.feature_metadata).baseline_by + if record.features.feature_metadata + else None + ) + expected_hash = compute_config_hash(config.columns, config.segment_by, trained_baseline_by) if expected_hash != record.segmentation.config_hash: raise InvalidParameterError( @@ -152,10 +163,12 @@ def score_global_model( f" Trained columns: {record.training.columns}\n" f" Provided columns: {config.columns}\n" f" Trained segment_by: {record.segmentation.segment_by}\n" - f" Provided segment_by: {config.segment_by}\n\n" - f"This model was trained with a different configuration. Either:\n" - f" 1. Use the correct columns/segments that match the trained model\n" - f" 2. Retrain the model with the new configuration" + f" Provided segment_by: {config.segment_by}\n" + f" Trained baseline_by: {trained_baseline_by or None}\n\n" + f"This model was trained with a different configuration, or by a DQX version before\n" + f"baseline_by became part of the configuration hash (0.17.0). Either:\n" + f" 1. Use the columns that match the trained model\n" + f" 2. Retrain the model — required for any model registered before 0.17.0" ) check_model_staleness(record, config.model_name) diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index cbbcc6765..558666ba7 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -579,7 +579,7 @@ def _save_training_record( segment_values=self._stringify_dict(artifacts.segment_values) if artifacts.segment_values else None, is_global_model=segment_by is None, sklearn_version=sklearn.__version__, - config_hash=compute_config_hash(context.columns, segment_by), + config_hash=compute_config_hash(context.columns, segment_by, context.baseline_by), ), ) registry = AnomalyModelRegistry(context.spark) diff --git a/tests/unit/test_anomaly_model_registry.py b/tests/unit/test_anomaly_model_registry.py index 5cef3bb9d..a493ede54 100644 --- a/tests/unit/test_anomaly_model_registry.py +++ b/tests/unit/test_anomaly_model_registry.py @@ -30,6 +30,30 @@ def test_compute_config_hash_handles_none_segment_by() -> None: assert hash_a == hash_b +def test_compute_config_hash_distinguishes_baseline_by() -> None: + """The gap this closed: the same name retrained with a different grouping used to hash alike. + + Collision detection exists to catch "same model_name, different configuration", and the grouping + is part of the configuration -- it changes the feature list and the persisted baselines. + """ + ungrouped = compute_config_hash(["a", "b"], None) + grouped = compute_config_hash(["a", "b"], None, ["region"]) + grouped_wider = compute_config_hash(["a", "b"], None, ["region", "product"]) + + assert ungrouped != grouped + assert grouped != grouped_wider + + +def test_compute_config_hash_is_baseline_order_independent() -> None: + """Baseline columns are a set: the key is built from them sorted, so listing order cannot matter.""" + assert compute_config_hash(["a"], None, ["p", "c"]) == compute_config_hash(["a"], None, ["c", "p"]) + + +def test_compute_config_hash_treats_empty_baseline_as_ungrouped() -> None: + """``baseline_by=[]`` is how a caller asks for whole-table comparison, which is the ungrouped case.""" + assert compute_config_hash(["a"], None, []) == compute_config_hash(["a"], None, None) + + def test_compute_config_hash_different_columns_produce_different_hash() -> None: """Different column sets should produce different hashes.""" hash_a = compute_config_hash(["col1", "col2"], None) From df362826cda0d1b1b71b211e1978ffc4c8762483 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 19:06:10 +0100 Subject: [PATCH 017/107] Stop fitting a scaler that cannot affect the model RobustScaler sat in front of the IsolationForest and could not have changed a single prediction. It is an affine per-feature transform, and Isolation Forest splits on per-feature thresholds drawn uniformly between each feature's min and max, so the induced partitions are identical with or without it. Measured rather than argued, across five ADBench datasets at five seeds each -- PR-AUC with the scaler and without it agreed to four decimal places every time: covertype 0.0572 / 0.0572 shuttle 0.9789 / 0.9789 mnist 0.2740 / 0.2740 fraud 0.1926 / 0.1926 cardio 0.5766 / 0.5766 What it did cost: a fit and a transform on every training run, a transform on every scoring pass, and bulk inside every pickled artifact. On every path, grouped or not. The single-step Pipeline stays. named_steps["model"] is how the SHAP explainer reaches the tree model, keeping it avoids churning eight -> Pipeline annotations across core.py and ensemble_training.py, and it leaves a slot for a transform that genuinely does something. One real hazard fixed alongside: the second SHAP call site indexed named_steps["scaler"] directly, so it would have raised KeyError on any model trained by this version. It now looks the step up, which also keeps working for older models that do carry a scaler. --- src/databricks/labs/dqx/anomaly/core.py | 23 ++++++++++++++----- .../labs/dqx/anomaly/explainability.py | 8 +++++-- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/core.py b/src/databricks/labs/dqx/anomaly/core.py index d0b9de0fc..36d57e3d2 100644 --- a/src/databricks/labs/dqx/anomaly/core.py +++ b/src/databricks/labs/dqx/anomaly/core.py @@ -23,7 +23,6 @@ from pyspark.sql.types import DoubleType, IntegerType, StructField, StructType from sklearn.ensemble import IsolationForest from sklearn.pipeline import Pipeline -from sklearn.preprocessing import RobustScaler from databricks.labs.dqx.anomaly.segment_utils import BASELINE_KEY_COLUMN, with_baseline_key from databricks.labs.dqx.anomaly.transformers import ( @@ -128,10 +127,22 @@ def prepare_training_features( def fit_sklearn_model(train_pandas: pd.DataFrame, params: AnomalyParams) -> tuple[Pipeline, dict[str, Any]]: - """Train sklearn IsolationForest pipeline on pre-engineered pandas DataFrame. + """Train the IsolationForest pipeline on pre-engineered pandas features. + + No feature scaling. ``RobustScaler`` used to sit in front of the forest, and it could not have + made any difference: it is an affine per-feature transform, and Isolation Forest splits on + per-feature thresholds drawn uniformly between each feature's min and max, so the induced + partitions are identical either way. Measured across five ADBench datasets, PR-AUC with and + without it agreed to four decimal places -- covertype 0.0572/0.0572, mnist 0.2740/0.2740, cardio + 0.5766/0.5766, shuttle 0.9789/0.9789, fraud 0.1926/0.1926. It cost a fit and a transform on every + training run and every scoring pass, and shipped inside every pickled artifact. + + The single-step ``Pipeline`` is kept deliberately: ``named_steps["model"]`` is how the SHAP + explainer reaches the tree model, and it leaves somewhere for a transform that genuinely does + something to go later. Returns: - - pipeline: sklearn Pipeline (RobustScaler + IsolationForest) + - pipeline: sklearn Pipeline wrapping the fitted IsolationForest - hyperparams: Model configuration for MLflow tracking """ algo_cfg = params.algorithm_config or IsolationForestConfig() @@ -146,7 +157,7 @@ def fit_sklearn_model(train_pandas: pd.DataFrame, params: AnomalyParams) -> tupl n_jobs=-1, ) - pipeline = Pipeline([('scaler', RobustScaler()), ('model', iso_forest)]) + pipeline = Pipeline([('model', iso_forest)]) pipeline.fit(train_pandas) hyperparams: dict[str, Any] = { @@ -154,7 +165,7 @@ def fit_sklearn_model(train_pandas: pd.DataFrame, params: AnomalyParams) -> tupl "num_trees": algo_cfg.num_trees, "max_samples": algo_cfg.subsampling_rate, "random_seed": algo_cfg.random_seed, - "feature_scaling": "RobustScaler", + "feature_scaling": "none", } return pipeline, hyperparams @@ -168,7 +179,7 @@ def fit_isolation_forest( Feature engineering runs on Spark, then the model trains on the driver. Returns: - - pipeline: sklearn Pipeline (RobustScaler + IsolationForest) + - pipeline: sklearn Pipeline wrapping the fitted IsolationForest - hyperparams: Model configuration for MLflow tracking - feature_metadata: Transformation metadata for distributed scoring """ diff --git a/src/databricks/labs/dqx/anomaly/explainability.py b/src/databricks/labs/dqx/anomaly/explainability.py index fdc63c7d4..b0e2eef73 100644 --- a/src/databricks/labs/dqx/anomaly/explainability.py +++ b/src/databricks/labs/dqx/anomaly/explainability.py @@ -192,10 +192,14 @@ def compute_contributions_for_matrix( """Compute normalized SHAP contributions for a feature matrix.""" # If model is a Pipeline (due to feature scaling), extract components # SHAP's TreeExplainer only supports tree models, not pipelines + # A Pipeline no longer necessarily contains a scaler: DQX fits the forest without one, since an + # affine per-feature transform cannot change axis-parallel splits. Models trained before that + # still carry a RobustScaler, so the step is looked up rather than assumed -- indexing + # named_steps["scaler"] directly would raise KeyError on anything trained by this version. if isinstance(model_local, Pipeline): - scaler = model_local.named_steps["scaler"] + scaler = model_local.named_steps.get("scaler") tree_model = model_local.named_steps["model"] - needs_scaling = True + needs_scaling = scaler is not None else: scaler = None tree_model = model_local From b0658bcf778bccc87143e9a6bc724051f806c944 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 19:12:07 +0100 Subject: [PATCH 018/107] Make baseline_by a queryable registry column baseline_by lived only inside the features.feature_metadata JSON blob. That is where scoring reads it, and functionally it was enough -- but it made the first question worth asking when detection looks wrong unanswerable in SQL. That cost real time this cycle. Diagnosing why zero-config conditioning was not engaging needed a script to pull a model record, parse the JSON, and print baseline_by and the size of baseline_medians, just to tell "conditioning never engaged" apart from "conditioning engaged on a 3-group basis where 90 were available". Both look identical from the outside: one model, is_global_model true. It now sits in the segmentation struct beside training.columns, so: SELECT identity.model_name, training.columns, segmentation.baseline_by FROM WHERE identity.status = 'active' answers it. The metadata blob stays the source of truth for scoring; this is a projection for humans, and the duplication is noted where the field is declared. The registry write gains mergeSchema. Without it a table created by an earlier DQX would reject the wider struct, which matters more than usual right now: the configuration-hash change tells users to retrain, retraining writes to their existing registry table, and a remedy that fails is not a remedy. --- src/databricks/labs/dqx/anomaly/model_config.py | 12 ++++++++++-- src/databricks/labs/dqx/anomaly/model_registry.py | 9 +++++++-- src/databricks/labs/dqx/anomaly/training_service.py | 1 + 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/model_config.py b/src/databricks/labs/dqx/anomaly/model_config.py index 39554eb0d..49cef2e84 100644 --- a/src/databricks/labs/dqx/anomaly/model_config.py +++ b/src/databricks/labs/dqx/anomaly/model_config.py @@ -58,10 +58,18 @@ class FeatureEngineering: @dataclass class SegmentationConfig: - """Segmentation configuration (5 fields).""" + """How a model relates to groups in the data (6 fields). + + ``baseline_by`` is duplicated here from the feature metadata on purpose. It also lives inside the + ``features.feature_metadata`` JSON blob, which is where scoring reads it, but a JSON blob is not + queryable: answering "is this model conditioned, and on what" meant parsing it. It is the first + question worth asking when detection looks wrong, so it belongs in a column next to + ``training.columns``. + """ segment_by: list[str] | None = None segment_values: dict[str, str] | None = None + baseline_by: list[str] | None = None is_global_model: bool = True sklearn_version: str | None = None config_hash: str | None = None @@ -75,7 +83,7 @@ class AnomalyModelRecord: - identity: Core model identification (5 fields) - training: Training configuration and metrics (6 fields) - features: Feature engineering metadata (5 fields) - - segmentation: Segmentation configuration (5 fields) + - segmentation: Grouping configuration (6 fields) Stored as nested structs in Delta tables (no flattening needed). """ diff --git a/src/databricks/labs/dqx/anomaly/model_registry.py b/src/databricks/labs/dqx/anomaly/model_registry.py index 5a2b1c43f..e44509302 100644 --- a/src/databricks/labs/dqx/anomaly/model_registry.py +++ b/src/databricks/labs/dqx/anomaly/model_registry.py @@ -32,7 +32,7 @@ "features struct, feature_metadata:string, " "feature_importance:map, temporal_config:map>, " "segmentation struct, segment_values:map, " - "is_global_model:boolean, sklearn_version:string, config_hash:string>" + "baseline_by:array, is_global_model:boolean, sklearn_version:string, config_hash:string>" ) @@ -88,6 +88,7 @@ def build_model_df(spark: SparkSession, record: AnomalyModelRecord) -> DataFrame "segmentation": { "segment_by": record.segmentation.segment_by, "segment_values": record.segmentation.segment_values, + "baseline_by": record.segmentation.baseline_by, "is_global_model": record.segmentation.is_global_model, "sklearn_version": record.segmentation.sklearn_version, "config_hash": record.segmentation.config_hash, @@ -108,7 +109,11 @@ def save_model(self, record: AnomalyModelRecord, table: str) -> None: self._archive_previous(table, record.identity.model_name) df = self.build_model_df(self.spark, record) - save_dataframe_as_table(df, OutputConfig(location=table, mode="append")) + # mergeSchema so a registry table created by an earlier DQX gains new struct fields on the + # next write instead of failing. Without it, adding `baseline_by` to the segmentation struct + # would make retraining fail against an existing table -- and retraining is exactly what the + # configuration-hash error tells the user to do, so the remedy has to work. + save_dataframe_as_table(df, OutputConfig(location=table, mode="append", options={"mergeSchema": "true"})) def get_active_model(self, table: str, model_name: str) -> AnomalyModelRecord | None: """Fetch the active model for a given name.""" diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index 558666ba7..6b6021f4a 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -576,6 +576,7 @@ def _save_training_record( ), segmentation=SegmentationConfig( segment_by=segment_by, + baseline_by=context.baseline_by, segment_values=self._stringify_dict(artifacts.segment_values) if artifacts.segment_values else None, is_global_model=segment_by is None, sklearn_version=sklearn.__version__, From 175533ca915e21e6e0621e304cdaf3acf219cc0c Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 19:14:40 +0100 Subject: [PATCH 019/107] Match the shipped forest configuration in the harness The harness fitted IsolationForest(n_estimators=100, contamination="auto") while DQX ships IsolationForestConfig(num_trees=200) with contamination taken from expected_anomaly_rate (0.02 by default). So every absolute figure it published described a lighter model than the product. contamination is the harmless half: it only shifts the predict/offset_ threshold and cannot reorder score_samples, so PR-AUC was unaffected by that difference. Tree count is not neutral -- a held-out measurement at 200 trees put fraud at 0.2607 where the 100-tree fit-and-score run had reported 0.1926. Paired comparisons were never affected, since the configuration was held constant on both sides of each comparison. It is the absolute per-dataset column that was understated. Found while mapping the end-to-end heuristic flow, which is the sort of thing that exercise is for. --- benchmarks/anomaly_conditioning/conditioning.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/benchmarks/anomaly_conditioning/conditioning.py b/benchmarks/anomaly_conditioning/conditioning.py index 505fa7316..468ab5e2e 100644 --- a/benchmarks/anomaly_conditioning/conditioning.py +++ b/benchmarks/anomaly_conditioning/conditioning.py @@ -11,6 +11,9 @@ ``pooled`` One model over the raw metrics. No notion of a group at all. This is DQX before #1484. +Note the estimator is configured to match the shipped defaults (see :data:`N_TREES`), so the absolute +figures are comparable with the product rather than with a lighter stand-in. + ``relative`` One model over the raw metrics *plus* each metric's deviation from its own group's baseline. This is what ``baseline_by`` does. @@ -83,9 +86,18 @@ class FitResult: seconds: float +# Mirrors what DQX ships: IsolationForestConfig(num_trees=200), and contamination taken from +# expected_anomaly_rate (default 0.02) rather than sklearn's "auto". An earlier version of this +# harness used 100 trees and "auto", which understated the product -- contamination only moves the +# predict/offset_ threshold and cannot change score_samples ranking, so PR-AUC was unaffected by that +# half, but tree count is not neutral. +N_TREES = 200 +CONTAMINATION = 0.02 + + def _forest(seed: int) -> IsolationForest: """One estimator configuration for every cell, so comparisons are not confounded by tuning.""" - return IsolationForest(n_estimators=100, contamination="auto", random_state=seed, n_jobs=-1) + return IsolationForest(n_estimators=N_TREES, contamination=CONTAMINATION, random_state=seed, n_jobs=-1) def fit_pooled(values: np.ndarray, groups: np.ndarray, seed: int) -> FitResult: From 5ee02194131473d4b60ae08c3dba5334280de36a Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 19:19:31 +0100 Subject: [PATCH 020/107] Record why a second detector is not added to the ensemble Isolation Forest loses to a four-line z-score on covertype and mnist under every hyperparameter tried, so the gap is inductive bias rather than tuning. The obvious remedy is to put a cheap complementary scorer in the ensemble. This commits the measurement that says not to, so the idea is not re-proposed from scratch and re-derived by whoever notices the same gap next. Held-out splits (70/30), 5 seeds, the shipped forest configuration, each member calibrated to percentiles against its own training distribution: mean-of-percentiles wins 5/10 median -0.0028 worst -0.1409 (thyroid) max-of-percentiles wins 3/10 median -0.0253 worst -0.0690 It helps exactly where predicted (covertype +0.0295, mnist +0.0724) and hurts where the forest is genuinely stronger (thyroid, fraud, shuttle, spambase, satellite). A coin flip on average with a 0.14 worst case is not a default. Two things the script records that are easy to miss when re-litigating this: * Naive blending is not even implementable as-is. DQX averages ensemble member scores raw and the scales are incomparable -- the forest sits around 0.4-0.7, max-abs-z is unbounded -- so a real version has to persist each member's training-score distribution. Ranking inside the scoring UDF would be wrong, since the UDF sees a pandas batch and ranks would depend on partitioning. * Choosing the better detector per dataset requires labels, and DQX is unsupervised. So the honest options are to expose the choice or document the limitation; guessing is not one. --- .../complementary_detector.py | 158 ++++++++++++++++++ .../results/complementary-detector.json | 102 +++++++++++ 2 files changed, 260 insertions(+) create mode 100644 benchmarks/anomaly_conditioning/complementary_detector.py create mode 100644 benchmarks/anomaly_conditioning/results/complementary-detector.json diff --git a/benchmarks/anomaly_conditioning/complementary_detector.py b/benchmarks/anomaly_conditioning/complementary_detector.py new file mode 100644 index 000000000..d1b746177 --- /dev/null +++ b/benchmarks/anomaly_conditioning/complementary_detector.py @@ -0,0 +1,158 @@ +"""Does a second detector alongside Isolation Forest help? Measured answer: not as a default. + +Isolation Forest loses to a four-line z-score on two of ten classical benchmarks, and loses under +every hyperparameter setting tried -- more trees, more samples per tree, all samples. So the gap is +inductive bias, not tuning: axis-parallel random splits dilute when an anomaly is one extreme feature +among few dimensions, and degrade in high dimension. + +The obvious remedy is to add a cheap complementary scorer to the ensemble. This module exists so that +proposal stays measured rather than re-argued. **It is not wired into DQX**, and the numbers below are +why. + +## What was measured + +Held-out splits (70/30), 5 seeds, the shipped forest configuration. Each member's scores are mapped to +a percentile against its own *training* distribution before combining -- the calibration a real +implementation would have to persist, since DQX averages member scores raw and the two scales are +incomparable (Isolation Forest sits around 0.4-0.7, max-abs-z is unbounded). Ranking inside the +scoring UDF would be wrong: the UDF sees a pandas batch, not the frame, so ranks would depend on +partitioning. + +## Result + + mean-of-percentiles wins 5/10 median -0.0028 worst -0.1409 (thyroid) + max-of-percentiles wins 3/10 median -0.0253 worst -0.0690 + +Per dataset, where the two disagree most: + + cover IF 0.0509 z 0.1009 mean +0.0121 max +0.0295 + mnist IF 0.2609 z 0.3641 mean +0.0475 max +0.0724 + thyroid IF 0.5446 z 0.2939 mean -0.1409 max -0.0690 + fraud IF 0.2607 z 0.1249 mean -0.0371 max -0.0598 + shuttle IF 0.9764 z 0.8948 mean -0.0176 max -0.0556 + +It helps exactly where predicted and hurts where Isolation Forest is genuinely stronger. A coin flip +on average, with a 0.14 worst case, is not a default. + +## Why this cannot be fixed by choosing per dataset + +Picking the better detector requires knowing which regime you are in, and that means labels. DQX is +unsupervised: there are none. So the honest options are to expose the choice to a user who does know +their anomalies are single-feature extremes, or to document the limitation. Guessing is not among +them. + +Run it with: + + uv run python benchmarks/anomaly_conditioning/complementary_detector.py +""" + +import json +import pathlib +import sys + +import numpy as np +from sklearn.ensemble import IsolationForest +from sklearn.metrics import average_precision_score + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) + +from conditioning import CONTAMINATION, N_TREES # noqa: E402 +from datasets import tabular # noqa: E402 + +SEEDS = 5 +TRAIN_FRACTION = 0.7 +RESULTS = pathlib.Path(__file__).resolve().parent / "results" + + +def percentile_of(reference: np.ndarray, values: np.ndarray) -> np.ndarray: + """Map *values* onto their percentile within a *reference* distribution. + + This is the calibration a real implementation would persist: the reference is the member's + training scores, computed once at training. Partition-independent by construction. + """ + order = np.sort(reference) + return np.searchsorted(order, values, side="right") / max(1, len(order)) + + +def max_abs_z(train: np.ndarray, values: np.ndarray) -> np.ndarray: + """Largest absolute z-score across features, standardised on the training split.""" + means, stds = np.nanmean(train, axis=0), np.nanstd(train, axis=0) + stds = np.where(stds == 0, 1.0, stds) + return np.nanmax(np.abs((values - means) / stds), axis=1) + + +def measure_dataset(values: np.ndarray, labels: np.ndarray) -> dict[str, float] | None: + """Median PR-AUC per strategy over *SEEDS* held-out splits, or None if unusable.""" + per_strategy: dict[str, list[float]] = {k: [] for k in ("iforest", "zscore", "mean_pct", "max_pct")} + for seed in range(SEEDS): + rng = np.random.default_rng(seed) + idx = rng.permutation(len(values)) + cut = int(TRAIN_FRACTION * len(values)) + train_idx, test_idx = idx[:cut], idx[cut:] + if len(np.unique(labels[test_idx])) < 2: + continue + + forest = IsolationForest(n_estimators=N_TREES, contamination=CONTAMINATION, random_state=seed, n_jobs=-1).fit( + values[train_idx] + ) + if_train = -forest.score_samples(values[train_idx]) + if_test = -forest.score_samples(values[test_idx]) + z_train = max_abs_z(values[train_idx], values[train_idx]) + z_test = max_abs_z(values[train_idx], values[test_idx]) + + if_pct, z_pct = percentile_of(if_train, if_test), percentile_of(z_train, z_test) + held_out = labels[test_idx] + per_strategy["iforest"].append(average_precision_score(held_out, if_test)) + per_strategy["zscore"].append(average_precision_score(held_out, z_test)) + per_strategy["mean_pct"].append(average_precision_score(held_out, (if_pct + z_pct) / 2)) + per_strategy["max_pct"].append(average_precision_score(held_out, np.maximum(if_pct, z_pct))) + + if not per_strategy["iforest"]: + return None + out = {k: float(np.median(v)) for k, v in per_strategy.items()} + out["mean_vs_iforest"] = out["mean_pct"] - out["iforest"] + out["max_vs_iforest"] = out["max_pct"] - out["iforest"] + return out + + +def main() -> int: + results: list[dict[str, object]] = [] + # Deltas are accumulated as floats alongside the rows rather than read back out of them: the rows + # mix a dataset name with numbers, so indexing them yields object and the statistics below would + # need casts to satisfy the type checker. + deltas: dict[str, list[float]] = {"mean": [], "max": []} + for name, values, labels in tabular.iter_datasets(): + measured = measure_dataset(values, labels) + if measured is None: + continue + row: dict[str, object] = {"dataset": name, "base_rate": float(labels.mean()), **measured} + results.append(row) + deltas["mean"].append(measured["mean_vs_iforest"]) + deltas["max"].append(measured["max_vs_iforest"]) + print( + f"{name:<12} base {float(labels.mean()):>7.3%} IF {measured['iforest']:.4f} " + f"z {measured['zscore']:.4f} " + f"mean {measured['mean_pct']:.4f} ({measured['mean_vs_iforest']:+.4f}) " + f"max {measured['max_pct']:.4f} ({measured['max_vs_iforest']:+.4f})", + flush=True, + ) + + print("\n--- verdict ---") + for strategy, values_for_strategy in deltas.items(): + print( + f"{strategy}-of-percentiles: " + f"wins {sum(d > 0 for d in values_for_strategy)}/{len(values_for_strategy)}, " + f"median {np.median(values_for_strategy):+.4f}, " + f"worst {min(values_for_strategy):+.4f}, best {max(values_for_strategy):+.4f}" + ) + print("\nNot a default. See the module docstring.") + + RESULTS.mkdir(parents=True, exist_ok=True) + path = RESULTS / "complementary-detector.json" + path.write_text(json.dumps(results, indent=2), encoding="utf-8") + print(f"wrote {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/anomaly_conditioning/results/complementary-detector.json b/benchmarks/anomaly_conditioning/results/complementary-detector.json new file mode 100644 index 000000000..ad7f252fa --- /dev/null +++ b/benchmarks/anomaly_conditioning/results/complementary-detector.json @@ -0,0 +1,102 @@ +[ + { + "dataset": "campaign", + "base_rate": 0.11266666666666666, + "iforest": 0.29189044522788793, + "zscore": 0.24907974586715767, + "mean_pct": 0.30525931138248596, + "max_pct": 0.28026469397768916, + "mean_vs_iforest": 0.013368866154598036, + "max_vs_iforest": -0.011625751250198768 + }, + { + "dataset": "cardio", + "base_rate": 0.0961223375204806, + "iforest": 0.5531411979830949, + "zscore": 0.5362761184610526, + "mean_pct": 0.6028574471263684, + "max_pct": 0.5403439969678784, + "mean_vs_iforest": 0.04971624914327344, + "max_vs_iforest": -0.01279720101521653 + }, + { + "dataset": "covertype", + "base_rate": 0.0096, + "iforest": 0.05085714016382059, + "zscore": 0.10085460150242194, + "mean_pct": 0.06297738498037918, + "max_pct": 0.08035853810631731, + "mean_vs_iforest": 0.012120244816558587, + "max_vs_iforest": 0.029501397942496718 + }, + { + "dataset": "fraud", + "base_rate": 0.0017333333333333333, + "iforest": 0.2606639933273247, + "zscore": 0.12492024471197495, + "mean_pct": 0.22356101154283578, + "max_pct": 0.2008899193829753, + "mean_vs_iforest": -0.037102981784488925, + "max_vs_iforest": -0.059774073944349415 + }, + { + "dataset": "mammography", + "base_rate": 0.023249575248144506, + "iforest": 0.16615148687632164, + "zscore": 0.1714218533444644, + "mean_pct": 0.18154326702612636, + "max_pct": 0.18128521174190282, + "mean_vs_iforest": 0.015391780149804718, + "max_vs_iforest": 0.015133724865581177 + }, + { + "dataset": "mnist", + "base_rate": 0.09206892016309351, + "iforest": 0.26090881659743215, + "zscore": 0.36405621393428067, + "mean_pct": 0.3084076801597011, + "max_pct": 0.3333332896133295, + "mean_vs_iforest": 0.047498863562268956, + "max_vs_iforest": 0.07242447301589733 + }, + { + "dataset": "satellite", + "base_rate": 0.3163947163947164, + "iforest": 0.674511197000149, + "zscore": 0.5898788542045754, + "mean_pct": 0.635511558286259, + "max_pct": 0.6366432388776665, + "mean_vs_iforest": -0.038999638713890006, + "max_vs_iforest": -0.03786795812248256 + }, + { + "dataset": "shuttle", + "base_rate": 0.0715, + "iforest": 0.9764084411952229, + "zscore": 0.8948204770684615, + "mean_pct": 0.9587705767318155, + "max_pct": 0.9207599245713749, + "mean_vs_iforest": -0.017637864463407316, + "max_vs_iforest": -0.055648516623847954 + }, + { + "dataset": "spambase", + "base_rate": 0.39909674352270025, + "iforest": 0.5103787257980769, + "zscore": 0.42751074893538277, + "mean_pct": 0.4888560907968301, + "max_pct": 0.4419059745028603, + "mean_vs_iforest": -0.021522635001246737, + "max_vs_iforest": -0.06847275129521657 + }, + { + "dataset": "thyroid", + "base_rate": 0.024655355249204668, + "iforest": 0.5446078938409467, + "zscore": 0.29394329833995414, + "mean_pct": 0.40374103378311116, + "max_pct": 0.4756158967397111, + "mean_vs_iforest": -0.1408668600578355, + "max_vs_iforest": -0.06899199710123555 + } +] \ No newline at end of file From bbe9d9ec35abb225fd489888a2af99955003281f Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 19:21:02 +0100 Subject: [PATCH 021/107] Document the heuristic path and the compatibility debt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two dev pages, both written because this cycle found its defects by accident and that is not a repeatable way to work. **anomaly_heuristic_map** — every decision made between train(df) and a flagged row, as a mermaid flowchart, with the governing default at each stage and the three questions that isolate most failures: what was chosen (stages 1-2, which decide silently and held both defects found while drawing this), did conditioning actually engage and on what, and is the comparison even valid given that a 30% sample drawn with .sample is partition-dependent under Spark Connect. It also records the measured weak points rather than only the design. Drawing it immediately surfaced that the harness modelled a 100-tree forest where DQX ships 200, which is the kind of thing the exercise is for. **anomaly_compatibility** — every affordance that exists only to keep old behaviour alive, marked so `git grep "COMPAT(anomaly-v1)"` finds them, with the registry-schema item called out because it touches persisted data rather than code. It deliberately separates that debt from permanent runtime fallbacks -- unseen-group and missing-quantile handling look identical in a diff and must not be deleted alongside. --- docs/dqx/docs/dev/anomaly_heuristic_map.mdx | 108 ++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 docs/dqx/docs/dev/anomaly_heuristic_map.mdx diff --git a/docs/dqx/docs/dev/anomaly_heuristic_map.mdx b/docs/dqx/docs/dev/anomaly_heuristic_map.mdx new file mode 100644 index 000000000..b19c6a9cc --- /dev/null +++ b/docs/dqx/docs/dev/anomaly_heuristic_map.mdx @@ -0,0 +1,108 @@ +--- + +title: Anomaly heuristic map + +sidebar_position: 645 + +--- + +# Anomaly Detection Heuristic Map + +Every decision row anomaly detection makes on the way from a DataFrame to a flagged row, with the +default that governs it. It exists to make a bad result diagnosable by working down a path, rather +than by reading the module. + +Training and scoring are two passes over the same feature-engineering code: training persists what +scoring later reads back. Diamonds are branches. + +```mermaid +flowchart TD + A["train(df)"] --> B{"columns given?"} + + B -- no --> C["1 pick metrics
numeric, stddev>0, nulls<50%
low-card categoricals also eligible"] + B -- yes --> D["use the caller's columns"] + C --> E["2 pick a grouping
nulls<10%, not id-like
baseline: ≥30 rows/group, ≤5000 groups
segmented: one lowest-cardinality column"] + D --> E + + E --> F{"baseline_by or
segment_by declared?"} + F -- both --> X1["error"] + F -- "segment_by" --> G["3 one model per segment
>50 error, >100 warn
baseline_by cleared"] + F -- "neither / baseline_by" --> H["4 one model,
grouping to baseline_by"] + + G --> I["5 sample 30%, split 80/20"] + H --> I + I --> J["6 feature engineering
onehot card≤20 else frequency
+ metric_rel_baseline per metric"] + J --> K["7 IsolationForest
200 trees, 256 rows/tree,
contamination 0.02, no scaling"] + K --> L["8 quantiles: global
+ per baseline group"] + L --> M["register: MLflow + registry row
+ feature_metadata JSON"] + + M -.->|"persisted baselines, medians, quantiles"| N + + N["score(df)"] --> O["9 re-engineer from metadata
baselines broadcast-joined
unseen group → global median"] + O --> P["pandas UDF: anomaly score"] + P --> Q["10 mark unseen baselines
isin if ≤200 keys, else join"] + Q --> R["11 severity percentile
per-group quantiles if present,
else global"] + R --> S{"baseline unseen?"} + S -- yes --> T["null score, not flagged
is_new_baseline = true"] + S -- no --> U{"severity ≥ threshold 95?"} + U -- yes --> V["flagged"] + U -- no --> W["passes"] +``` + +## Reading a bad result + +Working down the path, three questions separate most failures. + +**What was chosen?** Stages 1 and 2 decide silently, and both defects found while building this map +were there. + +```sql +SELECT identity.model_name, training.columns, segmentation.baseline_by +FROM +WHERE identity.status = 'active' +``` + +**Did conditioning engage, and on what?** Look for `_rel_baseline` in +`engineered_feature_names`, and count the entries in `baseline_medians`. Three groups where the data +has ninety means stage 2 chose too coarsely — a model can be conditioned and still be conditioned on +almost nothing. + +**Is the comparison even valid?** Training samples 30% of rows by default via `.sample`, and splits +80/20 via `.randomSplit`; under Spark Connect both depend on partition ordering. Two runs of identical +code can therefore train on different rows. If you are comparing two builds, set +`sample_fraction=1.0` first — otherwise you are measuring the sampler. Suspect stage 5 before +suspecting the design. + +## Defaults in one place + +| Default | Value | Governs | +|---|---|---| +| `MIN_ROWS_PER_BASELINE_GROUP` | 30 | rows/group needed to trust a baseline median | +| `MAX_BASELINE_GROUPS` | 5000 | ceiling on total baseline groups | +| `MAX_BASELINE_COLUMN_CARDINALITY` | 50 | per-column ceiling for a baseline column | +| `MAX_AUTO_GROUP_COUNT` | 20 | per-column ceiling on the legacy segmented path | +| `MIN_ROWS_PER_SEGMENT` | 100 | rows/segment needed to train a per-segment model | +| `MAX_SEGMENT_MODELS` | 50 | hard ceiling on the legacy segmented path | +| `DEFAULT_SAMPLE_FRACTION` | 0.3 | share of rows used for training | +| `DEFAULT_TRAIN_RATIO` | 0.8 | train/validation split | +| `categorical_cardinality_threshold` | 20 | one-hot below, frequency-encode above | +| `num_trees` | 200 | Isolation Forest size | +| `max_samples` | 256 | sklearn "auto"; rows per tree | +| `expected_anomaly_rate` | 0.02 | becomes contamination when unset | +| `threshold` | 95.0 | severity percentile that flags a row | + +## Known weak points + +Measured, with the evidence in `benchmarks/anomaly_conditioning/`: + +- **Stage 1 can pick dimensions as metrics.** Low-cardinality categoricals qualify as features, so a + table keyed by several dimensions can produce a dozen one-hot columns beside a single real metric, + diluting the signal. Largely avoided when those columns are recognised as a grouping instead, since + the profiler removes the grouping from the feature list — but a categorical that is *not* selected + as a grouping can still land in features. +- **Stage 7 has a blind spot.** Isolation Forest loses to a max-abs-z baseline where anomalies are + single-feature extremes in few dimensions, and in high dimension. Not fixable by tuning, and adding + a complementary scorer measured worse on average — see `complementary_detector.py`. +- **Stage 9's fallback is silent by design.** A group absent at training falls back to the global + baseline, which makes the row read as ordinary rather than extreme. `is_new_baseline` is how that + case is detected; it is the conservative direction, not a bug. From 57bcebbabfb84d2660cd927853ec272fb81c0841 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 19:37:22 +0100 Subject: [PATCH 022/107] Publish the comparison against the previous release The sweep in this page measured mechanisms against each other; it never measured this build against the one users are running. That is the question that actually matters, and the answer now leads the page: v0.16.0 and this build, through the real Spark/MLflow/Unity Catalog path, over the same Delta tables. contextual, zero config 0.1885 -> 0.5703 contextual, explicit columns 0.0376 -> 0.5703 global, zero config 0.3743 -> 1.0000 global, explicit columns 1.0000 -> 1.0000 Better on three, identical on the fourth, worse on none. The 0.0376 -> 0.5703 row is the cleanest statement of what conditioning buys, since v0.16.0 has no conditioning mechanism at all; the two exact 1.0000s are the regression check, on the path this work does not touch. The page also records how the measurement went wrong the first time, because it will catch anyone who repeats it: training samples 30% of rows by default via .sample, which under Spark Connect depends on partition ordering, so an uncontrolled run had the two builds training on different rows and showed the new build losing three of four cells. With sample_fraction=1.0 every cell reproduces to four decimals. A single-point comparison of this pipeline is not evidence. --- .../reference/anomaly_detection_quality.mdx | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index 20050b3cb..a55c7cad7 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -44,6 +44,39 @@ Comparisons are **paired by seed** and tested with Wilcoxon signed-rank. The see data draw and the forest, and dominates the variance between configurations, so unpaired means would largely measure the seed. +## Against the previous release + +The comparison that matters most: the released v0.16.0 build and this one, run through the **real DQX +pipeline** — Spark, MLflow, Unity Catalog — over the same Delta tables, so neither side can differ by +anything except the code. + +| scenario | how it was called | v0.16.0 | current | change | +|---|---|---|---|---| +| contextual anomaly | zero config | 0.1885 | **0.5703** | **+0.3818** | +| contextual anomaly | explicit `columns` | 0.0376 | **0.5703** | **+0.5327** | +| globally extreme anomaly | zero config | 0.3743 | **1.0000** | **+0.6257** | +| globally extreme anomaly | explicit `columns` | 1.0000 | 1.0000 | +0.0000 | + +Better on three, identical on the fourth, worse on none. Two rows are worth reading closely: + +- **`contextual` / explicit columns, 0.0376 → 0.5703.** v0.16.0 has no conditioning mechanism at all, + so this is the cleanest statement of what conditioning buys: a fifteenfold improvement on an anomaly + that is invisible to a whole-table comparison. +- **`global` / explicit columns, both exactly 1.0000.** This path is untouched by the change, and the + two builds agree exactly. That is the regression check: nothing was traded away to get the rows + above. + +Getting this measurement right took two attempts, and the reason is worth recording because it applies +to anyone repeating it. The first run showed the *new* build losing on three of four cells. It was +measuring the sampler: training samples 30% of rows by default via `.sample`, which under Spark +Connect depends on partition ordering, so the two builds trained on different rows. With +`sample_fraction=1.0` every cell is reproducible to four decimal places across repeats, and the +picture inverts. **A single-point comparison of this pipeline is not evidence.** + +Both `RobustScaler` removal and the configuration-hash change landed after these figures were +measured. Neither can move them: the scaler is provably a no-op (see below), and the hash affects +model identity rather than scores. + ## Results 1,395 cells: a synthetic two-factor sweep plus the Server Machine Dataset (28 entities) and NSL-KDD, @@ -161,7 +194,12 @@ Every configuration sees identical rows, so neither affects the comparison. - The harness reimplements the baseline-relative transform in numpy rather than calling DQX, so a sweep of a few thousand fits needs no Spark session. It therefore measures the **mechanism**, not DQX's implementation of it. Pipeline fidelity is asserted separately in - `tests/integration_anomaly/test_anomaly_quality.py`, which runs the real Spark and MLflow path. + `tests/integration_anomaly/test_anomaly_quality.py`, which runs the real Spark and MLflow path, and + the release comparison above runs the pipeline end to end. +- The harness fits the same estimator configuration DQX ships (200 trees, contamination from + `expected_anomaly_rate`). An earlier version used 100 trees, which understated the product; paired + comparisons were unaffected, since the configuration is held constant on both sides, but absolute + per-dataset figures moved. - Each configuration is fitted and scored on the same rows. This measures separability, not generalisation to unseen data. - Results are indicative. Always benchmark against your own data and environment. From 774e5a298e052e4e2af9dd6852aaf4ee77ffb001 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 20:04:39 +0100 Subject: [PATCH 023/107] Correct the real-data finding, and publish the tabular regime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerating the sweep with the corrected 200-tree harness reversed a published claim, so it is corrected here rather than left standing. baseline-relative minus pooled, real datasets was (100 trees): -0.0010 p = 3.05e-03 "marginally worse" now (200 trees): +0.0025 p = 1.75e-02 marginally better The page keeps the old number and says why it moved: the harness fitted a lighter forest than DQX ships, which understated it on exactly the datasets where the forest does the work. The contextual (+0.0742) and global (+0.0000) rows barely moved, because a paired comparison holds configuration constant on both sides — it was the real-data row, where the two mechanisms are closest, that a fidelity gap could flip. The guide and changelog carried the same figure and are updated too. Also adds the plain-tabular section the sweep has been producing but the page never showed: ten ADBench benchmarks spanning 1.8k to 30k rows, 6 to 100 features, and base rates from 0.17% to 40%. DQX beats the random floor on all ten and max-abs-z on eight, with fraud at 115x the floor. The two it loses get their own subsection rather than a footnote. covertype (0.0534 vs 0.1122) and mnist (0.2766 vs 0.3367) are Isolation Forest's inductive bias: it splits one random feature at a time, so a single extreme value among few dimensions is diluted and in 100 dimensions the split rarely lands on the informative axis. Tuning does not recover it and the obvious knob trades shuttle and cardio away for fraud. Publishing the losses with the diagnosis is more use to a reader than a table of wins, and anyone with ADBench reproduces covertype in three lines regardless. Supersedes the two partial results files from earlier runs. --- CHANGELOG.md | 2 +- .../results/2026-08-25-55ebd5ca.md | 77 - ...55ebd5ca.json => 2026-08-25-f9c703a1.json} | 14677 +++++++++------- .../results/2026-08-25-f9c703a1.md | 99 + .../guide/row_anomaly_detection/index.mdx | 12 +- .../reference/anomaly_detection_quality.mdx | 106 +- 6 files changed, 8804 insertions(+), 6169 deletions(-) delete mode 100644 benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.md rename benchmarks/anomaly_conditioning/results/{2026-08-25-55ebd5ca.json => 2026-08-25-f9c703a1.json} (69%) create mode 100644 benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cc2df999..a600e8f49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 0.16.0 -* Added baseline conditioning to row anomaly detection ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). Anomaly detection could not detect a **contextual** anomaly — a value that is unremarkable across the table but wrong for its own group. On the measurements in the issue, one group's volume dropping 80% behind a flat daily total scored 45.1, the 45th percentile, so no threshold recovered it. `AnomalyEngine.train()` now takes `baseline_by`: each numeric metric gains its deviation from that metric's own baseline within the row's group, as a signed log-ratio, on a **single** pooled model — so the cost does not grow with the group count, and the same collapse scores above 95. Measured offline in the unit suite, a contextual collapse goes from PR-AUC 0.0028 (chance) to 0.6962, while an anomaly that was already globally extreme is unchanged at 1.0000, so conditioning costs nothing measurable when there is nothing to gain. Baseline columns must be string, integral, boolean or date; floating-point and decimal types are rejected because Spark and Python format them differently, which would silently break the key lookup that matches persisted baselines to rows. Grouping auto-discovery is no longer coupled to column discovery, so passing explicit `columns` no longer silently gives up conditioning, and a discovered grouping is used as `baseline_by` rather than turning into one model per group. Across a wider sweep — 1,395 configurations over synthetic data, the Server Machine Dataset and NSL-KDD — conditioning is worth a median +0.0734 PR-AUC where anomalies are contextual and −0.0010 where they are not, and beats one-model-per-group in every case measured; see [Anomaly detection quality](https://databrickslabs.github.io/dqx/docs/reference/anomaly_detection_quality) for the full results, the licences, and what these numbers do not mean. +* Added baseline conditioning to row anomaly detection ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). Anomaly detection could not detect a **contextual** anomaly — a value that is unremarkable across the table but wrong for its own group. On the measurements in the issue, one group's volume dropping 80% behind a flat daily total scored 45.1, the 45th percentile, so no threshold recovered it. `AnomalyEngine.train()` now takes `baseline_by`: each numeric metric gains its deviation from that metric's own baseline within the row's group, as a signed log-ratio, on a **single** pooled model — so the cost does not grow with the group count, and the same collapse scores above 95. Measured offline in the unit suite, a contextual collapse goes from PR-AUC 0.0028 (chance) to 0.6962, while an anomaly that was already globally extreme is unchanged at 1.0000, so conditioning costs nothing measurable when there is nothing to gain. Baseline columns must be string, integral, boolean or date; floating-point and decimal types are rejected because Spark and Python format them differently, which would silently break the key lookup that matches persisted baselines to rows. Grouping auto-discovery is no longer coupled to column discovery, so passing explicit `columns` no longer silently gives up conditioning, and a discovered grouping is used as `baseline_by` rather than turning into one model per group. Measured against v0.16.0 on identical tables through the real pipeline, a contextual collapse goes from PR-AUC 0.0376 to 0.5703, and the untouched ungrouped path scores identically on both builds. Across a wider sweep — 1,545 configurations over synthetic data, the Server Machine Dataset, NSL-KDD and ten classical tabular benchmarks — conditioning is worth a median +0.0742 PR-AUC where anomalies are contextual and nothing measurable where they are not, and beats one-model-per-group in every case measured; see [Anomaly detection quality](https://databrickslabs.github.io/dqx/docs/reference/anomaly_detection_quality) for the full results, the licences, and what these numbers do not mean. * Added a pluggable actions and alerting subsystem ([#1289](https://github.com/databrickslabs/dqx/issues/1289)). DQX now supports extensible *actions* that run when checked data violates an optional condition evaluated against the summary metrics produced by `DQMetricsObserver`. The built-in `DQAlert` action can send notifications to Slack, Microsoft Teams, a generic HTTPS webhook, or the log, so pipelines can react to data quality regressions without custom plumbing. You can create your own custom actions as well, and custom alerting is possible via the callback destination, which invokes an in-process Python callable for each alert. * Added an MCP (Model Context Protocol) server for DQX ([#1252](https://github.com/databrickslabs/dqx/issues/1252)). The server exposes DQX's data quality capabilities as tools that any MCP-compatible AI agent (Claude, Genie Code, Cursor, Mosaic AI) can discover and orchestrate. It runs as a Databricks App with on-behalf-of (OBO) authentication, so all data access is governed by the calling user's Unity Catalog permissions. * Added support for summary metrics in Lakeflow Declarative Pipelines (LDP/DLT) ([#1301](https://github.com/databrickslabs/dqx/issues/1301)). A new `DQEngine.compute_summary_metrics(...)` produces the same row counts, per-check breakdown, and custom observer metrics as a lazy aggregation over the results DataFrame, so metrics can be computed inside Spark Declarative Pipelines where the observer- and streaming-listener-based paths cannot be used. diff --git a/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.md b/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.md deleted file mode 100644 index 0305678a5..000000000 --- a/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.md +++ /dev/null @@ -1,77 +0,0 @@ -# Anomaly conditioning experiment — 2026-08-25 - -DQX `55ebd5ca` · datasets: synthetic, smd, nslkdd · seeds per cell: 15 · cells: 1395 - -PR-AUC is the primary metric. No point-adjusted F1 appears anywhere; see `metrics.py`. -DQX scores rows independently, so figures on time-series data are not comparable with -published sequence-model results — that is a different task, not a worse implementation. - -## Does removing the heterogeneity gate cost anything? - -Rules fixed before running: **no gate** if the worst delta below eta-squared 0.1 exceeds -0.01; **gate needed** if any such cell reaches -0.02. - -- **Pre-registered rule (worst single cell): gate needed; refit the threshold from this sweep** -- **Variance-robust companion (worst per-grouping median): no gate needed** - -The pre-registered rule takes a minimum over individual cells, so it is maximally sensitive to estimator variance. It is reported unchanged, alongside the median form of the same question, so the criterion set in advance and the answer it gave are both visible. Where the two disagree, the per-grouping deltas below show why. - -| statistic | value | -|---|---| -| n_low_eta_cells | 90 | -| n_low_eta_groupings | 4 | -| worst_delta_single_cell | -0.0744 | -| n_harmful_cells | 3 | -| worst_median_delta_per_grouping | +0.0000 | -| n_harmful_groupings | 0 | -| median_delta_below_threshold | +0.0000 | - -### Does eta-squared predict the benefit at all? - -Spearman rho(eta-squared, delta) = **+0.597** (bootstrap 95% CI +0.523 to +0.668). - -Least squares `delta ~ 1 + eta_squared + is_contextual`, which asks whether eta-squared still carries signal once the anomaly mechanism is controlled for: - -| term | coefficient | -|---|---| -| intercept | -0.0998 | -| eta_squared | +0.2760 | -| is_contextual | +0.0830 | -| R-squared | 0.561 (n=390) | - -### Every grouping below eta-squared 0.1, seed by seed - -| dataset | grouping | eta-squared | median delta | min | max | seeds agree? | -|---|---|---|---|---|---|---| -| nslkdd | protocol_type | 0.0509 | +0.0285 | -0.0744 | +0.1003 | **no** | -| smd | machine_family | 0.0669 | +0.0033 | -0.0047 | +0.0125 | **no** | -| synthetic | spread=0.000 | 0.0008 | +0.0000 | -0.0019 | +0.0283 | **no** | -| synthetic | spread=0.050 | 0.0593 | +0.0000 | -0.0019 | +0.0169 | **no** | - -## Paired comparisons - -### baseline-relative minus pooled (PR-AUC) - -| mechanism | n | median delta | IQR | Wilcoxon p | -|---|---|---|---|---| -| contextual | 195 | +0.0734 | +0.0068 to +0.2649 | 5.28e-32 | -| global | 195 | +0.0000 | -0.0000 to +0.0000 | 2.01e-01 | -| real | 75 | -0.0010 | -0.1622 to +0.0110 | 3.05e-03 | -| *all* | 465 | +0.0000 | +0.0000 to +0.0466 | 1.07e-23 | - -### baseline-relative minus per-group (PR-AUC) - -| mechanism | n | median delta | IQR | Wilcoxon p | -|---|---|---|---|---| -| contextual | 195 | +0.0274 | +0.0128 to +0.0413 | 1.20e-33 | -| global | 195 | +0.0006 | +0.0000 to +0.0017 | 3.35e-26 | -| real | 75 | +0.0123 | -0.0043 to +0.2785 | 3.24e-05 | -| *all* | 465 | +0.0066 | +0.0004 to +0.0302 | 7.95e-55 | - -## Cost - -| config | median models | median seconds | -|---|---|---| -| pooled | 1 | 0.08 | -| relative | 1 | 0.08 | -| per_group | 12 | 0.73 | - diff --git a/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.json b/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.json similarity index 69% rename from benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.json rename to benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.json index d45798352..8e4d17b25 100644 --- a/benchmarks/anomaly_conditioning/results/2026-08-25-55ebd5ca.json +++ b/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.json @@ -1,36 +1,2587 @@ { "generated": "2026-08-25", - "git_sha": "55ebd5ca", + "git_sha": "f9c703a1", "seeds": 15, "datasets": [ "synthetic", "smd", - "nslkdd" + "nslkdd", + "tabular" ], "verdict_pre_registered": "gate needed; refit the threshold from this sweep", "verdict_robust": "no gate needed", "evidence": { "n_low_eta_cells": 90.0, "n_low_eta_groupings": 4.0, - "worst_delta_single_cell": -0.07436193395506441, - "n_harmful_cells": 3.0, + "worst_delta_single_cell": -0.049771945237090054, + "n_harmful_cells": 1.0, "worst_median_delta_per_grouping": 0.0, "n_harmful_groupings": 0.0, - "median_delta_below_threshold": 1.6653345369377348e-16 + "median_delta_below_threshold": 2.220446049250313e-16 }, "spearman": { - "rho": 0.5973468499523505, - "ci_low": 0.5227401597548323, - "ci_high": 0.6681276119704511 + "rho": 0.5705144506753026, + "ci_low": 0.4897270783992741, + "ci_high": 0.6453979051966328 }, "regression": { - "intercept": -0.09975022217842222, - "eta_squared": 0.2760137159852486, - "is_contextual": 0.08302997541505142, - "r_squared": 0.5605703092383967, + "intercept": -0.10148377259307864, + "eta_squared": 0.2807518647753969, + "is_contextual": 0.0849219328806492, + "r_squared": 0.578282695858033, "n": 390.0 }, "cells": [ + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.28654276534927003, + "roc_auc": 0.712279662486274, + "precision_at_n": 0.31863905325443787, + "macro_pr_auc": 0.28654276534927003, + "worst_group_fpr": 0.0318557475582269, + "n_models": 1, + "seconds": 0.45140175000415184, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2731203426957805, + "roc_auc": 0.7090630348672307, + "precision_at_n": 0.3168639053254438, + "macro_pr_auc": 0.2731203426957805, + "worst_group_fpr": 0.0325694966190834, + "n_models": 1, + "seconds": 0.43827241599501576, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.28308659403148606, + "roc_auc": 0.7062419033604665, + "precision_at_n": 0.32337278106508877, + "macro_pr_auc": 0.28308659403148606, + "worst_group_fpr": 0.03102930127723516, + "n_models": 1, + "seconds": 0.46153479199711, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.29427162993120104, + "roc_auc": 0.7042722526996208, + "precision_at_n": 0.3289940828402367, + "macro_pr_auc": 0.29427162993120104, + "worst_group_fpr": 0.030315552216378664, + "n_models": 1, + "seconds": 0.444575542001985, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2963478420199089, + "roc_auc": 0.7207483362155962, + "precision_at_n": 0.32781065088757394, + "macro_pr_auc": 0.2963478420199089, + "worst_group_fpr": 0.030277986476333583, + "n_models": 1, + "seconds": 0.4534223330047098, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2967663272625093, + "roc_auc": 0.711231883977434, + "precision_at_n": 0.32751479289940827, + "macro_pr_auc": 0.2967663272625093, + "worst_group_fpr": 0.030015026296018033, + "n_models": 1, + "seconds": 0.4450537090015132, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2985561324179786, + "roc_auc": 0.7120676994651883, + "precision_at_n": 0.32662721893491126, + "macro_pr_auc": 0.2985561324179786, + "worst_group_fpr": 0.029827197595792637, + "n_models": 1, + "seconds": 0.47069399999600137, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2900148776746608, + "roc_auc": 0.7138835639884591, + "precision_at_n": 0.32514792899408285, + "macro_pr_auc": 0.2900148776746608, + "worst_group_fpr": 0.030991735537190084, + "n_models": 1, + "seconds": 0.49100608300068416, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.30683358269183825, + "roc_auc": 0.7184997766061021, + "precision_at_n": 0.3375739644970414, + "macro_pr_auc": 0.30683358269183825, + "worst_group_fpr": 0.029601803155522164, + "n_models": 1, + "seconds": 0.4592341250026948, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.31648638158245596, + "roc_auc": 0.7182507146381909, + "precision_at_n": 0.33579881656804733, + "macro_pr_auc": 0.31648638158245596, + "worst_group_fpr": 0.028437265214124718, + "n_models": 1, + "seconds": 0.4374390829980257, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2837749575539722, + "roc_auc": 0.7155108607222402, + "precision_at_n": 0.32751479289940827, + "macro_pr_auc": 0.2837749575539722, + "worst_group_fpr": 0.032719759579263714, + "n_models": 1, + "seconds": 0.4527861250026035, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2801629860263534, + "roc_auc": 0.7052293788538226, + "precision_at_n": 0.3210059171597633, + "macro_pr_auc": 0.2801629860263534, + "worst_group_fpr": 0.031893313298271976, + "n_models": 1, + "seconds": 0.4540731669985689, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.27636889431427625, + "roc_auc": 0.7038439865919206, + "precision_at_n": 0.3103550295857988, + "macro_pr_auc": 0.27636889431427625, + "worst_group_fpr": 0.03155522163786627, + "n_models": 1, + "seconds": 0.46590912499959813, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2794685845810904, + "roc_auc": 0.7071127727961801, + "precision_at_n": 0.3210059171597633, + "macro_pr_auc": 0.2794685845810904, + "worst_group_fpr": 0.032607062359128476, + "n_models": 1, + "seconds": 0.46779604200128233, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:campaign", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.28674470272020847, + "roc_auc": 0.7181384619830264, + "precision_at_n": 0.3224852071005917, + "macro_pr_auc": 0.28674470272020847, + "worst_group_fpr": 0.03204357625845229, + "n_models": 1, + "seconds": 0.4490214580000611, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5747040318516179, + "roc_auc": 0.9381694589398516, + "precision_at_n": 0.5227272727272727, + "macro_pr_auc": 0.5747040318516179, + "worst_group_fpr": 0.025377643504531724, + "n_models": 1, + "seconds": 0.12044458300078986, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5817186829361747, + "roc_auc": 0.9349491897830265, + "precision_at_n": 0.5340909090909091, + "macro_pr_auc": 0.5817186829361747, + "worst_group_fpr": 0.02175226586102719, + "n_models": 1, + "seconds": 0.12850574999902165, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5160671100710018, + "roc_auc": 0.9197370227959352, + "precision_at_n": 0.4772727272727273, + "macro_pr_auc": 0.5160671100710018, + "worst_group_fpr": 0.025377643504531724, + "n_models": 1, + "seconds": 0.1251544160040794, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5991840858390056, + "roc_auc": 0.9371017577588575, + "precision_at_n": 0.5227272727272727, + "macro_pr_auc": 0.5991840858390056, + "worst_group_fpr": 0.022356495468277945, + "n_models": 1, + "seconds": 0.12043650000123307, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5970158568934664, + "roc_auc": 0.9373146113705026, + "precision_at_n": 0.5625, + "macro_pr_auc": 0.5970158568934664, + "worst_group_fpr": 0.02175226586102719, + "n_models": 1, + "seconds": 0.12548333399900002, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5599964596064102, + "roc_auc": 0.9303041746772864, + "precision_at_n": 0.48863636363636365, + "macro_pr_auc": 0.5599964596064102, + "worst_group_fpr": 0.022356495468277945, + "n_models": 1, + "seconds": 0.11849504099518526, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5637472053356579, + "roc_auc": 0.9244129360065916, + "precision_at_n": 0.48863636363636365, + "macro_pr_auc": 0.5637472053356579, + "worst_group_fpr": 0.023564954682779457, + "n_models": 1, + "seconds": 0.11911787500139326, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6131736550587313, + "roc_auc": 0.9400302114803626, + "precision_at_n": 0.5284090909090909, + "macro_pr_auc": 0.6131736550587313, + "worst_group_fpr": 0.02175226586102719, + "n_models": 1, + "seconds": 0.1293290829999023, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6008414383264314, + "roc_auc": 0.9334695138698159, + "precision_at_n": 0.5454545454545454, + "macro_pr_auc": 0.6008414383264314, + "worst_group_fpr": 0.02054380664652568, + "n_models": 1, + "seconds": 0.15402783400350017, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6082662112640778, + "roc_auc": 0.9389178797033781, + "precision_at_n": 0.5625, + "macro_pr_auc": 0.6082662112640778, + "worst_group_fpr": 0.022356495468277945, + "n_models": 1, + "seconds": 0.14646341700427, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5694346979625783, + "roc_auc": 0.9276435045317221, + "precision_at_n": 0.5284090909090909, + "macro_pr_auc": 0.5694346979625783, + "worst_group_fpr": 0.022356495468277945, + "n_models": 1, + "seconds": 0.12798587499855785, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5843174027183013, + "roc_auc": 0.9226826421312827, + "precision_at_n": 0.5340909090909091, + "macro_pr_auc": 0.5843174027183013, + "worst_group_fpr": 0.02054380664652568, + "n_models": 1, + "seconds": 0.13026841700047953, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5242593935607399, + "roc_auc": 0.925233452348256, + "precision_at_n": 0.4943181818181818, + "macro_pr_auc": 0.5242593935607399, + "worst_group_fpr": 0.024773413897280966, + "n_models": 1, + "seconds": 0.1208520000000135, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5800825625288935, + "roc_auc": 0.928906893710519, + "precision_at_n": 0.5340909090909091, + "macro_pr_auc": 0.5800825625288935, + "worst_group_fpr": 0.021148036253776436, + "n_models": 1, + "seconds": 0.1326180409960216, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:cardio", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5810785900141355, + "roc_auc": 0.9325048063718758, + "precision_at_n": 0.5227272727272727, + "macro_pr_auc": 0.5810785900141355, + "worst_group_fpr": 0.021148036253776436, + "n_models": 1, + "seconds": 0.13559600000007777, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.05863077331032433, + "roc_auc": 0.8966191176030635, + "precision_at_n": 0.08680555555555555, + "macro_pr_auc": 0.05863077331032433, + "worst_group_fpr": 0.047489229940764675, + "n_models": 1, + "seconds": 0.4108022920045187, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.05197638212386508, + "roc_auc": 0.8538906371537127, + "precision_at_n": 0.09027777777777778, + "macro_pr_auc": 0.05197638212386508, + "worst_group_fpr": 0.04796042003231018, + "n_models": 1, + "seconds": 0.41273645799810765, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.07405783756103791, + "roc_auc": 0.8923926640190271, + "precision_at_n": 0.14583333333333334, + "macro_pr_auc": 0.07405783756103791, + "worst_group_fpr": 0.047287291330102316, + "n_models": 1, + "seconds": 0.43597833400417585, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.04102903280600602, + "roc_auc": 0.8618620703195117, + "precision_at_n": 0.059027777777777776, + "macro_pr_auc": 0.04102903280600602, + "worst_group_fpr": 0.048263327948303715, + "n_models": 1, + "seconds": 0.4380415420018835, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.06665877389720262, + "roc_auc": 0.8846589294261952, + "precision_at_n": 0.10069444444444445, + "macro_pr_auc": 0.06665877389720262, + "worst_group_fpr": 0.0473546042003231, + "n_models": 1, + "seconds": 0.4297401669973624, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.04091324210776709, + "roc_auc": 0.8409143284793275, + "precision_at_n": 0.08333333333333333, + "macro_pr_auc": 0.04091324210776709, + "worst_group_fpr": 0.0483642972536349, + "n_models": 1, + "seconds": 0.4420878340024501, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.06156281724480017, + "roc_auc": 0.8851713720232753, + "precision_at_n": 0.10069444444444445, + "macro_pr_auc": 0.06156281724480017, + "worst_group_fpr": 0.04752288637587507, + "n_models": 1, + "seconds": 0.4307920840001316, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.057937136441490775, + "roc_auc": 0.8834884334051336, + "precision_at_n": 0.09027777777777778, + "macro_pr_auc": 0.057937136441490775, + "worst_group_fpr": 0.04775848142164782, + "n_models": 1, + "seconds": 0.4298634170045261, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.07550091529958756, + "roc_auc": 0.9120703428842816, + "precision_at_n": 0.10416666666666667, + "macro_pr_auc": 0.07550091529958756, + "worst_group_fpr": 0.04691707054388799, + "n_models": 1, + "seconds": 0.4283183329971507, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.062231373450948146, + "roc_auc": 0.9062794493807216, + "precision_at_n": 0.08680555555555555, + "macro_pr_auc": 0.062231373450948146, + "worst_group_fpr": 0.04752288637587507, + "n_models": 1, + "seconds": 0.4181232920018374, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.04935501629482199, + "roc_auc": 0.8635736402800216, + "precision_at_n": 0.08333333333333333, + "macro_pr_auc": 0.04935501629482199, + "worst_group_fpr": 0.047893107162089395, + "n_models": 1, + "seconds": 0.4182043330001761, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.050823837097595845, + "roc_auc": 0.8820772003829355, + "precision_at_n": 0.0798611111111111, + "macro_pr_auc": 0.050823837097595845, + "worst_group_fpr": 0.047859450726978996, + "n_models": 1, + "seconds": 0.42386550000082934, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.0529526540163899, + "roc_auc": 0.8707557832974332, + "precision_at_n": 0.09027777777777778, + "macro_pr_auc": 0.0529526540163899, + "worst_group_fpr": 0.047691168551427035, + "n_models": 1, + "seconds": 0.42577320800046436, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.05343059863532222, + "roc_auc": 0.876983392419075, + "precision_at_n": 0.08680555555555555, + "macro_pr_auc": 0.05343059863532222, + "worst_group_fpr": 0.04802773290253096, + "n_models": 1, + "seconds": 0.4220345419962541, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:covertype", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.04440980646344468, + "roc_auc": 0.8639258642224615, + "precision_at_n": 0.08680555555555555, + "macro_pr_auc": 0.04440980646344468, + "worst_group_fpr": 0.04802773290253096, + "n_models": 1, + "seconds": 0.42304295899521094, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2465692235779982, + "roc_auc": 0.9758299000318501, + "precision_at_n": 0.34615384615384615, + "macro_pr_auc": 0.2465692235779982, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4254823329974897, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.25947397991604965, + "roc_auc": 0.9779560212059878, + "precision_at_n": 0.36538461538461536, + "macro_pr_auc": 0.25947397991604965, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4116599170010886, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.18388773085249868, + "roc_auc": 0.9758241207837174, + "precision_at_n": 0.28846153846153844, + "macro_pr_auc": 0.18388773085249868, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.41671454099559924, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.1795807148953323, + "roc_auc": 0.9794400036987189, + "precision_at_n": 0.3269230769230769, + "macro_pr_auc": 0.1795807148953323, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4084197500051232, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.16856047663142879, + "roc_auc": 0.9700095550235793, + "precision_at_n": 0.3076923076923077, + "macro_pr_auc": 0.16856047663142879, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.39896358300029533, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.24094286403702606, + "roc_auc": 0.975582034500827, + "precision_at_n": 0.36538461538461536, + "macro_pr_auc": 0.24094286403702606, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.3959473330032779, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.1635245883381066, + "roc_auc": 0.9734578397427336, + "precision_at_n": 0.3269230769230769, + "macro_pr_auc": 0.1635245883381066, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4198383749971981, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.23708364780494726, + "roc_auc": 0.9753919614511306, + "precision_at_n": 0.36538461538461536, + "macro_pr_auc": 0.23708364780494726, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4060974160020123, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.18755996109452192, + "roc_auc": 0.9757496326966743, + "precision_at_n": 0.3269230769230769, + "macro_pr_auc": 0.18755996109452192, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4110126249943278, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.24933591270925248, + "roc_auc": 0.9780427099279777, + "precision_at_n": 0.36538461538461536, + "macro_pr_auc": 0.24933591270925248, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4057716250026715, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.1931317946202666, + "roc_auc": 0.9727046110694435, + "precision_at_n": 0.3076923076923077, + "macro_pr_auc": 0.1931317946202666, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.3979578749931534, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.24973837478550712, + "roc_auc": 0.975131895285161, + "precision_at_n": 0.34615384615384615, + "macro_pr_auc": 0.24973837478550712, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4164667919976637, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.22402651624566144, + "roc_auc": 0.976911261571339, + "precision_at_n": 0.34615384615384615, + "macro_pr_auc": 0.22402651624566144, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4011265419976553, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.20220615974700726, + "roc_auc": 0.9764277311442398, + "precision_at_n": 0.34615384615384615, + "macro_pr_auc": 0.20220615974700726, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.40274466700066114, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:fraud", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.24498820618018466, + "roc_auc": 0.9742598709558106, + "precision_at_n": 0.34615384615384615, + "macro_pr_auc": 0.24498820618018466, + "worst_group_fpr": 0.04855082142380126, + "n_models": 1, + "seconds": 0.4057962079969002, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2238807780411677, + "roc_auc": 0.8635803773266009, + "precision_at_n": 0.25769230769230766, + "macro_pr_auc": 0.2238807780411677, + "worst_group_fpr": 0.04321157191247826, + "n_models": 1, + "seconds": 0.24048349999793572, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.21930579778341633, + "roc_auc": 0.8656585609757814, + "precision_at_n": 0.23076923076923078, + "macro_pr_auc": 0.21930579778341633, + "worst_group_fpr": 0.04266227226952302, + "n_models": 1, + "seconds": 0.2333686670026509, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.182872933288566, + "roc_auc": 0.8549151754589821, + "precision_at_n": 0.18461538461538463, + "macro_pr_auc": 0.182872933288566, + "worst_group_fpr": 0.04403552137691111, + "n_models": 1, + "seconds": 0.2408585410012165, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.17941433606707546, + "roc_auc": 0.8555120106479622, + "precision_at_n": 0.19230769230769232, + "macro_pr_auc": 0.17941433606707546, + "worst_group_fpr": 0.04366932161494095, + "n_models": 1, + "seconds": 0.23315041699970607, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.21586806123972352, + "roc_auc": 0.8570884302002126, + "precision_at_n": 0.24615384615384617, + "macro_pr_auc": 0.21586806123972352, + "worst_group_fpr": 0.042845372150508106, + "n_models": 1, + "seconds": 0.22282941699813819, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.26213192835196936, + "roc_auc": 0.8787290755568701, + "precision_at_n": 0.27307692307692305, + "macro_pr_auc": 0.26213192835196936, + "worst_group_fpr": 0.041289023162134945, + "n_models": 1, + "seconds": 0.22768337500019697, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.21348555051172324, + "roc_auc": 0.859291614729681, + "precision_at_n": 0.23846153846153847, + "macro_pr_auc": 0.21348555051172324, + "worst_group_fpr": 0.0433031218529708, + "n_models": 1, + "seconds": 0.24882616600370966, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2697302415338744, + "roc_auc": 0.8680913245867928, + "precision_at_n": 0.26153846153846155, + "macro_pr_auc": 0.2697302415338744, + "worst_group_fpr": 0.04220452256706033, + "n_models": 1, + "seconds": 0.2434860830035177, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.22921292704887455, + "roc_auc": 0.8652096141522123, + "precision_at_n": 0.24615384615384617, + "macro_pr_auc": 0.22921292704887455, + "worst_group_fpr": 0.04257072232903049, + "n_models": 1, + "seconds": 0.2308954170002835, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.24865535675164857, + "roc_auc": 0.8644905950041901, + "precision_at_n": 0.2692307692307692, + "macro_pr_auc": 0.24865535675164857, + "worst_group_fpr": 0.041929872745582714, + "n_models": 1, + "seconds": 0.24952804199710954, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.17923859209213988, + "roc_auc": 0.8493904886654131, + "precision_at_n": 0.18461538461538463, + "macro_pr_auc": 0.17923859209213988, + "worst_group_fpr": 0.04412707131740364, + "n_models": 1, + "seconds": 0.22745279200171353, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.23452842739895857, + "roc_auc": 0.8661916633215727, + "precision_at_n": 0.2692307692307692, + "macro_pr_auc": 0.23452842739895857, + "worst_group_fpr": 0.042021422686075255, + "n_models": 1, + "seconds": 0.243230667001626, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.23844682989086965, + "roc_auc": 0.8721990999936617, + "precision_at_n": 0.25384615384615383, + "macro_pr_auc": 0.23844682989086965, + "worst_group_fpr": 0.042753822210015564, + "n_models": 1, + "seconds": 0.23289158300030977, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.21194472435609962, + "roc_auc": 0.8622377622377622, + "precision_at_n": 0.23461538461538461, + "macro_pr_auc": 0.21194472435609962, + "worst_group_fpr": 0.04257072232903049, + "n_models": 1, + "seconds": 0.2331420420014183, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mammography", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2014456994766733, + "roc_auc": 0.8547007373291361, + "precision_at_n": 0.21153846153846154, + "macro_pr_auc": 0.2014456994766733, + "worst_group_fpr": 0.04412707131740364, + "n_models": 1, + "seconds": 0.23307858299813233, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.26341171282325326, + "roc_auc": 0.801063098859709, + "precision_at_n": 0.2842857142857143, + "macro_pr_auc": 0.26341171282325326, + "worst_group_fpr": 0.03766478342749529, + "n_models": 1, + "seconds": 0.20396720899589127, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2942781064028406, + "roc_auc": 0.8192200078640757, + "precision_at_n": 0.3442857142857143, + "macro_pr_auc": 0.2942781064028406, + "worst_group_fpr": 0.035057221497899464, + "n_models": 1, + "seconds": 0.21104345899948385, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.28390131830542825, + "roc_auc": 0.813244759007471, + "precision_at_n": 0.32142857142857145, + "macro_pr_auc": 0.28390131830542825, + "worst_group_fpr": 0.035781544256120526, + "n_models": 1, + "seconds": 0.21832416699908208, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2766453432404812, + "roc_auc": 0.8005322737526128, + "precision_at_n": 0.31, + "macro_pr_auc": 0.2766453432404812, + "worst_group_fpr": 0.036216137911053166, + "n_models": 1, + "seconds": 0.2121904590021586, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.24913424002114803, + "roc_auc": 0.7965526375695867, + "precision_at_n": 0.2757142857142857, + "macro_pr_auc": 0.24913424002114803, + "worst_group_fpr": 0.03969288715051427, + "n_models": 1, + "seconds": 0.20452533399657113, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.25279398706118017, + "roc_auc": 0.7974207901326544, + "precision_at_n": 0.28285714285714286, + "macro_pr_auc": 0.25279398706118017, + "worst_group_fpr": 0.03795451253078372, + "n_models": 1, + "seconds": 0.21117137499823002, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.28021063871464763, + "roc_auc": 0.8046619482212702, + "precision_at_n": 0.2985714285714286, + "macro_pr_auc": 0.28021063871464763, + "worst_group_fpr": 0.03491235694625525, + "n_models": 1, + "seconds": 0.22707304199866485, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2627035589597312, + "roc_auc": 0.7974106496140394, + "precision_at_n": 0.2842857142857143, + "macro_pr_auc": 0.2627035589597312, + "worst_group_fpr": 0.03751991887585108, + "n_models": 1, + "seconds": 0.21374645800096914, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.28187582019142704, + "roc_auc": 0.8174472796506694, + "precision_at_n": 0.3142857142857143, + "macro_pr_auc": 0.28187582019142704, + "worst_group_fpr": 0.03679559611763002, + "n_models": 1, + "seconds": 0.23416804099542787, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2979922289822242, + "roc_auc": 0.8243186192338734, + "precision_at_n": 0.33, + "macro_pr_auc": 0.2979922289822242, + "worst_group_fpr": 0.035781544256120526, + "n_models": 1, + "seconds": 0.22203474999696482, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2530813480265501, + "roc_auc": 0.7928681111731958, + "precision_at_n": 0.27285714285714285, + "macro_pr_auc": 0.2530813480265501, + "worst_group_fpr": 0.03983775170215848, + "n_models": 1, + "seconds": 0.2217996250037686, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2917199103505091, + "roc_auc": 0.8195225678276524, + "precision_at_n": 0.31857142857142856, + "macro_pr_auc": 0.2917199103505091, + "worst_group_fpr": 0.034767492394611035, + "n_models": 1, + "seconds": 0.22975045799830696, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.29507231326733696, + "roc_auc": 0.8193398315432214, + "precision_at_n": 0.3171428571428571, + "macro_pr_auc": 0.29507231326733696, + "worst_group_fpr": 0.035202086049543675, + "n_models": 1, + "seconds": 0.22104950000357348, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.2743140461522794, + "roc_auc": 0.8030489849133916, + "precision_at_n": 0.3157142857142857, + "macro_pr_auc": 0.2743140461522794, + "worst_group_fpr": 0.035346950601187886, + "n_models": 1, + "seconds": 0.20849816699774237, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:mnist", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.26139822373735183, + "roc_auc": 0.8049419507046626, + "precision_at_n": 0.30428571428571427, + "macro_pr_auc": 0.26139822373735183, + "worst_group_fpr": 0.03694046066927423, + "n_models": 1, + "seconds": 0.22115241600113222, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6403791801793419, + "roc_auc": 0.668364193326667, + "precision_at_n": 0.5545186640471512, + "macro_pr_auc": 0.6403791801793419, + "worst_group_fpr": 0.003864514662423278, + "n_models": 1, + "seconds": 0.179841375000251, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6623979377499488, + "roc_auc": 0.6969444296815092, + "precision_at_n": 0.5697445972495089, + "macro_pr_auc": 0.6623979377499488, + "worst_group_fpr": 0.0040918390543305296, + "n_models": 1, + "seconds": 0.18470820799848298, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6644205260710849, + "roc_auc": 0.7087140495853005, + "precision_at_n": 0.5805500982318271, + "macro_pr_auc": 0.6644205260710849, + "worst_group_fpr": 0.003864514662423278, + "n_models": 1, + "seconds": 0.17875091599853477, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.654998919438973, + "roc_auc": 0.6876682323317811, + "precision_at_n": 0.5667976424361493, + "macro_pr_auc": 0.654998919438973, + "worst_group_fpr": 0.0036371902705160265, + "n_models": 1, + "seconds": 0.19897937499627005, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6763875719164962, + "roc_auc": 0.711991942265857, + "precision_at_n": 0.5756385068762279, + "macro_pr_auc": 0.6763875719164962, + "worst_group_fpr": 0.0036371902705160265, + "n_models": 1, + "seconds": 0.21004141600133153, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.64718089577694, + "roc_auc": 0.6859499010982582, + "precision_at_n": 0.5491159135559921, + "macro_pr_auc": 0.64718089577694, + "worst_group_fpr": 0.004319163446237781, + "n_models": 1, + "seconds": 0.20022020900069037, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6915081633260632, + "roc_auc": 0.7071973626797661, + "precision_at_n": 0.587426326129666, + "macro_pr_auc": 0.6915081633260632, + "worst_group_fpr": 0.0015912707433507615, + "n_models": 1, + "seconds": 0.20368483300262596, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6533914685598003, + "roc_auc": 0.6998173589193115, + "precision_at_n": 0.5741650294695482, + "macro_pr_auc": 0.6533914685598003, + "worst_group_fpr": 0.004546487838145033, + "n_models": 1, + "seconds": 0.20090770899696508, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6668336717138339, + "roc_auc": 0.7130578882233907, + "precision_at_n": 0.5697445972495089, + "macro_pr_auc": 0.6668336717138339, + "worst_group_fpr": 0.003864514662423278, + "n_models": 1, + "seconds": 0.20171112500247546, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6792339193334375, + "roc_auc": 0.7134395163037143, + "precision_at_n": 0.5957760314341847, + "macro_pr_auc": 0.6792339193334375, + "worst_group_fpr": 0.0040918390543305296, + "n_models": 1, + "seconds": 0.18887791700399248, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6979689965372294, + "roc_auc": 0.7070788994283841, + "precision_at_n": 0.6051080550098232, + "macro_pr_auc": 0.6979689965372294, + "worst_group_fpr": 0.00136394635144351, + "n_models": 1, + "seconds": 0.1881111660040915, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.652471383397541, + "roc_auc": 0.6978401056500161, + "precision_at_n": 0.5677799607072691, + "macro_pr_auc": 0.652471383397541, + "worst_group_fpr": 0.004773812230052284, + "n_models": 1, + "seconds": 0.18804724999790778, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6697610641381991, + "roc_auc": 0.6952732157826547, + "precision_at_n": 0.5830058939096268, + "macro_pr_auc": 0.6697610641381991, + "worst_group_fpr": 0.003864514662423278, + "n_models": 1, + "seconds": 0.18602595900301822, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6670674623851404, + "roc_auc": 0.6951316404737458, + "precision_at_n": 0.5677799607072691, + "macro_pr_auc": 0.6670674623851404, + "worst_group_fpr": 0.0036371902705160265, + "n_models": 1, + "seconds": 0.19453533400519518, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:satellite", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6772328057647212, + "roc_auc": 0.7089098879857942, + "precision_at_n": 0.5849705304518664, + "macro_pr_auc": 0.6772328057647212, + "worst_group_fpr": 0.003409865878608775, + "n_models": 1, + "seconds": 0.20645341600175016, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9811631815989674, + "roc_auc": 0.9970048858578746, + "precision_at_n": 0.9655011655011655, + "macro_pr_auc": 0.9811631815989674, + "worst_group_fpr": 0.0003590019745108598, + "n_models": 1, + "seconds": 0.4364367080052034, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9788869653101445, + "roc_auc": 0.997158361294064, + "precision_at_n": 0.958041958041958, + "macro_pr_auc": 0.9788869653101445, + "worst_group_fpr": 0.0003949021719619458, + "n_models": 1, + "seconds": 0.4340969580007368, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9817910384582027, + "roc_auc": 0.9976532986549141, + "precision_at_n": 0.9599067599067599, + "macro_pr_auc": 0.9817910384582027, + "worst_group_fpr": 0.0003590019745108598, + "n_models": 1, + "seconds": 0.4223874160015839, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9819669316060807, + "roc_auc": 0.9971915166745539, + "precision_at_n": 0.965034965034965, + "macro_pr_auc": 0.9819669316060807, + "worst_group_fpr": 0.00032310177705977385, + "n_models": 1, + "seconds": 0.44787437500053784, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9818810429358318, + "roc_auc": 0.9975047772786061, + "precision_at_n": 0.9645687645687646, + "macro_pr_auc": 0.9818810429358318, + "worst_group_fpr": 0.00028720157960868787, + "n_models": 1, + "seconds": 0.4450239169964334, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9837926373440424, + "roc_auc": 0.9978865411498691, + "precision_at_n": 0.9687645687645687, + "macro_pr_auc": 0.9837926373440424, + "worst_group_fpr": 0.00028720157960868787, + "n_models": 1, + "seconds": 0.42829704099858645, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9842447280553007, + "roc_auc": 0.9979000978677877, + "precision_at_n": 0.9696969696969697, + "macro_pr_auc": 0.9842447280553007, + "worst_group_fpr": 0.00028720157960868787, + "n_models": 1, + "seconds": 0.4368969589995686, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9729041160727758, + "roc_auc": 0.9967768484731327, + "precision_at_n": 0.9333333333333333, + "macro_pr_auc": 0.9729041160727758, + "worst_group_fpr": 0.0007180039490217197, + "n_models": 1, + "seconds": 0.4372796669995296, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9779109874894807, + "roc_auc": 0.9961613065328736, + "precision_at_n": 0.9622377622377623, + "macro_pr_auc": 0.9779109874894807, + "worst_group_fpr": 0.00028720157960868787, + "n_models": 1, + "seconds": 0.4329318750023958, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9759659554541371, + "roc_auc": 0.9964731277816901, + "precision_at_n": 0.9505827505827505, + "macro_pr_auc": 0.9759659554541371, + "worst_group_fpr": 0.0006103033566684617, + "n_models": 1, + "seconds": 0.42724345899478067, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9687688224385602, + "roc_auc": 0.9967668231965485, + "precision_at_n": 0.9212121212121213, + "macro_pr_auc": 0.9687688224385602, + "worst_group_fpr": 0.0007898043439238916, + "n_models": 1, + "seconds": 0.4360776250032359, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9778553282619138, + "roc_auc": 0.996779442659895, + "precision_at_n": 0.9603729603729604, + "macro_pr_auc": 0.9778553282619138, + "worst_group_fpr": 0.0002154011847065159, + "n_models": 1, + "seconds": 0.4356054999952903, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9748345309853084, + "roc_auc": 0.9971416915520308, + "precision_at_n": 0.9445221445221446, + "macro_pr_auc": 0.9748345309853084, + "worst_group_fpr": 0.0006821037515706336, + "n_models": 1, + "seconds": 0.42429037500551203, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9788403825355294, + "roc_auc": 0.9969967350904346, + "precision_at_n": 0.9627039627039627, + "macro_pr_auc": 0.9788403825355294, + "worst_group_fpr": 0.00028720157960868787, + "n_models": 1, + "seconds": 0.42620766699837986, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:shuttle", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.9775212081798447, + "roc_auc": 0.9969381064696089, + "precision_at_n": 0.9473193473193473, + "macro_pr_auc": 0.9775212081798447, + "worst_group_fpr": 0.0005744031592173757, + "n_models": 1, + "seconds": 0.4447026660054689, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4996823139165014, + "roc_auc": 0.6505501692538507, + "precision_at_n": 0.5247170935080405, + "macro_pr_auc": 0.4996823139165014, + "worst_group_fpr": 0.04865506329113924, + "n_models": 1, + "seconds": 0.16026191700075287, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4637123206381993, + "roc_auc": 0.6162852172405214, + "precision_at_n": 0.50565812983919, + "macro_pr_auc": 0.4637123206381993, + "worst_group_fpr": 0.06131329113924051, + "n_models": 1, + "seconds": 0.1401472920042579, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4635364518748895, + "roc_auc": 0.616683378442563, + "precision_at_n": 0.4973198332340679, + "macro_pr_auc": 0.4635364518748895, + "worst_group_fpr": 0.05617088607594937, + "n_models": 1, + "seconds": 0.14207695799996145, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4773874757151399, + "roc_auc": 0.6245720356073914, + "precision_at_n": 0.5080405002977963, + "macro_pr_auc": 0.4773874757151399, + "worst_group_fpr": 0.049841772151898736, + "n_models": 1, + "seconds": 0.13609958300366998, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5115427344895851, + "roc_auc": 0.65513067226574, + "precision_at_n": 0.5306730196545563, + "macro_pr_auc": 0.5115427344895851, + "worst_group_fpr": 0.0446993670886076, + "n_models": 1, + "seconds": 0.13952529199741548, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4677266779695209, + "roc_auc": 0.62460513717478, + "precision_at_n": 0.5020845741512805, + "macro_pr_auc": 0.4677266779695209, + "worst_group_fpr": 0.06091772151898734, + "n_models": 1, + "seconds": 0.145223999999871, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.469784426449968, + "roc_auc": 0.6214238527303021, + "precision_at_n": 0.4973198332340679, + "macro_pr_auc": 0.469784426449968, + "worst_group_fpr": 0.0557753164556962, + "n_models": 1, + "seconds": 0.1338729169947328, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4818956661574424, + "roc_auc": 0.6314401985434368, + "precision_at_n": 0.5116140559857058, + "macro_pr_auc": 0.4818956661574424, + "worst_group_fpr": 0.049841772151898736, + "n_models": 1, + "seconds": 0.14036466600373387, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4798119762992656, + "roc_auc": 0.6289304871796805, + "precision_at_n": 0.5068493150684932, + "macro_pr_auc": 0.4798119762992656, + "worst_group_fpr": 0.048259493670886076, + "n_models": 1, + "seconds": 0.14640716700523626, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.47533003058254375, + "roc_auc": 0.6280062348745863, + "precision_at_n": 0.509827278141751, + "macro_pr_auc": 0.47533003058254375, + "worst_group_fpr": 0.05814873417721519, + "n_models": 1, + "seconds": 0.13803812499827472, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.48377517349008564, + "roc_auc": 0.6226054962266568, + "precision_at_n": 0.5074449076831448, + "macro_pr_auc": 0.48377517349008564, + "worst_group_fpr": 0.0446993670886076, + "n_models": 1, + "seconds": 0.13034129200241296, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.47324342257073737, + "roc_auc": 0.6320707775122323, + "precision_at_n": 0.5080405002977963, + "macro_pr_auc": 0.47324342257073737, + "worst_group_fpr": 0.06131329113924051, + "n_models": 1, + "seconds": 0.1449920410013874, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4758315595303919, + "roc_auc": 0.6310042238071185, + "precision_at_n": 0.521143537820131, + "macro_pr_auc": 0.4758315595303919, + "worst_group_fpr": 0.05617088607594937, + "n_models": 1, + "seconds": 0.14752429100190056, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4179513915431128, + "roc_auc": 0.557428981235063, + "precision_at_n": 0.44371649791542583, + "macro_pr_auc": 0.4179513915431128, + "worst_group_fpr": 0.0668512658227848, + "n_models": 1, + "seconds": 0.1595240000024205, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:spambase", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4824987826933403, + "roc_auc": 0.6376413825664763, + "precision_at_n": 0.5181655747468732, + "macro_pr_auc": 0.4824987826933403, + "worst_group_fpr": 0.053401898734177215, + "n_models": 1, + "seconds": 0.15749891699670115, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 0, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.49972632444908194, + "roc_auc": 0.9777230254831988, + "precision_at_n": 0.5483870967741935, + "macro_pr_auc": 0.49972632444908194, + "worst_group_fpr": 0.031530307148681706, + "n_models": 1, + "seconds": 0.15497462500206893, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 1, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5444216351253587, + "roc_auc": 0.979564339304451, + "precision_at_n": 0.5806451612903226, + "macro_pr_auc": 0.5444216351253587, + "worst_group_fpr": 0.03098668116335961, + "n_models": 1, + "seconds": 0.14914366700395476, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 2, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5070328987360753, + "roc_auc": 0.9773109219136805, + "precision_at_n": 0.5376344086021505, + "macro_pr_auc": 0.5070328987360753, + "worst_group_fpr": 0.03180212014134275, + "n_models": 1, + "seconds": 0.15282645899424097, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 3, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5017291279412852, + "roc_auc": 0.9771969358199838, + "precision_at_n": 0.5913978494623656, + "macro_pr_auc": 0.5017291279412852, + "worst_group_fpr": 0.03180212014134275, + "n_models": 1, + "seconds": 0.15333070900669554, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 4, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5177825020575512, + "roc_auc": 0.9788395046573548, + "precision_at_n": 0.6021505376344086, + "macro_pr_auc": 0.5177825020575512, + "worst_group_fpr": 0.031530307148681706, + "n_models": 1, + "seconds": 0.1534228329983307, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 5, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4980836054374582, + "roc_auc": 0.9761330656121491, + "precision_at_n": 0.5483870967741935, + "macro_pr_auc": 0.4980836054374582, + "worst_group_fpr": 0.032345746126664854, + "n_models": 1, + "seconds": 0.14515966599719832, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 6, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6499322129066327, + "roc_auc": 0.9818294475766263, + "precision_at_n": 0.6236559139784946, + "macro_pr_auc": 0.6499322129066327, + "worst_group_fpr": 0.03071486817069856, + "n_models": 1, + "seconds": 0.1430729589992552, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 7, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.4889107086450289, + "roc_auc": 0.9777844026105738, + "precision_at_n": 0.5161290322580645, + "macro_pr_auc": 0.4889107086450289, + "worst_group_fpr": 0.03207393313400381, + "n_models": 1, + "seconds": 0.14029387499613222, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 8, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.43467165817772785, + "roc_auc": 0.9722341566636562, + "precision_at_n": 0.5053763440860215, + "macro_pr_auc": 0.43467165817772785, + "worst_group_fpr": 0.032889372111986954, + "n_models": 1, + "seconds": 0.1374471250019269, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 9, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.6443810112816749, + "roc_auc": 0.9829108541065682, + "precision_at_n": 0.6344086021505376, + "macro_pr_auc": 0.6443810112816749, + "worst_group_fpr": 0.02989942919271541, + "n_models": 1, + "seconds": 0.14336575000197627, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 10, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5769111665754879, + "roc_auc": 0.9802365649852257, + "precision_at_n": 0.5913978494623656, + "macro_pr_auc": 0.5769111665754879, + "worst_group_fpr": 0.031530307148681706, + "n_models": 1, + "seconds": 0.14559333299985155, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 11, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5389354598908428, + "roc_auc": 0.9779889930351574, + "precision_at_n": 0.5483870967741935, + "macro_pr_auc": 0.5389354598908428, + "worst_group_fpr": 0.03180212014134275, + "n_models": 1, + "seconds": 0.13968100000056438, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 12, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.5466419419531122, + "roc_auc": 0.9763201197146256, + "precision_at_n": 0.5483870967741935, + "macro_pr_auc": 0.5466419419531122, + "worst_group_fpr": 0.03207393313400381, + "n_models": 1, + "seconds": 0.14631679200101644, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 13, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.586136736438214, + "roc_auc": 0.9802394877055769, + "precision_at_n": 0.6021505376344086, + "macro_pr_auc": 0.586136736438214, + "worst_group_fpr": 0.03098668116335961, + "n_models": 1, + "seconds": 0.14696887500031153, + "eta_squared": 0.0, + "warnings": [] + }, + { + "dataset": "tabular:thyroid", + "grouping": "none", + "config": "pooled", + "seed": 14, + "mechanism": "tabular", + "level_spread": NaN, + "pr_auc": 0.575796106634768, + "roc_auc": 0.9809321724288098, + "precision_at_n": 0.5913978494623656, + "macro_pr_auc": 0.575796106634768, + "worst_group_fpr": 0.03098668116335961, + "n_models": 1, + "seconds": 0.14394399999582674, + "eta_squared": 0.0, + "warnings": [] + }, { "dataset": "synthetic", "grouping": "spread=0.000", @@ -38,13 +2589,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 1, - "seconds": 0.09368937500039465, + "seconds": 0.18409758299821988, "eta_squared": 0.0008371029862412371, "warnings": [] }, @@ -55,13 +2606,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07720820900067338, + "seconds": 0.18579829199734377, "eta_squared": 0.0008371029862412371, "warnings": [] }, @@ -72,13 +2623,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9997841047394043, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7040970829984872, + "seconds": 1.288182541000424, "eta_squared": 0.0008371029862412371, "warnings": [] }, @@ -89,13 +2640,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.0718564589988091, + "seconds": 0.18637058299646014, "eta_squared": 0.0012807601285772677, "warnings": [] }, @@ -106,13 +2657,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9995636400137604, - "roc_auc": 0.9999911422902495, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07067412499964121, + "seconds": 0.1840459169980022, "eta_squared": 0.0012807601285772677, "warnings": [] }, @@ -123,13 +2674,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9962594841774015, - "roc_auc": 0.999926923894558, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9975862799450697, + "roc_auc": 0.9999512825963719, + "precision_at_n": 0.96875, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7128532499991707, + "seconds": 1.3135257909962093, "eta_squared": 0.0012807601285772677, "warnings": [] }, @@ -140,13 +2691,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07275233299878892, + "seconds": 0.18191958300303668, "eta_squared": 0.00097235471194274, "warnings": [] }, @@ -157,13 +2708,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9980505965884341, - "roc_auc": 0.9999623547335601, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, + "pr_auc": 0.9988727861917877, + "roc_auc": 0.9999778557256236, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07040591600161861, + "seconds": 0.19064775000151712, "eta_squared": 0.00097235471194274, "warnings": [] }, @@ -174,13 +2725,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9979498197998601, - "roc_auc": 0.9999557114512473, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9997874149659862, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7100916669987782, + "seconds": 1.3351120419974905, "eta_squared": 0.00097235471194274, "warnings": [] }, @@ -191,13 +2742,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 1.0, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.06776700000045821, + "seconds": 0.18484791700029746, "eta_squared": 0.00043327407293021155, "warnings": [] }, @@ -208,13 +2759,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07179583299875958, + "seconds": 0.1640156669964199, "eta_squared": 0.00043327407293021155, "warnings": [] }, @@ -225,13 +2776,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7085450409977057, + "seconds": 1.3733154159999685, "eta_squared": 0.00043327407293021155, "warnings": [] }, @@ -242,13 +2793,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.0739716669995687, + "seconds": 0.18438108299596934, "eta_squared": 0.0011831306529618937, "warnings": [] }, @@ -259,13 +2810,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07306791700102622, + "seconds": 0.17161412499990547, "eta_squared": 0.0011831306529618937, "warnings": [] }, @@ -276,13 +2827,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9988727861917877, - "roc_auc": 0.9999778557256236, + "pr_auc": 0.9986319303600137, + "roc_auc": 0.9999734268707483, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, + "macro_pr_auc": 0.9988425925925926, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7117822090003756, + "seconds": 1.3089136660055374, "eta_squared": 0.0011831306529618937, "warnings": [] }, @@ -293,13 +2844,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9997841047394043, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07129791599800228, + "seconds": 0.17085050000605406, "eta_squared": 0.001278796513683895, "warnings": [] }, @@ -316,7 +2867,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.0722847079996427, + "seconds": 0.18761091699707322, "eta_squared": 0.001278796513683895, "warnings": [] }, @@ -327,13 +2878,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9985712607506163, - "roc_auc": 0.9999712124433106, + "pr_auc": 0.996576575419816, + "roc_auc": 0.9999357816043083, "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03826530612244898, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.6925027090001095, + "seconds": 1.3391146669964655, "eta_squared": 0.001278796513683895, "warnings": [] }, @@ -344,13 +2895,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, + "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.0716216250002617, + "seconds": 0.1871012499977951, "eta_squared": 0.0013720012875030802, "warnings": [] }, @@ -361,13 +2912,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.08743787500134204, + "seconds": 0.18300454199925298, "eta_squared": 0.0013720012875030802, "warnings": [] }, @@ -378,13 +2929,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9946789809149659, - "roc_auc": 0.999913637329932, + "pr_auc": 0.9959089137856371, + "roc_auc": 0.9999291383219955, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "macro_pr_auc": 0.9931588955026455, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.710910709000018, + "seconds": 1.3320764579984825, "eta_squared": 0.0013720012875030802, "warnings": [] }, @@ -395,13 +2946,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.06848508300026879, + "seconds": 0.17152695799450157, "eta_squared": 0.0017954572306766912, "warnings": [] }, @@ -412,13 +2963,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "pr_auc": 1.0, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07606524999937392, + "seconds": 0.1670167499978561, "eta_squared": 0.0017954572306766912, "warnings": [] }, @@ -429,13 +2980,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9973931657464519, - "roc_auc": 0.9999468537414967, - "precision_at_n": 0.96875, + "pr_auc": 0.998003812170808, + "roc_auc": 0.9999601403061225, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7286392080022779, + "seconds": 1.3436600409986568, "eta_squared": 0.0017954572306766912, "warnings": [] }, @@ -446,13 +2997,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9986808847878315, - "roc_auc": 0.9999734268707483, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9963831018518517, + "pr_auc": 0.9996744556165973, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.08059941700048512, + "seconds": 0.1912815000032424, "eta_squared": 0.00084717659914683, "warnings": [] }, @@ -467,9 +3018,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07574087500324822, + "seconds": 0.16627637499914272, "eta_squared": 0.00084717659914683, "warnings": [] }, @@ -480,13 +3031,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9950291456983933, - "roc_auc": 0.9999180661848074, + "pr_auc": 0.9960141105034992, + "roc_auc": 0.9999313527494331, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7295693329979258, + "seconds": 1.3255488340000738, "eta_squared": 0.00084717659914683, "warnings": [] }, @@ -497,13 +3048,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07634137499917415, + "seconds": 0.16260399999737274, "eta_squared": 0.0012853540416852677, "warnings": [] }, @@ -514,13 +3065,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.0719384589974652, + "seconds": 0.1862242499992135, "eta_squared": 0.0012853540416852677, "warnings": [] }, @@ -531,13 +3082,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.731033625001146, + "seconds": 1.3023840000023483, "eta_squared": 0.0012853540416852677, "warnings": [] }, @@ -548,13 +3099,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07527325000046403, + "seconds": 0.1731478750007227, "eta_squared": 0.0012397194238675945, "warnings": [] }, @@ -565,13 +3116,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07464133299799869, + "seconds": 0.18117912500019884, "eta_squared": 0.0012397194238675945, "warnings": [] }, @@ -582,13 +3133,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9980517045519615, - "roc_auc": 0.9999601403061226, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04591836734693878, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.699046958998224, + "seconds": 1.3403592499962542, "eta_squared": 0.0012397194238675945, "warnings": [] }, @@ -599,13 +3150,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07438695900054881, + "seconds": 0.18963520800025435, "eta_squared": 0.0003383620246718858, "warnings": [] }, @@ -616,13 +3167,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.08109362500181305, + "seconds": 0.17530620799516328, "eta_squared": 0.0003383620246718858, "warnings": [] }, @@ -633,13 +3184,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9948596494505617, - "roc_auc": 0.9999158517573696, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9940263209142927, + "roc_auc": 0.9998959219104309, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9897734788359788, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7164103750001232, + "seconds": 1.3515108339997823, "eta_squared": 0.0003383620246718858, "warnings": [] }, @@ -656,7 +3207,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 1, - "seconds": 0.07404937500177766, + "seconds": 0.1935919579991605, "eta_squared": 0.0010503666837600957, "warnings": [] }, @@ -667,13 +3218,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07101924999733455, + "seconds": 0.18339049999485724, "eta_squared": 0.0010503666837600957, "warnings": [] }, @@ -684,13 +3235,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7057310419986607, + "seconds": 1.3044878329965286, "eta_squared": 0.0010503666837600957, "warnings": [] }, @@ -701,13 +3252,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.08904483299920685, + "seconds": 0.18961091699748067, "eta_squared": 0.002415635561632408, "warnings": [] }, @@ -718,13 +3269,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9993384082076204, - "roc_auc": 0.9999867134353743, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07063283300158218, + "seconds": 0.16607283300254494, "eta_squared": 0.002415635561632408, "warnings": [] }, @@ -735,13 +3286,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.999572638333684, - "roc_auc": 0.9999911422902494, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, + "pr_auc": 0.9985267336421512, + "roc_auc": 0.9999712124433106, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7176964999998745, + "seconds": 1.3830415830016136, "eta_squared": 0.002415635561632408, "warnings": [] }, @@ -752,13 +3303,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07436258300003828, + "seconds": 0.19411041699640919, "eta_squared": 0.0009561098242806195, "warnings": [] }, @@ -773,9 +3324,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.03826530612244898, "n_models": 1, - "seconds": 0.07921012500082725, + "seconds": 0.182575042003009, "eta_squared": 0.0009561098242806195, "warnings": [] }, @@ -786,13 +3337,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.0, - "pr_auc": 0.9985638435893296, - "roc_auc": 0.9999712124433107, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9948950066137566, - "worst_group_fpr": 0.04591836734693878, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7040092079987517, + "seconds": 1.3324455840047449, "eta_squared": 0.0009561098242806195, "warnings": [] }, @@ -803,13 +3354,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07160279199888464, + "seconds": 0.18907358399883378, "eta_squared": 0.0007681373849954036, "warnings": [] }, @@ -821,12 +3372,12 @@ "mechanism": "contextual", "level_spread": 0.0, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07975612500013085, + "seconds": 0.1886240000021644, "eta_squared": 0.0007681373849954036, "warnings": [] }, @@ -837,13 +3388,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6898896250022517, + "seconds": 1.3481295829988085, "eta_squared": 0.0007681373849954036, "warnings": [] }, @@ -854,13 +3405,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, + "pr_auc": 0.9989935202589898, + "roc_auc": 0.9999800701530612, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07450095899912412, + "seconds": 0.18981487499695504, "eta_squared": 0.0023345860714670385, "warnings": [] }, @@ -871,13 +3422,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07206262500039884, + "seconds": 0.1884202910005115, "eta_squared": 0.0023345860714670385, "warnings": [] }, @@ -888,13 +3439,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9523907508279281, - "roc_auc": 0.9992980265022675, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.6941066250001313, + "seconds": 1.4001940000016475, "eta_squared": 0.0023345860714670385, "warnings": [] }, @@ -909,9 +3460,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07249937500091619, + "seconds": 0.18420899999910034, "eta_squared": 0.0013075922239467576, "warnings": [] }, @@ -922,13 +3473,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.998136673741906, - "roc_auc": 0.9999645691609977, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.0823573329980718, + "seconds": 0.1899541250022594, "eta_squared": 0.0013075922239467576, "warnings": [] }, @@ -939,13 +3490,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767568942402173, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7552631670005212, + "seconds": 1.337920917001611, "eta_squared": 0.0013075922239467576, "warnings": [] }, @@ -956,13 +3507,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9944819614427851, - "roc_auc": 0.9999092084750566, + "pr_auc": 0.991638404762116, + "roc_auc": 0.9998693487811791, "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.04591836734693878, + "macro_pr_auc": 0.9856812169312169, + "worst_group_fpr": 0.03826530612244898, "n_models": 1, - "seconds": 0.07165954100128147, + "seconds": 0.19650370800081873, "eta_squared": 0.0010029218229787487, "warnings": [] }, @@ -973,13 +3524,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725625, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.0729617920005694, + "seconds": 0.1894969589993707, "eta_squared": 0.0010029218229787487, "warnings": [] }, @@ -990,13 +3541,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7429075420004665, + "seconds": 1.3478906250020373, "eta_squared": 0.0010029218229787487, "warnings": [] }, @@ -1011,9 +3562,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07104595800046809, + "seconds": 0.17953670800488908, "eta_squared": 0.0017217081640612903, "warnings": [] }, @@ -1028,9 +3579,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07243112500145799, + "seconds": 0.18294712500210153, "eta_squared": 0.0017217081640612903, "warnings": [] }, @@ -1041,13 +3592,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7270605000012438, + "seconds": 1.3613609160020133, "eta_squared": 0.0017217081640612903, "warnings": [] }, @@ -1058,13 +3609,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9970752459494735, - "roc_auc": 0.9999468537414967, + "pr_auc": 0.9958642320763798, + "roc_auc": 0.9999291383219954, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07321583299926715, + "seconds": 0.19453262500610435, "eta_squared": 0.0012261319468726715, "warnings": [] }, @@ -1075,13 +3626,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9991129213154376, - "roc_auc": 0.9999822845804989, + "pr_auc": 0.9931509781675918, + "roc_auc": 0.9998959219104309, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07625950000146986, + "seconds": 0.1858277499995893, "eta_squared": 0.0012261319468726715, "warnings": [] }, @@ -1092,13 +3643,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7202732090008794, + "seconds": 1.372646875002829, "eta_squared": 0.0012261319468726715, "warnings": [] }, @@ -1109,13 +3660,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9964798833343682, - "roc_auc": 0.9999313527494331, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9976851851851851, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07321233300172025, + "seconds": 0.18463604200223926, "eta_squared": 0.0012204410626207138, "warnings": [] }, @@ -1126,13 +3677,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9972031494854697, - "roc_auc": 0.9999490681689343, + "pr_auc": 0.9978764404259796, + "roc_auc": 0.9999601403061225, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04336734693877551, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07305512500170153, + "seconds": 0.18741529199905926, "eta_squared": 0.0012204410626207138, "warnings": [] }, @@ -1143,13 +3694,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7081770830009191, + "seconds": 1.346244165993994, "eta_squared": 0.0012204410626207138, "warnings": [] }, @@ -1160,13 +3711,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.992820838204789, - "roc_auc": 0.9998804209183673, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9906045148405054, + "roc_auc": 0.9998715632086168, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9896288029100528, "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.07393937500091852, + "seconds": 0.20375691699882736, "eta_squared": 0.0019679556564418787, "warnings": [] }, @@ -1177,13 +3728,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9974816945212338, - "roc_auc": 0.9999534970238095, + "pr_auc": 0.9994516328453016, + "roc_auc": 0.9999889278628119, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.06988745899798232, + "seconds": 0.2048590830017929, "eta_squared": 0.0019679556564418787, "warnings": [] }, @@ -1194,13 +3745,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7241048750001937, + "seconds": 1.3410939999957918, "eta_squared": 0.0019679556564418787, "warnings": [] }, @@ -1215,9 +3766,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07015695900190622, + "seconds": 0.17705245799879776, "eta_squared": 0.001183606140824988, "warnings": [] }, @@ -1228,13 +3779,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, + "pr_auc": 0.9993384082076205, + "roc_auc": 0.9999867134353742, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.08318279200102552, + "seconds": 0.1958594579991768, "eta_squared": 0.001183606140824988, "warnings": [] }, @@ -1245,13 +3796,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7309281660018314, + "seconds": 1.3489050830030465, "eta_squared": 0.001183606140824988, "warnings": [] }, @@ -1263,12 +3814,12 @@ "mechanism": "contextual", "level_spread": 0.0, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07772495899916976, + "seconds": 0.18971562499791617, "eta_squared": 0.0010477149599897315, "warnings": [] }, @@ -1285,7 +3836,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07547470799909206, + "seconds": 0.19377075000375044, "eta_squared": 0.0010477149599897315, "warnings": [] }, @@ -1296,13 +3847,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7729518750020361, + "seconds": 1.3623629170033382, "eta_squared": 0.0010477149599897315, "warnings": [] }, @@ -1313,13 +3864,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.997490523278894, - "roc_auc": 0.9999534970238095, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07022299999880488, + "seconds": 0.19366850000369595, "eta_squared": 0.0015314052300759431, "warnings": [] }, @@ -1330,13 +3881,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9996744556165971, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07097679100115784, + "seconds": 0.18420812499971362, "eta_squared": 0.0015314052300759431, "warnings": [] }, @@ -1347,13 +3898,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7067097500003001, + "seconds": 1.3384155000021565, "eta_squared": 0.0015314052300759431, "warnings": [] }, @@ -1364,13 +3915,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9713348315763177, - "roc_auc": 0.9997231965702948, + "pr_auc": 0.9658179986738644, + "roc_auc": 0.9996545493197279, "precision_at_n": 0.96875, - "macro_pr_auc": 0.9783895502645503, - "worst_group_fpr": 0.04591836734693878, + "macro_pr_auc": 0.9711557539682539, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07170770800075843, + "seconds": 0.190427916997578, "eta_squared": 0.0008561578874620307, "warnings": [] }, @@ -1381,13 +3932,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9996744556165974, - "roc_auc": 0.9999933567176872, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9956960258104145, + "roc_auc": 0.9999269238945578, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.0818978329989477, + "seconds": 0.19694179199723294, "eta_squared": 0.0008561578874620307, "warnings": [] }, @@ -1398,13 +3949,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.961741641669088, - "roc_auc": 0.9994884672619048, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7422807080001803, + "seconds": 1.3296376250000321, "eta_squared": 0.0008561578874620307, "warnings": [] }, @@ -1421,7 +3972,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.08475429100144538, + "seconds": 0.19709537499875296, "eta_squared": 0.0009042790866601086, "warnings": [] }, @@ -1432,13 +3983,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.0698327079990122, + "seconds": 0.19135083400033182, "eta_squared": 0.0009042790866601086, "warnings": [] }, @@ -1449,13 +4000,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7110866250004619, + "seconds": 1.3695947080050246, "eta_squared": 0.0009042790866601086, "warnings": [] }, @@ -1466,13 +4017,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9929780154537662, - "roc_auc": 0.9998937074829932, + "pr_auc": 0.9924349918271719, + "roc_auc": 0.9998870642006803, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9896288029100528, "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07018608399812365, + "seconds": 0.18747533400164684, "eta_squared": 0.0023824674825981534, "warnings": [] }, @@ -1483,13 +4034,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9995636400137602, - "roc_auc": 0.9999911422902494, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.08228349999990314, + "seconds": 0.18687129199679475, "eta_squared": 0.0023824674825981534, "warnings": [] }, @@ -1500,13 +4051,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9082358809082258, + "roc_auc": 0.9985362634637187, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9244201689514191, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7147403329981898, + "seconds": 1.35008045900031, "eta_squared": 0.0023824674825981534, "warnings": [] }, @@ -1523,7 +4074,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07568112499939161, + "seconds": 0.19990237500314834, "eta_squared": 0.0017212676917411547, "warnings": [] }, @@ -1534,13 +4085,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.9997841047394044, - "roc_auc": 0.9999955711451246, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07336058399960166, + "seconds": 0.18206070899759652, "eta_squared": 0.0017212676917411547, "warnings": [] }, @@ -1551,13 +4102,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.0, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7232982089990401, + "seconds": 1.309598540996376, "eta_squared": 0.0017212676917411547, "warnings": [] }, @@ -1572,9 +4123,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, + "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.07654600000023493, + "seconds": 0.17353662499954225, "eta_squared": 0.03955842865464311, "warnings": [] }, @@ -1585,13 +4136,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07850200000029872, + "seconds": 0.17513391700049397, "eta_squared": 0.03955842865464311, "warnings": [] }, @@ -1602,13 +4153,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9997841047394043, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7702025409998896, + "seconds": 1.339590750001662, "eta_squared": 0.03955842865464311, "warnings": [] }, @@ -1623,9 +4174,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.07867550000082701, + "seconds": 0.17290741600299953, "eta_squared": 0.035766956730376886, "warnings": [] }, @@ -1636,13 +4187,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.08078166600171244, + "seconds": 0.18505200000072364, "eta_squared": 0.035766956730376886, "warnings": [] }, @@ -1653,13 +4204,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9968128111867416, - "roc_auc": 0.999937996031746, + "pr_auc": 0.9982768210701308, + "roc_auc": 0.9999645691609977, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7648674589981965, + "seconds": 1.3701164169979165, "eta_squared": 0.035766956730376886, "warnings": [] }, @@ -1674,9 +4225,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.08140275000187103, + "seconds": 0.20308208300411934, "eta_squared": 0.0395464466000243, "warnings": [] }, @@ -1687,13 +4238,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.999019577035818, - "roc_auc": 0.9999800701530612, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9976851851851851, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9997841047394043, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07521029200142948, + "seconds": 0.19559391700022388, "eta_squared": 0.0395464466000243, "warnings": [] }, @@ -1704,13 +4255,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9979533443255417, - "roc_auc": 0.9999557114512471, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9994661873670918, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7182910410010663, + "seconds": 1.3472885000010137, "eta_squared": 0.0395464466000243, "warnings": [] }, @@ -1725,9 +4276,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07188695800141431, + "seconds": 0.18855170800088672, "eta_squared": 0.038621152227959823, "warnings": [] }, @@ -1738,13 +4289,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07432549999793991, + "seconds": 0.16623616699507693, "eta_squared": 0.038621152227959823, "warnings": [] }, @@ -1755,13 +4306,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7132319590018597, + "seconds": 1.4932873750003637, "eta_squared": 0.038621152227959823, "warnings": [] }, @@ -1776,9 +4327,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07170508299896028, + "seconds": 0.18449270899873227, "eta_squared": 0.0391862568727475, "warnings": [] }, @@ -1789,13 +4340,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07496162499955972, + "seconds": 0.21453958300116938, "eta_squared": 0.0391862568727475, "warnings": [] }, @@ -1806,13 +4357,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9989911574039089, - "roc_auc": 0.9999800701530611, + "pr_auc": 0.998991157403909, + "roc_auc": 0.9999800701530612, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7146162909994018, + "seconds": 1.4944277079994208, "eta_squared": 0.0391862568727475, "warnings": [] }, @@ -1823,13 +4374,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07139183300023433, + "seconds": 0.1983925839958829, "eta_squared": 0.03294941366121079, "warnings": [] }, @@ -1840,13 +4391,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07158858299953863, + "seconds": 0.19444470799498959, "eta_squared": 0.03294941366121079, "warnings": [] }, @@ -1857,13 +4408,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9983257405764419, - "roc_auc": 0.9999667835884354, + "pr_auc": 0.9961542267012345, + "roc_auc": 0.9999291383219954, "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7419408749992726, + "seconds": 1.3954163340022205, "eta_squared": 0.03294941366121079, "warnings": [] }, @@ -1874,13 +4425,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9991081986024108, - "roc_auc": 0.9999822845804989, + "pr_auc": 0.9970566287270859, + "roc_auc": 0.9999468537414966, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07530224999936763, + "seconds": 0.19073270899389172, "eta_squared": 0.03415590740792779, "warnings": [] }, @@ -1891,13 +4442,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, + "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.07935620799980825, + "seconds": 0.19973162499809405, "eta_squared": 0.03415590740792779, "warnings": [] }, @@ -1908,13 +4459,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9963227694148249, - "roc_auc": 0.9999357816043084, + "pr_auc": 0.9969139346631589, + "roc_auc": 0.999944639314059, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7186046250026266, + "seconds": 1.407847500006028, "eta_squared": 0.03415590740792779, "warnings": [] }, @@ -1925,13 +4476,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, + "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.07252712499757763, + "seconds": 0.18568058300297707, "eta_squared": 0.04191038320105364, "warnings": [] }, @@ -1948,7 +4499,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.07778999999936786, + "seconds": 0.1967608749982901, "eta_squared": 0.04191038320105364, "warnings": [] }, @@ -1959,13 +4510,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9982416315100598, - "roc_auc": 0.9999645691609977, - "precision_at_n": 0.96875, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "pr_auc": 0.9980038121708081, + "roc_auc": 0.9999601403061225, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7112914999997884, + "seconds": 1.671937874998548, "eta_squared": 0.04191038320105364, "warnings": [] }, @@ -1976,13 +4527,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, + "worst_group_fpr": 0.06377551020408163, "n_models": 1, - "seconds": 0.07115170800170745, + "seconds": 0.29679591600142885, "eta_squared": 0.04475252345121021, "warnings": [] }, @@ -1993,13 +4544,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07341995900060283, + "seconds": 0.3207631660043262, "eta_squared": 0.04475252345121021, "warnings": [] }, @@ -2010,13 +4561,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9960141105034992, - "roc_auc": 0.9999313527494331, + "pr_auc": 0.9967692587372329, + "roc_auc": 0.9999424248866213, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6964845830007107, + "seconds": 2.5013937080002506, "eta_squared": 0.04475252345121021, "warnings": [] }, @@ -2027,13 +4578,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07056295799702639, + "seconds": 0.23139412500313483, "eta_squared": 0.03474882738233468, "warnings": [] }, @@ -2044,13 +4595,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07717137500003446, + "seconds": 0.3954302500060294, "eta_squared": 0.03474882738233468, "warnings": [] }, @@ -2065,9 +4616,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.6975382499986154, + "seconds": 2.4514795420036535, "eta_squared": 0.03474882738233468, "warnings": [] }, @@ -2082,9 +4633,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.06700420800189022, + "seconds": 0.25115487500443123, "eta_squared": 0.04213584867477234, "warnings": [] }, @@ -2095,13 +4646,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07658679099768051, + "seconds": 0.2550350419987808, "eta_squared": 0.04213584867477234, "warnings": [] }, @@ -2112,13 +4663,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9984977607821051, - "roc_auc": 0.999968998015873, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04591836734693878, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7315555829991354, + "seconds": 1.995787833002396, "eta_squared": 0.04213584867477234, "warnings": [] }, @@ -2129,13 +4680,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.07062958299866295, + "seconds": 0.24527554200176382, "eta_squared": 0.03654307103775154, "warnings": [] }, @@ -2146,13 +4697,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.06992241599800764, + "seconds": 0.22836079199623782, "eta_squared": 0.03654307103775154, "warnings": [] }, @@ -2163,13 +4714,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9953729381748984, - "roc_auc": 0.9999224950396824, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9949478368548235, + "roc_auc": 0.9999092084750567, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.991075562169312, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7172980830000597, + "seconds": 2.451732458001061, "eta_squared": 0.03654307103775154, "warnings": [] }, @@ -2180,13 +4731,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.06928816700019524, + "seconds": 0.22093716700328514, "eta_squared": 0.03988094020033876, "warnings": [] }, @@ -2197,13 +4748,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 1, - "seconds": 0.0711717080012022, + "seconds": 0.24318262500310084, "eta_squared": 0.03988094020033876, "warnings": [] }, @@ -2214,13 +4765,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7151609159991494, + "seconds": 2.112241249997169, "eta_squared": 0.03988094020033876, "warnings": [] }, @@ -2231,13 +4782,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9984216555199152, - "roc_auc": 0.999968998015873, - "precision_at_n": 0.9791666666666666, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.06899037500261329, + "seconds": 0.29302350000216393, "eta_squared": 0.038582085800609046, "warnings": [] }, @@ -2248,13 +4799,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9988727861917879, - "roc_auc": 0.9999778557256236, + "pr_auc": 0.9998926116838486, + "roc_auc": 0.9999977855725622, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07521916699988651, + "seconds": 0.27036920899990946, "eta_squared": 0.038582085800609046, "warnings": [] }, @@ -2265,13 +4816,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.9995726383336838, - "roc_auc": 0.9999911422902493, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.998526733642151, + "roc_auc": 0.9999712124433106, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7040710830005992, + "seconds": 2.3249247089988785, "eta_squared": 0.038582085800609046, "warnings": [] }, @@ -2288,7 +4839,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07072458299808204, + "seconds": 0.3159535419981694, "eta_squared": 0.036627617689678975, "warnings": [] }, @@ -2300,12 +4851,12 @@ "mechanism": "global", "level_spread": 0.05, "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.07034900000144262, + "seconds": 0.3032592090021353, "eta_squared": 0.036627617689678975, "warnings": [] }, @@ -2316,13 +4867,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.05, - "pr_auc": 0.999244808841958, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04591836734693878, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7096689579993836, + "seconds": 2.4489135420008097, "eta_squared": 0.036627617689678975, "warnings": [] }, @@ -2333,13 +4884,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.06995812500099419, + "seconds": 0.3446114580001449, "eta_squared": 0.059293082499552834, "warnings": [] }, @@ -2354,9 +4905,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07958750000034343, + "seconds": 0.37284204199386295, "eta_squared": 0.059293082499552834, "warnings": [] }, @@ -2367,13 +4918,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7175909170000523, + "seconds": 2.9106780420042924, "eta_squared": 0.059293082499552834, "warnings": [] }, @@ -2384,13 +4935,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9875576233231009, - "roc_auc": 0.9998494189342404, + "pr_auc": 0.9987530543910216, + "roc_auc": 0.999975641298186, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.06887755102040816, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.09570874999917578, + "seconds": 0.32288629099639365, "eta_squared": 0.05670231707537825, "warnings": [] }, @@ -2401,13 +4952,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.10129179100113106, + "seconds": 0.38123066700063646, "eta_squared": 0.05670231707537825, "warnings": [] }, @@ -2418,13 +4969,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9523907508279281, - "roc_auc": 0.9992980265022675, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7102379169991764, + "seconds": 3.0213677079955232, "eta_squared": 0.05670231707537825, "warnings": [] }, @@ -2435,13 +4986,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9960506815613993, - "roc_auc": 0.9999247094671202, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9950810185185185, - "worst_group_fpr": 0.04846938775510204, + "pr_auc": 0.997231086663878, + "roc_auc": 0.9999490681689343, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9948950066137566, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07414150000113295, + "seconds": 0.3023284579976462, "eta_squared": 0.05921841304093976, "warnings": [] }, @@ -2452,13 +5003,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07848229100272874, + "seconds": 0.3231065829968429, "eta_squared": 0.05921841304093976, "warnings": [] }, @@ -2469,13 +5020,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7420237920014188, + "seconds": 2.9427785000007134, "eta_squared": 0.05921841304093976, "warnings": [] }, @@ -2486,13 +5037,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9994596097759277, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07248145799894701, + "seconds": 0.3356336669967277, "eta_squared": 0.058057615078346766, "warnings": [] }, @@ -2503,13 +5054,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07383162499900209, + "seconds": 0.3980491250040359, "eta_squared": 0.058057615078346766, "warnings": [] }, @@ -2520,13 +5071,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7287121660010598, + "seconds": 2.9782467920013005, "eta_squared": 0.058057615078346766, "warnings": [] }, @@ -2537,13 +5088,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9992239393431516, - "roc_auc": 0.9999844990079365, + "pr_auc": 0.9958588297035126, + "roc_auc": 0.9999291383219955, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07586733299831394, + "seconds": 0.3495809579981142, "eta_squared": 0.05544497562800828, "warnings": [] }, @@ -2554,13 +5105,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07230329199956032, + "seconds": 0.31225608300155727, "eta_squared": 0.05544497562800828, "warnings": [] }, @@ -2571,13 +5122,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.6943149999970046, + "seconds": 2.862608999996155, "eta_squared": 0.05544497562800828, "warnings": [] }, @@ -2588,13 +5139,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9986319303600137, - "roc_auc": 0.9999734268707483, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "pr_auc": 0.9969838574473947, + "roc_auc": 0.9999424248866213, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.06787462500142283, + "seconds": 0.3957704999993439, "eta_squared": 0.051197551161426436, "warnings": [] }, @@ -2605,13 +5156,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, + "pr_auc": 0.9974733447852491, + "roc_auc": 0.9999534970238095, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.07406333299877588, + "seconds": 0.34860474999732105, "eta_squared": 0.051197551161426436, "warnings": [] }, @@ -2622,13 +5173,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6935085420009273, + "seconds": 2.8150589580036467, "eta_squared": 0.051197551161426436, "warnings": [] }, @@ -2639,13 +5190,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9958579726891336, - "roc_auc": 0.9999291383219955, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.07717508299901965, + "seconds": 0.28609125000366475, "eta_squared": 0.050492980765247095, "warnings": [] }, @@ -2656,13 +5207,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07736512500196113, + "seconds": 0.38692079199972795, "eta_squared": 0.050492980765247095, "warnings": [] }, @@ -2673,13 +5224,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7311062079970725, + "seconds": 3.080347499999334, "eta_squared": 0.050492980765247095, "warnings": [] }, @@ -2690,13 +5241,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9996744556165972, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.07627204099844676, + "seconds": 0.304620499999146, "eta_squared": 0.05910198864179911, "warnings": [] }, @@ -2707,13 +5258,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9992239393431517, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, + "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.07063824999931967, + "seconds": 0.34869220899417996, "eta_squared": 0.05910198864179911, "warnings": [] }, @@ -2724,13 +5275,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.695684500002244, + "seconds": 2.949169083003653, "eta_squared": 0.05910198864179911, "warnings": [] }, @@ -2741,13 +5292,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9974764602423118, + "roc_auc": 0.9999534970238096, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.06994925000253716, + "seconds": 0.31692675000522286, "eta_squared": 0.06438107327632228, "warnings": [] }, @@ -2758,13 +5309,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9993384082076204, + "roc_auc": 0.999986713435374, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07746212500205729, + "seconds": 0.27923370899952715, "eta_squared": 0.06438107327632228, "warnings": [] }, @@ -2775,13 +5326,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.6991803749988321, + "seconds": 2.7171240000025136, "eta_squared": 0.06438107327632228, "warnings": [] }, @@ -2792,13 +5343,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, + "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.07582712500152411, + "seconds": 0.3716905830005999, "eta_squared": 0.054805584022504586, "warnings": [] }, @@ -2815,7 +5366,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 1, - "seconds": 0.07222833300329512, + "seconds": 0.34220654100499814, "eta_squared": 0.054805584022504586, "warnings": [] }, @@ -2826,13 +5377,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7066742080023687, + "seconds": 2.906894082996587, "eta_squared": 0.054805584022504586, "warnings": [] }, @@ -2843,13 +5394,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9937045293446132, - "roc_auc": 0.9998848497732427, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.061224489795918366, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.0901148339980864, + "seconds": 0.33631362500455, "eta_squared": 0.06303098656943196, "warnings": [] }, @@ -2860,13 +5411,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.08605449999959092, + "seconds": 0.377403124999546, "eta_squared": 0.06303098656943196, "warnings": [] }, @@ -2877,13 +5428,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7075918749978882, + "seconds": 3.042988957997295, "eta_squared": 0.06303098656943196, "warnings": [] }, @@ -2894,13 +5445,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9983853734038979, - "roc_auc": 0.999968998015873, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, + "pr_auc": 0.9763155400681044, + "roc_auc": 0.9996634070294784, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9861565806878306, + "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.07811904200207209, + "seconds": 0.33773637500416953, "eta_squared": 0.05142686270297512, "warnings": [] }, @@ -2911,13 +5462,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9964737355983997, - "roc_auc": 0.999937996031746, + "pr_auc": 0.9985093813404057, + "roc_auc": 0.9999712124433107, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.07093691699992632, + "seconds": 0.3668642499978887, "eta_squared": 0.05142686270297512, "warnings": [] }, @@ -2928,13 +5479,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9619580084145981, - "roc_auc": 0.99949289611678, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7219730830001936, + "seconds": 2.82656241600489, "eta_squared": 0.05142686270297512, "warnings": [] }, @@ -2945,13 +5496,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07191874999989523, + "seconds": 0.32261537500016857, "eta_squared": 0.05895449280709131, "warnings": [] }, @@ -2968,7 +5519,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07913745800033212, + "seconds": 0.3600661249947734, "eta_squared": 0.05895449280709131, "warnings": [] }, @@ -2979,13 +5530,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.712019375001546, + "seconds": 3.1006452910005464, "eta_squared": 0.05895449280709131, "warnings": [] }, @@ -2996,13 +5547,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.983066282557806, - "roc_auc": 0.9997386975623582, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.07142857142857142, + "pr_auc": 0.9866811716792745, + "roc_auc": 0.9998139880952381, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9903687169312169, + "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.07618983300199034, + "seconds": 0.26628616700327257, "eta_squared": 0.052277832661798654, "warnings": [] }, @@ -3013,13 +5564,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.05102040816326531, "n_models": 1, - "seconds": 0.07268458399994415, + "seconds": 0.3878202920022886, "eta_squared": 0.052277832661798654, "warnings": [] }, @@ -3030,13 +5581,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7438016249980137, + "seconds": 2.709965332993306, "eta_squared": 0.052277832661798654, "warnings": [] }, @@ -3047,13 +5598,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.9997841047394044, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07378295799935586, + "seconds": 0.2931164580004406, "eta_squared": 0.05617290915117752, "warnings": [] }, @@ -3068,9 +5619,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07609516699812957, + "seconds": 0.29841250000026776, "eta_squared": 0.05617290915117752, "warnings": [] }, @@ -3081,13 +5632,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.05, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7129640410021238, + "seconds": 1.633176374998584, "eta_squared": 0.05617290915117752, "warnings": [] }, @@ -3098,13 +5649,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07423887499680859, + "seconds": 0.2699645419997978, "eta_squared": 0.12687309144122869, "warnings": [] }, @@ -3116,12 +5667,12 @@ "mechanism": "global", "level_spread": 0.1, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07873454200307606, + "seconds": 0.3061248330050148, "eta_squared": 0.12687309144122869, "warnings": [] }, @@ -3132,13 +5683,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9997841047394043, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6979090000022552, + "seconds": 3.4195997919960064, "eta_squared": 0.12687309144122869, "warnings": [] }, @@ -3149,13 +5700,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, + "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.07222316700062947, + "seconds": 0.3291228750022128, "eta_squared": 0.12119391519465993, "warnings": [] }, @@ -3166,13 +5717,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07347983400177327, + "seconds": 0.32965295799658634, "eta_squared": 0.12119391519465993, "warnings": [] }, @@ -3183,13 +5734,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9968373110910685, - "roc_auc": 0.999937996031746, + "pr_auc": 0.9988154954031649, + "roc_auc": 0.999975641298186, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.6928048749978188, + "seconds": 3.1566792910016375, "eta_squared": 0.12119391519465993, "warnings": [] }, @@ -3200,13 +5751,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9902144031663376, - "roc_auc": 0.9998339179421768, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9922401094276094, - "worst_group_fpr": 0.06887755102040816, + "pr_auc": 0.9986723262321067, + "roc_auc": 0.9999734268707482, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.07798179199744482, + "seconds": 0.39594216600380605, "eta_squared": 0.12540942706236655, "warnings": [] }, @@ -3217,13 +5768,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9997841047394043, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04846938775510204, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.0712847499999043, + "seconds": 0.35103045800497057, "eta_squared": 0.12540942706236655, "warnings": [] }, @@ -3234,13 +5785,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.998047536533169, - "roc_auc": 0.9999579258786848, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, + "pr_auc": 0.999787414965986, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7157716250003432, + "seconds": 1.784541499997431, "eta_squared": 0.12540942706236655, "warnings": [] }, @@ -3251,13 +5802,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07014016600078321, + "seconds": 0.17943770899728406, "eta_squared": 0.12527474321356308, "warnings": [] }, @@ -3268,13 +5819,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.04591836734693878, "n_models": 1, - "seconds": 0.07505729100012104, + "seconds": 0.21587554100551642, "eta_squared": 0.12527474321356308, "warnings": [] }, @@ -3285,13 +5836,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7035883330026991, + "seconds": 1.4354623749968596, "eta_squared": 0.12527474321356308, "warnings": [] }, @@ -3302,13 +5853,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07179441699918243, + "seconds": 0.17310749999887776, "eta_squared": 0.12512147981398453, "warnings": [] }, @@ -3319,13 +5870,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.06887755102040816, "n_models": 1, - "seconds": 0.07887549999941257, + "seconds": 0.17312124999443768, "eta_squared": 0.12512147981398453, "warnings": [] }, @@ -3336,13 +5887,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9986319303600136, - "roc_auc": 0.9999734268707483, + "pr_auc": 0.9992239393431517, + "roc_auc": 0.9999844990079365, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04336734693877551, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7104391660031979, + "seconds": 1.4705428340021172, "eta_squared": 0.12512147981398453, "warnings": [] }, @@ -3353,13 +5904,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, + "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.07062662500175065, + "seconds": 0.17702162500063423, "eta_squared": 0.1151837730978297, "warnings": [] }, @@ -3374,9 +5925,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07220429199878708, + "seconds": 0.1687370839936193, "eta_squared": 0.1151837730978297, "warnings": [] }, @@ -3387,13 +5938,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9990030018845484, - "roc_auc": 0.9999800701530611, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9967136368233248, + "roc_auc": 0.9999379960317459, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7412907910002104, + "seconds": 1.4160712910015718, "eta_squared": 0.1151837730978297, "warnings": [] }, @@ -3404,13 +5955,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9950557391870531, - "roc_auc": 0.9999158517573695, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.09438775510204081, + "pr_auc": 0.9958581590975477, + "roc_auc": 0.9999291383219955, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.07358395899791503, + "seconds": 0.2092109170043841, "eta_squared": 0.11684378654258522, "warnings": [] }, @@ -3421,13 +5972,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, + "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.07783791700057918, + "seconds": 0.18265700000483776, "eta_squared": 0.11684378654258522, "warnings": [] }, @@ -3438,13 +5989,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9966225451222093, - "roc_auc": 0.9999402104591838, + "pr_auc": 0.9965173484043472, + "roc_auc": 0.999937996031746, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9948950066137566, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7159545840004284, + "seconds": 1.4119265420013107, "eta_squared": 0.11684378654258522, "warnings": [] }, @@ -3455,13 +6006,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9992401664395811, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.08673469387755102, "n_models": 1, - "seconds": 0.07142070799818612, + "seconds": 0.17269949999899836, "eta_squared": 0.12961477677268454, "warnings": [] }, @@ -3472,13 +6023,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07209687499926076, + "seconds": 0.17104045800078893, "eta_squared": 0.12961477677268454, "warnings": [] }, @@ -3489,13 +6040,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9985539550199853, - "roc_auc": 0.9999712124433107, + "pr_auc": 0.9980038121708081, + "roc_auc": 0.9999601403061225, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7082592910010135, + "seconds": 1.4036386660009157, "eta_squared": 0.12961477677268454, "warnings": [] }, @@ -3510,9 +6061,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.07899929099949077, + "seconds": 0.1794141250065877, "eta_squared": 0.13340934644786664, "warnings": [] }, @@ -3527,9 +6078,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, + "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.07169262499883189, + "seconds": 0.18372316699969815, "eta_squared": 0.13340934644786664, "warnings": [] }, @@ -3540,13 +6091,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9955346576239892, - "roc_auc": 0.9999247094671201, + "pr_auc": 0.9970566287270857, + "roc_auc": 0.9999468537414966, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7164824999999837, + "seconds": 1.4117635419970611, "eta_squared": 0.13340934644786664, "warnings": [] }, @@ -3561,9 +6112,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, + "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.07427245800136006, + "seconds": 0.17737312499957625, "eta_squared": 0.11917540465192976, "warnings": [] }, @@ -3574,13 +6125,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07583237500148243, + "seconds": 0.18114274999970803, "eta_squared": 0.11917540465192976, "warnings": [] }, @@ -3591,13 +6142,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7166787919995841, + "seconds": 1.4187604999970063, "eta_squared": 0.11917540465192976, "warnings": [] }, @@ -3609,12 +6160,12 @@ "mechanism": "global", "level_spread": 0.1, "pr_auc": 1.0, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.06855162499778089, + "seconds": 0.17882358300266787, "eta_squared": 0.12876131345125413, "warnings": [] }, @@ -3625,13 +6176,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07894258400119725, + "seconds": 0.17535625000164146, "eta_squared": 0.12876131345125413, "warnings": [] }, @@ -3642,13 +6193,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9986062434248825, - "roc_auc": 0.9999712124433106, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9988425925925926, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7314287919980416, + "seconds": 1.4296091249998426, "eta_squared": 0.12876131345125413, "warnings": [] }, @@ -3659,13 +6210,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "pr_auc": 0.9981739069981774, + "roc_auc": 0.9999645691609979, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9976851851851851, + "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.07099874999767053, + "seconds": 0.17627454199828207, "eta_squared": 0.1235703476453848, "warnings": [] }, @@ -3676,13 +6227,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07508233299813583, + "seconds": 0.17232879099901766, "eta_squared": 0.1235703476453848, "warnings": [] }, @@ -3693,13 +6244,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9943210112718865, - "roc_auc": 0.9999092084750567, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9940096746750579, + "roc_auc": 0.9998981363378685, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.991075562169312, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7117179999986547, + "seconds": 1.4056731669988949, "eta_squared": 0.1235703476453848, "warnings": [] }, @@ -3714,9 +6265,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07244870900103706, + "seconds": 0.17102750000049127, "eta_squared": 0.12854427437882665, "warnings": [] }, @@ -3733,7 +6284,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 1, - "seconds": 0.07468970800255192, + "seconds": 0.19475837500067428, "eta_squared": 0.12854427437882665, "warnings": [] }, @@ -3744,13 +6295,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7214789999998175, + "seconds": 1.4142136660011602, "eta_squared": 0.12854427437882665, "warnings": [] }, @@ -3761,13 +6312,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9981218997092937, - "roc_auc": 0.99996235473356, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.08673469387755102, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.07330120799815631, + "seconds": 0.17834583300282247, "eta_squared": 0.1246847039613907, "warnings": [] }, @@ -3778,13 +6329,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9980007756782858, - "roc_auc": 0.9999601403061226, - "precision_at_n": 0.96875, + "pr_auc": 0.9992332114897581, + "roc_auc": 0.9999844990079365, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.061224489795918366, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.08027399999991758, + "seconds": 0.1776951249994454, "eta_squared": 0.1246847039613907, "warnings": [] }, @@ -3795,13 +6346,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9995726383336838, - "roc_auc": 0.9999911422902493, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9985267336421512, + "roc_auc": 0.9999712124433106, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7322338340018177, + "seconds": 1.4102792910052813, "eta_squared": 0.1246847039613907, "warnings": [] }, @@ -3816,9 +6367,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, + "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.06885337499988964, + "seconds": 0.182436791001237, "eta_squared": 0.12175002598109345, "warnings": [] }, @@ -3835,7 +6386,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.06756683399726171, + "seconds": 0.17618145800224738, "eta_squared": 0.12175002598109345, "warnings": [] }, @@ -3846,13 +6397,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.1, - "pr_auc": 0.9992448088419582, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04591836734693878, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7063922909983376, + "seconds": 1.42044850000093, "eta_squared": 0.12175002598109345, "warnings": [] }, @@ -3863,13 +6414,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9992239393431517, + "roc_auc": 0.9999844990079364, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.07291979100045864, + "seconds": 0.17877445799967973, "eta_squared": 0.19451621854380768, "warnings": [] }, @@ -3880,13 +6431,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.07728233300076681, + "seconds": 0.17992270800459664, "eta_squared": 0.19451621854380768, "warnings": [] }, @@ -3897,13 +6448,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7823825830018905, + "seconds": 1.4257903329998953, "eta_squared": 0.19451621854380768, "warnings": [] }, @@ -3914,13 +6465,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9821201179067449, + "pr_auc": 0.9821091676743208, "roc_auc": 0.9998228458049887, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9861565806878306, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.08046337500127265, + "seconds": 0.1845532090010238, "eta_squared": 0.1898993994842026, "warnings": [] }, @@ -3931,13 +6482,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07804666700030793, + "seconds": 0.17791879199648974, "eta_squared": 0.1898993994842026, "warnings": [] }, @@ -3948,13 +6499,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.999289168792517, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7099072080018232, + "seconds": 1.460088291001739, "eta_squared": 0.1898993994842026, "warnings": [] }, @@ -3965,13 +6516,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9812650841708556, - "roc_auc": 0.9997874149659864, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9797908399470899, - "worst_group_fpr": 0.07142857142857142, + "pr_auc": 0.9963227694148251, + "roc_auc": 0.9999357816043084, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.0738493329990888, + "seconds": 0.1762644589980482, "eta_squared": 0.1936752703132149, "warnings": [] }, @@ -3983,12 +6534,12 @@ "mechanism": "contextual", "level_spread": 0.1, "pr_auc": 1.0, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.072544791000837, + "seconds": 0.17437245800101664, "eta_squared": 0.1936752703132149, "warnings": [] }, @@ -3999,13 +6550,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7283174579970364, + "seconds": 1.4415905409987317, "eta_squared": 0.1936752703132149, "warnings": [] }, @@ -4016,13 +6567,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07802712500051712, + "seconds": 0.18049354200047674, "eta_squared": 0.1934640118312397, "warnings": [] }, @@ -4037,9 +6588,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04846938775510204, "n_models": 1, - "seconds": 0.07160441700034426, + "seconds": 0.17590970799938077, "eta_squared": 0.1934640118312397, "warnings": [] }, @@ -4050,13 +6601,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7246251670003403, + "seconds": 1.423768916996778, "eta_squared": 0.1934640118312397, "warnings": [] }, @@ -4067,13 +6618,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "pr_auc": 0.9939120074223009, + "roc_auc": 0.9998937074829932, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9948950066137566, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.07575679200090235, + "seconds": 0.1685598750045756, "eta_squared": 0.18861888416601494, "warnings": [] }, @@ -4084,13 +6635,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.06377551020408163, "n_models": 1, - "seconds": 0.07119949999832897, + "seconds": 0.17661041599785676, "eta_squared": 0.18861888416601494, "warnings": [] }, @@ -4101,13 +6652,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.6969671250008105, + "seconds": 1.3883059590007178, "eta_squared": 0.18861888416601494, "warnings": [] }, @@ -4118,13 +6669,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9985428054545235, - "roc_auc": 0.9999712124433107, + "pr_auc": 0.9973980281707969, + "roc_auc": 0.9999490681689343, "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, + "macro_pr_auc": 0.9975405092592592, "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.0693282909996924, + "seconds": 0.1700266250045388, "eta_squared": 0.1805813085726683, "warnings": [] }, @@ -4136,12 +6687,12 @@ "mechanism": "contextual", "level_spread": 0.1, "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.06892870800220408, + "seconds": 0.18030274999910034, "eta_squared": 0.1805813085726683, "warnings": [] }, @@ -4152,13 +6703,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6985914169999887, + "seconds": 1.4230401249951683, "eta_squared": 0.1805813085726683, "warnings": [] }, @@ -4169,13 +6720,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9928499229185896, - "roc_auc": 0.9998693487811791, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9918568121693121, - "worst_group_fpr": 0.09438775510204081, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07400295900151832, + "seconds": 0.17946370899881003, "eta_squared": 0.17980556561098363, "warnings": [] }, @@ -4186,13 +6737,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725622, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, + "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.08185012499961886, + "seconds": 0.17470899999898393, "eta_squared": 0.17980556561098363, "warnings": [] }, @@ -4203,13 +6754,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7189363329998741, + "seconds": 1.4238765409972984, "eta_squared": 0.17980556561098363, "warnings": [] }, @@ -4220,13 +6771,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07125499999892781, + "seconds": 0.17319483399478486, "eta_squared": 0.1935769586972524, "warnings": [] }, @@ -4237,13 +6788,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.0792962500017893, + "seconds": 0.16585537500213832, "eta_squared": 0.1935769586972524, "warnings": [] }, @@ -4254,13 +6805,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.741542916999606, + "seconds": 1.3955539160015178, "eta_squared": 0.1935769586972524, "warnings": [] }, @@ -4271,13 +6822,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.999233211489758, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, + "pr_auc": 0.9959283238067923, + "roc_auc": 0.9999247094671202, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.08608595800251351, + "seconds": 0.17348287499771686, "eta_squared": 0.20068556264939091, "warnings": [] }, @@ -4292,9 +6843,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, + "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.08244766599818831, + "seconds": 0.17678399999567773, "eta_squared": 0.20068556264939091, "warnings": [] }, @@ -4305,13 +6856,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7124477079996723, + "seconds": 1.3915143749982235, "eta_squared": 0.20068556264939091, "warnings": [] }, @@ -4322,13 +6873,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.999118742625289, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.07079854100084049, + "seconds": 0.17100987500452902, "eta_squared": 0.1874671200172165, "warnings": [] }, @@ -4343,9 +6894,9 @@ "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, + "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.0786639580001065, + "seconds": 0.18211825000616955, "eta_squared": 0.1874671200172165, "warnings": [] }, @@ -4356,13 +6907,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7268245829982334, + "seconds": 1.4138579169957666, "eta_squared": 0.1874671200172165, "warnings": [] }, @@ -4373,13 +6924,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9911302410778361, - "roc_auc": 0.9998405612244897, - "precision_at_n": 0.96875, + "pr_auc": 0.9956974507990147, + "roc_auc": 0.9999269238945578, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.06961750000118627, + "seconds": 0.1727192499965895, "eta_squared": 0.19900625679340145, "warnings": [] }, @@ -4391,12 +6942,12 @@ "mechanism": "contextual", "level_spread": 0.1, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07949883399851387, + "seconds": 0.16367533300217474, "eta_squared": 0.19900625679340145, "warnings": [] }, @@ -4407,13 +6958,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.703285749998031, + "seconds": 1.3640864999979385, "eta_squared": 0.19900625679340145, "warnings": [] }, @@ -4424,13 +6975,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9906410394177838, - "roc_auc": 0.9998604910714286, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.08928571428571429, + "pr_auc": 0.9555740461597554, + "roc_auc": 0.9993533871882085, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9783966901154401, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07260295900050551, + "seconds": 0.1687240000028396, "eta_squared": 0.1831366419552775, "warnings": [] }, @@ -4441,13 +6992,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9953721770912125, - "roc_auc": 0.9999224950396826, + "pr_auc": 0.9960141105034992, + "roc_auc": 0.9999313527494331, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07940979200066067, + "seconds": 0.16969283299840754, "eta_squared": 0.1831366419552775, "warnings": [] }, @@ -4458,13 +7009,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.961741641669088, - "roc_auc": 0.9994884672619048, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7247961670000223, + "seconds": 1.37561216600443, "eta_squared": 0.1831366419552775, "warnings": [] }, @@ -4475,13 +7026,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9991106879479051, - "roc_auc": 0.9999822845804989, + "pr_auc": 0.9971987550814965, + "roc_auc": 0.9999490681689341, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.07304641699738568, + "seconds": 0.17607562500052154, "eta_squared": 0.1962296761690674, "warnings": [] }, @@ -4492,13 +7043,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07132141700276406, + "seconds": 0.17710829200223088, "eta_squared": 0.1962296761690674, "warnings": [] }, @@ -4509,13 +7060,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.6932837919994199, + "seconds": 1.382764624999254, "eta_squared": 0.1962296761690674, "warnings": [] }, @@ -4526,13 +7077,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9726214262302605, - "roc_auc": 0.9996700503117913, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.991075562169312, - "worst_group_fpr": 0.09438775510204081, + "pr_auc": 0.9720807360651503, + "roc_auc": 0.9996789080215419, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9882853835978835, + "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.06829887500134646, + "seconds": 0.18297341699508252, "eta_squared": 0.18278781206386457, "warnings": [] }, @@ -4543,13 +7094,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, + "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.07537133300138521, + "seconds": 0.1765655830022297, "eta_squared": 0.18278781206386457, "warnings": [] }, @@ -4560,13 +7111,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.6942515420014388, + "seconds": 1.3746274999939487, "eta_squared": 0.18278781206386457, "warnings": [] }, @@ -4577,13 +7128,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.99846047105138, + "roc_auc": 0.999968998015873, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.06851670900141471, + "seconds": 0.17902783400495537, "eta_squared": 0.18987545660900912, "warnings": [] }, @@ -4595,12 +7146,12 @@ "mechanism": "contextual", "level_spread": 0.1, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.07073066699740593, + "seconds": 0.20602816700557014, "eta_squared": 0.18987545660900912, "warnings": [] }, @@ -4611,13 +7162,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.1, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7188157089985907, + "seconds": 1.3872172090050299, "eta_squared": 0.18987545660900912, "warnings": [] }, @@ -4632,9 +7183,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.06851633399855928, + "seconds": 0.16560841599857667, "eta_squared": 0.21842060879328182, "warnings": [] }, @@ -4649,9 +7200,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07292166599654593, + "seconds": 0.16980620899994392, "eta_squared": 0.21842060879328182, "warnings": [] }, @@ -4662,13 +7213,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9997841047394043, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6916868750013236, + "seconds": 1.3945893329946557, "eta_squared": 0.21842060879328182, "warnings": [] }, @@ -4683,9 +7234,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.07799687499937136, + "seconds": 0.18620050000026822, "eta_squared": 0.2129441367707082, "warnings": [] }, @@ -4700,9 +7251,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, + "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.07319541599645163, + "seconds": 0.17440516699571162, "eta_squared": 0.2129441367707082, "warnings": [] }, @@ -4713,13 +7264,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9968326319040454, - "roc_auc": 0.999937996031746, + "pr_auc": 0.9980209770081038, + "roc_auc": 0.9999601403061225, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7092204580003454, + "seconds": 1.4028902919963002, "eta_squared": 0.2129441367707082, "warnings": [] }, @@ -4730,13 +7281,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9993556244447951, - "roc_auc": 0.9999867134353742, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07785770800182945, + "seconds": 0.17391474999749335, "eta_squared": 0.21607807398685436, "warnings": [] }, @@ -4748,12 +7299,12 @@ "mechanism": "global", "level_spread": 0.15, "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.06997070800207439, + "seconds": 0.17599245800374774, "eta_squared": 0.21607807398685436, "warnings": [] }, @@ -4764,13 +7315,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9985636101934156, - "roc_auc": 0.9999689980158731, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7070650839996233, + "seconds": 1.386537333994056, "eta_squared": 0.21607807398685436, "warnings": [] }, @@ -4781,13 +7332,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "worst_group_fpr": 0.1096938775510204, "n_models": 1, - "seconds": 0.0689469579992874, + "seconds": 0.17551283400098328, "eta_squared": 0.2167824088169093, "warnings": [] }, @@ -4798,13 +7349,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, + "worst_group_fpr": 0.05357142857142857, "n_models": 1, - "seconds": 0.0699157919989375, + "seconds": 0.17358104099548655, "eta_squared": 0.2167824088169093, "warnings": [] }, @@ -4819,9 +7370,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7044350830001349, + "seconds": 1.4019418329990003, "eta_squared": 0.2167824088169093, "warnings": [] }, @@ -4833,12 +7384,12 @@ "mechanism": "global", "level_spread": 0.15, "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.06935874999908265, + "seconds": 0.17289695799991023, "eta_squared": 0.2158502792872289, "warnings": [] }, @@ -4853,9 +7404,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.06596420899950317, + "seconds": 0.17523695799900452, "eta_squared": 0.2158502792872289, "warnings": [] }, @@ -4866,13 +7417,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9993384082076204, - "roc_auc": 0.9999867134353742, + "pr_auc": 0.9994516328453017, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.709873374999006, + "seconds": 1.3947917919940664, "eta_squared": 0.2158502792872289, "warnings": [] }, @@ -4889,7 +7440,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.07122862499818439, + "seconds": 0.17122699999890756, "eta_squared": 0.20579994667793172, "warnings": [] }, @@ -4900,13 +7451,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.07308333300170489, + "seconds": 0.16683429099794012, "eta_squared": 0.20579994667793172, "warnings": [] }, @@ -4917,13 +7468,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9988859606860464, - "roc_auc": 0.9999778557256235, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, + "pr_auc": 0.9973565105014646, + "roc_auc": 0.9999490681689343, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7061194160014566, + "seconds": 1.3734197079975274, "eta_squared": 0.20579994667793172, "warnings": [] }, @@ -4934,13 +7485,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07734245900064707, + "seconds": 0.1721250830014469, "eta_squared": 0.20697832970174135, "warnings": [] }, @@ -4951,13 +7502,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.07908163265306123, "n_models": 1, - "seconds": 0.07056250000096043, + "seconds": 0.1647393750026822, "eta_squared": 0.20697832970174135, "warnings": [] }, @@ -4968,13 +7519,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9969139346631588, - "roc_auc": 0.999944639314059, + "pr_auc": 0.996769258737233, + "roc_auc": 0.9999424248866213, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7153266250024899, + "seconds": 1.36176987500221, "eta_squared": 0.20697832970174135, "warnings": [] }, @@ -4989,9 +7540,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.06616024999675574, + "seconds": 0.16589779199421173, "eta_squared": 0.2209510705543195, "warnings": [] }, @@ -5002,13 +7553,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.06887755102040816, "n_models": 1, - "seconds": 0.07109645799937425, + "seconds": 0.1698608750011772, "eta_squared": 0.2209510705543195, "warnings": [] }, @@ -5019,13 +7570,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9987072535258925, - "roc_auc": 0.9999734268707484, - "precision_at_n": 0.96875, + "pr_auc": 0.9985762117921207, + "roc_auc": 0.9999712124433107, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.6969909580002422, + "seconds": 1.3871745409996947, "eta_squared": 0.2209510705543195, "warnings": [] }, @@ -5036,13 +7587,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.07319562500197208, + "seconds": 0.16837633400427876, "eta_squared": 0.223914333840144, "warnings": [] }, @@ -5054,12 +7605,12 @@ "mechanism": "global", "level_spread": 0.15, "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.0793693749983504, + "seconds": 0.1724463749997085, "eta_squared": 0.223914333840144, "warnings": [] }, @@ -5070,13 +7621,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9956960258104144, - "roc_auc": 0.9999269238945577, + "pr_auc": 0.9977421731790774, + "roc_auc": 0.9999579258786848, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7002469580002071, + "seconds": 1.3297875420030323, "eta_squared": 0.223914333840144, "warnings": [] }, @@ -5087,13 +7638,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.1096938775510204, "n_models": 1, - "seconds": 0.0686213749977469, + "seconds": 0.1701794160035206, "eta_squared": 0.21066498526646607, "warnings": [] }, @@ -5108,9 +7659,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, + "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.07141987499926472, + "seconds": 0.17412170799798332, "eta_squared": 0.21066498526646607, "warnings": [] }, @@ -5125,9 +7676,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.682583000001614, + "seconds": 1.3639297500048997, "eta_squared": 0.21066498526646607, "warnings": [] }, @@ -5138,13 +7689,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.07908163265306123, "n_models": 1, - "seconds": 0.07027670800016494, + "seconds": 0.16910462499799905, "eta_squared": 0.21894049322652165, "warnings": [] }, @@ -5155,13 +7706,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "pr_auc": 1.0, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07136733299921616, + "seconds": 0.16824579200329026, "eta_squared": 0.21894049322652165, "warnings": [] }, @@ -5172,13 +7723,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9990329939892202, - "roc_auc": 0.9999800701530611, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7137086670009012, + "seconds": 1.3689260000028298, "eta_squared": 0.21894049322652165, "warnings": [] }, @@ -5189,13 +7740,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9993651772660819, + "pr_auc": 0.9993384082076204, "roc_auc": 0.9999867134353742, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.07226612500016927, + "seconds": 0.16919154200149933, "eta_squared": 0.2159597863844126, "warnings": [] }, @@ -5210,9 +7761,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.07840720899912412, + "seconds": 0.17326112500450108, "eta_squared": 0.2159597863844126, "warnings": [] }, @@ -5223,13 +7774,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9939428770383998, - "roc_auc": 0.9999047796201814, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9940096746750579, + "roc_auc": 0.9998981363378685, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.991075562169312, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6842563329992117, + "seconds": 1.3610072499941452, "eta_squared": 0.2159597863844126, "warnings": [] }, @@ -5240,13 +7791,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.07295425000120304, + "seconds": 0.1723165000003064, "eta_squared": 0.22105758156006608, "warnings": [] }, @@ -5261,9 +7812,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07012737499826471, + "seconds": 0.17333112499909475, "eta_squared": 0.22105758156006608, "warnings": [] }, @@ -5274,13 +7825,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7247925829979067, + "seconds": 1.3661107500010985, "eta_squared": 0.22105758156006608, "warnings": [] }, @@ -5291,13 +7842,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9987684430207726, - "roc_auc": 0.9999734268707483, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.10204081632653061, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.07108045800123364, + "seconds": 0.16165904099761974, "eta_squared": 0.21621224561046687, "warnings": [] }, @@ -5309,12 +7860,12 @@ "mechanism": "global", "level_spread": 0.15, "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.06887755102040816, "n_models": 1, - "seconds": 0.07759091600019019, + "seconds": 0.1595212089960114, "eta_squared": 0.21621224561046687, "warnings": [] }, @@ -5325,13 +7876,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9996789080215416, - "roc_auc": 0.9999933567176871, + "pr_auc": 0.9987675894739253, + "roc_auc": 0.9999756412981858, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7220111249989714, + "seconds": 1.3435670410035527, "eta_squared": 0.21621224561046687, "warnings": [] }, @@ -5348,7 +7899,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07145308300096076, + "seconds": 0.1719130419951398, "eta_squared": 0.21276020362000367, "warnings": [] }, @@ -5365,7 +7916,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.0916170830023475, + "seconds": 0.16653300000325544, "eta_squared": 0.21276020362000367, "warnings": [] }, @@ -5376,30 +7927,30 @@ "seed": 14, "mechanism": "global", "level_spread": 0.15, - "pr_auc": 0.9990243190307221, - "roc_auc": 0.9999800701530612, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04591836734693878, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7191717089990561, + "seconds": 1.3313044579990674, "eta_squared": 0.21276020362000367, "warnings": [] }, { "dataset": "synthetic", "grouping": "spread=0.150", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, + "config": "pooled", + "seed": 0, + "mechanism": "contextual", + "level_spread": 0.15, + "pr_auc": 0.9975142413573642, + "roc_auc": 0.9999490681689343, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.07261966599980951, + "seconds": 0.16716983399965102, "eta_squared": 0.3454157417517599, "warnings": [] }, @@ -5414,9 +7965,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.0727134999979171, + "seconds": 0.16568862499843817, "eta_squared": 0.3454157417517599, "warnings": [] }, @@ -5427,13 +7978,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7135545409983024, + "seconds": 1.3630879589982214, "eta_squared": 0.3454157417517599, "warnings": [] }, @@ -5444,13 +7995,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9801921610275169, + "pr_auc": 0.9780919278280775, "roc_auc": 0.9997541985544217, "precision_at_n": 0.96875, - "macro_pr_auc": 0.988702876984127, - "worst_group_fpr": 0.13520408163265307, + "macro_pr_auc": 0.9896288029100528, + "worst_group_fpr": 0.1326530612244898, "n_models": 1, - "seconds": 0.07139412499964237, + "seconds": 0.163802625000244, "eta_squared": 0.34102334572506343, "warnings": [] }, @@ -5461,13 +8012,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07210879099875456, + "seconds": 0.16768716699880315, "eta_squared": 0.34102334572506343, "warnings": [] }, @@ -5478,13 +8029,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.9992891687925171, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7238602500001434, + "seconds": 1.3686439580051228, "eta_squared": 0.34102334572506343, "warnings": [] }, @@ -5495,13 +8046,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9997874149659862, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.08066958299968974, + "seconds": 0.16696858299837913, "eta_squared": 0.34453696882915624, "warnings": [] }, @@ -5516,9 +8067,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, + "worst_group_fpr": 0.07908163265306123, "n_models": 1, - "seconds": 0.07343699999910314, + "seconds": 0.16658858400478493, "eta_squared": 0.34453696882915624, "warnings": [] }, @@ -5529,13 +8080,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7385112909978488, + "seconds": 1.3819815000024391, "eta_squared": 0.34453696882915624, "warnings": [] }, @@ -5546,13 +8097,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725625, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.07873783299874049, + "seconds": 0.17264137500023935, "eta_squared": 0.3456480927214205, "warnings": [] }, @@ -5563,13 +8114,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.0889895410000463, + "seconds": 0.16875304200220853, "eta_squared": 0.3456480927214205, "warnings": [] }, @@ -5580,13 +8131,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7152433749979537, + "seconds": 1.3209401249987422, "eta_squared": 0.3456480927214205, "warnings": [] }, @@ -5597,13 +8148,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9993395503859831, - "roc_auc": 0.9999867134353743, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "pr_auc": 0.975828562183688, + "roc_auc": 0.9997054811507937, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9818617724867723, + "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.07209704100023373, + "seconds": 0.1670529590046499, "eta_squared": 0.34002683577528714, "warnings": [] }, @@ -5614,13 +8165,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07163737500013667, + "seconds": 0.1904448749992298, "eta_squared": 0.34002683577528714, "warnings": [] }, @@ -5631,13 +8182,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7301543749999837, + "seconds": 1.3737939999991795, "eta_squared": 0.34002683577528714, "warnings": [] }, @@ -5648,13 +8199,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.971795730603734, - "roc_auc": 0.9995593289399093, + "pr_auc": 0.9719844664016125, + "roc_auc": 0.9995460423752835, "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.1377551020408163, + "macro_pr_auc": 0.985890903078403, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.07648383299965644, + "seconds": 0.16656937500374625, "eta_squared": 0.3305219088439165, "warnings": [] }, @@ -5665,13 +8216,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9998926116838486, - "roc_auc": 0.9999977855725625, - "precision_at_n": 0.9895833333333334, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07291724999959115, + "seconds": 0.1735560000015539, "eta_squared": 0.3305219088439165, "warnings": [] }, @@ -5682,13 +8233,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7337241659988649, + "seconds": 1.3466816660002223, "eta_squared": 0.3305219088439165, "warnings": [] }, @@ -5699,13 +8250,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9997874149659864, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.08109112499732873, + "seconds": 0.16413079199992353, "eta_squared": 0.32973445787453604, "warnings": [] }, @@ -5722,7 +8273,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.075488000002224, + "seconds": 0.1655842920008581, "eta_squared": 0.32973445787453604, "warnings": [] }, @@ -5733,13 +8284,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7605138750004699, + "seconds": 1.3275695420015836, "eta_squared": 0.32973445787453604, "warnings": [] }, @@ -5750,13 +8301,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9764156875457186, - "roc_auc": 0.9996545493197279, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9951264880952381, - "worst_group_fpr": 0.125, + "pr_auc": 0.9971973944928514, + "roc_auc": 0.9999490681689343, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.13010204081632654, "n_models": 1, - "seconds": 0.07583337500182097, + "seconds": 0.17034458300622646, "eta_squared": 0.3446151723429949, "warnings": [] }, @@ -5771,9 +8322,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07259120800154051, + "seconds": 0.16753458400489762, "eta_squared": 0.3446151723429949, "warnings": [] }, @@ -5784,13 +8335,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7225472089994582, + "seconds": 1.3903519999948912, "eta_squared": 0.3446151723429949, "warnings": [] }, @@ -5801,13 +8352,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9992239393431516, - "roc_auc": 0.9999844990079366, + "pr_auc": 0.9997841047394043, + "roc_auc": 0.9999955711451247, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.07207162499980768, + "seconds": 0.1745359999986249, "eta_squared": 0.3507338333119392, "warnings": [] }, @@ -5818,13 +8369,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, + "worst_group_fpr": 0.05612244897959184, "n_models": 1, - "seconds": 0.07437016599942581, + "seconds": 0.1656812919973163, "eta_squared": 0.3507338333119392, "warnings": [] }, @@ -5835,13 +8386,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.6980373340011283, + "seconds": 1.354504041999462, "eta_squared": 0.3507338333119392, "warnings": [] }, @@ -5852,13 +8403,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9777801087320123, - "roc_auc": 0.9997918438208617, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9861565806878306, + "pr_auc": 0.9987530543910215, + "roc_auc": 0.999975641298186, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.07867308300046716, + "seconds": 0.17035816599673126, "eta_squared": 0.33856980964700606, "warnings": [] }, @@ -5869,13 +8420,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.07843920899904333, + "seconds": 0.16907495800114702, "eta_squared": 0.33856980964700606, "warnings": [] }, @@ -5886,13 +8437,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7272615830006544, + "seconds": 1.3467641250026645, "eta_squared": 0.33856980964700606, "warnings": [] }, @@ -5903,13 +8454,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9991094704786827, - "roc_auc": 0.9999822845804989, + "pr_auc": 0.9981328388755406, + "roc_auc": 0.9999645691609976, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, + "macro_pr_auc": 0.9988425925925926, "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.07898945800116053, + "seconds": 0.17213825000362704, "eta_squared": 0.3494306148914419, "warnings": [] }, @@ -5920,13 +8471,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07821712499935529, + "seconds": 0.1726834580040304, "eta_squared": 0.3494306148914419, "warnings": [] }, @@ -5937,13 +8488,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7282064579994767, + "seconds": 1.3647090419981396, "eta_squared": 0.3494306148914419, "warnings": [] }, @@ -5954,13 +8505,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9936280415960002, - "roc_auc": 0.9998914930555556, - "precision_at_n": 0.96875, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "pr_auc": 0.9503620548777874, + "roc_auc": 0.999320170776644, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9767184493746993, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.07279424999796902, + "seconds": 0.16600250000192318, "eta_squared": 0.33458322121718237, "warnings": [] }, @@ -5971,13 +8522,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9991081986024111, + "pr_auc": 0.9991081986024108, "roc_auc": 0.9999822845804989, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.07553291600197554, + "seconds": 0.16288820800400572, "eta_squared": 0.33458322121718237, "warnings": [] }, @@ -5988,13 +8539,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9619580084145981, - "roc_auc": 0.99949289611678, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7148141659999965, + "seconds": 1.5557989579974674, "eta_squared": 0.33458322121718237, "warnings": [] }, @@ -6005,13 +8556,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "pr_auc": 0.9857423520840152, + "roc_auc": 0.9997962726757371, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9918568121693121, + "worst_group_fpr": 0.13010204081632654, "n_models": 1, - "seconds": 0.07439137499750359, + "seconds": 0.19819208300032187, "eta_squared": 0.3490755282832429, "warnings": [] }, @@ -6026,9 +8577,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, + "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.08225108300030115, + "seconds": 0.1862407090011402, "eta_squared": 0.3490755282832429, "warnings": [] }, @@ -6039,13 +8590,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7257115419997717, + "seconds": 1.4572139169977163, "eta_squared": 0.3490755282832429, "warnings": [] }, @@ -6056,13 +8607,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9238049983388068, - "roc_auc": 0.9992094494047619, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9592280693843195, - "worst_group_fpr": 0.11224489795918367, + "pr_auc": 0.9274545831517811, + "roc_auc": 0.9992714533730159, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9602749969937469, + "worst_group_fpr": 0.1096938775510204, "n_models": 1, - "seconds": 0.06664079099937226, + "seconds": 0.16133904100570362, "eta_squared": 0.3333216444268115, "warnings": [] }, @@ -6073,13 +8624,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.07816883299892652, + "seconds": 0.1783835839960375, "eta_squared": 0.3333216444268115, "warnings": [] }, @@ -6090,13 +8641,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9082358809082258, + "roc_auc": 0.9985362634637187, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9244201689514191, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7100955829992017, + "seconds": 1.348618667005212, "eta_squared": 0.3333216444268115, "warnings": [] }, @@ -6107,13 +8658,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.9864046213824539, - "roc_auc": 0.9998361323696145, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.986146811966324, + "roc_auc": 0.999796272675737, + "precision_at_n": 0.96875, "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.07676120799806085, + "seconds": 0.16729599999962375, "eta_squared": 0.34143017232311446, "warnings": [] }, @@ -6128,9 +8679,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07141820800097776, + "seconds": 0.16671287499775644, "eta_squared": 0.34143017232311446, "warnings": [] }, @@ -6141,13 +8692,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.15, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7144282080007542, + "seconds": 1.4168364169963752, "eta_squared": 0.34143017232311446, "warnings": [] }, @@ -6162,9 +8713,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "worst_group_fpr": 0.1377551020408163, "n_models": 1, - "seconds": 0.07433545900130412, + "seconds": 0.16023679100180743, "eta_squared": 0.2935663417355178, "warnings": [] }, @@ -6175,13 +8726,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07685870799832628, + "seconds": 0.1728855420005857, "eta_squared": 0.2935663417355178, "warnings": [] }, @@ -6192,13 +8743,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7047439999987546, + "seconds": 1.361629166000057, "eta_squared": 0.2935663417355178, "warnings": [] }, @@ -6209,13 +8760,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.06890383399877464, + "seconds": 0.16881350000039674, "eta_squared": 0.288953232932902, "warnings": [] }, @@ -6226,13 +8777,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.07123420800053282, + "seconds": 0.17742975000146544, "eta_squared": 0.288953232932902, "warnings": [] }, @@ -6243,13 +8794,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9972700593819733, - "roc_auc": 0.9999468537414966, + "pr_auc": 0.9988020366397061, + "roc_auc": 0.999975641298186, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7110342499981925, + "seconds": 1.4125322500040056, "eta_squared": 0.288953232932902, "warnings": [] }, @@ -6260,13 +8811,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9995636400137603, - "roc_auc": 0.9999911422902493, + "pr_auc": 0.9997874149659862, + "roc_auc": 0.9999955711451247, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.11479591836734694, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.07179974999962724, + "seconds": 0.18944229099724907, "eta_squared": 0.29120655697879977, "warnings": [] }, @@ -6277,13 +8828,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, + "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.07621354099683231, + "seconds": 0.1854238749947399, "eta_squared": 0.29120655697879977, "warnings": [] }, @@ -6294,13 +8845,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9984301094341246, - "roc_auc": 0.9999667835884354, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7028574159994605, + "seconds": 1.4347965830020257, "eta_squared": 0.29120655697879977, "warnings": [] }, @@ -6311,13 +8862,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.07623379199867486, + "seconds": 0.17028995799773838, "eta_squared": 0.2923445749968756, "warnings": [] }, @@ -6332,9 +8883,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.08033066699863411, + "seconds": 0.17227354199712863, "eta_squared": 0.2923445749968756, "warnings": [] }, @@ -6345,13 +8896,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6950934169981338, + "seconds": 1.3645374170009745, "eta_squared": 0.2923445749968756, "warnings": [] }, @@ -6366,9 +8917,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.06914320799842244, + "seconds": 0.1613175830061664, "eta_squared": 0.29078029700670166, "warnings": [] }, @@ -6379,13 +8930,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.07281108300230699, + "seconds": 0.16172791699500522, "eta_squared": 0.29078029700670166, "warnings": [] }, @@ -6396,13 +8947,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9993384082076205, - "roc_auc": 0.9999867134353742, + "pr_auc": 0.9994516328453015, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7129051669980981, + "seconds": 1.345396291995712, "eta_squared": 0.29078029700670166, "warnings": [] }, @@ -6417,9 +8968,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.06847087500136695, + "seconds": 0.17803733300388558, "eta_squared": 0.282343864590447, "warnings": [] }, @@ -6430,13 +8981,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, + "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.07125195800108486, + "seconds": 0.16395645899319788, "eta_squared": 0.282343864590447, "warnings": [] }, @@ -6447,13 +8998,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9985281258723928, - "roc_auc": 0.9999712124433106, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, + "pr_auc": 0.9968256172342184, + "roc_auc": 0.9999402104591837, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7077696249980363, + "seconds": 1.3976950829965062, "eta_squared": 0.282343864590447, "warnings": [] }, @@ -6464,13 +9015,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9991081986024108, - "roc_auc": 0.9999822845804989, + "pr_auc": 0.9980276421576781, + "roc_auc": 0.9999623547335601, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.06953916599741206, + "seconds": 0.17328049999923678, "eta_squared": 0.2827224241416107, "warnings": [] }, @@ -6485,9 +9036,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.07077208299961057, + "seconds": 0.1606982919984148, "eta_squared": 0.2827224241416107, "warnings": [] }, @@ -6498,13 +9049,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9973362833817405, - "roc_auc": 0.9999512825963719, + "pr_auc": 0.9970566287270858, + "roc_auc": 0.9999468537414966, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7117570829977922, + "seconds": 1.3746588750000228, "eta_squared": 0.2827224241416107, "warnings": [] }, @@ -6515,13 +9066,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.07288400000106776, + "seconds": 0.16199141600372968, "eta_squared": 0.2957817245569126, "warnings": [] }, @@ -6536,9 +9087,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.06842374999905587, + "seconds": 0.1711746669971035, "eta_squared": 0.2957817245569126, "warnings": [] }, @@ -6549,13 +9100,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9986862958989656, - "roc_auc": 0.9999734268707483, + "pr_auc": 0.9984642313812274, + "roc_auc": 0.999968998015873, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.718693249997159, + "seconds": 1.3538777079957072, "eta_squared": 0.2957817245569126, "warnings": [] }, @@ -6570,9 +9121,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.07095466699684039, + "seconds": 0.16894041700288653, "eta_squared": 0.2975501660008078, "warnings": [] }, @@ -6583,13 +9134,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, + "worst_group_fpr": 0.058673469387755105, "n_models": 1, - "seconds": 0.0700917499998468, + "seconds": 0.1791822500017588, "eta_squared": 0.2975501660008078, "warnings": [] }, @@ -6600,13 +9151,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9958562822206711, - "roc_auc": 0.9999291383219955, + "pr_auc": 0.9977421731790774, + "roc_auc": 0.9999579258786848, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.703653917000338, + "seconds": 1.3465448749993811, "eta_squared": 0.2975501660008078, "warnings": [] }, @@ -6621,9 +9172,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07769012500284589, + "seconds": 0.1691537079968839, "eta_squared": 0.28690079185598694, "warnings": [] }, @@ -6634,13 +9185,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.06761479199849418, + "seconds": 0.16600350000226172, "eta_squared": 0.28690079185598694, "warnings": [] }, @@ -6651,13 +9202,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7115977500034205, + "seconds": 1.4231247080024332, "eta_squared": 0.28690079185598694, "warnings": [] }, @@ -6668,13 +9219,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.10467395900195697, + "seconds": 0.19377870799507946, "eta_squared": 0.2932065449666956, "warnings": [] }, @@ -6685,13 +9236,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.07908163265306123, "n_models": 1, - "seconds": 0.07399174999954994, + "seconds": 0.19929208300163737, "eta_squared": 0.2932065449666956, "warnings": [] }, @@ -6702,13 +9253,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9988204781635737, - "roc_auc": 0.9999756412981858, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04591836734693878, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7070473329986271, + "seconds": 1.414609458995983, "eta_squared": 0.2932065449666956, "warnings": [] }, @@ -6719,13 +9270,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.06950833299924852, + "seconds": 0.1728825419995701, "eta_squared": 0.2919163375746864, "warnings": [] }, @@ -6736,13 +9287,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.07142857142857142, "n_models": 1, - "seconds": 0.07058487500034971, + "seconds": 0.17778083399753086, "eta_squared": 0.2919163375746864, "warnings": [] }, @@ -6753,13 +9304,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9950307939051444, - "roc_auc": 0.9999180661848073, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9947936280692428, + "roc_auc": 0.9999092084750567, + "precision_at_n": 0.96875, "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7124103749993083, + "seconds": 1.4157218749969616, "eta_squared": 0.2919163375746864, "warnings": [] }, @@ -6770,13 +9321,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9995636400137603, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07182687499880558, + "seconds": 0.18477799999527633, "eta_squared": 0.2964580999866939, "warnings": [] }, @@ -6793,7 +9344,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.061224489795918366, "n_models": 1, - "seconds": 0.07359462499880465, + "seconds": 0.1686459999982617, "eta_squared": 0.2964580999866939, "warnings": [] }, @@ -6804,13 +9355,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.6911854169993603, + "seconds": 1.3955319169981522, "eta_squared": 0.2964580999866939, "warnings": [] }, @@ -6821,13 +9372,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.07328529200094636, + "seconds": 0.1688257080022595, "eta_squared": 0.29175866652881005, "warnings": [] }, @@ -6838,13 +9389,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9998926116838486, - "roc_auc": 0.9999977855725625, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, + "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.07719345800069277, + "seconds": 0.16431325000303332, "eta_squared": 0.29175866652881005, "warnings": [] }, @@ -6855,13 +9406,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9995726383336836, - "roc_auc": 0.9999911422902494, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9986478576731588, + "roc_auc": 0.9999734268707482, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7007327499995881, + "seconds": 1.4059872499929043, "eta_squared": 0.29175866652881005, "warnings": [] }, @@ -6872,13 +9423,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.06992137500128592, + "seconds": 0.18456200000218814, "eta_squared": 0.28829596838598986, "warnings": [] }, @@ -6889,13 +9440,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, + "worst_group_fpr": 0.06377551020408163, "n_models": 1, - "seconds": 0.0699008750016219, + "seconds": 0.17934149999928195, "eta_squared": 0.28829596838598986, "warnings": [] }, @@ -6906,13 +9457,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.2, - "pr_auc": 0.9992448088419582, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04846938775510204, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7195295839992468, + "seconds": 1.3846435000014026, "eta_squared": 0.28829596838598986, "warnings": [] }, @@ -6923,13 +9474,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9826354837074835, - "roc_auc": 0.9996766935941044, - "precision_at_n": 0.9270833333333334, + "pr_auc": 0.9651287441700821, + "roc_auc": 0.999399890164399, + "precision_at_n": 0.90625, "macro_pr_auc": 0.9892361111111111, - "worst_group_fpr": 0.16071428571428573, + "worst_group_fpr": 0.16581632653061223, "n_models": 1, - "seconds": 0.07226437500139582, + "seconds": 0.17997529199783457, "eta_squared": 0.4778476892757978, "warnings": [] }, @@ -6940,13 +9491,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.07525420900128665, + "seconds": 0.16583841700048652, "eta_squared": 0.4778476892757978, "warnings": [] }, @@ -6957,13 +9508,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7041030830005184, + "seconds": 1.3948089169934974, "eta_squared": 0.4778476892757978, "warnings": [] }, @@ -6974,13 +9525,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9686266973751816, - "roc_auc": 0.9996611926020408, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.981712962962963, + "pr_auc": 0.9690309915092666, + "roc_auc": 0.9996988378684808, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9844730790043291, "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.07087304200103972, + "seconds": 0.17768600000272272, "eta_squared": 0.4744098790454379, "warnings": [] }, @@ -6991,13 +9542,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.06929333399966708, + "seconds": 0.17603745899396017, "eta_squared": 0.4744098790454379, "warnings": [] }, @@ -7008,13 +9559,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.9992891687925171, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7102770840028825, + "seconds": 1.3801205000054324, "eta_squared": 0.4744098790454379, "warnings": [] }, @@ -7025,13 +9576,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9419522669342344, - "roc_auc": 0.9991651608560091, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9726416523291522, - "worst_group_fpr": 0.1377551020408163, + "pr_auc": 0.9818643368287174, + "roc_auc": 0.9996855513038548, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9876602564102565, + "worst_group_fpr": 0.16071428571428573, "n_models": 1, - "seconds": 0.07590425000307732, + "seconds": 0.19598387500445824, "eta_squared": 0.4772875504667477, "warnings": [] }, @@ -7042,13 +9593,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.07962629200119409, + "seconds": 0.16780179199849954, "eta_squared": 0.4772875504667477, "warnings": [] }, @@ -7059,13 +9610,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7056990419987414, + "seconds": 1.3479372919973684, "eta_squared": 0.4772875504667477, "warnings": [] }, @@ -7076,13 +9627,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9816622918044869, - "roc_auc": 0.9997364831349207, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9950810185185185, - "worst_group_fpr": 0.15816326530612246, + "pr_auc": 0.9974745112652791, + "roc_auc": 0.9999534970238095, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.07474441699741874, + "seconds": 0.16843795799650252, "eta_squared": 0.47920852546400317, "warnings": [] }, @@ -7093,13 +9644,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07266374999744585, + "seconds": 0.1729825840011472, "eta_squared": 0.47920852546400317, "warnings": [] }, @@ -7110,13 +9661,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7114176249997399, + "seconds": 1.3807125000021188, "eta_squared": 0.47920852546400317, "warnings": [] }, @@ -7127,13 +9678,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9736044464091245, - "roc_auc": 0.999696623441043, + "pr_auc": 0.9652492766036631, + "roc_auc": 0.9996501204648526, "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9818617724867723, - "worst_group_fpr": 0.13520408163265307, + "macro_pr_auc": 0.967104828042328, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07567225000093458, + "seconds": 0.16445862500404473, "eta_squared": 0.47375497337598166, "warnings": [] }, @@ -7144,13 +9695,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.07178654200106394, + "seconds": 0.16723541599640157, "eta_squared": 0.47375497337598166, "warnings": [] }, @@ -7161,13 +9712,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7036598749982659, + "seconds": 1.3959956249964307, "eta_squared": 0.47375497337598166, "warnings": [] }, @@ -7178,13 +9729,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9172572822230664, - "roc_auc": 0.9992648100907029, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.96766587000962, - "worst_group_fpr": 0.15306122448979592, + "pr_auc": 0.9246003273961511, + "roc_auc": 0.9992116638321995, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9771111411736411, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.07218012499652104, + "seconds": 0.17050491699774284, "eta_squared": 0.4645660409992461, "warnings": [] }, @@ -7199,9 +9750,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.07338595900000655, + "seconds": 0.17489291699894238, "eta_squared": 0.4645660409992461, "warnings": [] }, @@ -7212,13 +9763,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6958567920009955, + "seconds": 1.431837334006559, "eta_squared": 0.4645660409992461, "warnings": [] }, @@ -7229,13 +9780,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9976369764612152, - "roc_auc": 0.9999557114512472, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.07794729200031725, + "seconds": 0.17707912499463418, "eta_squared": 0.46373875898462746, "warnings": [] }, @@ -7250,9 +9801,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.08673469387755102, "n_models": 1, - "seconds": 0.07319979200110538, + "seconds": 0.16968387499946402, "eta_squared": 0.46373875898462746, "warnings": [] }, @@ -7263,13 +9814,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7167679579979449, + "seconds": 1.4110775420049322, "eta_squared": 0.46373875898462746, "warnings": [] }, @@ -7280,13 +9831,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9861848812887417, - "roc_auc": 0.9998007015306123, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9929976851851853, + "pr_auc": 0.9835976221998108, + "roc_auc": 0.9997519841269842, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9922401094276094, "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.06588654200095334, + "seconds": 0.16894850000244332, "eta_squared": 0.47745063229238155, "warnings": [] }, @@ -7298,12 +9849,12 @@ "mechanism": "contextual", "level_spread": 0.2, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.08673469387755102, "n_models": 1, - "seconds": 0.07609766699897591, + "seconds": 0.157957333001832, "eta_squared": 0.47745063229238155, "warnings": [] }, @@ -7314,13 +9865,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7205418749981618, + "seconds": 1.378067874997214, "eta_squared": 0.47745063229238155, "warnings": [] }, @@ -7331,13 +9882,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9895686047987848, - "roc_auc": 0.999820631377551, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.14795918367346939, + "pr_auc": 0.9768890335189591, + "roc_auc": 0.9996501204648526, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9937375992063492, + "worst_group_fpr": 0.15306122448979592, "n_models": 1, - "seconds": 0.0699456670008658, + "seconds": 0.16433320800570073, "eta_squared": 0.48200866700409517, "warnings": [] }, @@ -7354,7 +9905,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.07046799999807263, + "seconds": 0.1653154169980553, "eta_squared": 0.48200866700409517, "warnings": [] }, @@ -7365,13 +9916,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.6927875830006087, + "seconds": 1.3574518329987768, "eta_squared": 0.48200866700409517, "warnings": [] }, @@ -7382,13 +9933,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9800971167255851, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9941137566137566, - "worst_group_fpr": 0.1683673469387755, + "pr_auc": 0.9893053104296298, + "roc_auc": 0.9998117736678005, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.16071428571428573, "n_models": 1, - "seconds": 0.07665895799800637, + "seconds": 0.16279591700003948, "eta_squared": 0.4722715287711093, "warnings": [] }, @@ -7399,13 +9950,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.07793999999921652, + "seconds": 0.17586354200466303, "eta_squared": 0.4722715287711093, "warnings": [] }, @@ -7416,13 +9967,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7258374170014577, + "seconds": 1.394391750000068, "eta_squared": 0.4722715287711093, "warnings": [] }, @@ -7433,13 +9984,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9659399603165795, - "roc_auc": 0.9997121244331066, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9888186177248678, - "worst_group_fpr": 0.13010204081632654, + "pr_auc": 0.9855322722251981, + "roc_auc": 0.9998361323696145, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07170687499819905, + "seconds": 0.17189950000465615, "eta_squared": 0.4809671284581783, "warnings": [] }, @@ -7450,13 +10001,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.07701929199902224, + "seconds": 0.17399262500111945, "eta_squared": 0.4809671284581783, "warnings": [] }, @@ -7467,13 +10018,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7040943329993752, + "seconds": 1.3464244999995572, "eta_squared": 0.4809671284581783, "warnings": [] }, @@ -7484,13 +10035,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9251687155968192, - "roc_auc": 0.9991939484126984, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9708276966089465, - "worst_group_fpr": 0.1377551020408163, + "pr_auc": 0.9115082120378382, + "roc_auc": 0.999085441468254, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9604564995189994, + "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.07568179200097802, + "seconds": 0.17778216599981533, "eta_squared": 0.469061169841381, "warnings": [] }, @@ -7502,12 +10053,12 @@ "mechanism": "contextual", "level_spread": 0.2, "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.0663265306122449, "n_models": 1, - "seconds": 0.07196716600083164, + "seconds": 0.17343408299348084, "eta_squared": 0.469061169841381, "warnings": [] }, @@ -7518,13 +10069,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9619580084145981, - "roc_auc": 0.99949289611678, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6845039580002776, + "seconds": 1.38340474999859, "eta_squared": 0.469061169841381, "warnings": [] }, @@ -7535,13 +10086,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9993384082076205, - "roc_auc": 0.9999867134353742, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, + "pr_auc": 0.992342976560897, + "roc_auc": 0.9998671343537415, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.07087045799926273, + "seconds": 0.16625833300349768, "eta_squared": 0.4822963153336835, "warnings": [] }, @@ -7552,13 +10103,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.0727752499988128, + "seconds": 0.16770204199565342, "eta_squared": 0.4822963153336835, "warnings": [] }, @@ -7569,13 +10120,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7130109160025313, + "seconds": 1.3579438329979894, "eta_squared": 0.4822963153336835, "warnings": [] }, @@ -7586,13 +10137,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9398813761507946, - "roc_auc": 0.9993223852040816, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.988420664983165, - "worst_group_fpr": 0.14285714285714285, + "pr_auc": 0.9224361945178026, + "roc_auc": 0.9990876558956916, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9789299242424242, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.07730920900212368, + "seconds": 0.16845174999616574, "eta_squared": 0.4673396279875538, "warnings": [] }, @@ -7607,9 +10158,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.07188620799934142, + "seconds": 0.16399837499920977, "eta_squared": 0.4673396279875538, "warnings": [] }, @@ -7620,13 +10171,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9088206216751, + "roc_auc": 0.9985429067460317, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.705412957999215, + "seconds": 1.363436792002176, "eta_squared": 0.4673396279875538, "warnings": [] }, @@ -7637,13 +10188,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.9820778190927275, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, + "pr_auc": 0.974059049230118, + "roc_auc": 0.9996833368764173, + "precision_at_n": 0.9479166666666666, "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.15051020408163265, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.06871708399921772, + "seconds": 0.17070308300026227, "eta_squared": 0.4750062627828297, "warnings": [] }, @@ -7655,12 +10206,12 @@ "mechanism": "contextual", "level_spread": 0.2, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.08673469387755102, "n_models": 1, - "seconds": 0.07639395799924387, + "seconds": 0.16764712500298629, "eta_squared": 0.4750062627828297, "warnings": [] }, @@ -7671,13 +10222,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.2, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7285070000034466, + "seconds": 1.3769224999996368, "eta_squared": 0.4750062627828297, "warnings": [] }, @@ -7692,9 +10243,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1836734693877551, + "worst_group_fpr": 0.18877551020408162, "n_models": 1, - "seconds": 0.0747485419997247, + "seconds": 0.16629520800051978, "eta_squared": 0.3927094336787097, "warnings": [] }, @@ -7705,13 +10256,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.07151725000221631, + "seconds": 0.16802812500100117, "eta_squared": 0.3927094336787097, "warnings": [] }, @@ -7722,13 +10273,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6960212080011843, + "seconds": 1.365570124995429, "eta_squared": 0.3927094336787097, "warnings": [] }, @@ -7739,13 +10290,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.07316204100061441, + "seconds": 0.17283054199651815, "eta_squared": 0.38945558020781024, "warnings": [] }, @@ -7756,13 +10307,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.1096938775510204, "n_models": 1, - "seconds": 0.06780370799970115, + "seconds": 0.1684383340034401, "eta_squared": 0.38945558020781024, "warnings": [] }, @@ -7773,13 +10324,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.997984202808983, - "roc_auc": 0.9999601403061225, + "pr_auc": 0.999137094907938, + "roc_auc": 0.9999822845804989, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7270031250009197, + "seconds": 1.3717942499933997, "eta_squared": 0.38945558020781024, "warnings": [] }, @@ -7790,13 +10341,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9997841047394044, - "roc_auc": 0.9999955711451246, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9994629892108767, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, + "worst_group_fpr": 0.16581632653061223, "n_models": 1, - "seconds": 0.06545379100134596, + "seconds": 0.1766787500018836, "eta_squared": 0.391417879351872, "warnings": [] }, @@ -7807,13 +10358,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07216316600170103, + "seconds": 0.1647379580026609, "eta_squared": 0.391417879351872, "warnings": [] }, @@ -7824,13 +10375,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9980440120074874, - "roc_auc": 0.9999579258786848, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, + "pr_auc": 0.9996843434343431, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7017973340007302, + "seconds": 1.3938766670034966, "eta_squared": 0.391417879351872, "warnings": [] }, @@ -7845,9 +10396,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20408163265306123, + "worst_group_fpr": 0.18112244897959184, "n_models": 1, - "seconds": 0.06790637499943841, + "seconds": 0.17747354199673282, "eta_squared": 0.3925992743462009, "warnings": [] }, @@ -7862,9 +10413,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.07908163265306123, "n_models": 1, - "seconds": 0.07433179200234008, + "seconds": 0.1662358329995186, "eta_squared": 0.3925992743462009, "warnings": [] }, @@ -7875,13 +10426,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7195739169983426, + "seconds": 1.3559329999989131, "eta_squared": 0.3925992743462009, "warnings": [] }, @@ -7892,13 +10443,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, + "worst_group_fpr": 0.13520408163265307, "n_models": 1, - "seconds": 0.07371741600218229, + "seconds": 0.172857166005997, "eta_squared": 0.39013809068873967, "warnings": [] }, @@ -7909,13 +10460,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.06969729200136499, + "seconds": 0.17408391600474715, "eta_squared": 0.39013809068873967, "warnings": [] }, @@ -7926,13 +10477,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9994516328453016, + "pr_auc": 0.9994516328453015, "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7018840829987312, + "seconds": 1.3925372079975205, "eta_squared": 0.39013809068873967, "warnings": [] }, @@ -7943,13 +10494,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.06848408400037442, + "seconds": 0.16980783300095936, "eta_squared": 0.385337437465984, "warnings": [] }, @@ -7960,13 +10511,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.06798533400069573, + "seconds": 0.17061158300202806, "eta_squared": 0.385337437465984, "warnings": [] }, @@ -7977,13 +10528,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9984204639542932, - "roc_auc": 0.999968998015873, + "pr_auc": 0.9973971597648099, + "roc_auc": 0.9999512825963718, "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.710395709000295, + "seconds": 1.390012999996543, "eta_squared": 0.385337437465984, "warnings": [] }, @@ -7994,13 +10545,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9991081986024108, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, + "worst_group_fpr": 0.15816326530612246, "n_models": 1, - "seconds": 0.07513691599888261, + "seconds": 0.17900604200258385, "eta_squared": 0.38448012991671376, "warnings": [] }, @@ -8012,12 +10563,12 @@ "mechanism": "global", "level_spread": 0.3, "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.06959566600198741, + "seconds": 0.16348675000335788, "eta_squared": 0.38448012991671376, "warnings": [] }, @@ -8028,13 +10579,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9982598713958658, - "roc_auc": 0.9999667835884354, + "pr_auc": 0.9978740297191621, + "roc_auc": 0.9999601403061225, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7011397080023016, + "seconds": 1.386373208995792, "eta_squared": 0.38448012991671376, "warnings": [] }, @@ -8045,13 +10596,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.17346938775510204, "n_models": 1, - "seconds": 0.06881287499709288, + "seconds": 0.16611620799812954, "eta_squared": 0.3945500497199951, "warnings": [] }, @@ -8062,13 +10613,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.06850195799779613, + "seconds": 0.17003979199944297, "eta_squared": 0.3945500497199951, "warnings": [] }, @@ -8079,13 +10630,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9982623386901864, - "roc_auc": 0.9999645691609979, - "precision_at_n": 0.96875, + "pr_auc": 0.9984642313812273, + "roc_auc": 0.9999689980158729, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7089162079973903, + "seconds": 1.4195754999964265, "eta_squared": 0.3945500497199951, "warnings": [] }, @@ -8100,9 +10651,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17857142857142858, + "worst_group_fpr": 0.16326530612244897, "n_models": 1, - "seconds": 0.07024066699887044, + "seconds": 0.17566954100038856, "eta_squared": 0.39452480740459633, "warnings": [] }, @@ -8113,13 +10664,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999997, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.07908163265306123, "n_models": 1, - "seconds": 0.07150866699885228, + "seconds": 0.16447487500408897, "eta_squared": 0.39452480740459633, "warnings": [] }, @@ -8130,13 +10681,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9960141105034992, - "roc_auc": 0.9999313527494331, + "pr_auc": 0.9977421731790774, + "roc_auc": 0.9999579258786848, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, + "macro_pr_auc": 0.9975405092592592, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7044262499985052, + "seconds": 1.3837716659982107, "eta_squared": 0.39452480740459633, "warnings": [] }, @@ -8147,13 +10698,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, + "worst_group_fpr": 0.15306122448979592, "n_models": 1, - "seconds": 0.0690706669993233, + "seconds": 0.16298408299917355, "eta_squared": 0.3881032222309504, "warnings": [] }, @@ -8164,13 +10715,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07449799999812967, + "seconds": 0.1777255420020083, "eta_squared": 0.3881032222309504, "warnings": [] }, @@ -8181,13 +10732,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.6935719579996658, + "seconds": 1.3902658750012051, "eta_squared": 0.3881032222309504, "warnings": [] }, @@ -8198,13 +10749,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.0743379579980683, + "seconds": 0.17622975000267616, "eta_squared": 0.39197938133375715, "warnings": [] }, @@ -8215,13 +10766,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.07260508299805224, + "seconds": 0.15857129199866904, "eta_squared": 0.39197938133375715, "warnings": [] }, @@ -8232,13 +10783,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9990474418934239, - "roc_auc": 0.9999800701530612, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.6825684169998567, + "seconds": 1.3659195410000393, "eta_squared": 0.39197938133375715, "warnings": [] }, @@ -8249,13 +10800,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.0703274160005094, + "seconds": 0.1675723329972243, "eta_squared": 0.39171866046305087, "warnings": [] }, @@ -8266,13 +10817,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.06744299999991199, + "seconds": 0.1658329169949866, "eta_squared": 0.39171866046305087, "warnings": [] }, @@ -8283,13 +10834,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9961709131539347, - "roc_auc": 0.9999335671768708, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9953829558217713, + "roc_auc": 0.9999180661848073, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9943163029100529, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7208609170011187, + "seconds": 1.3530025000000023, "eta_squared": 0.39171866046305087, "warnings": [] }, @@ -8300,13 +10851,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, + "worst_group_fpr": 0.16326530612244897, "n_models": 1, - "seconds": 0.07486599999901955, + "seconds": 0.1654347920048167, "eta_squared": 0.39513960651559954, "warnings": [] }, @@ -8317,13 +10868,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.07409995800117031, + "seconds": 0.17483304099732777, "eta_squared": 0.39513960651559954, "warnings": [] }, @@ -8334,13 +10885,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7254400829988299, + "seconds": 1.4074117500058492, "eta_squared": 0.39513960651559954, "warnings": [] }, @@ -8357,7 +10908,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.06718825000280049, + "seconds": 0.16447770800004946, "eta_squared": 0.39167748869152313, "warnings": [] }, @@ -8368,13 +10919,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.06884524999986752, + "seconds": 0.1615245409993804, "eta_squared": 0.39167748869152313, "warnings": [] }, @@ -8385,13 +10936,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9996789080215419, - "roc_auc": 0.9999933567176871, + "pr_auc": 0.9990030018845482, + "roc_auc": 0.9999800701530611, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, + "macro_pr_auc": 0.9943163029100529, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7007277500015334, + "seconds": 1.376486749999458, "eta_squared": 0.39167748869152313, "warnings": [] }, @@ -8402,13 +10953,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.17346938775510204, "n_models": 1, - "seconds": 0.07341037500009406, + "seconds": 0.170403834003082, "eta_squared": 0.38870468849861617, "warnings": [] }, @@ -8419,13 +10970,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, + "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.06719725000220933, + "seconds": 0.16837604100146564, "eta_squared": 0.38870468849861617, "warnings": [] }, @@ -8436,13 +10987,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.3, - "pr_auc": 0.9996789080215419, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04846938775510204, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.716530290999799, + "seconds": 1.379614165998646, "eta_squared": 0.38870468849861617, "warnings": [] }, @@ -8453,13 +11004,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9263657652290938, - "roc_auc": 0.9989968643707483, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9875038156288157, - "worst_group_fpr": 0.20918367346938777, + "pr_auc": 0.8685479265604363, + "roc_auc": 0.9984078266723357, + "precision_at_n": 0.875, + "macro_pr_auc": 0.9758969907407407, + "worst_group_fpr": 0.1989795918367347, "n_models": 1, - "seconds": 0.07047358400086523, + "seconds": 0.16000670899666147, "eta_squared": 0.6640971283653139, "warnings": [] }, @@ -8470,13 +11021,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.07679395900049713, + "seconds": 0.17192883300594985, "eta_squared": 0.6640971283653139, "warnings": [] }, @@ -8487,13 +11038,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7105053749983199, + "seconds": 1.367972500003816, "eta_squared": 0.6640971283653139, "warnings": [] }, @@ -8504,13 +11055,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9265826092568507, - "roc_auc": 0.9990920847505669, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9703647336459836, + "pr_auc": 0.9435913175655869, + "roc_auc": 0.9994109623015873, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9786458333333333, "worst_group_fpr": 0.18622448979591838, "n_models": 1, - "seconds": 0.06875366700114682, + "seconds": 0.17479149999417132, "eta_squared": 0.6621686277077374, "warnings": [] }, @@ -8525,9 +11076,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.07715662500049802, + "seconds": 0.17065937499864958, "eta_squared": 0.6621686277077374, "warnings": [] }, @@ -8538,13 +11089,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.999289168792517, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.6922855829980108, + "seconds": 1.3877288329967996, "eta_squared": 0.6621686277077374, "warnings": [] }, @@ -8555,13 +11106,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9363247964140696, - "roc_auc": 0.9990677260487528, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9933903769841269, - "worst_group_fpr": 0.1760204081632653, + "pr_auc": 0.9567273961682181, + "roc_auc": 0.9993378861961452, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9937375992063492, + "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.06781591699837008, + "seconds": 0.16829045800113818, "eta_squared": 0.6640597584313623, "warnings": [] }, @@ -8576,9 +11127,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.06943708300241269, + "seconds": 0.16952591599692823, "eta_squared": 0.6640597584313623, "warnings": [] }, @@ -8589,13 +11140,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.6979062919999706, + "seconds": 1.3698817499971483, "eta_squared": 0.6640597584313623, "warnings": [] }, @@ -8606,13 +11157,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9676978194854132, - "roc_auc": 0.9994176055839001, - "precision_at_n": 0.8854166666666666, - "macro_pr_auc": 0.9979166666666667, + "pr_auc": 0.9937583637293692, + "roc_auc": 0.999895921910431, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9988425925925926, "worst_group_fpr": 0.18112244897959184, "n_models": 1, - "seconds": 0.07692316700195079, + "seconds": 0.1770604170014849, "eta_squared": 0.6662381870514966, "warnings": [] }, @@ -8623,13 +11174,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.07189474999904633, + "seconds": 0.1717158749961527, "eta_squared": 0.6662381870514966, "warnings": [] }, @@ -8640,13 +11191,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7306125830000383, + "seconds": 1.3958179999972344, "eta_squared": 0.6662381870514966, "warnings": [] }, @@ -8657,13 +11208,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9907521318855751, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.15816326530612246, + "pr_auc": 0.9587057434347515, + "roc_auc": 0.9995150403911565, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9827824374699374, + "worst_group_fpr": 0.16326530612244897, "n_models": 1, - "seconds": 0.07344716700026765, + "seconds": 0.17357199999969453, "eta_squared": 0.6620305481663469, "warnings": [] }, @@ -8674,13 +11225,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.08673469387755102, "n_models": 1, - "seconds": 0.06924670800071908, + "seconds": 0.17166254200128606, "eta_squared": 0.6620305481663469, "warnings": [] }, @@ -8691,13 +11242,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7220302500027174, + "seconds": 1.3588842079989263, "eta_squared": 0.6620305481663469, "warnings": [] }, @@ -8708,13 +11259,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9048015569661748, - "roc_auc": 0.9989968643707483, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9823273659211158, + "pr_auc": 0.9309252336755022, + "roc_auc": 0.9992913832199548, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9934027777777779, "worst_group_fpr": 0.1989795918367347, "n_models": 1, - "seconds": 0.0702958750007383, + "seconds": 0.1715607500009355, "eta_squared": 0.6550141848809792, "warnings": [] }, @@ -8725,13 +11276,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.07197683300182689, + "seconds": 0.1753054999935557, "eta_squared": 0.6550141848809792, "warnings": [] }, @@ -8742,13 +11293,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7107337910019851, + "seconds": 1.4161360419966513, "eta_squared": 0.6550141848809792, "warnings": [] }, @@ -8759,13 +11310,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9884679414612956, - "roc_auc": 0.9998117736678005, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9959788857666181, + "roc_auc": 0.9999224950396824, + "precision_at_n": 0.96875, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18112244897959184, + "worst_group_fpr": 0.17346938775510204, "n_models": 1, - "seconds": 0.06940833300177474, + "seconds": 0.17546308400051203, "eta_squared": 0.6541683708984072, "warnings": [] }, @@ -8776,13 +11327,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.07447241599948029, + "seconds": 0.17512133400305174, "eta_squared": 0.6541683708984072, "warnings": [] }, @@ -8793,13 +11344,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7033861659983813, + "seconds": 1.368081250002433, "eta_squared": 0.6541683708984072, "warnings": [] }, @@ -8810,13 +11361,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9679302253727047, - "roc_auc": 0.9995681866496597, + "pr_auc": 0.9710655795109718, + "roc_auc": 0.9996014030612246, "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9916087962962963, - "worst_group_fpr": 0.18622448979591838, + "macro_pr_auc": 0.9951264880952381, + "worst_group_fpr": 0.19642857142857142, "n_models": 1, - "seconds": 0.06832433399904403, + "seconds": 0.16607929199381033, "eta_squared": 0.6641774551533273, "warnings": [] }, @@ -8827,13 +11378,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.06596158300089883, + "seconds": 0.17151229199953377, "eta_squared": 0.6641774551533273, "warnings": [] }, @@ -8844,13 +11395,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6846688330006145, + "seconds": 1.3807262499976787, "eta_squared": 0.6641774551533273, "warnings": [] }, @@ -8861,13 +11412,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9711104008701593, - "roc_auc": 0.9996124751984127, - "precision_at_n": 0.9583333333333334, + "pr_auc": 0.983198469390041, + "roc_auc": 0.9997896293934241, + "precision_at_n": 0.96875, "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.18112244897959184, + "worst_group_fpr": 0.19387755102040816, "n_models": 1, - "seconds": 0.07261400000061258, + "seconds": 0.17668508300266694, "eta_squared": 0.666609652326715, "warnings": [] }, @@ -8882,9 +11433,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.08673469387755102, "n_models": 1, - "seconds": 0.0710116249974817, + "seconds": 0.17483558299863944, "eta_squared": 0.666609652326715, "warnings": [] }, @@ -8895,13 +11446,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7163267500000075, + "seconds": 1.3628304160010885, "eta_squared": 0.666609652326715, "warnings": [] }, @@ -8912,13 +11463,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9659613202417214, - "roc_auc": 0.9997143388605442, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9842179232804232, - "worst_group_fpr": 0.1913265306122449, + "pr_auc": 0.9881947804279476, + "roc_auc": 0.9998339179421769, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.17091836734693877, "n_models": 1, - "seconds": 0.07235275000130059, + "seconds": 0.17356795799423708, "eta_squared": 0.6607993707927986, "warnings": [] }, @@ -8929,13 +11480,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, + "worst_group_fpr": 0.13010204081632654, "n_models": 1, - "seconds": 0.07504649999827961, + "seconds": 0.17469100000016624, "eta_squared": 0.6607993707927986, "warnings": [] }, @@ -8946,13 +11497,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7094101250004314, + "seconds": 1.3728247500039288, "eta_squared": 0.6607993707927986, "warnings": [] }, @@ -8963,13 +11514,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.941703041760084, - "roc_auc": 0.9992714533730158, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9908966901154401, - "worst_group_fpr": 0.1760204081632653, + "pr_auc": 0.959482208444036, + "roc_auc": 0.9995283269557823, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9926669973544974, + "worst_group_fpr": 0.15816326530612246, "n_models": 1, - "seconds": 0.07642904200110934, + "seconds": 0.16373237499647075, "eta_squared": 0.6655080804629401, "warnings": [] }, @@ -8980,13 +11531,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07010483300109627, + "seconds": 0.17313954099518014, "eta_squared": 0.6655080804629401, "warnings": [] }, @@ -8997,13 +11548,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7144434169968008, + "seconds": 1.3540781249976135, "eta_squared": 0.6655080804629401, "warnings": [] }, @@ -9014,13 +11565,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9367880988062811, - "roc_auc": 0.999209449404762, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9861137415824915, + "pr_auc": 0.8950772036231616, + "roc_auc": 0.9989415036848073, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9716874849687348, "worst_group_fpr": 0.16071428571428573, "n_models": 1, - "seconds": 0.0789701249996142, + "seconds": 0.17875016600009985, "eta_squared": 0.6590680406076661, "warnings": [] }, @@ -9031,13 +11582,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.07323458300015773, + "seconds": 0.16972233300475636, "eta_squared": 0.6590680406076661, "warnings": [] }, @@ -9048,13 +11599,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9619580084145981, - "roc_auc": 0.99949289611678, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7030437920002441, + "seconds": 1.350839624996297, "eta_squared": 0.6590680406076661, "warnings": [] }, @@ -9065,13 +11616,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9796856156394346, - "roc_auc": 0.999734268707483, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9976851851851851, - "worst_group_fpr": 0.18622448979591838, + "pr_auc": 0.9697032509277987, + "roc_auc": 0.9995991886337869, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9950810185185185, + "worst_group_fpr": 0.18877551020408162, "n_models": 1, - "seconds": 0.06972791699809022, + "seconds": 0.1595385000036913, "eta_squared": 0.6680621080435837, "warnings": [] }, @@ -9082,13 +11633,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.07147479199920781, + "seconds": 0.16372979099833174, "eta_squared": 0.6680621080435837, "warnings": [] }, @@ -9099,13 +11650,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7273979999990843, + "seconds": 1.3488865839972277, "eta_squared": 0.6680621080435837, "warnings": [] }, @@ -9116,13 +11667,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9004911739406816, - "roc_auc": 0.9989105017006803, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9697574705387204, - "worst_group_fpr": 0.17346938775510204, + "pr_auc": 0.8854033842732889, + "roc_auc": 0.9987665639172335, + "precision_at_n": 0.875, + "macro_pr_auc": 0.9717581319143819, + "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.0703660420003871, + "seconds": 0.1792242500014254, "eta_squared": 0.6571877895615115, "warnings": [] }, @@ -9133,13 +11684,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.07190329200238921, + "seconds": 0.17895816700183786, "eta_squared": 0.6571877895615115, "warnings": [] }, @@ -9150,13 +11701,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7175949580014276, + "seconds": 1.3684294999984559, "eta_squared": 0.6571877895615115, "warnings": [] }, @@ -9167,13 +11718,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.9726997232911961, - "roc_auc": 0.9996944090136054, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.990549467893218, - "worst_group_fpr": 0.18877551020408162, + "pr_auc": 0.9656305339974596, + "roc_auc": 0.9996036174886621, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9923776455026455, + "worst_group_fpr": 0.1913265306122449, "n_models": 1, - "seconds": 0.07027662499967846, + "seconds": 0.17626054200081853, "eta_squared": 0.6627275317190392, "warnings": [] }, @@ -9188,9 +11739,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.07170183299967903, + "seconds": 0.16547554099815898, "eta_squared": 0.6627275317190392, "warnings": [] }, @@ -9201,13 +11752,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.3, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7343082910010708, + "seconds": 1.4033026670003892, "eta_squared": 0.6627275317190392, "warnings": [] }, @@ -9218,13 +11769,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "pr_auc": 1.0, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1989795918367347, + "worst_group_fpr": 0.21173469387755103, "n_models": 1, - "seconds": 0.06916733300022315, + "seconds": 0.1722321670022211, "eta_squared": 0.44964427142161606, "warnings": [] }, @@ -9235,13 +11786,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.07196804200066254, + "seconds": 0.17166645899851574, "eta_squared": 0.44964427142161606, "warnings": [] }, @@ -9252,13 +11803,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7126680000001215, + "seconds": 1.3952178340041428, "eta_squared": 0.44964427142161606, "warnings": [] }, @@ -9269,13 +11820,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, + "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.07681637499990757, + "seconds": 0.1660061250004219, "eta_squared": 0.4469631140734328, "warnings": [] }, @@ -9286,13 +11837,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.0704585000021325, + "seconds": 0.16690162499435246, "eta_squared": 0.4469631140734328, "warnings": [] }, @@ -9303,13 +11854,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9982327768083502, - "roc_auc": 0.9999645691609977, + "pr_auc": 0.9989319838882099, + "roc_auc": 0.9999778557256236, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7689973329979694, + "seconds": 1.3839427919956506, "eta_squared": 0.4469631140734328, "warnings": [] }, @@ -9320,13 +11871,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, + "worst_group_fpr": 0.18622448979591838, "n_models": 1, - "seconds": 0.06915779200062389, + "seconds": 0.17876699999760604, "eta_squared": 0.449549676187598, "warnings": [] }, @@ -9337,13 +11888,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.07148883300033049, + "seconds": 0.17536795899650315, "eta_squared": 0.449549676187598, "warnings": [] }, @@ -9354,13 +11905,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.99935217360804, - "roc_auc": 0.999986713435374, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7156301670001994, + "seconds": 1.4285302920034155, "eta_squared": 0.449549676187598, "warnings": [] }, @@ -9371,13 +11922,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23214285714285715, + "worst_group_fpr": 0.20918367346938777, "n_models": 1, - "seconds": 0.07223383399832528, + "seconds": 0.1689662920034607, "eta_squared": 0.45043297969341634, "warnings": [] }, @@ -9389,12 +11940,12 @@ "mechanism": "global", "level_spread": 0.4, "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07250633300282061, + "seconds": 0.16670979099581018, "eta_squared": 0.45043297969341634, "warnings": [] }, @@ -9405,13 +11956,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7102636250019714, + "seconds": 1.4172910420020344, "eta_squared": 0.45043297969341634, "warnings": [] }, @@ -9426,9 +11977,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17091836734693877, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.07457300000169198, + "seconds": 0.17847641700063832, "eta_squared": 0.4473170350696837, "warnings": [] }, @@ -9439,13 +11990,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "pr_auc": 1.0, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.07632920900141471, + "seconds": 0.18600770799821476, "eta_squared": 0.4473170350696837, "warnings": [] }, @@ -9456,13 +12007,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9995636400137603, - "roc_auc": 0.9999911422902494, + "pr_auc": 0.9994516328453015, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7027896670006157, + "seconds": 1.3435870000030263, "eta_squared": 0.4473170350696837, "warnings": [] }, @@ -9473,13 +12024,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18112244897959184, + "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.07392875000005006, + "seconds": 0.17356312499759952, "eta_squared": 0.4449952275639174, "warnings": [] }, @@ -9490,13 +12041,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.07942145800188882, + "seconds": 0.16124245800165227, "eta_squared": 0.4449952275639174, "warnings": [] }, @@ -9507,13 +12058,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9990030018845484, - "roc_auc": 0.9999800701530612, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, + "pr_auc": 0.997792771646775, + "roc_auc": 0.9999579258786848, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7168002500002331, + "seconds": 1.3857422920045792, "eta_squared": 0.4449952275639174, "warnings": [] }, @@ -9524,13 +12075,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9994516328453015, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.07417616699967766, + "seconds": 0.1697561249966384, "eta_squared": 0.4435643816373198, "warnings": [] }, @@ -9541,13 +12092,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.07187375000285101, + "seconds": 0.15726891700614942, "eta_squared": 0.4435643816373198, "warnings": [] }, @@ -9558,13 +12109,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9981328388755407, - "roc_auc": 0.9999645691609979, + "pr_auc": 0.9978990413346327, + "roc_auc": 0.9999601403061225, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9948950066137566, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.8298017499982961, + "seconds": 1.3263949579995824, "eta_squared": 0.4435643816373198, "warnings": [] }, @@ -9579,9 +12130,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18622448979591838, + "worst_group_fpr": 0.19642857142857142, "n_models": 1, - "seconds": 0.07912500000020373, + "seconds": 0.16596974999993108, "eta_squared": 0.4514009747359702, "warnings": [] }, @@ -9592,13 +12143,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.07275858399952995, + "seconds": 0.16418912500375882, "eta_squared": 0.4514009747359702, "warnings": [] }, @@ -9609,13 +12160,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9985821949802475, - "roc_auc": 0.9999712124433106, + "pr_auc": 0.9984642313812274, + "roc_auc": 0.9999689980158731, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7548948750009004, + "seconds": 1.3550333340026555, "eta_squared": 0.4514009747359702, "warnings": [] }, @@ -9632,7 +12183,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.1913265306122449, "n_models": 1, - "seconds": 0.07519174999833922, + "seconds": 0.1597828340018168, "eta_squared": 0.4503463925177883, "warnings": [] }, @@ -9643,13 +12194,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.07878029199855519, + "seconds": 0.16455620900524082, "eta_squared": 0.4503463925177883, "warnings": [] }, @@ -9660,13 +12211,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9955908290925521, - "roc_auc": 0.9999247094671203, + "pr_auc": 0.9973362833817404, + "roc_auc": 0.9999512825963719, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9931588955026456, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7416448750009295, + "seconds": 1.4011888330060174, "eta_squared": 0.4503463925177883, "warnings": [] }, @@ -9681,9 +12232,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18877551020408162, + "worst_group_fpr": 0.1683673469387755, "n_models": 1, - "seconds": 0.06985316700229305, + "seconds": 0.16978279199975077, "eta_squared": 0.4460711711615254, "warnings": [] }, @@ -9695,12 +12246,12 @@ "mechanism": "global", "level_spread": 0.4, "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.0776381669966213, + "seconds": 0.17047620799712604, "eta_squared": 0.4460711711615254, "warnings": [] }, @@ -9711,13 +12262,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7081212079974648, + "seconds": 1.3659657090029214, "eta_squared": 0.4460711711615254, "warnings": [] }, @@ -9728,13 +12279,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, + "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.07157841700245626, + "seconds": 0.16213725000125123, "eta_squared": 0.44924129916852445, "warnings": [] }, @@ -9749,9 +12300,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.07996595800068462, + "seconds": 0.16343062500527594, "eta_squared": 0.44924129916852445, "warnings": [] }, @@ -9762,13 +12313,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9991450011576791, - "roc_auc": 0.9999822845804989, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.721172416000627, + "seconds": 1.3577794580050977, "eta_squared": 0.44924129916852445, "warnings": [] }, @@ -9779,13 +12330,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, + "pr_auc": 0.9996789080215417, + "roc_auc": 0.9999933567176871, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.14795918367346939, "n_models": 1, - "seconds": 0.06692212500274763, + "seconds": 0.18308925000019372, "eta_squared": 0.44859304750127427, "warnings": [] }, @@ -9796,13 +12347,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.07908163265306123, "n_models": 1, - "seconds": 0.07617491600103676, + "seconds": 0.15952333299355814, "eta_squared": 0.44859304750127427, "warnings": [] }, @@ -9813,13 +12364,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.996324417621576, - "roc_auc": 0.9999357816043084, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9950648711286866, + "roc_auc": 0.999913637329932, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7081812920005177, + "seconds": 1.3622961250002845, "eta_squared": 0.44859304750127427, "warnings": [] }, @@ -9834,9 +12385,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.19642857142857142, + "worst_group_fpr": 0.18112244897959184, "n_models": 1, - "seconds": 0.06883758299954934, + "seconds": 0.17267058399738744, "eta_squared": 0.4514629140297388, "warnings": [] }, @@ -9853,7 +12404,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.07397959183673469, "n_models": 1, - "seconds": 0.06850795800346532, + "seconds": 0.16739333299483405, "eta_squared": 0.4514629140297388, "warnings": [] }, @@ -9864,13 +12415,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7134700000024168, + "seconds": 1.3257128330005798, "eta_squared": 0.4514629140297388, "warnings": [] }, @@ -9881,13 +12432,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9997841047394044, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, + "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.07810383400283172, + "seconds": 0.165811291000864, "eta_squared": 0.4491049550478841, "warnings": [] }, @@ -9898,13 +12449,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07709904199873563, + "seconds": 0.17129149999527726, "eta_squared": 0.4491049550478841, "warnings": [] }, @@ -9915,13 +12466,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9996789080215419, - "roc_auc": 0.9999933567176872, + "pr_auc": 0.999233211489758, + "roc_auc": 0.9999844990079364, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7326332919983543, + "seconds": 1.3227324589970522, "eta_squared": 0.4491049550478841, "warnings": [] }, @@ -9932,13 +12483,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18877551020408162, + "worst_group_fpr": 0.1913265306122449, "n_models": 1, - "seconds": 0.07244779199754703, + "seconds": 0.16286304200184532, "eta_squared": 0.44655634205910016, "warnings": [] }, @@ -9950,12 +12501,12 @@ "mechanism": "global", "level_spread": 0.4, "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07034012499934761, + "seconds": 0.17305812499398598, "eta_squared": 0.44655634205910016, "warnings": [] }, @@ -9966,13 +12517,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.4, - "pr_auc": 0.9996744556165973, - "roc_auc": 0.999993356717687, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7143351250015257, + "seconds": 1.3466750830048113, "eta_squared": 0.44655634205910016, "warnings": [] }, @@ -9983,13 +12534,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.919494070332322, - "roc_auc": 0.9988728564342403, + "pr_auc": 0.8767362769226793, + "roc_auc": 0.9984986181972789, "precision_at_n": 0.8854166666666666, - "macro_pr_auc": 0.9884958791208791, - "worst_group_fpr": 0.25255102040816324, + "macro_pr_auc": 0.9861111111111112, + "worst_group_fpr": 0.23214285714285715, "n_models": 1, - "seconds": 0.07150508300037473, + "seconds": 0.16523579199565575, "eta_squared": 0.7723857444189095, "warnings": [] }, @@ -10000,13 +12551,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.16071428571428573, "n_models": 1, - "seconds": 0.08206308399894624, + "seconds": 0.16723662499862257, "eta_squared": 0.7723857444189095, "warnings": [] }, @@ -10017,13 +12568,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7402828750018671, + "seconds": 1.3786440420008148, "eta_squared": 0.7723857444189095, "warnings": [] }, @@ -10034,13 +12585,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9222452035785629, - "roc_auc": 0.999171804138322, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9704387626262626, - "worst_group_fpr": 0.19387755102040816, + "pr_auc": 0.9248102078838638, + "roc_auc": 0.999282525510204, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9731714466089466, + "worst_group_fpr": 0.2066326530612245, "n_models": 1, - "seconds": 0.08093129199914983, + "seconds": 0.1626497079996625, "eta_squared": 0.7711687637922273, "warnings": [] }, @@ -10051,13 +12602,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07064316700052586, + "seconds": 0.1611572920010076, "eta_squared": 0.7711687637922273, "warnings": [] }, @@ -10068,13 +12619,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9523907508279281, - "roc_auc": 0.9992980265022675, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7176182500006689, + "seconds": 1.4567947500036098, "eta_squared": 0.7711687637922273, "warnings": [] }, @@ -10085,13 +12636,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9502640920205745, - "roc_auc": 0.9993002409297052, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9971590909090909, - "worst_group_fpr": 0.22193877551020408, + "pr_auc": 0.8950775411242928, + "roc_auc": 0.998813066893424, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9839725378787879, + "worst_group_fpr": 0.2193877551020408, "n_models": 1, - "seconds": 0.07786358299927088, + "seconds": 0.16679279199888697, "eta_squared": 0.772529544751337, "warnings": [] }, @@ -10102,13 +12653,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.07384237499718438, + "seconds": 0.15794870800164063, "eta_squared": 0.772529544751337, "warnings": [] }, @@ -10119,13 +12670,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767568942402173, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7333828750015527, + "seconds": 1.380221416002314, "eta_squared": 0.772529544751337, "warnings": [] }, @@ -10136,13 +12687,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.8310530351619684, - "roc_auc": 0.9972474666950113, - "precision_at_n": 0.7708333333333334, - "macro_pr_auc": 0.9805530118030119, - "worst_group_fpr": 0.24489795918367346, + "pr_auc": 0.8759932708819158, + "roc_auc": 0.9980269451530612, + "precision_at_n": 0.7916666666666666, + "macro_pr_auc": 0.9796400534851623, + "worst_group_fpr": 0.23469387755102042, "n_models": 1, - "seconds": 0.0734578750016226, + "seconds": 0.17111045800265856, "eta_squared": 0.7743402728249429, "warnings": [] }, @@ -10153,13 +12704,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.07175804199869162, + "seconds": 0.16481958399526775, "eta_squared": 0.7743402728249429, "warnings": [] }, @@ -10170,13 +12721,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7108539579967328, + "seconds": 1.3571281249969616, "eta_squared": 0.7743402728249429, "warnings": [] }, @@ -10187,13 +12738,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9504117563479513, - "roc_auc": 0.9995128259637187, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9752645502645502, - "worst_group_fpr": 0.20918367346938777, + "pr_auc": 0.909387757786931, + "roc_auc": 0.9989791489512472, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9601960828523328, + "worst_group_fpr": 0.19642857142857142, "n_models": 1, - "seconds": 0.06905545900008292, + "seconds": 0.16465991699806182, "eta_squared": 0.7712908836823555, "warnings": [] }, @@ -10204,13 +12755,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.07561912500023027, + "seconds": 0.166659125003207, "eta_squared": 0.7712908836823555, "warnings": [] }, @@ -10221,13 +12772,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.741537582998717, + "seconds": 1.398124542000005, "eta_squared": 0.7712908836823555, "warnings": [] }, @@ -10238,13 +12789,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9157899293207066, - "roc_auc": 0.9991784474206349, - "precision_at_n": 0.9375, + "pr_auc": 0.9090194390578653, + "roc_auc": 0.9991053713151927, + "precision_at_n": 0.9270833333333334, "macro_pr_auc": 0.992205710955711, - "worst_group_fpr": 0.23979591836734693, + "worst_group_fpr": 0.23214285714285715, "n_models": 1, - "seconds": 0.07177312500061817, + "seconds": 0.16571400000248104, "eta_squared": 0.7661755359039057, "warnings": [] }, @@ -10255,13 +12806,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.07254974999887054, + "seconds": 0.17354329100635368, "eta_squared": 0.7661755359039057, "warnings": [] }, @@ -10272,13 +12823,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7203230000013718, + "seconds": 1.4361895830006688, "eta_squared": 0.7661755359039057, "warnings": [] }, @@ -10289,13 +12840,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9714622579844718, - "roc_auc": 0.9994818239795918, - "precision_at_n": 0.90625, + "pr_auc": 0.9886393372479427, + "roc_auc": 0.9997940582482994, + "precision_at_n": 0.9583333333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18877551020408162, + "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.07678883300104644, + "seconds": 0.18052250000619097, "eta_squared": 0.765390043535227, "warnings": [] }, @@ -10306,13 +12857,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.06896087499990244, + "seconds": 0.173466083004314, "eta_squared": 0.765390043535227, "warnings": [] }, @@ -10323,13 +12874,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.72620470900074, + "seconds": 1.9007839999976568, "eta_squared": 0.765390043535227, "warnings": [] }, @@ -10340,13 +12891,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.8365449171531172, - "roc_auc": 0.997674851190476, - "precision_at_n": 0.8020833333333334, - "macro_pr_auc": 0.9779265873015873, - "worst_group_fpr": 0.21683673469387754, + "pr_auc": 0.867928879095778, + "roc_auc": 0.9982063137755103, + "precision_at_n": 0.8333333333333334, + "macro_pr_auc": 0.983447570947571, + "worst_group_fpr": 0.22448979591836735, "n_models": 1, - "seconds": 0.06944787499742233, + "seconds": 0.178036917001009, "eta_squared": 0.7725358121772077, "warnings": [] }, @@ -10357,13 +12908,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.0715800420002779, + "seconds": 0.15969991699967068, "eta_squared": 0.7725358121772077, "warnings": [] }, @@ -10374,13 +12925,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7381474160029029, + "seconds": 1.3786372500035213, "eta_squared": 0.7725358121772077, "warnings": [] }, @@ -10391,13 +12942,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9533739644744637, + "pr_auc": 0.946962425632692, "roc_auc": 0.9993644593253969, "precision_at_n": 0.9375, - "macro_pr_auc": 0.9951264880952381, - "worst_group_fpr": 0.2193877551020408, + "macro_pr_auc": 0.9911789021164021, + "worst_group_fpr": 0.22193877551020408, "n_models": 1, - "seconds": 0.07220670799870277, + "seconds": 0.15973129200574476, "eta_squared": 0.7740110430937635, "warnings": [] }, @@ -10408,13 +12959,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.06808333399749245, + "seconds": 0.1722696659999201, "eta_squared": 0.7740110430937635, "warnings": [] }, @@ -10425,13 +12976,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7301062500009721, + "seconds": 1.4021838749977178, "eta_squared": 0.7740110430937635, "warnings": [] }, @@ -10442,13 +12993,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9609648437754886, - "roc_auc": 0.9993556016156463, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9879133597883598, - "worst_group_fpr": 0.21428571428571427, + "pr_auc": 0.9838845010577177, + "roc_auc": 0.9997298398526078, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9979166666666667, + "worst_group_fpr": 0.19642857142857142, "n_models": 1, - "seconds": 0.07667612499790266, + "seconds": 0.16683854199800408, "eta_squared": 0.7703322802500099, "warnings": [] }, @@ -10463,9 +13014,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.08043412499682745, + "seconds": 0.16663541599700693, "eta_squared": 0.7703322802500099, "warnings": [] }, @@ -10476,13 +13027,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7481633329989563, + "seconds": 1.3506795840003178, "eta_squared": 0.7703322802500099, "warnings": [] }, @@ -10493,13 +13044,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.923013720180947, - "roc_auc": 0.999109800170068, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9822616041366041, - "worst_group_fpr": 0.19642857142857142, + "pr_auc": 0.9422513950453302, + "roc_auc": 0.9994153911564626, + "precision_at_n": 0.9270833333333334, + "macro_pr_auc": 0.9862959956709956, + "worst_group_fpr": 0.18877551020408162, "n_models": 1, - "seconds": 0.06946004199926392, + "seconds": 0.16146966700034682, "eta_squared": 0.7727219503559253, "warnings": [] }, @@ -10514,9 +13065,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.07302362500195159, + "seconds": 0.16613545799918938, "eta_squared": 0.7727219503559253, "warnings": [] }, @@ -10527,13 +13078,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7149331670007086, + "seconds": 1.3736709590011742, "eta_squared": 0.7727219503559253, "warnings": [] }, @@ -10544,13 +13095,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.8567167741821362, - "roc_auc": 0.998764349489796, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9580439814814815, + "pr_auc": 0.847124125101395, + "roc_auc": 0.9986802012471656, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9559132996632996, "worst_group_fpr": 0.1913265306122449, "n_models": 1, - "seconds": 0.07800854200104368, + "seconds": 0.17574216699722456, "eta_squared": 0.769516047151419, "warnings": [] }, @@ -10561,13 +13112,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07188725000014529, + "seconds": 0.16795120800088625, "eta_squared": 0.769516047151419, "warnings": [] }, @@ -10578,13 +13129,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9619580084145981, - "roc_auc": 0.99949289611678, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7189485419985431, + "seconds": 1.3588489169997047, "eta_squared": 0.769516047151419, "warnings": [] }, @@ -10595,13 +13146,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9995692588987349, - "roc_auc": 0.9999911422902494, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9870162732643589, + "roc_auc": 0.9997918438208616, + "precision_at_n": 0.96875, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20918367346938777, + "worst_group_fpr": 0.21683673469387754, "n_models": 1, - "seconds": 0.07393587499973364, + "seconds": 0.1588478749981732, "eta_squared": 0.7753234694608296, "warnings": [] }, @@ -10616,9 +13167,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.06857591600055457, + "seconds": 0.1671964580018539, "eta_squared": 0.7753234694608296, "warnings": [] }, @@ -10629,13 +13180,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7238538749988948, + "seconds": 1.3748501669979305, "eta_squared": 0.7753234694608296, "warnings": [] }, @@ -10646,13 +13197,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9323072531442229, - "roc_auc": 0.9991297300170068, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9873594576719578, - "worst_group_fpr": 0.20153061224489796, + "pr_auc": 0.8824466841761875, + "roc_auc": 0.9982483878968255, + "precision_at_n": 0.8125, + "macro_pr_auc": 0.988420664983165, + "worst_group_fpr": 0.21173469387755103, "n_models": 1, - "seconds": 0.07103625000308966, + "seconds": 0.15993070900003659, "eta_squared": 0.7678020127711631, "warnings": [] }, @@ -10663,13 +13214,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07522237499870243, + "seconds": 0.16797495899663772, "eta_squared": 0.7678020127711631, "warnings": [] }, @@ -10680,13 +13231,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.9329070122690577, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7089806250005495, + "seconds": 1.383623333000287, "eta_squared": 0.7678020127711631, "warnings": [] }, @@ -10697,13 +13248,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.8998203771520226, - "roc_auc": 0.9989857922335601, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9762887286324786, - "worst_group_fpr": 0.20153061224489796, + "pr_auc": 0.9087040584314307, + "roc_auc": 0.9991275155895691, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.978131764069264, + "worst_group_fpr": 0.20918367346938777, "n_models": 1, - "seconds": 0.07112204199802363, + "seconds": 0.17611595900234533, "eta_squared": 0.7715725614681234, "warnings": [] }, @@ -10718,9 +13269,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.07155020799837075, + "seconds": 0.17582254100125283, "eta_squared": 0.7715725614681234, "warnings": [] }, @@ -10731,13 +13282,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.4, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7092746669986809, + "seconds": 1.3735412919995724, "eta_squared": 0.7715725614681234, "warnings": [] }, @@ -10748,13 +13299,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21683673469387754, + "worst_group_fpr": 0.22959183673469388, "n_models": 1, - "seconds": 0.07030904099883628, + "seconds": 0.17748766699514817, "eta_squared": 0.4857308726173069, "warnings": [] }, @@ -10765,13 +13316,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.07049504200040246, + "seconds": 0.16957866700249724, "eta_squared": 0.4857308726173069, "warnings": [] }, @@ -10782,13 +13333,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6890660829994886, + "seconds": 1.3515388750020065, "eta_squared": 0.4857308726173069, "warnings": [] }, @@ -10803,9 +13354,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, + "worst_group_fpr": 0.1989795918367347, "n_models": 1, - "seconds": 0.06879945900072926, + "seconds": 0.1621778330008965, "eta_squared": 0.4832253415278028, "warnings": [] }, @@ -10816,13 +13367,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.06889812500230619, + "seconds": 0.1764309160062112, "eta_squared": 0.4832253415278028, "warnings": [] }, @@ -10833,13 +13384,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9979007406391652, - "roc_auc": 0.9999579258786848, + "pr_auc": 0.998701026538696, + "roc_auc": 0.9999734268707483, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7033612500017625, + "seconds": 1.350652999994054, "eta_squared": 0.4832253415278028, "warnings": [] }, @@ -10850,13 +13401,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23979591836734693, + "worst_group_fpr": 0.22193877551020408, "n_models": 1, - "seconds": 0.0700648749989341, + "seconds": 0.16132041699893307, "eta_squared": 0.48660199721043473, "warnings": [] }, @@ -10867,13 +13418,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.07226320799964014, + "seconds": 0.17449829100223724, "eta_squared": 0.48660199721043473, "warnings": [] }, @@ -10884,13 +13435,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9992448088419579, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7190990830022201, + "seconds": 1.371311083996261, "eta_squared": 0.48660199721043473, "warnings": [] }, @@ -10901,13 +13452,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25255102040816324, + "worst_group_fpr": 0.2193877551020408, "n_models": 1, - "seconds": 0.07167383399792016, + "seconds": 0.1692836670044926, "eta_squared": 0.4871724603545716, "warnings": [] }, @@ -10924,7 +13475,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.07642195800144691, + "seconds": 0.17022404100134736, "eta_squared": 0.4871724603545716, "warnings": [] }, @@ -10935,13 +13486,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7078079580023768, + "seconds": 1.4007270829970366, "eta_squared": 0.4871724603545716, "warnings": [] }, @@ -10952,13 +13503,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16326530612244897, + "worst_group_fpr": 0.1836734693877551, "n_models": 1, - "seconds": 0.07651633400018909, + "seconds": 0.16118583300703904, "eta_squared": 0.4835220144166391, "warnings": [] }, @@ -10969,13 +13520,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, + "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.0694544580001093, + "seconds": 0.17091954199713655, "eta_squared": 0.4835220144166391, "warnings": [] }, @@ -10986,13 +13537,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9995636400137602, - "roc_auc": 0.9999911422902494, + "pr_auc": 0.9994516328453015, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.6862827090008068, + "seconds": 1.3620432499956223, "eta_squared": 0.4835220144166391, "warnings": [] }, @@ -11003,13 +13554,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9998926116838486, - "roc_auc": 0.9999977855725624, + "pr_auc": 0.9997874149659862, + "roc_auc": 0.9999955711451247, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1989795918367347, + "worst_group_fpr": 0.19387755102040816, "n_models": 1, - "seconds": 0.07201791599800345, + "seconds": 0.17449883299559588, "eta_squared": 0.4827570725669257, "warnings": [] }, @@ -11024,9 +13575,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.06927183299922035, + "seconds": 0.16171612500329502, "eta_squared": 0.4827570725669257, "warnings": [] }, @@ -11037,13 +13588,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9994516328453016, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9982979149346853, + "roc_auc": 0.9999667835884353, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.727608708999469, + "seconds": 1.357203749998007, "eta_squared": 0.4827570725669257, "warnings": [] }, @@ -11054,13 +13605,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999997, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.19642857142857142, + "worst_group_fpr": 0.1836734693877551, "n_models": 1, - "seconds": 0.07113816700075404, + "seconds": 0.17087583299871767, "eta_squared": 0.4811183421127033, "warnings": [] }, @@ -11071,13 +13622,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07138120899981004, + "seconds": 0.16551808299846016, "eta_squared": 0.4811183421127033, "warnings": [] }, @@ -11088,13 +13639,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9980042380524952, - "roc_auc": 0.9999623547335601, + "pr_auc": 0.9976086261705307, + "roc_auc": 0.9999557114512472, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7304766249981185, + "seconds": 1.3765239999993355, "eta_squared": 0.4811183421127033, "warnings": [] }, @@ -11105,13 +13656,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18622448979591838, + "worst_group_fpr": 0.20408163265306123, "n_models": 1, - "seconds": 0.06869233299948974, + "seconds": 0.1601575419990695, "eta_squared": 0.48752906081921704, "warnings": [] }, @@ -11122,13 +13673,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.07400212500215275, + "seconds": 0.16759574999741744, "eta_squared": 0.48752906081921704, "warnings": [] }, @@ -11139,13 +13690,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9983640883002555, - "roc_auc": 0.9999667835884354, + "pr_auc": 0.9984642313812274, + "roc_auc": 0.999968998015873, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7849107920010283, + "seconds": 1.3940407500049332, "eta_squared": 0.48752906081921704, "warnings": [] }, @@ -11157,12 +13708,12 @@ "mechanism": "global", "level_spread": 0.5, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2423469387755102, + "worst_group_fpr": 0.22704081632653061, "n_models": 1, - "seconds": 0.08595991699985461, + "seconds": 0.16296787500323262, "eta_squared": 0.4858291404456496, "warnings": [] }, @@ -11177,9 +13728,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.08572004199959338, + "seconds": 0.175631334001082, "eta_squared": 0.4858291404456496, "warnings": [] }, @@ -11190,13 +13741,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9973362833817403, - "roc_auc": 0.9999512825963719, + "pr_auc": 0.9982598713958657, + "roc_auc": 0.9999667835884354, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, + "macro_pr_auc": 0.9975405092592592, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7693029170004593, + "seconds": 1.3584863340001903, "eta_squared": 0.4858291404456496, "warnings": [] }, @@ -11207,13 +13758,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18877551020408162, + "worst_group_fpr": 0.1836734693877551, "n_models": 1, - "seconds": 0.08046441700207652, + "seconds": 0.16996641700097825, "eta_squared": 0.4825689121407673, "warnings": [] }, @@ -11224,13 +13775,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07818054200106417, + "seconds": 0.17013387499901, "eta_squared": 0.4825689121407673, "warnings": [] }, @@ -11241,13 +13792,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7497013749998587, + "seconds": 1.3519318329999805, "eta_squared": 0.4825689121407673, "warnings": [] }, @@ -11258,13 +13809,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, + "worst_group_fpr": 0.1683673469387755, "n_models": 1, - "seconds": 0.07646537500113482, + "seconds": 0.1744903750004596, "eta_squared": 0.48575387175214657, "warnings": [] }, @@ -11275,13 +13826,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.07978204200117034, + "seconds": 0.17946716699952958, "eta_squared": 0.48575387175214657, "warnings": [] }, @@ -11292,13 +13843,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7509155000007013, + "seconds": 1.3484732500000973, "eta_squared": 0.48575387175214657, "warnings": [] }, @@ -11313,9 +13864,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18622448979591838, + "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.08028016699972795, + "seconds": 0.17183425000257557, "eta_squared": 0.4844186055700644, "warnings": [] }, @@ -11326,13 +13877,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, + "worst_group_fpr": 0.08163265306122448, "n_models": 1, - "seconds": 0.0883259160000307, + "seconds": 0.171107000001939, "eta_squared": 0.4844186055700644, "warnings": [] }, @@ -11343,13 +13894,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9939481572022735, - "roc_auc": 0.9999047796201813, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9945643187830688, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.993313456184553, + "roc_auc": 0.9998914930555556, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 1.1186415419979312, + "seconds": 1.37635274999775, "eta_squared": 0.4844186055700644, "warnings": [] }, @@ -11360,13 +13911,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21173469387755103, + "worst_group_fpr": 0.19387755102040816, "n_models": 1, - "seconds": 0.143989625001268, + "seconds": 0.16494270800467348, "eta_squared": 0.48710566059576565, "warnings": [] }, @@ -11377,13 +13928,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.08673469387755102, "n_models": 1, - "seconds": 0.08993958299834048, + "seconds": 0.17022479099978227, "eta_squared": 0.48710566059576565, "warnings": [] }, @@ -11394,13 +13945,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7706514999990759, + "seconds": 1.388493833001121, "eta_squared": 0.48710566059576565, "warnings": [] }, @@ -11415,9 +13966,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.16326530612244897, "n_models": 1, - "seconds": 0.07688920899818186, + "seconds": 0.16758466600003885, "eta_squared": 0.4855240340488236, "warnings": [] }, @@ -11429,12 +13980,12 @@ "mechanism": "global", "level_spread": 0.5, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.08556545800092863, + "seconds": 0.16482770800212165, "eta_squared": 0.4855240340488236, "warnings": [] }, @@ -11445,13 +13996,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9996789080215418, - "roc_auc": 0.9999933567176871, + "pr_auc": 0.9993384082076202, + "roc_auc": 0.9999867134353742, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7525589169999876, + "seconds": 1.3600303330022143, "eta_squared": 0.4855240340488236, "warnings": [] }, @@ -11466,9 +14017,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20408163265306123, + "worst_group_fpr": 0.2066326530612245, "n_models": 1, - "seconds": 0.08668029100226704, + "seconds": 0.16833912500442239, "eta_squared": 0.48320943890358836, "warnings": [] }, @@ -11479,13 +14030,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.08095079199847532, + "seconds": 0.16610700000455836, "eta_squared": 0.48320943890358836, "warnings": [] }, @@ -11496,13 +14047,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.5, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7630562089980231, + "seconds": 1.346856915995886, "eta_squared": 0.48320943890358836, "warnings": [] }, @@ -11513,13 +14064,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8205571978459951, - "roc_auc": 0.9977169253117913, - "precision_at_n": 0.8020833333333334, - "macro_pr_auc": 0.9679937394781145, - "worst_group_fpr": 0.25, + "pr_auc": 0.822434544724292, + "roc_auc": 0.9979582979024944, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9709017255892256, + "worst_group_fpr": 0.25255102040816324, "n_models": 1, - "seconds": 0.07825237499855575, + "seconds": 0.1645924170006765, "eta_squared": 0.8366350521930952, "warnings": [] }, @@ -11534,9 +14085,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, + "worst_group_fpr": 0.16071428571428573, "n_models": 1, - "seconds": 0.08690137499797856, + "seconds": 0.16921308300516102, "eta_squared": 0.8366350521930952, "warnings": [] }, @@ -11547,13 +14098,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7718835000014224, + "seconds": 1.3897101670008851, "eta_squared": 0.8366350521930952, "warnings": [] }, @@ -11564,13 +14115,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8921185258048079, - "roc_auc": 0.9989392892573696, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9646262591575092, - "worst_group_fpr": 0.21683673469387754, + "pr_auc": 0.9257995892782048, + "roc_auc": 0.9992958120748299, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9727839052287582, + "worst_group_fpr": 0.22704081632653061, "n_models": 1, - "seconds": 0.08617970899649663, + "seconds": 0.1708017499986454, "eta_squared": 0.8357517463861228, "warnings": [] }, @@ -11581,13 +14132,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9997841047394042, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.07478454199736007, + "seconds": 0.16030687500460772, "eta_squared": 0.8357517463861228, "warnings": [] }, @@ -11598,13 +14149,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.999289168792517, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7834026669988816, + "seconds": 1.3927423749992158, "eta_squared": 0.8357517463861228, "warnings": [] }, @@ -11615,13 +14166,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9112206394512736, - "roc_auc": 0.9989791489512472, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9868371212121212, - "worst_group_fpr": 0.24744897959183673, + "pr_auc": 0.897939798991374, + "roc_auc": 0.9987488484977324, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9859623015873016, + "worst_group_fpr": 0.24489795918367346, "n_models": 1, - "seconds": 0.08341195799948764, + "seconds": 0.16611995799758006, "eta_squared": 0.8368167743920167, "warnings": [] }, @@ -11636,9 +14187,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.08615370800180244, + "seconds": 0.17989779200433986, "eta_squared": 0.8368167743920167, "warnings": [] }, @@ -11649,13 +14200,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7578562080016127, + "seconds": 1.3759323339982075, "eta_squared": 0.8368167743920167, "warnings": [] }, @@ -11666,13 +14217,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8788587192526313, - "roc_auc": 0.9977080676020409, - "precision_at_n": 0.7916666666666666, - "macro_pr_auc": 0.9920460037647537, - "worst_group_fpr": 0.2780612244897959, + "pr_auc": 0.8883344359076468, + "roc_auc": 0.998146524234694, + "precision_at_n": 0.8125, + "macro_pr_auc": 0.9882265593203092, + "worst_group_fpr": 0.2602040816326531, "n_models": 1, - "seconds": 0.07655620900186477, + "seconds": 0.172415250002814, "eta_squared": 0.8382354298920958, "warnings": [] }, @@ -11683,13 +14234,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 1.0, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.13520408163265307, "n_models": 1, - "seconds": 0.08366674999706447, + "seconds": 0.17383608299860498, "eta_squared": 0.8382354298920958, "warnings": [] }, @@ -11700,13 +14251,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7506139580000308, + "seconds": 1.4798644580005202, "eta_squared": 0.8382354298920958, "warnings": [] }, @@ -11717,13 +14268,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9772469572383853, - "roc_auc": 0.999796272675737, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.1913265306122449, + "pr_auc": 0.9580139997216633, + "roc_auc": 0.9995349702380952, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9852306547619047, + "worst_group_fpr": 0.19387755102040816, "n_models": 1, - "seconds": 0.07961420900028315, + "seconds": 0.17657287499605445, "eta_squared": 0.8359843649672052, "warnings": [] }, @@ -11734,13 +14285,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.08680995800023084, + "seconds": 0.17390804200113053, "eta_squared": 0.8359843649672052, "warnings": [] }, @@ -11751,13 +14302,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7827762499982782, + "seconds": 1.4002181249961723, "eta_squared": 0.8359843649672052, "warnings": [] }, @@ -11768,13 +14319,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8981325648851586, - "roc_auc": 0.9987931370464852, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9958570075757577, - "worst_group_fpr": 0.2729591836734694, + "pr_auc": 0.8633742318610074, + "roc_auc": 0.9983901112528345, + "precision_at_n": 0.875, + "macro_pr_auc": 0.9907986111111112, + "worst_group_fpr": 0.2576530612244898, "n_models": 1, - "seconds": 0.07629529100086074, + "seconds": 0.17919729099958204, "eta_squared": 0.8321590306687349, "warnings": [] }, @@ -11785,13 +14336,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.08843254100065678, + "seconds": 0.16210970800602809, "eta_squared": 0.8321590306687349, "warnings": [] }, @@ -11802,13 +14353,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7516903329997149, + "seconds": 1.3868609579949407, "eta_squared": 0.8321590306687349, "warnings": [] }, @@ -11819,13 +14370,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9138970246678486, - "roc_auc": 0.9986912733843537, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9971590909090909, - "worst_group_fpr": 0.20918367346938777, + "pr_auc": 0.9367261157789226, + "roc_auc": 0.999165160856009, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9952256944444445, + "worst_group_fpr": 0.19642857142857142, "n_models": 1, - "seconds": 0.0761759160013753, + "seconds": 0.16814787500334205, "eta_squared": 0.8314517992187488, "warnings": [] }, @@ -11840,9 +14391,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.1326530612244898, "n_models": 1, - "seconds": 0.0872115419988404, + "seconds": 0.1755587919979007, "eta_squared": 0.8314517992187488, "warnings": [] }, @@ -11853,13 +14404,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7406817920018511, + "seconds": 1.385932999997749, "eta_squared": 0.8314517992187488, "warnings": [] }, @@ -11870,13 +14421,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.7751847692274307, - "roc_auc": 0.9966407135770975, - "precision_at_n": 0.7395833333333334, - "macro_pr_auc": 0.9787990944240944, + "pr_auc": 0.8402001419290229, + "roc_auc": 0.9976859233276644, + "precision_at_n": 0.8020833333333334, + "macro_pr_auc": 0.980082335964689, "worst_group_fpr": 0.2423469387755102, "n_models": 1, - "seconds": 0.08818870799950673, + "seconds": 0.17706429099780507, "eta_squared": 0.8367353195642477, "warnings": [] }, @@ -11887,13 +14438,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, + "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.07610624999870197, + "seconds": 0.16533629199693678, "eta_squared": 0.8367353195642477, "warnings": [] }, @@ -11904,13 +14455,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7761402079995605, + "seconds": 1.368197916999634, "eta_squared": 0.8367353195642477, "warnings": [] }, @@ -11921,13 +14472,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9286330254336417, - "roc_auc": 0.9988285678854876, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9966145833333333, - "worst_group_fpr": 0.22448979591836735, + "pr_auc": 0.9448111276375721, + "roc_auc": 0.9991585175736962, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.23214285714285715, "n_models": 1, - "seconds": 0.0843877919978695, + "seconds": 0.16717541700199945, "eta_squared": 0.83775825552584, "warnings": [] }, @@ -11939,12 +14490,12 @@ "mechanism": "contextual", "level_spread": 0.5, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, + "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.07827283399819862, + "seconds": 0.16732737499842187, "eta_squared": 0.83775825552584, "warnings": [] }, @@ -11955,13 +14506,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7463412920005794, + "seconds": 1.3741866670025047, "eta_squared": 0.83775825552584, "warnings": [] }, @@ -11972,13 +14523,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9293409565936561, - "roc_auc": 0.999171804138322, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9826007326007327, - "worst_group_fpr": 0.2423469387755102, + "pr_auc": 0.9629432775616903, + "roc_auc": 0.9995083971088435, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9958570075757577, + "worst_group_fpr": 0.22704081632653061, "n_models": 1, - "seconds": 0.08487620899904869, + "seconds": 0.16956324999773642, "eta_squared": 0.8352327138393587, "warnings": [] }, @@ -11989,13 +14540,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.08720804199765553, + "seconds": 0.16260958399652736, "eta_squared": 0.8352327138393587, "warnings": [] }, @@ -12006,13 +14557,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7998721670010127, + "seconds": 1.3512121249950724, "eta_squared": 0.8352327138393587, "warnings": [] }, @@ -12023,13 +14574,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8742404129640526, - "roc_auc": 0.998782064909297, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9756076388888889, - "worst_group_fpr": 0.20918367346938777, + "pr_auc": 0.9365376334289945, + "roc_auc": 0.9993489583333333, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.987453403078403, + "worst_group_fpr": 0.21683673469387754, "n_models": 1, - "seconds": 0.0876146249975136, + "seconds": 0.1605099579974194, "eta_squared": 0.8363958210386342, "warnings": [] }, @@ -12040,13 +14591,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.087231375000556, + "seconds": 0.16847175000293646, "eta_squared": 0.8363958210386342, "warnings": [] }, @@ -12057,13 +14608,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7806689999997616, + "seconds": 1.3879215829947498, "eta_squared": 0.8363958210386342, "warnings": [] }, @@ -12074,13 +14625,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8110908446160179, - "roc_auc": 0.9978320755385488, - "precision_at_n": 0.8125, - "macro_pr_auc": 0.9634441368816368, - "worst_group_fpr": 0.22959183673469388, + "pr_auc": 0.7836205496803595, + "roc_auc": 0.9974224064625851, + "precision_at_n": 0.7916666666666666, + "macro_pr_auc": 0.9519114906063435, + "worst_group_fpr": 0.21683673469387754, "n_models": 1, - "seconds": 0.07668337500217604, + "seconds": 0.16175441600353224, "eta_squared": 0.8349061810858989, "warnings": [] }, @@ -12091,13 +14642,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.07496341700243647, + "seconds": 0.1734445840047556, "eta_squared": 0.8349061810858989, "warnings": [] }, @@ -12108,13 +14659,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.961741641669088, - "roc_auc": 0.9994884672619048, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7732104579990846, + "seconds": 1.3815032909988076, "eta_squared": 0.8349061810858989, "warnings": [] }, @@ -12125,13 +14676,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9929563180122893, - "roc_auc": 0.999875992063492, - "precision_at_n": 0.96875, + "pr_auc": 0.9605736093684417, + "roc_auc": 0.9993556016156463, + "precision_at_n": 0.90625, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2423469387755102, + "worst_group_fpr": 0.25510204081632654, "n_models": 1, - "seconds": 0.07441083300000173, + "seconds": 0.17814487499708775, "eta_squared": 0.838772846460863, "warnings": [] }, @@ -12142,13 +14693,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.07725229200150352, + "seconds": 0.16565112500393298, "eta_squared": 0.838772846460863, "warnings": [] }, @@ -12159,13 +14710,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7636144169991894, + "seconds": 1.3763967080012662, "eta_squared": 0.838772846460863, "warnings": [] }, @@ -12176,13 +14727,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8345176660614, - "roc_auc": 0.9977545705782312, - "precision_at_n": 0.7916666666666666, - "macro_pr_auc": 0.9732875631313131, - "worst_group_fpr": 0.23214285714285715, + "pr_auc": 0.7758628084809992, + "roc_auc": 0.996569851899093, + "precision_at_n": 0.7395833333333334, + "macro_pr_auc": 0.9730602152477151, + "worst_group_fpr": 0.2372448979591837, "n_models": 1, - "seconds": 0.08674941700155614, + "seconds": 0.15988479099905817, "eta_squared": 0.8333960800737457, "warnings": [] }, @@ -12197,9 +14748,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.07854441699964809, + "seconds": 0.16298129099595826, "eta_squared": 0.8333960800737457, "warnings": [] }, @@ -12210,13 +14761,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7687328330030141, + "seconds": 1.365707333003229, "eta_squared": 0.8333960800737457, "warnings": [] }, @@ -12227,13 +14778,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.8730680702206932, - "roc_auc": 0.9985761231575963, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9735504079254079, - "worst_group_fpr": 0.23469387755102042, + "pr_auc": 0.8645601833146894, + "roc_auc": 0.9986159828514739, + "precision_at_n": 0.8854166666666666, + "macro_pr_auc": 0.9683420745920746, + "worst_group_fpr": 0.23214285714285715, "n_models": 1, - "seconds": 0.08224779199736076, + "seconds": 0.17296958299994003, "eta_squared": 0.8360376953619889, "warnings": [] }, @@ -12244,13 +14795,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.0859369579993654, + "seconds": 0.16544758300005924, "eta_squared": 0.8360376953619889, "warnings": [] }, @@ -12261,13 +14812,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.5, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7832443750012317, + "seconds": 1.356297917001939, "eta_squared": 0.8360376953619889, "warnings": [] }, @@ -12278,13 +14829,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25, + "worst_group_fpr": 0.25255102040816324, "n_models": 1, - "seconds": 0.08491629099808051, + "seconds": 0.16672174999985145, "eta_squared": 0.5109379001151101, "warnings": [] }, @@ -12296,12 +14847,12 @@ "mechanism": "global", "level_spread": 0.6, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.08680275000006077, + "seconds": 0.16512512500048615, "eta_squared": 0.5109379001151101, "warnings": [] }, @@ -12312,13 +14863,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7681977909996931, + "seconds": 1.3814167080054176, "eta_squared": 0.5109379001151101, "warnings": [] }, @@ -12329,13 +14880,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1913265306122449, + "worst_group_fpr": 0.21683673469387754, "n_models": 1, - "seconds": 0.0772125830008008, + "seconds": 0.15334912500111386, "eta_squared": 0.5084467520162148, "warnings": [] }, @@ -12346,13 +14897,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.08866916699844296, + "seconds": 0.16486483300104737, "eta_squared": 0.5084467520162148, "warnings": [] }, @@ -12363,13 +14914,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9980151809221449, - "roc_auc": 0.9999601403061225, + "pr_auc": 0.9988083913047778, + "roc_auc": 0.999975641298186, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7772392080005375, + "seconds": 1.3967122499961988, "eta_squared": 0.5084467520162148, "warnings": [] }, @@ -12380,13 +14931,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.24489795918367346, + "worst_group_fpr": 0.22959183673469388, "n_models": 1, - "seconds": 0.08360429200183717, + "seconds": 0.17508479100069962, "eta_squared": 0.512559151132954, "warnings": [] }, @@ -12398,12 +14949,12 @@ "mechanism": "global", "level_spread": 0.6, "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.08503254199968069, + "seconds": 0.15553166700556176, "eta_squared": 0.512559151132954, "warnings": [] }, @@ -12414,13 +14965,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9993599176792338, - "roc_auc": 0.9999867134353743, - "precision_at_n": 0.9791666666666666, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.776030165998236, + "seconds": 1.3449602910040994, "eta_squared": 0.512559151132954, "warnings": [] }, @@ -12435,9 +14986,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.26785714285714285, + "worst_group_fpr": 0.22704081632653061, "n_models": 1, - "seconds": 0.09037379099754617, + "seconds": 0.16223812499811174, "eta_squared": 0.5128629688830891, "warnings": [] }, @@ -12448,13 +14999,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.07780999999886262, + "seconds": 0.16368945799331414, "eta_squared": 0.5128629688830891, "warnings": [] }, @@ -12465,13 +15016,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7654231659980724, + "seconds": 1.3854607079993002, "eta_squared": 0.5128629688830891, "warnings": [] }, @@ -12482,13 +15033,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18622448979591838, + "worst_group_fpr": 0.21173469387755103, "n_models": 1, - "seconds": 0.08365862499704235, + "seconds": 0.16409433299850207, "eta_squared": 0.508762898773635, "warnings": [] }, @@ -12499,13 +15050,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.13010204081632654, "n_models": 1, - "seconds": 0.08557658300196636, + "seconds": 0.16383304099872475, "eta_squared": 0.508762898773635, "warnings": [] }, @@ -12516,13 +15067,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9994516328453017, - "roc_auc": 0.9999889278628118, + "pr_auc": 0.9991081986024108, + "roc_auc": 0.9999822845804989, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7562834170021233, + "seconds": 1.3557975420044386, "eta_squared": 0.508762898773635, "warnings": [] }, @@ -12533,13 +15084,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23469387755102042, + "worst_group_fpr": 0.22704081632653061, "n_models": 1, - "seconds": 0.07370516599985422, + "seconds": 0.1634810409959755, "eta_squared": 0.5090176444765433, "warnings": [] }, @@ -12550,13 +15101,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.08651979199930793, + "seconds": 0.16000712499953806, "eta_squared": 0.5090176444765433, "warnings": [] }, @@ -12567,13 +15118,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9994516328453016, - "roc_auc": 0.9999889278628119, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, + "pr_auc": 0.9980484049901451, + "roc_auc": 0.9999623547335601, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.763468124998326, + "seconds": 1.3558610420004698, "eta_squared": 0.5090176444765433, "warnings": [] }, @@ -12584,13 +15135,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838487, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, + "worst_group_fpr": 0.22193877551020408, "n_models": 1, - "seconds": 0.08445399999982328, + "seconds": 0.17023445799713954, "eta_squared": 0.5073389623798141, "warnings": [] }, @@ -12601,13 +15152,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.13520408163265307, "n_models": 1, - "seconds": 0.07331941600205027, + "seconds": 0.15853700000297977, "eta_squared": 0.5073389623798141, "warnings": [] }, @@ -12618,13 +15169,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9983853734038979, - "roc_auc": 0.999968998015873, + "pr_auc": 0.9981328388755407, + "roc_auc": 0.9999645691609977, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7771152919995075, + "seconds": 1.3721635000038077, "eta_squared": 0.5073389623798141, "warnings": [] }, @@ -12635,13 +15186,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2372448979591837, + "worst_group_fpr": 0.23979591836734693, "n_models": 1, - "seconds": 0.08142979200056288, + "seconds": 0.17407870800525416, "eta_squared": 0.5128210303210285, "warnings": [] }, @@ -12656,9 +15207,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.12244897959183673, "n_models": 1, - "seconds": 0.08359362500050338, + "seconds": 0.16557087500405032, "eta_squared": 0.5128210303210285, "warnings": [] }, @@ -12669,13 +15220,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9986895597463293, - "roc_auc": 0.9999734268707484, + "pr_auc": 0.9984642313812274, + "roc_auc": 0.999968998015873, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7754584160029481, + "seconds": 1.3749887919984758, "eta_squared": 0.5128210303210285, "warnings": [] }, @@ -12686,13 +15237,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2372448979591837, + "worst_group_fpr": 0.2423469387755102, "n_models": 1, - "seconds": 0.07261362500139512, + "seconds": 0.16553512500104262, "eta_squared": 0.5106686466317674, "warnings": [] }, @@ -12703,13 +15254,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.07893954100291012, + "seconds": 0.1711596669993014, "eta_squared": 0.5106686466317674, "warnings": [] }, @@ -12720,13 +15271,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9976086261705304, - "roc_auc": 0.9999557114512472, + "pr_auc": 0.9986319303600135, + "roc_auc": 0.9999734268707483, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 0.9988425925925926, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7453919169965957, + "seconds": 1.3863047909981105, "eta_squared": 0.5106686466317674, "warnings": [] }, @@ -12737,13 +15288,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22193877551020408, + "worst_group_fpr": 0.21428571428571427, "n_models": 1, - "seconds": 0.07460391700078617, + "seconds": 0.1531346659976407, "eta_squared": 0.5078990358901121, "warnings": [] }, @@ -12754,13 +15305,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.08502054199925624, + "seconds": 0.1592212079995079, "eta_squared": 0.5078990358901121, "warnings": [] }, @@ -12772,12 +15323,12 @@ "mechanism": "global", "level_spread": 0.6, "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7404959579980641, + "seconds": 1.3302593340049498, "eta_squared": 0.5078990358901121, "warnings": [] }, @@ -12788,13 +15339,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1836734693877551, + "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.07493704200169304, + "seconds": 0.1550901660011732, "eta_squared": 0.5113451805013186, "warnings": [] }, @@ -12805,13 +15356,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.078347250000661, + "seconds": 0.17638620900106616, "eta_squared": 0.5113451805013186, "warnings": [] }, @@ -12822,13 +15373,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9994695668020408, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9791666666666666, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7691615000003367, + "seconds": 1.5315749589935876, "eta_squared": 0.5113451805013186, "warnings": [] }, @@ -12839,13 +15390,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9996744556165973, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.22448979591836735, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.20153061224489796, "n_models": 1, - "seconds": 0.07297562500025379, + "seconds": 0.18538216700108023, "eta_squared": 0.5093438660938511, "warnings": [] }, @@ -12856,13 +15407,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, + "worst_group_fpr": 0.07908163265306123, "n_models": 1, - "seconds": 0.07727287499801605, + "seconds": 0.18358962500497, "eta_squared": 0.5093438660938511, "warnings": [] }, @@ -12873,13 +15424,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9931567828976611, - "roc_auc": 0.9998959219104309, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9926243465670476, + "roc_auc": 0.9998870642006803, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7367227909999201, + "seconds": 1.4807712080000783, "eta_squared": 0.5093438660938511, "warnings": [] }, @@ -12890,13 +15441,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2193877551020408, + "worst_group_fpr": 0.21428571428571427, "n_models": 1, - "seconds": 0.07862629200099036, + "seconds": 0.1798074589969474, "eta_squared": 0.5120210648371795, "warnings": [] }, @@ -12911,9 +15462,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, + "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.07654770799854305, + "seconds": 0.18321233300230233, "eta_squared": 0.5120210648371795, "warnings": [] }, @@ -12924,13 +15475,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7411699999975099, + "seconds": 1.5098842079969472, "eta_squared": 0.5120210648371795, "warnings": [] }, @@ -12941,13 +15492,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9996843434343433, - "roc_auc": 0.999993356717687, + "pr_auc": 0.9998926116838486, + "roc_auc": 0.9999977855725624, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, + "worst_group_fpr": 0.19387755102040816, "n_models": 1, - "seconds": 0.08959908299948438, + "seconds": 0.18296558300062316, "eta_squared": 0.5109849469679029, "warnings": [] }, @@ -12958,13 +15509,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, + "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.08650095900156884, + "seconds": 0.18526066599588376, "eta_squared": 0.5109849469679029, "warnings": [] }, @@ -12975,13 +15526,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9998926116838486, - "roc_auc": 0.9999977855725624, + "pr_auc": 0.9994516328453016, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, + "macro_pr_auc": 0.9975405092592592, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7631417499978852, + "seconds": 1.4333265829991433, "eta_squared": 0.5109849469679029, "warnings": [] }, @@ -12996,9 +15547,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2372448979591837, + "worst_group_fpr": 0.22704081632653061, "n_models": 1, - "seconds": 0.08152608300224529, + "seconds": 0.15811366600246402, "eta_squared": 0.5087750149271244, "warnings": [] }, @@ -13009,13 +15560,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.08800775000054273, + "seconds": 0.1583179170047515, "eta_squared": 0.5087750149271244, "warnings": [] }, @@ -13026,13 +15577,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.6, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7683398330009368, + "seconds": 1.3897468329960248, "eta_squared": 0.5087750149271244, "warnings": [] }, @@ -13043,13 +15594,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.7885567270879117, - "roc_auc": 0.9967957234977325, - "precision_at_n": 0.75, - "macro_pr_auc": 0.978984302054155, - "worst_group_fpr": 0.28061224489795916, + "pr_auc": 0.7582860409826389, + "roc_auc": 0.9965012046485261, + "precision_at_n": 0.7395833333333334, + "macro_pr_auc": 0.9725567256817257, + "worst_group_fpr": 0.2729591836734694, "n_models": 1, - "seconds": 0.0855373749982391, + "seconds": 0.18037091699807206, "eta_squared": 0.8766830602256049, "warnings": [] }, @@ -13060,13 +15611,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, + "worst_group_fpr": 0.1683673469387755, "n_models": 1, - "seconds": 0.08905991600113339, + "seconds": 0.1764629169992986, "eta_squared": 0.8766830602256049, "warnings": [] }, @@ -13077,13 +15628,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7801802079993649, + "seconds": 1.380092749997857, "eta_squared": 0.8766830602256049, "warnings": [] }, @@ -13094,13 +15645,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9263263993422348, - "roc_auc": 0.9991031568877551, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9799768518518519, - "worst_group_fpr": 0.22193877551020408, + "pr_auc": 0.8995843945355146, + "roc_auc": 0.9988019947562359, + "precision_at_n": 0.875, + "macro_pr_auc": 0.9790426587301587, + "worst_group_fpr": 0.24489795918367346, "n_models": 1, - "seconds": 0.0870615000021644, + "seconds": 0.1653334160000668, "eta_squared": 0.8759756368485012, "warnings": [] }, @@ -13111,13 +15662,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9983880098609269, - "roc_auc": 0.999968998015873, + "pr_auc": 0.9996744556165971, + "roc_auc": 0.9999933567176871, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, + "worst_group_fpr": 0.15306122448979592, "n_models": 1, - "seconds": 0.08714395900096861, + "seconds": 0.1696655420018942, "eta_squared": 0.8759756368485012, "warnings": [] }, @@ -13128,13 +15679,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.999289168792517, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7651897499999905, + "seconds": 1.3898012920035399, "eta_squared": 0.8759756368485012, "warnings": [] }, @@ -13145,13 +15696,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.87515182259879, - "roc_auc": 0.9986536281179138, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9836749188311688, - "worst_group_fpr": 0.28061224489795916, + "pr_auc": 0.8990980106960048, + "roc_auc": 0.9986802012471655, + "precision_at_n": 0.8854166666666666, + "macro_pr_auc": 0.989186507936508, + "worst_group_fpr": 0.2755102040816326, "n_models": 1, - "seconds": 0.07952075000139303, + "seconds": 0.1781659589978517, "eta_squared": 0.8768603441135048, "warnings": [] }, @@ -13166,9 +15717,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, + "worst_group_fpr": 0.1760204081632653, "n_models": 1, - "seconds": 0.08885616699990351, + "seconds": 0.1716181250012596, "eta_squared": 0.8768603441135048, "warnings": [] }, @@ -13179,13 +15730,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7935098329980974, + "seconds": 1.3796445839980152, "eta_squared": 0.8768603441135048, "warnings": [] }, @@ -13196,13 +15747,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.5919385078950008, - "roc_auc": 0.9934165072278911, - "precision_at_n": 0.625, - "macro_pr_auc": 0.9361432189170192, - "worst_group_fpr": 0.288265306122449, + "pr_auc": 0.7447341731386434, + "roc_auc": 0.9962133290816326, + "precision_at_n": 0.7291666666666666, + "macro_pr_auc": 0.9704103284832452, + "worst_group_fpr": 0.2780612244897959, "n_models": 1, - "seconds": 0.08872162500119884, + "seconds": 0.1653214170000865, "eta_squared": 0.8779733389869431, "warnings": [] }, @@ -13213,13 +15764,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, + "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.07963954099977855, + "seconds": 0.17273270900477655, "eta_squared": 0.8779733389869431, "warnings": [] }, @@ -13230,13 +15781,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7658361249996233, + "seconds": 1.384525624998787, "eta_squared": 0.8779733389869431, "warnings": [] }, @@ -13247,13 +15798,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9737943240947925, - "roc_auc": 0.9997852005385487, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9896288029100528, - "worst_group_fpr": 0.22448979591836735, + "pr_auc": 0.9477638935300299, + "roc_auc": 0.9993201707766439, + "precision_at_n": 0.90625, + "macro_pr_auc": 0.9879453012265512, + "worst_group_fpr": 0.23469387755102042, "n_models": 1, - "seconds": 0.08150162500169245, + "seconds": 0.1635339170024963, "eta_squared": 0.8762481102453108, "warnings": [] }, @@ -13264,13 +15815,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9964766874443567, - "roc_auc": 0.9999379960317459, + "pr_auc": 0.9950325742344305, + "roc_auc": 0.9999180661848073, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.086150874998566, + "seconds": 0.1647775419987738, "eta_squared": 0.8762481102453108, "warnings": [] }, @@ -13281,13 +15832,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7922221669978171, + "seconds": 1.48115091699583, "eta_squared": 0.8762481102453108, "warnings": [] }, @@ -13298,13 +15849,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.7774689882969182, - "roc_auc": 0.9966982886904763, - "precision_at_n": 0.75, - "macro_pr_auc": 0.9810831529581531, - "worst_group_fpr": 0.2780612244897959, + "pr_auc": 0.6834088297506767, + "roc_auc": 0.9952544820011339, + "precision_at_n": 0.6770833333333334, + "macro_pr_auc": 0.9591807208994708, + "worst_group_fpr": 0.28061224489795916, "n_models": 1, - "seconds": 0.08492241700150771, + "seconds": 0.17019391700159758, "eta_squared": 0.8732821441751656, "warnings": [] }, @@ -13315,13 +15866,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.15306122448979592, "n_models": 1, - "seconds": 0.08977879200028838, + "seconds": 0.17503504199703457, "eta_squared": 0.8732821441751656, "warnings": [] }, @@ -13332,13 +15883,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7473138329987705, + "seconds": 1.3845166249957401, "eta_squared": 0.8732821441751656, "warnings": [] }, @@ -13349,13 +15900,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.8118769820654939, - "roc_auc": 0.9974999114229025, - "precision_at_n": 0.8333333333333334, - "macro_pr_auc": 0.9899305555555555, - "worst_group_fpr": 0.2653061224489796, + "pr_auc": 0.8956222767758246, + "roc_auc": 0.9986359126984127, + "precision_at_n": 0.875, + "macro_pr_auc": 0.990625, + "worst_group_fpr": 0.2423469387755102, "n_models": 1, - "seconds": 0.08858470899940585, + "seconds": 0.16629595900303684, "eta_squared": 0.8726482673910156, "warnings": [] }, @@ -13370,9 +15921,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.0864155839990417, + "seconds": 0.16443587499816203, "eta_squared": 0.8726482673910156, "warnings": [] }, @@ -13383,13 +15934,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7839988749983604, + "seconds": 1.3696010829953593, "eta_squared": 0.8726482673910156, "warnings": [] }, @@ -13400,13 +15951,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.6566644460928512, - "roc_auc": 0.9937996031746031, - "precision_at_n": 0.6041666666666666, - "macro_pr_auc": 0.9704588779956427, - "worst_group_fpr": 0.24489795918367346, + "pr_auc": 0.6513235336107632, + "roc_auc": 0.9946477288832201, + "precision_at_n": 0.6458333333333334, + "macro_pr_auc": 0.9570489183290719, + "worst_group_fpr": 0.25510204081632654, "n_models": 1, - "seconds": 0.08813191700028256, + "seconds": 0.16151991600054316, "eta_squared": 0.8767223767771402, "warnings": [] }, @@ -13417,13 +15968,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.08840037499976461, + "seconds": 0.16672608299995773, "eta_squared": 0.8767223767771402, "warnings": [] }, @@ -13434,13 +15985,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7705785830003151, + "seconds": 1.3941900830031955, "eta_squared": 0.8767223767771402, "warnings": [] }, @@ -13451,13 +16002,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9132700475497844, - "roc_auc": 0.9985141191893425, - "precision_at_n": 0.8333333333333334, - "macro_pr_auc": 0.9951388888888889, - "worst_group_fpr": 0.2576530612244898, + "pr_auc": 0.8992649074239921, + "roc_auc": 0.9984299709467122, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9932034111721612, + "worst_group_fpr": 0.2627551020408163, "n_models": 1, - "seconds": 0.08638079199954518, + "seconds": 0.17306258300232003, "eta_squared": 0.8774993715157073, "warnings": [] }, @@ -13468,13 +16019,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, + "worst_group_fpr": 0.1326530612244898, "n_models": 1, - "seconds": 0.08703520799826947, + "seconds": 0.16032320899830665, "eta_squared": 0.8774993715157073, "warnings": [] }, @@ -13485,13 +16036,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7560972079991188, + "seconds": 1.3418790419964353, "eta_squared": 0.8774993715157073, "warnings": [] }, @@ -13502,13 +16053,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.8440371582166274, - "roc_auc": 0.9977567850056689, - "precision_at_n": 0.8333333333333334, - "macro_pr_auc": 0.9803240740740741, - "worst_group_fpr": 0.2653061224489796, + "pr_auc": 0.9445820394815596, + "roc_auc": 0.9992183071145124, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9958570075757577, + "worst_group_fpr": 0.25, "n_models": 1, - "seconds": 0.07792279199929908, + "seconds": 0.1642562079941854, "eta_squared": 0.8756459899827929, "warnings": [] }, @@ -13519,13 +16070,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, + "worst_group_fpr": 0.15306122448979592, "n_models": 1, - "seconds": 0.08649587500258349, + "seconds": 0.16373504199873423, "eta_squared": 0.8756459899827929, "warnings": [] }, @@ -13536,13 +16087,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7773372499977995, + "seconds": 1.3671592500031693, "eta_squared": 0.8756459899827929, "warnings": [] }, @@ -13553,13 +16104,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.7648853117072381, - "roc_auc": 0.9973183283730158, - "precision_at_n": 0.8125, - "macro_pr_auc": 0.96149172008547, + "pr_auc": 0.8382088298719095, + "roc_auc": 0.9983258928571428, + "precision_at_n": 0.875, + "macro_pr_auc": 0.9708979677729678, "worst_group_fpr": 0.24744897959183673, "n_models": 1, - "seconds": 0.07740291699883528, + "seconds": 0.1684582079979009, "eta_squared": 0.8761496699086859, "warnings": [] }, @@ -13570,13 +16121,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.08015466599681531, + "seconds": 0.1701111249931273, "eta_squared": 0.8761496699086859, "warnings": [] }, @@ -13587,13 +16138,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7577294590009842, + "seconds": 1.4356458749971353, "eta_squared": 0.8761496699086859, "warnings": [] }, @@ -13604,13 +16155,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.7016601981681742, - "roc_auc": 0.9958634495464853, - "precision_at_n": 0.7395833333333334, - "macro_pr_auc": 0.9485367063492064, - "worst_group_fpr": 0.2780612244897959, + "pr_auc": 0.6902864089412835, + "roc_auc": 0.9956420068027211, + "precision_at_n": 0.7291666666666666, + "macro_pr_auc": 0.9454585017326057, + "worst_group_fpr": 0.2653061224489796, "n_models": 1, - "seconds": 0.08493816700138268, + "seconds": 0.1596362499985844, "eta_squared": 0.8755759250473935, "warnings": [] }, @@ -13621,13 +16172,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.08781279100003303, + "seconds": 0.16964354200172238, "eta_squared": 0.8755759250473935, "warnings": [] }, @@ -13638,13 +16189,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9619580084145981, - "roc_auc": 0.99949289611678, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7508680829996592, + "seconds": 1.3637192919995869, "eta_squared": 0.8755759250473935, "warnings": [] }, @@ -13655,13 +16206,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9406836090385928, - "roc_auc": 0.9991275155895691, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.25, + "pr_auc": 0.8934586737191591, + "roc_auc": 0.9983037485827665, + "precision_at_n": 0.8333333333333334, + "macro_pr_auc": 0.9951264880952381, + "worst_group_fpr": 0.2627551020408163, "n_models": 1, - "seconds": 0.08873591600058717, + "seconds": 0.16413462499622256, "eta_squared": 0.8782757121696583, "warnings": [] }, @@ -13672,13 +16223,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.1377551020408163, "n_models": 1, - "seconds": 0.08785258399802842, + "seconds": 0.16557108300185064, "eta_squared": 0.8782757121696583, "warnings": [] }, @@ -13689,13 +16240,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7660697499995877, + "seconds": 1.358468708996952, "eta_squared": 0.8782757121696583, "warnings": [] }, @@ -13706,13 +16257,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.8008599649452659, - "roc_auc": 0.9969285891439909, - "precision_at_n": 0.75, - "macro_pr_auc": 0.9716713263588264, - "worst_group_fpr": 0.24489795918367346, + "pr_auc": 0.726617627727997, + "roc_auc": 0.9955445719954649, + "precision_at_n": 0.6979166666666666, + "macro_pr_auc": 0.9701088263588263, + "worst_group_fpr": 0.24744897959183673, "n_models": 1, - "seconds": 0.07465600000068662, + "seconds": 0.1707825419944129, "eta_squared": 0.8742451489593618, "warnings": [] }, @@ -13723,13 +16274,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.9944359305718173, - "roc_auc": 0.9998937074829932, - "precision_at_n": 0.96875, + "pr_auc": 0.994913149994302, + "roc_auc": 0.9999092084750566, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.07024633399851155, + "seconds": 0.16297733299870742, "eta_squared": 0.8742451489593618, "warnings": [] }, @@ -13740,13 +16291,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8164289589985856, + "seconds": 1.3907979169962346, "eta_squared": 0.8742451489593618, "warnings": [] }, @@ -13757,13 +16308,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.8042180619734371, - "roc_auc": 0.9975663442460317, - "precision_at_n": 0.8229166666666666, - "macro_pr_auc": 0.9657957782957783, - "worst_group_fpr": 0.25510204081632654, + "pr_auc": 0.853190539545772, + "roc_auc": 0.9984964037698413, + "precision_at_n": 0.8854166666666666, + "macro_pr_auc": 0.9689207782957783, + "worst_group_fpr": 0.2602040816326531, "n_models": 1, - "seconds": 0.08552541699827998, + "seconds": 0.19305558299674885, "eta_squared": 0.8761879838657488, "warnings": [] }, @@ -13778,9 +16329,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, + "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.08105375000013737, + "seconds": 0.17603737500030547, "eta_squared": 0.8761879838657488, "warnings": [] }, @@ -13791,13 +16342,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.6, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7777734590017644, + "seconds": 1.3474013329978334, "eta_squared": 0.8761879838657488, "warnings": [] }, @@ -13812,9 +16363,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, + "worst_group_fpr": 0.25, "n_models": 1, - "seconds": 0.08777550000013434, + "seconds": 0.16116825000062818, "eta_squared": 0.5298890398187, "warnings": [] }, @@ -13829,9 +16380,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.08882937500311527, + "seconds": 0.17193729199789232, "eta_squared": 0.5298890398187, "warnings": [] }, @@ -13842,13 +16393,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7712032080016797, + "seconds": 1.378871082997648, "eta_squared": 0.5298890398187, "warnings": [] }, @@ -13859,13 +16410,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.19387755102040816, + "worst_group_fpr": 0.22704081632653061, "n_models": 1, - "seconds": 0.08165000000008149, + "seconds": 0.16921825000463286, "eta_squared": 0.52735131605007, "warnings": [] }, @@ -13876,13 +16427,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.07766783300030511, + "seconds": 0.1556395000006887, "eta_squared": 0.52735131605007, "warnings": [] }, @@ -13893,13 +16444,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9984607720576867, - "roc_auc": 0.999968998015873, + "pr_auc": 0.9992589075782236, + "roc_auc": 0.9999844990079365, "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7554894579989195, + "seconds": 1.3399752499972237, "eta_squared": 0.52735131605007, "warnings": [] }, @@ -13910,13 +16461,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.26785714285714285, + "worst_group_fpr": 0.27040816326530615, "n_models": 1, - "seconds": 0.07812595799987321, + "seconds": 0.16008912499819417, "eta_squared": 0.5320991054125361, "warnings": [] }, @@ -13931,9 +16482,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.08511112500127638, + "seconds": 0.16749195900047198, "eta_squared": 0.5320991054125361, "warnings": [] }, @@ -13944,13 +16495,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9993685567010306, - "roc_auc": 0.9999867134353742, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7884545829983836, + "seconds": 1.3525973750001867, "eta_squared": 0.5320991054125361, "warnings": [] }, @@ -13961,13 +16512,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29081632653061223, + "worst_group_fpr": 0.25255102040816324, "n_models": 1, - "seconds": 0.0858608330017887, + "seconds": 0.16665166699385736, "eta_squared": 0.5321841551665233, "warnings": [] }, @@ -13978,13 +16529,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.08419258399953833, + "seconds": 0.16000091600290034, "eta_squared": 0.5321841551665233, "warnings": [] }, @@ -13995,13 +16546,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7621752499981085, + "seconds": 1.3520715839986224, "eta_squared": 0.5321841551665233, "warnings": [] }, @@ -14013,12 +16564,12 @@ "mechanism": "global", "level_spread": 0.7, "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1836734693877551, + "worst_group_fpr": 0.21428571428571427, "n_models": 1, - "seconds": 0.08351683300134027, + "seconds": 0.15576579199841945, "eta_squared": 0.5277037636692434, "warnings": [] }, @@ -14033,9 +16584,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.08679024999946705, + "seconds": 0.1596036670016474, "eta_squared": 0.5277037636692434, "warnings": [] }, @@ -14046,13 +16597,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9995636400137602, - "roc_auc": 0.9999911422902494, + "pr_auc": 0.9994516328453017, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7911076249984035, + "seconds": 1.3927896660024999, "eta_squared": 0.5277037636692434, "warnings": [] }, @@ -14063,13 +16614,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25255102040816324, + "worst_group_fpr": 0.23469387755102042, "n_models": 1, - "seconds": 0.08264058299755561, + "seconds": 0.16862491599749774, "eta_squared": 0.5286644856867957, "warnings": [] }, @@ -14080,13 +16631,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.07730950000041048, + "seconds": 0.16028712499974063, "eta_squared": 0.5286644856867957, "warnings": [] }, @@ -14097,13 +16648,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9994516328453017, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, + "pr_auc": 0.9980484049901452, + "roc_auc": 0.99996235473356, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8215333750013087, + "seconds": 1.3658714170014719, "eta_squared": 0.5286644856867957, "warnings": [] }, @@ -14114,13 +16665,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999997, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.24489795918367346, + "worst_group_fpr": 0.2372448979591837, "n_models": 1, - "seconds": 0.08576008300224203, + "seconds": 0.1669780419979361, "eta_squared": 0.5270205162013888, "warnings": [] }, @@ -14131,13 +16682,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.08757841600163374, + "seconds": 0.15942808399995556, "eta_squared": 0.5270205162013888, "warnings": [] }, @@ -14148,13 +16699,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9981328388755404, + "pr_auc": 0.9981328388755407, "roc_auc": 0.9999645691609977, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04336734693877551, + "macro_pr_auc": 0.9960524140211641, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7834615419997135, + "seconds": 1.3874943750051898, "eta_squared": 0.5270205162013888, "warnings": [] }, @@ -14165,13 +16716,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2372448979591837, + "worst_group_fpr": 0.24489795918367346, "n_models": 1, - "seconds": 0.07499008299782872, + "seconds": 0.16428512499987846, "eta_squared": 0.5318672897267391, "warnings": [] }, @@ -14182,13 +16733,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.08617700000104378, + "seconds": 0.15578358300263062, "eta_squared": 0.5318672897267391, "warnings": [] }, @@ -14199,13 +16750,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.999243364595796, - "roc_auc": 0.9999844990079364, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9986906806565897, + "roc_auc": 0.9999734268707483, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.77146250000078, + "seconds": 1.3920086250000168, "eta_squared": 0.5318672897267391, "warnings": [] }, @@ -14220,9 +16771,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2576530612244898, + "worst_group_fpr": 0.26785714285714285, "n_models": 1, - "seconds": 0.0849276250010007, + "seconds": 0.15572579099534778, "eta_squared": 0.5293730129243481, "warnings": [] }, @@ -14234,12 +16785,12 @@ "mechanism": "global", "level_spread": 0.7, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.08418367346938775, "n_models": 1, - "seconds": 0.07352420799725223, + "seconds": 0.17435733399906894, "eta_squared": 0.5293730129243481, "warnings": [] }, @@ -14250,13 +16801,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9967692587372328, - "roc_auc": 0.9999424248866213, + "pr_auc": 0.9980042380524953, + "roc_auc": 0.9999623547335601, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, + "macro_pr_auc": 0.9975405092592592, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7896272920006595, + "seconds": 1.3670541669998784, "eta_squared": 0.5293730129243481, "warnings": [] }, @@ -14267,13 +16818,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, + "worst_group_fpr": 0.21173469387755103, "n_models": 1, - "seconds": 0.08258941600070102, + "seconds": 0.1595689159948961, "eta_squared": 0.5268459802946263, "warnings": [] }, @@ -14284,13 +16835,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.1326530612244898, "n_models": 1, - "seconds": 0.08991237499867566, + "seconds": 0.1730810000008205, "eta_squared": 0.5268459802946263, "warnings": [] }, @@ -14301,13 +16852,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7764590839979064, + "seconds": 1.3595376250013942, "eta_squared": 0.5268459802946263, "warnings": [] }, @@ -14318,13 +16869,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21428571428571427, + "worst_group_fpr": 0.2066326530612245, "n_models": 1, - "seconds": 0.08458404100019834, + "seconds": 0.16157433300395496, "eta_squared": 0.5306182183453104, "warnings": [] }, @@ -14335,13 +16886,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.09078204199977336, + "seconds": 0.15203350000228966, "eta_squared": 0.5306182183453104, "warnings": [] }, @@ -14352,13 +16903,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.794608000000153, + "seconds": 1.3960167920013191, "eta_squared": 0.5306182183453104, "warnings": [] }, @@ -14369,13 +16920,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725625, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2627551020408163, + "worst_group_fpr": 0.22704081632653061, "n_models": 1, - "seconds": 0.08935416700114729, + "seconds": 0.16133270799764432, "eta_squared": 0.5280422190863733, "warnings": [] }, @@ -14386,13 +16937,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.07653061224489796, "n_models": 1, - "seconds": 0.09058883300167508, + "seconds": 0.17179475000011735, "eta_squared": 0.5280422190863733, "warnings": [] }, @@ -14403,13 +16954,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9935541423625902, - "roc_auc": 0.9999003507653061, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9945023148148149, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.993499468089315, + "roc_auc": 0.9998937074829932, + "precision_at_n": 0.96875, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7893917919973319, + "seconds": 1.3524868749955203, "eta_squared": 0.5280422190863733, "warnings": [] }, @@ -14420,13 +16971,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.2423469387755102, "n_models": 1, - "seconds": 0.07543029099906562, + "seconds": 0.16467408400058048, "eta_squared": 0.5307835609735325, "warnings": [] }, @@ -14438,12 +16989,12 @@ "mechanism": "global", "level_spread": 0.7, "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.09438775510204081, "n_models": 1, - "seconds": 0.07623579099890776, + "seconds": 0.17430150000291178, "eta_squared": 0.5307835609735325, "warnings": [] }, @@ -14454,13 +17005,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7468186670012074, + "seconds": 1.3547277499965276, "eta_squared": 0.5307835609735325, "warnings": [] }, @@ -14471,13 +17022,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17346938775510204, + "worst_group_fpr": 0.2066326530612245, "n_models": 1, - "seconds": 0.08429450000039651, + "seconds": 0.16530779100139625, "eta_squared": 0.5301452136480338, "warnings": [] }, @@ -14488,13 +17039,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.08776687499994296, + "seconds": 0.16847454099479364, "eta_squared": 0.5301452136480338, "warnings": [] }, @@ -14505,13 +17056,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9996789080215417, - "roc_auc": 0.9999933567176871, + "pr_auc": 0.9994516328453015, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7628876249982568, + "seconds": 1.371672416004003, "eta_squared": 0.5301452136480338, "warnings": [] }, @@ -14522,13 +17073,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2193877551020408, + "worst_group_fpr": 0.23214285714285715, "n_models": 1, - "seconds": 0.08443741699738894, + "seconds": 0.16930608300026506, "eta_squared": 0.5279659930517174, "warnings": [] }, @@ -14543,9 +17094,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.08273758400173392, + "seconds": 0.1642920000012964, "eta_squared": 0.5279659930517174, "warnings": [] }, @@ -14556,13 +17107,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7746504579990869, + "seconds": 1.3668617090006592, "eta_squared": 0.5279659930517174, "warnings": [] }, @@ -14573,13 +17124,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.6472915324390636, - "roc_auc": 0.9940609056122449, - "precision_at_n": 0.6875, - "macro_pr_auc": 0.9523411195286196, - "worst_group_fpr": 0.29081632653061223, + "pr_auc": 0.6681460572719677, + "roc_auc": 0.9949223178854875, + "precision_at_n": 0.6979166666666666, + "macro_pr_auc": 0.9583931021660801, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.07696279200172285, + "seconds": 0.1728503330014064, "eta_squared": 0.9029294791735885, "warnings": [] }, @@ -14590,13 +17141,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17091836734693877, + "worst_group_fpr": 0.1760204081632653, "n_models": 1, - "seconds": 0.09040825000192854, + "seconds": 0.1650128340043011, "eta_squared": 0.9029294791735885, "warnings": [] }, @@ -14607,13 +17158,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8112011669982166, + "seconds": 1.347333207995689, "eta_squared": 0.9029294791735885, "warnings": [] }, @@ -14624,13 +17175,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.7335148768545713, - "roc_auc": 0.996383839994331, - "precision_at_n": 0.7604166666666666, - "macro_pr_auc": 0.9474263583638584, - "worst_group_fpr": 0.25255102040816324, + "pr_auc": 0.7308688714872446, + "roc_auc": 0.9965919961734694, + "precision_at_n": 0.7291666666666666, + "macro_pr_auc": 0.9465434419381787, + "worst_group_fpr": 0.25510204081632654, "n_models": 1, - "seconds": 0.08776487499926588, + "seconds": 0.16908604199852562, "eta_squared": 0.9023260653166297, "warnings": [] }, @@ -14641,13 +17192,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9987543844046409, - "roc_auc": 0.999975641298186, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725624, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.1326530612244898, "n_models": 1, - "seconds": 0.08751541599849588, + "seconds": 0.17620766600157367, "eta_squared": 0.9023260653166297, "warnings": [] }, @@ -14658,13 +17209,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.999289168792517, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7803810410005099, + "seconds": 1.3685417080050684, "eta_squared": 0.9023260653166297, "warnings": [] }, @@ -14675,13 +17226,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.7566046046890987, - "roc_auc": 0.996273118622449, - "precision_at_n": 0.71875, - "macro_pr_auc": 0.9692839318698493, - "worst_group_fpr": 0.3010204081632653, + "pr_auc": 0.7983959157429952, + "roc_auc": 0.997039310515873, + "precision_at_n": 0.7708333333333334, + "macro_pr_auc": 0.9768409014732544, + "worst_group_fpr": 0.29846938775510207, "n_models": 1, - "seconds": 0.08632062499964377, + "seconds": 0.1641099159969599, "eta_squared": 0.9030920988447193, "warnings": [] }, @@ -14692,13 +17243,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "worst_group_fpr": 0.15816326530612246, "n_models": 1, - "seconds": 0.08881295900209807, + "seconds": 0.16917108299821848, "eta_squared": 0.9030920988447193, "warnings": [] }, @@ -14709,13 +17260,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7908264580000832, + "seconds": 1.34316495800158, "eta_squared": 0.9030920988447193, "warnings": [] }, @@ -14726,13 +17277,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.5444770662299433, - "roc_auc": 0.9901413690476191, - "precision_at_n": 0.4479166666666667, - "macro_pr_auc": 0.9579326923076922, - "worst_group_fpr": 0.3112244897959184, + "pr_auc": 0.6840190010656133, + "roc_auc": 0.9945259353741497, + "precision_at_n": 0.65625, + "macro_pr_auc": 0.9668358262108262, + "worst_group_fpr": 0.3010204081632653, "n_models": 1, - "seconds": 0.08835808400181122, + "seconds": 0.16739337499893736, "eta_squared": 0.9039807536694324, "warnings": [] }, @@ -14743,13 +17294,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, + "worst_group_fpr": 0.17091836734693877, "n_models": 1, - "seconds": 0.08825104200150236, + "seconds": 0.16177070799312787, "eta_squared": 0.9039807536694324, "warnings": [] }, @@ -14760,13 +17311,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7938991660012107, + "seconds": 1.3828187500039348, "eta_squared": 0.9039807536694324, "warnings": [] }, @@ -14777,13 +17328,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.907544427474411, - "roc_auc": 0.9989636479591837, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9743055555555555, - "worst_group_fpr": 0.2423469387755102, + "pr_auc": 0.7868633343573769, + "roc_auc": 0.9970127373866214, + "precision_at_n": 0.78125, + "macro_pr_auc": 0.9630104993386244, + "worst_group_fpr": 0.2576530612244898, "n_models": 1, - "seconds": 0.07891408299838076, + "seconds": 0.16557145900151227, "eta_squared": 0.9026069122444567, "warnings": [] }, @@ -14794,13 +17345,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9995636400137604, - "roc_auc": 0.9999911422902495, + "pr_auc": 0.9961735140694566, + "roc_auc": 0.9999335671768708, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.1377551020408163, "n_models": 1, - "seconds": 0.08629674999974668, + "seconds": 0.17229137499816716, "eta_squared": 0.9026069122444567, "warnings": [] }, @@ -14811,13 +17362,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7687257500001579, + "seconds": 1.395121375004237, "eta_squared": 0.9026069122444567, "warnings": [] }, @@ -14828,13 +17379,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.7374702609384092, - "roc_auc": 0.9964635593820861, - "precision_at_n": 0.7604166666666666, - "macro_pr_auc": 0.9770084422657952, + "pr_auc": 0.6933595662983045, + "roc_auc": 0.9956397923752834, + "precision_at_n": 0.7083333333333334, + "macro_pr_auc": 0.9530006740944241, "worst_group_fpr": 0.29336734693877553, "n_models": 1, - "seconds": 0.08752916600133176, + "seconds": 0.16351079200103413, "eta_squared": 0.9002286409429918, "warnings": [] }, @@ -14845,13 +17396,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.14795918367346939, "n_models": 1, - "seconds": 0.0856532090001565, + "seconds": 0.17506095900171204, "eta_squared": 0.9002286409429918, "warnings": [] }, @@ -14862,13 +17413,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7807869169992045, + "seconds": 1.3643528329994297, "eta_squared": 0.9002286409429918, "warnings": [] }, @@ -14879,13 +17430,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.832233935178333, - "roc_auc": 0.9975131979875284, - "precision_at_n": 0.8020833333333334, - "macro_pr_auc": 0.9936342592592592, - "worst_group_fpr": 0.23979591836734693, + "pr_auc": 0.8618262227577957, + "roc_auc": 0.9980911635487528, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9924107142857143, + "worst_group_fpr": 0.24489795918367346, "n_models": 1, - "seconds": 0.08214566700189607, + "seconds": 0.17302979200030677, "eta_squared": 0.8996587213625616, "warnings": [] }, @@ -14896,13 +17447,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, + "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.08768945899646496, + "seconds": 0.1768632500024978, "eta_squared": 0.8996587213625616, "warnings": [] }, @@ -14913,13 +17464,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8176077089992759, + "seconds": 1.4162449580035172, "eta_squared": 0.8996587213625616, "warnings": [] }, @@ -14930,13 +17481,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.6626887062555417, - "roc_auc": 0.9928097541099774, - "precision_at_n": 0.6041666666666666, - "macro_pr_auc": 0.9801338281601439, - "worst_group_fpr": 0.29081632653061223, + "pr_auc": 0.684205193809398, + "roc_auc": 0.9949156746031746, + "precision_at_n": 0.6354166666666666, + "macro_pr_auc": 0.969165774547367, + "worst_group_fpr": 0.29336734693877553, "n_models": 1, - "seconds": 0.08764775000236114, + "seconds": 0.16751866700360551, "eta_squared": 0.9029208108231173, "warnings": [] }, @@ -14947,13 +17498,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.0888431670027785, + "seconds": 0.1731683750040247, "eta_squared": 0.9029208108231173, "warnings": [] }, @@ -14964,13 +17515,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8138497500003723, + "seconds": 1.3723675000001094, "eta_squared": 0.9029208108231173, "warnings": [] }, @@ -14981,13 +17532,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.7680113801219064, - "roc_auc": 0.9959099525226758, + "pr_auc": 0.791864796418778, + "roc_auc": 0.9964458439625851, "precision_at_n": 0.7395833333333334, - "macro_pr_auc": 0.9742989417989417, - "worst_group_fpr": 0.288265306122449, + "macro_pr_auc": 0.987832190957191, + "worst_group_fpr": 0.29846938775510207, "n_models": 1, - "seconds": 0.08371254200028488, + "seconds": 0.16881570799887413, "eta_squared": 0.9035454354972019, "warnings": [] }, @@ -14998,13 +17549,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9982626909014108, - "roc_auc": 0.9999667835884353, - "precision_at_n": 0.9895833333333334, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.07779562499854364, + "seconds": 0.16889095900114626, "eta_squared": 0.9035454354972019, "warnings": [] }, @@ -15015,13 +17566,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7602968749997672, + "seconds": 1.3773021250017337, "eta_squared": 0.9035454354972019, "warnings": [] }, @@ -15032,13 +17583,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.7166776861324206, - "roc_auc": 0.9954714958900227, - "precision_at_n": 0.6875, - "macro_pr_auc": 0.9653441930955519, - "worst_group_fpr": 0.2857142857142857, + "pr_auc": 0.8040889343745524, + "roc_auc": 0.9969440901360545, + "precision_at_n": 0.7604166666666666, + "macro_pr_auc": 0.9774305555555555, + "worst_group_fpr": 0.28316326530612246, "n_models": 1, - "seconds": 0.08736404099909123, + "seconds": 0.1685613329973421, "eta_squared": 0.9021131136700368, "warnings": [] }, @@ -15053,9 +17604,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, + "worst_group_fpr": 0.15816326530612246, "n_models": 1, - "seconds": 0.08585641700119595, + "seconds": 0.17807291699864436, "eta_squared": 0.9021131136700368, "warnings": [] }, @@ -15066,13 +17617,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7692449580026732, + "seconds": 1.4345150419976562, "eta_squared": 0.9021131136700368, "warnings": [] }, @@ -15082,14 +17633,14 @@ "config": "pooled", "seed": 10, "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.6628779134839293, - "roc_auc": 0.9960029584750567, - "precision_at_n": 0.78125, - "macro_pr_auc": 0.9226900314723591, - "worst_group_fpr": 0.2729591836734694, + "level_spread": 0.7, + "pr_auc": 0.728479400397314, + "roc_auc": 0.9970215950963719, + "precision_at_n": 0.8020833333333334, + "macro_pr_auc": 0.9415345806930236, + "worst_group_fpr": 0.26785714285714285, "n_models": 1, - "seconds": 0.07446300000083284, + "seconds": 0.17713350000121864, "eta_squared": 0.9022494388744451, "warnings": [] }, @@ -15100,13 +17651,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.08461487499880604, + "seconds": 0.17489637499966193, "eta_squared": 0.9022494388744451, "warnings": [] }, @@ -15117,13 +17668,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7702019580028718, + "seconds": 1.3611302090066602, "eta_squared": 0.9022494388744451, "warnings": [] }, @@ -15134,13 +17685,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.750630516683807, - "roc_auc": 0.9969418757086168, - "precision_at_n": 0.7708333333333334, - "macro_pr_auc": 0.9536864755614755, - "worst_group_fpr": 0.29336734693877553, + "pr_auc": 0.611290735134805, + "roc_auc": 0.9935604450113379, + "precision_at_n": 0.6354166666666666, + "macro_pr_auc": 0.9403211805555557, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.08416654100074084, + "seconds": 0.16300062499794876, "eta_squared": 0.9021789744937543, "warnings": [] }, @@ -15155,9 +17706,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.07893762499952572, + "seconds": 0.17516062499635154, "eta_squared": 0.9021789744937543, "warnings": [] }, @@ -15168,13 +17719,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.961741641669088, - "roc_auc": 0.9994884672619048, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7485258340020664, + "seconds": 1.385379457999079, "eta_squared": 0.9021789744937543, "warnings": [] }, @@ -15185,13 +17736,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.7616804829455618, - "roc_auc": 0.9962930484693877, - "precision_at_n": 0.75, - "macro_pr_auc": 0.9817997685185186, - "worst_group_fpr": 0.29336734693877553, + "pr_auc": 0.704252633464105, + "roc_auc": 0.9954050630668934, + "precision_at_n": 0.6979166666666666, + "macro_pr_auc": 0.9739869852369852, + "worst_group_fpr": 0.30357142857142855, "n_models": 1, - "seconds": 0.08561912500226754, + "seconds": 0.17574295799568063, "eta_squared": 0.9041553072370441, "warnings": [] }, @@ -15202,13 +17753,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, + "worst_group_fpr": 0.14795918367346939, "n_models": 1, - "seconds": 0.08452362500247546, + "seconds": 0.17137891700258479, "eta_squared": 0.9041553072370441, "warnings": [] }, @@ -15219,13 +17770,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7471194579993607, + "seconds": 1.349269334001292, "eta_squared": 0.9041553072370441, "warnings": [] }, @@ -15236,13 +17787,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.7058328327915732, - "roc_auc": 0.995469281462585, - "precision_at_n": 0.6979166666666666, - "macro_pr_auc": 0.9647858796296296, - "worst_group_fpr": 0.2627551020408163, + "pr_auc": 0.5794175981215678, + "roc_auc": 0.9927610367063492, + "precision_at_n": 0.5833333333333334, + "macro_pr_auc": 0.9460557960557959, + "worst_group_fpr": 0.2729591836734694, "n_models": 1, - "seconds": 0.08218399999896064, + "seconds": 0.17066850000264822, "eta_squared": 0.9009953154147747, "warnings": [] }, @@ -15253,13 +17804,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9994516328453016, - "roc_auc": 0.9999889278628118, + "pr_auc": 0.9992239393431515, + "roc_auc": 0.9999844990079366, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.08120787500229198, + "seconds": 0.17392825000570156, "eta_squared": 0.9009953154147747, "warnings": [] }, @@ -15270,13 +17821,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7353911250029341, + "seconds": 1.3921578329973272, "eta_squared": 0.9009953154147747, "warnings": [] }, @@ -15287,13 +17838,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.7180977512797182, - "roc_auc": 0.9964657738095237, - "precision_at_n": 0.78125, - "macro_pr_auc": 0.9498210139318886, - "worst_group_fpr": 0.2857142857142857, + "pr_auc": 0.793114486939759, + "roc_auc": 0.9976194905045352, + "precision_at_n": 0.8333333333333334, + "macro_pr_auc": 0.9674167846042846, + "worst_group_fpr": 0.28061224489795916, "n_models": 1, - "seconds": 0.07534395799666527, + "seconds": 0.17824149999796646, "eta_squared": 0.9024948788179256, "warnings": [] }, @@ -15304,13 +17855,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.08118570800070302, + "seconds": 0.16386495799815748, "eta_squared": 0.9024948788179256, "warnings": [] }, @@ -15321,13 +17872,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.7, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7686429999994289, + "seconds": 1.4200830000045244, "eta_squared": 0.9024948788179256, "warnings": [] }, @@ -15338,13 +17889,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2576530612244898, + "worst_group_fpr": 0.2780612244897959, "n_models": 1, - "seconds": 0.08530229200187023, + "seconds": 0.17246666600112803, "eta_squared": 0.5448994609628225, "warnings": [] }, @@ -15355,13 +17906,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999997, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.08992466700146906, + "seconds": 0.15952704200026346, "eta_squared": 0.5448994609628225, "warnings": [] }, @@ -15372,13 +17923,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7847364580011345, + "seconds": 1.3783540420045028, "eta_squared": 0.5448994609628225, "warnings": [] }, @@ -15393,9 +17944,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20153061224489796, + "worst_group_fpr": 0.23979591836734693, "n_models": 1, - "seconds": 0.08345112499955576, + "seconds": 0.15861687500000698, "eta_squared": 0.5422953822643336, "warnings": [] }, @@ -15406,13 +17957,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.08259204199930537, + "seconds": 0.16462504199444083, "eta_squared": 0.5422953822643336, "warnings": [] }, @@ -15423,13 +17974,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.999130139958115, - "roc_auc": 0.9999822845804989, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9995726383336836, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7781143330030318, + "seconds": 1.3471067919963389, "eta_squared": 0.5422953822643336, "warnings": [] }, @@ -15440,13 +17991,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.28316326530612246, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.08957220799857168, + "seconds": 0.16979625000385568, "eta_squared": 0.5475800579356785, "warnings": [] }, @@ -15457,13 +18008,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.07663624999986496, + "seconds": 0.1656924579947372, "eta_squared": 0.5475800579356785, "warnings": [] }, @@ -15474,13 +18025,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9996789080215418, - "roc_auc": 0.999993356717687, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03316326530612245, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7863824170017324, + "seconds": 1.3735570419958094, "eta_squared": 0.5475800579356785, "warnings": [] }, @@ -15492,12 +18043,12 @@ "mechanism": "global", "level_spread": 0.8, "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29846938775510207, + "worst_group_fpr": 0.2653061224489796, "n_models": 1, - "seconds": 0.08090100000117673, + "seconds": 0.16967754200595664, "eta_squared": 0.5474858301661086, "warnings": [] }, @@ -15509,12 +18060,12 @@ "mechanism": "global", "level_spread": 0.8, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.11989795918367346, "n_models": 1, - "seconds": 0.07967862499936018, + "seconds": 0.1688615419989219, "eta_squared": 0.5474858301661086, "warnings": [] }, @@ -15529,9 +18080,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7925279999981285, + "seconds": 1.3673592920022202, "eta_squared": 0.5474858301661086, "warnings": [] }, @@ -15546,9 +18097,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2066326530612245, + "worst_group_fpr": 0.22959183673469388, "n_models": 1, - "seconds": 0.08738858300057473, + "seconds": 0.15921850000449922, "eta_squared": 0.5426833011841099, "warnings": [] }, @@ -15559,13 +18110,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.08409604099870194, + "seconds": 0.17815041600260884, "eta_squared": 0.5426833011841099, "warnings": [] }, @@ -15576,13 +18127,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9995636400137603, - "roc_auc": 0.9999911422902494, + "pr_auc": 0.9994516328453017, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7880617500013614, + "seconds": 1.3637191250018077, "eta_squared": 0.5426833011841099, "warnings": [] }, @@ -15593,13 +18144,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2653061224489796, + "worst_group_fpr": 0.25, "n_models": 1, - "seconds": 0.08894629199858173, + "seconds": 0.16293445799965411, "eta_squared": 0.5441570579629037, "warnings": [] }, @@ -15611,12 +18162,12 @@ "mechanism": "global", "level_spread": 0.8, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.08500491599988891, + "seconds": 0.1631988340013777, "eta_squared": 0.5441570579629037, "warnings": [] }, @@ -15627,13 +18178,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9994516328453017, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, + "pr_auc": 0.9980484049901452, + "roc_auc": 0.99996235473356, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7914371249971737, + "seconds": 1.4030500829976518, "eta_squared": 0.5441570579629037, "warnings": [] }, @@ -15644,13 +18195,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2729591836734694, + "worst_group_fpr": 0.2627551020408163, "n_models": 1, - "seconds": 0.08759733300030348, + "seconds": 0.16487941700324882, "eta_squared": 0.5425795873833327, "warnings": [] }, @@ -15661,13 +18212,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, + "worst_group_fpr": 0.13520408163265307, "n_models": 1, - "seconds": 0.09105041599832475, + "seconds": 0.16211316599947168, "eta_squared": 0.5425795873833327, "warnings": [] }, @@ -15678,13 +18229,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9976086261705306, - "roc_auc": 0.9999557114512472, + "pr_auc": 0.9969514320092234, + "roc_auc": 0.999944639314059, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9948950066137566, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7902605419985775, + "seconds": 1.3671795839982224, "eta_squared": 0.5425795873833327, "warnings": [] }, @@ -15695,13 +18246,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.24744897959183673, + "worst_group_fpr": 0.2602040816326531, "n_models": 1, - "seconds": 0.08753587500177673, + "seconds": 0.16315424999629613, "eta_squared": 0.5469699481347092, "warnings": [] }, @@ -15712,13 +18263,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, + "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.09185233400057768, + "seconds": 0.1662539579992881, "eta_squared": 0.5469699481347092, "warnings": [] }, @@ -15729,13 +18280,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9992433645957961, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9986906806565897, + "roc_auc": 0.9999734268707484, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7912287920007657, + "seconds": 1.3874576660018647, "eta_squared": 0.5469699481347092, "warnings": [] }, @@ -15750,9 +18301,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2780612244897959, + "worst_group_fpr": 0.2857142857142857, "n_models": 1, - "seconds": 0.08330637499966542, + "seconds": 0.16050458300014725, "eta_squared": 0.544205346444814, "warnings": [] }, @@ -15763,13 +18314,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.08133950000046752, + "seconds": 0.17303725000238046, "eta_squared": 0.544205346444814, "warnings": [] }, @@ -15780,13 +18331,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9966244734203707, - "roc_auc": 0.9999402104591837, + "pr_auc": 0.9986319303600136, + "roc_auc": 0.9999734268707483, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7714259999993374, + "seconds": 1.372127290997014, "eta_squared": 0.544205346444814, "warnings": [] }, @@ -15797,13 +18348,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25510204081632654, + "worst_group_fpr": 0.24489795918367346, "n_models": 1, - "seconds": 0.07723812499898486, + "seconds": 0.17187070799991488, "eta_squared": 0.5417981360356992, "warnings": [] }, @@ -15818,9 +18369,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, + "worst_group_fpr": 0.13520408163265307, "n_models": 1, - "seconds": 0.08279041700006928, + "seconds": 0.15840191699680872, "eta_squared": 0.5417981360356992, "warnings": [] }, @@ -15831,13 +18382,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.8046925830021792, + "seconds": 1.3960168749981676, "eta_squared": 0.5417981360356992, "warnings": [] }, @@ -15852,9 +18403,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21428571428571427, + "worst_group_fpr": 0.22448979591836735, "n_models": 1, - "seconds": 0.0878153330013447, + "seconds": 0.17027712499839254, "eta_squared": 0.5458940752362631, "warnings": [] }, @@ -15869,9 +18420,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.09355462500025169, + "seconds": 0.17035249999753432, "eta_squared": 0.5458940752362631, "warnings": [] }, @@ -15882,13 +18433,13 @@ "seed": 10, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.8644236660002207, + "seconds": 1.3717554160029977, "eta_squared": 0.5458940752362631, "warnings": [] }, @@ -15899,13 +18450,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9996744556165973, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.2653061224489796, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.2423469387755102, "n_models": 1, - "seconds": 0.09201187500002561, + "seconds": 0.16951775000052294, "eta_squared": 0.5428380555165477, "warnings": [] }, @@ -15916,13 +18467,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.08699370799877215, + "seconds": 0.16986883299978217, "eta_squared": 0.5428380555165477, "warnings": [] }, @@ -15933,13 +18484,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.994855534587282, - "roc_auc": 0.9999158517573696, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9940807443577351, + "roc_auc": 0.9999025651927437, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7936065409994626, + "seconds": 1.3802135000005364, "eta_squared": 0.5428380555165477, "warnings": [] }, @@ -15950,13 +18501,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2653061224489796, + "worst_group_fpr": 0.25510204081632654, "n_models": 1, - "seconds": 0.08785570799955167, + "seconds": 0.16807145799975842, "eta_squared": 0.5456713636299851, "warnings": [] }, @@ -15967,13 +18518,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.08340708299874677, + "seconds": 0.16689949999999953, "eta_squared": 0.5456713636299851, "warnings": [] }, @@ -15984,13 +18535,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.762391000000207, + "seconds": 1.3508785419980995, "eta_squared": 0.5456713636299851, "warnings": [] }, @@ -16001,13 +18552,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20408163265306123, + "worst_group_fpr": 0.23979591836734693, "n_models": 1, - "seconds": 0.08767304199864157, + "seconds": 0.16799091599386884, "eta_squared": 0.5453344397432399, "warnings": [] }, @@ -16018,13 +18569,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.09091595800055075, + "seconds": 0.17053383299935376, "eta_squared": 0.5453344397432399, "warnings": [] }, @@ -16035,13 +18586,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 0.9997874149659862, - "roc_auc": 0.9999955711451246, + "pr_auc": 0.9996744556165972, + "roc_auc": 0.9999933567176871, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, + "macro_pr_auc": 0.9975405092592592, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7814873749994149, + "seconds": 1.3389392909957678, "eta_squared": 0.5453344397432399, "warnings": [] }, @@ -16052,13 +18603,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25255102040816324, + "worst_group_fpr": 0.2576530612244898, "n_models": 1, - "seconds": 0.0868818340022699, + "seconds": 0.16087425000296207, "eta_squared": 0.5431459755082583, "warnings": [] }, @@ -16069,13 +18620,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.8, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.07824070900096558, + "seconds": 0.17221383399737533, "eta_squared": 0.5431459755082583, "warnings": [] }, @@ -16090,9 +18641,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.8067619579996972, + "seconds": 1.3914548750035465, "eta_squared": 0.5431459755082583, "warnings": [] }, @@ -16103,13 +18654,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.681798304119326, - "roc_auc": 0.9949909651360545, - "precision_at_n": 0.6875, - "macro_pr_auc": 0.9718088624338624, - "worst_group_fpr": 0.29336734693877553, + "pr_auc": 0.6205158398648214, + "roc_auc": 0.9936003047052154, + "precision_at_n": 0.625, + "macro_pr_auc": 0.9638013513243293, + "worst_group_fpr": 0.29846938775510207, "n_models": 1, - "seconds": 0.08836204200270004, + "seconds": 0.16608383399579907, "eta_squared": 0.9209097965562205, "warnings": [] }, @@ -16121,12 +18672,12 @@ "mechanism": "contextual", "level_spread": 0.8, "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18112244897959184, + "worst_group_fpr": 0.1760204081632653, "n_models": 1, - "seconds": 0.07975916700161179, + "seconds": 0.17551341599755688, "eta_squared": 0.9209097965562205, "warnings": [] }, @@ -16137,13 +18688,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8325424169997859, + "seconds": 1.384251082999981, "eta_squared": 0.9209097965562205, "warnings": [] }, @@ -16154,13 +18705,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.6975289453069624, - "roc_auc": 0.9948160253684807, + "pr_auc": 0.6365808137468482, + "roc_auc": 0.9937331703514739, "precision_at_n": 0.6354166666666666, - "macro_pr_auc": 0.951994825708061, - "worst_group_fpr": 0.29336734693877553, + "macro_pr_auc": 0.9516318369453045, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.08175404099893058, + "seconds": 0.1625109579981654, "eta_squared": 0.920373719914181, "warnings": [] }, @@ -16171,13 +18722,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9958578127329641, - "roc_auc": 0.9999291383219955, + "pr_auc": 0.9974733447852492, + "roc_auc": 0.9999534970238095, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.1377551020408163, "n_models": 1, - "seconds": 0.0871164169984695, + "seconds": 0.1630079590031528, "eta_squared": 0.920373719914181, "warnings": [] }, @@ -16188,13 +18739,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9523907508279281, - "roc_auc": 0.9992980265022675, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.8049216250001336, + "seconds": 1.3787873749970458, "eta_squared": 0.920373719914181, "warnings": [] }, @@ -16205,13 +18756,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.6518896249722297, - "roc_auc": 0.9932592828798186, - "precision_at_n": 0.5729166666666666, - "macro_pr_auc": 0.9735127005347594, - "worst_group_fpr": 0.32142857142857145, + "pr_auc": 0.6627120617322612, + "roc_auc": 0.9937929598922902, + "precision_at_n": 0.59375, + "macro_pr_auc": 0.97492784992785, + "worst_group_fpr": 0.3239795918367347, "n_models": 1, - "seconds": 0.08367216699843993, + "seconds": 0.16869162500370294, "eta_squared": 0.9210567957325315, "warnings": [] }, @@ -16222,13 +18773,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16071428571428573, + "worst_group_fpr": 0.1836734693877551, "n_models": 1, - "seconds": 0.08688112500021816, + "seconds": 0.1687801669977489, "eta_squared": 0.9210567957325315, "warnings": [] }, @@ -16239,13 +18790,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.8075408750009956, + "seconds": 1.4026547920002486, "eta_squared": 0.9210567957325315, "warnings": [] }, @@ -16256,13 +18807,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.4683855431996348, - "roc_auc": 0.9850481859410432, - "precision_at_n": 0.3645833333333333, - "macro_pr_auc": 0.9690972222222222, + "pr_auc": 0.5390877475746735, + "roc_auc": 0.989851279053288, + "precision_at_n": 0.4375, + "macro_pr_auc": 0.9653311965811966, "worst_group_fpr": 0.3086734693877551, "n_models": 1, - "seconds": 0.08949387499887962, + "seconds": 0.16397204100212548, "eta_squared": 0.9217812642726251, "warnings": [] }, @@ -16273,13 +18824,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.08769679200122482, + "seconds": 0.17485808300261851, "eta_squared": 0.9217812642726251, "warnings": [] }, @@ -16290,13 +18841,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7730962090026878, + "seconds": 1.4121400840012939, "eta_squared": 0.9217812642726251, "warnings": [] }, @@ -16307,13 +18858,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.920340014165772, - "roc_auc": 0.9986669146825397, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9895367364117363, - "worst_group_fpr": 0.25255102040816324, + "pr_auc": 0.7404323466969316, + "roc_auc": 0.9964281285430839, + "precision_at_n": 0.7604166666666666, + "macro_pr_auc": 0.9565724522796891, + "worst_group_fpr": 0.2755102040816326, "n_models": 1, - "seconds": 0.08265704099903814, + "seconds": 0.17475745800038567, "eta_squared": 0.920649352214294, "warnings": [] }, @@ -16324,13 +18875,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9998926116838486, - "roc_auc": 0.9999977855725625, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.13010204081632654, "n_models": 1, - "seconds": 0.09078566700191004, + "seconds": 0.17303454200009583, "eta_squared": 0.920649352214294, "warnings": [] }, @@ -16341,13 +18892,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.7948213329982536, + "seconds": 1.3783052500002668, "eta_squared": 0.920649352214294, "warnings": [] }, @@ -16358,13 +18909,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.7376304330422577, - "roc_auc": 0.9951326884920635, - "precision_at_n": 0.6666666666666666, - "macro_pr_auc": 0.9928685897435897, - "worst_group_fpr": 0.29591836734693877, + "pr_auc": 0.7417238194824436, + "roc_auc": 0.9957327983276645, + "precision_at_n": 0.6770833333333334, + "macro_pr_auc": 0.9907176157176157, + "worst_group_fpr": 0.29846938775510207, "n_models": 1, - "seconds": 0.08608991599976434, + "seconds": 0.17469762499968056, "eta_squared": 0.9186868472692276, "warnings": [] }, @@ -16375,13 +18926,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1683673469387755, + "worst_group_fpr": 0.16326530612244897, "n_models": 1, - "seconds": 0.08270133300175075, + "seconds": 0.16913675000250805, "eta_squared": 0.9186868472692276, "warnings": [] }, @@ -16392,13 +18943,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8256208329985384, + "seconds": 1.3586927500000456, "eta_squared": 0.9186868472692276, "warnings": [] }, @@ -16409,13 +18960,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.7419668543274931, - "roc_auc": 0.9959210246598639, - "precision_at_n": 0.7395833333333334, - "macro_pr_auc": 0.9858743686868686, - "worst_group_fpr": 0.2576530612244898, + "pr_auc": 0.6793935016179142, + "roc_auc": 0.9950286104024944, + "precision_at_n": 0.7083333333333334, + "macro_pr_auc": 0.9670454545454544, + "worst_group_fpr": 0.27040816326530615, "n_models": 1, - "seconds": 0.08848787500028266, + "seconds": 0.17535466700064717, "eta_squared": 0.9181718730647732, "warnings": [] }, @@ -16426,13 +18977,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.1377551020408163, "n_models": 1, - "seconds": 0.08740104199750931, + "seconds": 0.16288858400366735, "eta_squared": 0.9181718730647732, "warnings": [] }, @@ -16443,13 +18994,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8212779999994382, + "seconds": 1.3662952910017339, "eta_squared": 0.9181718730647732, "warnings": [] }, @@ -16460,13 +19011,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.5073131114931568, - "roc_auc": 0.9889832234977325, - "precision_at_n": 0.4791666666666667, - "macro_pr_auc": 0.9532295380625122, - "worst_group_fpr": 0.3010204081632653, + "pr_auc": 0.5471876899551111, + "roc_auc": 0.9916958971088435, + "precision_at_n": 0.5520833333333334, + "macro_pr_auc": 0.9524636243386242, + "worst_group_fpr": 0.30357142857142855, "n_models": 1, - "seconds": 0.08298383299916168, + "seconds": 0.17614049999974668, "eta_squared": 0.9208674523939366, "warnings": [] }, @@ -16477,13 +19028,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.16581632653061223, "n_models": 1, - "seconds": 0.0848398329981137, + "seconds": 0.16324841599998763, "eta_squared": 0.9208674523939366, "warnings": [] }, @@ -16494,13 +19045,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8311338749990682, + "seconds": 1.3721862920065178, "eta_squared": 0.9208674523939366, "warnings": [] }, @@ -16511,13 +19062,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.6895028537478385, - "roc_auc": 0.9936556653911566, - "precision_at_n": 0.6041666666666666, - "macro_pr_auc": 0.9805205098343684, - "worst_group_fpr": 0.29846938775510207, + "pr_auc": 0.6437263546024731, + "roc_auc": 0.9928850446428571, + "precision_at_n": 0.5520833333333334, + "macro_pr_auc": 0.9770419973544974, + "worst_group_fpr": 0.30612244897959184, "n_models": 1, - "seconds": 0.0786492500010354, + "seconds": 0.1704562500017346, "eta_squared": 0.9213876858018002, "warnings": [] }, @@ -16528,13 +19079,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9963283583334289, - "roc_auc": 0.9999357816043083, + "pr_auc": 0.9992239393431517, + "roc_auc": 0.9999844990079365, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, + "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.08939033299975563, + "seconds": 0.16489541699411348, "eta_squared": 0.9213876858018002, "warnings": [] }, @@ -16545,13 +19096,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.8267360840000038, + "seconds": 1.394481416005874, "eta_squared": 0.9213876858018002, "warnings": [] }, @@ -16562,13 +19113,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.6339613178668613, - "roc_auc": 0.9939656852324263, - "precision_at_n": 0.6458333333333334, - "macro_pr_auc": 0.9464195526695526, - "worst_group_fpr": 0.3086734693877551, + "pr_auc": 0.726224453983664, + "roc_auc": 0.9952699829931974, + "precision_at_n": 0.6979166666666666, + "macro_pr_auc": 0.9709099927849928, + "worst_group_fpr": 0.3010204081632653, "n_models": 1, - "seconds": 0.07994841700201505, + "seconds": 0.1655430829996476, "eta_squared": 0.9202352791959443, "warnings": [] }, @@ -16579,13 +19130,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 1.0, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.16326530612244897, "n_models": 1, - "seconds": 0.08697612499963725, + "seconds": 0.17286137499468168, "eta_squared": 0.9202352791959443, "warnings": [] }, @@ -16596,13 +19147,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.8108292079996318, + "seconds": 1.3797562909967382, "eta_squared": 0.9202352791959443, "warnings": [] }, @@ -16613,13 +19164,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.7125730237172959, - "roc_auc": 0.9963461947278911, - "precision_at_n": 0.7604166666666666, - "macro_pr_auc": 0.952378684088243, - "worst_group_fpr": 0.28316326530612246, + "pr_auc": 0.6766827706413012, + "roc_auc": 0.9955932893990931, + "precision_at_n": 0.71875, + "macro_pr_auc": 0.9492125496031747, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.0884319590004452, + "seconds": 0.16753829200024484, "eta_squared": 0.9201601632705698, "warnings": [] }, @@ -16630,13 +19181,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, + "pr_auc": 1.0, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.11479591836734694, "n_models": 1, - "seconds": 0.08454666699981317, + "seconds": 0.16889616700063925, "eta_squared": 0.9201601632705698, "warnings": [] }, @@ -16647,13 +19198,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7721837089993642, + "seconds": 1.364785457997641, "eta_squared": 0.9201601632705698, "warnings": [] }, @@ -16664,13 +19215,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.7410573923465887, - "roc_auc": 0.9964613449546484, - "precision_at_n": 0.75, - "macro_pr_auc": 0.9539373394636552, - "worst_group_fpr": 0.30357142857142855, + "pr_auc": 0.59167460136602, + "roc_auc": 0.9932304953231292, + "precision_at_n": 0.6041666666666666, + "macro_pr_auc": 0.9397987697762353, + "worst_group_fpr": 0.30612244897959184, "n_models": 1, - "seconds": 0.08881941599975107, + "seconds": 0.16093204200296896, "eta_squared": 0.9203742833505759, "warnings": [] }, @@ -16681,13 +19232,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9995636400137603, + "roc_auc": 0.9999911422902494, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, + "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.08157641700017848, + "seconds": 0.16490308300126344, "eta_squared": 0.9203742833505759, "warnings": [] }, @@ -16698,13 +19249,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9619580084145981, - "roc_auc": 0.99949289611678, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7682856249994074, + "seconds": 1.4029429169968353, "eta_squared": 0.9203742833505759, "warnings": [] }, @@ -16715,13 +19266,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.7621316150026999, - "roc_auc": 0.9956663655045351, - "precision_at_n": 0.6875, - "macro_pr_auc": 0.9945549242424243, - "worst_group_fpr": 0.29591836734693877, + "pr_auc": 0.6925685701050049, + "roc_auc": 0.9940210459183674, + "precision_at_n": 0.5833333333333334, + "macro_pr_auc": 0.9916200697450698, + "worst_group_fpr": 0.30357142857142855, "n_models": 1, - "seconds": 0.08588337499895715, + "seconds": 0.1594262910002726, "eta_squared": 0.9218843396708194, "warnings": [] }, @@ -16736,9 +19287,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.15051020408163265, "n_models": 1, - "seconds": 0.12737383299827343, + "seconds": 0.17699987500236603, "eta_squared": 0.9218843396708194, "warnings": [] }, @@ -16749,13 +19300,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7875508329998411, + "seconds": 1.3922457090011449, "eta_squared": 0.9218843396708194, "warnings": [] }, @@ -16766,13 +19317,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.5876624542741195, - "roc_auc": 0.9925993835034013, - "precision_at_n": 0.6354166666666666, - "macro_pr_auc": 0.9481195371667185, - "worst_group_fpr": 0.2755102040816326, + "pr_auc": 0.5450910945212057, + "roc_auc": 0.99144566680839, + "precision_at_n": 0.53125, + "macro_pr_auc": 0.9489377552047388, + "worst_group_fpr": 0.28316326530612246, "n_models": 1, - "seconds": 0.08751795900025172, + "seconds": 0.16409912499511847, "eta_squared": 0.9193097916370743, "warnings": [] }, @@ -16783,13 +19334,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9757123371101586, - "roc_auc": 0.9996014030612245, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9956018518518519, - "worst_group_fpr": 0.1326530612244898, + "pr_auc": 0.9886091097053284, + "roc_auc": 0.9998250602324263, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9975405092592592, + "worst_group_fpr": 0.1377551020408163, "n_models": 1, - "seconds": 0.08832674999939627, + "seconds": 0.16337608300091233, "eta_squared": 0.9193097916370743, "warnings": [] }, @@ -16800,13 +19351,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.9329070122690577, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8594744999973045, + "seconds": 1.3948482920022798, "eta_squared": 0.9193097916370743, "warnings": [] }, @@ -16817,13 +19368,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.7111010445901043, - "roc_auc": 0.9958878082482994, - "precision_at_n": 0.7291666666666666, - "macro_pr_auc": 0.9571602009102009, - "worst_group_fpr": 0.31887755102040816, + "pr_auc": 0.6187515878913341, + "roc_auc": 0.9938593927154195, + "precision_at_n": 0.6458333333333334, + "macro_pr_auc": 0.9463323577294166, + "worst_group_fpr": 0.3163265306122449, "n_models": 1, - "seconds": 0.07846191600037855, + "seconds": 0.17068183299852535, "eta_squared": 0.920517468315107, "warnings": [] }, @@ -16835,12 +19386,12 @@ "mechanism": "contextual", "level_spread": 0.8, "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17091836734693877, + "worst_group_fpr": 0.16326530612244897, "n_models": 1, - "seconds": 0.08422133400017628, + "seconds": 0.16075029200146673, "eta_squared": 0.920517468315107, "warnings": [] }, @@ -16851,13 +19402,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.8, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.780792792000284, + "seconds": 1.3838352919992758, "eta_squared": 0.920517468315107, "warnings": [] }, @@ -16868,13 +19419,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.288265306122449, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.07455820799805224, + "seconds": 0.17201291600213153, "eta_squared": 0.5572328878968438, "warnings": [] }, @@ -16889,9 +19440,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, + "worst_group_fpr": 0.1096938775510204, "n_models": 1, - "seconds": 0.07642225000017788, + "seconds": 0.16330987500259653, "eta_squared": 0.5572328878968438, "warnings": [] }, @@ -16902,13 +19453,13 @@ "seed": 0, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8349476659968786, + "seconds": 1.3859826250045444, "eta_squared": 0.5572328878968438, "warnings": [] }, @@ -16919,13 +19470,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21683673469387754, + "worst_group_fpr": 0.25255102040816324, "n_models": 1, - "seconds": 0.08829708299890626, + "seconds": 0.15904233300534543, "eta_squared": 0.5545595572982635, "warnings": [] }, @@ -16940,9 +19491,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.0792739580028865, + "seconds": 0.15868020800553495, "eta_squared": 0.5545595572982635, "warnings": [] }, @@ -16953,13 +19504,13 @@ "seed": 1, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9988020366397061, - "roc_auc": 0.9999767485119048, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9993553717642547, + "roc_auc": 0.9999867134353742, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.800436333000107, + "seconds": 1.3736786670051515, "eta_squared": 0.5545595572982635, "warnings": [] }, @@ -16970,13 +19521,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.30612244897959184, + "worst_group_fpr": 0.3010204081632653, "n_models": 1, - "seconds": 0.08130583399906754, + "seconds": 0.15516070900048362, "eta_squared": 0.5602958775475205, "warnings": [] }, @@ -16987,13 +19538,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.08753462499953457, + "seconds": 0.16194199999881675, "eta_squared": 0.5602958775475205, "warnings": [] }, @@ -17004,13 +19555,13 @@ "seed": 2, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.999575836489899, - "roc_auc": 0.9999911422902494, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7678894999990007, + "seconds": 1.3723653750057565, "eta_squared": 0.5602958775475205, "warnings": [] }, @@ -17021,13 +19572,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.3163265306122449, + "worst_group_fpr": 0.2729591836734694, "n_models": 1, - "seconds": 0.08538550000230316, + "seconds": 0.16464475000248058, "eta_squared": 0.5600535097729571, "warnings": [] }, @@ -17038,13 +19589,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, + "worst_group_fpr": 0.1326530612244898, "n_models": 1, - "seconds": 0.0778062499994121, + "seconds": 0.1710237499937648, "eta_squared": 0.5600535097729571, "warnings": [] }, @@ -17055,13 +19606,13 @@ "seed": 3, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7888367919986194, + "seconds": 1.3771845829978702, "eta_squared": 0.5600535097729571, "warnings": [] }, @@ -17076,9 +19627,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20918367346938777, + "worst_group_fpr": 0.23469387755102042, "n_models": 1, - "seconds": 0.0808924160010065, + "seconds": 0.15870858299604151, "eta_squared": 0.5549776244660766, "warnings": [] }, @@ -17093,9 +19644,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, + "worst_group_fpr": 0.13010204081632654, "n_models": 1, - "seconds": 0.07597116699980688, + "seconds": 0.15347879200271564, "eta_squared": 0.5549776244660766, "warnings": [] }, @@ -17106,13 +19657,13 @@ "seed": 4, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9995636400137602, - "roc_auc": 0.9999911422902494, + "pr_auc": 0.9994516328453016, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7883488749976095, + "seconds": 1.3591966250023688, "eta_squared": 0.5549776244660766, "warnings": [] }, @@ -17123,13 +19674,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2780612244897959, + "worst_group_fpr": 0.26785714285714285, "n_models": 1, - "seconds": 0.08512200000041048, + "seconds": 0.1636783750000177, "eta_squared": 0.5568389917399191, "warnings": [] }, @@ -17140,13 +19691,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.0915095830023347, + "seconds": 0.1575528339963057, "eta_squared": 0.5568389917399191, "warnings": [] }, @@ -17157,13 +19708,13 @@ "seed": 5, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9995636400137603, - "roc_auc": 0.9999911422902492, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, + "pr_auc": 0.9981739069981774, + "roc_auc": 0.9999645691609977, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8239208750019316, + "seconds": 1.373370834000525, "eta_squared": 0.5568389917399191, "warnings": [] }, @@ -17174,13 +19725,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838488, + "roc_auc": 0.9999977855725622, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29081632653061223, + "worst_group_fpr": 0.2780612244897959, "n_models": 1, - "seconds": 0.08209349999742699, + "seconds": 0.17131791599967983, "eta_squared": 0.55533969332111, "warnings": [] }, @@ -17191,13 +19742,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999997, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.14540816326530612, "n_models": 1, - "seconds": 0.08643829199718311, + "seconds": 0.15299862499523442, "eta_squared": 0.55533969332111, "warnings": [] }, @@ -17208,13 +19759,13 @@ "seed": 6, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9983853734038977, - "roc_auc": 0.999968998015873, + "pr_auc": 0.997899041334633, + "roc_auc": 0.9999601403061225, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, + "macro_pr_auc": 0.9948950066137566, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8057116249983665, + "seconds": 1.4097843749987078, "eta_squared": 0.55533969332111, "warnings": [] }, @@ -17225,13 +19776,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.28316326530612246, + "worst_group_fpr": 0.2653061224489796, "n_models": 1, - "seconds": 0.08546391700292588, + "seconds": 0.1579061250013183, "eta_squared": 0.5593879759814088, "warnings": [] }, @@ -17242,13 +19793,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, + "worst_group_fpr": 0.14285714285714285, "n_models": 1, - "seconds": 0.08844525000313297, + "seconds": 0.16645320800307672, "eta_squared": 0.5593879759814088, "warnings": [] }, @@ -17259,13 +19810,13 @@ "seed": 7, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9992332114897579, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9986906806565897, + "roc_auc": 0.9999734268707484, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7703120000005583, + "seconds": 1.3940077090010163, "eta_squared": 0.5593879759814088, "warnings": [] }, @@ -17280,9 +19831,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.28316326530612246, + "worst_group_fpr": 0.3010204081632653, "n_models": 1, - "seconds": 0.07417500000155997, + "seconds": 0.1680378750024829, "eta_squared": 0.5564035218173411, "warnings": [] }, @@ -17293,13 +19844,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.07529554099892266, + "seconds": 0.16570941700047115, "eta_squared": 0.5564035218173411, "warnings": [] }, @@ -17310,13 +19861,13 @@ "seed": 8, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9960159379888793, - "roc_auc": 0.9999313527494331, + "pr_auc": 0.9982598713958657, + "roc_auc": 0.9999667835884354, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7456938750001427, + "seconds": 1.3501350420046947, "eta_squared": 0.5564035218173411, "warnings": [] }, @@ -17327,13 +19878,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.24744897959183673, + "worst_group_fpr": 0.22959183673469388, "n_models": 1, - "seconds": 0.0773404580031638, + "seconds": 0.17021429200394778, "eta_squared": 0.5540528723307074, "warnings": [] }, @@ -17344,13 +19895,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.09114179100288311, + "seconds": 0.1611757079954259, "eta_squared": 0.5540528723307074, "warnings": [] }, @@ -17361,13 +19912,13 @@ "seed": 9, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.8689664580015233, + "seconds": 1.363703709001129, "eta_squared": 0.5540528723307074, "warnings": [] }, @@ -17382,9 +19933,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22193877551020408, + "worst_group_fpr": 0.23469387755102042, "n_models": 1, - "seconds": 0.07785095800136332, + "seconds": 0.163054208001995, "eta_squared": 0.5584465959181422, "warnings": [] }, @@ -17399,9 +19950,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.0865530420014693, + "seconds": 0.16169712499686284, "eta_squared": 0.5584465959181422, "warnings": [] }, @@ -17416,9 +19967,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.76247237500138, + "seconds": 1.3753051669991692, "eta_squared": 0.5584465959181422, "warnings": [] }, @@ -17429,13 +19980,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.288265306122449, + "worst_group_fpr": 0.27040816326530615, "n_models": 1, - "seconds": 0.08170925000013085, + "seconds": 0.15954833300202154, "eta_squared": 0.5549925397074991, "warnings": [] }, @@ -17446,13 +19997,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, + "worst_group_fpr": 0.09183673469387756, "n_models": 1, - "seconds": 0.08589354200012167, + "seconds": 0.16052020799543243, "eta_squared": 0.5549925397074991, "warnings": [] }, @@ -17463,13 +20014,13 @@ "seed": 11, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9953696119468665, - "roc_auc": 0.9999224950396826, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9938979958197234, + "roc_auc": 0.9999003507653061, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8043951660001767, + "seconds": 1.3658120420004707, "eta_squared": 0.5549925397074991, "warnings": [] }, @@ -17480,13 +20031,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999998, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2755102040816326, + "worst_group_fpr": 0.2602040816326531, "n_models": 1, - "seconds": 0.08595641700230772, + "seconds": 0.15997279099974548, "eta_squared": 0.5579244171259582, "warnings": [] }, @@ -17498,12 +20049,12 @@ "mechanism": "global", "level_spread": 0.9, "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.08596374999979162, + "seconds": 0.16077487500297138, "eta_squared": 0.5579244171259582, "warnings": [] }, @@ -17514,13 +20065,13 @@ "seed": 12, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7921546249999665, + "seconds": 1.3736999590037158, "eta_squared": 0.5579244171259582, "warnings": [] }, @@ -17531,13 +20082,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22193877551020408, + "worst_group_fpr": 0.2653061224489796, "n_models": 1, - "seconds": 0.0836649579978257, + "seconds": 0.1635333750018617, "eta_squared": 0.5578235062169735, "warnings": [] }, @@ -17548,13 +20099,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.11224489795918367, "n_models": 1, - "seconds": 0.08290887499970268, + "seconds": 0.16439820799860172, "eta_squared": 0.5578235062169735, "warnings": [] }, @@ -17565,13 +20116,13 @@ "seed": 13, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9996789080215419, - "roc_auc": 0.9999933567176871, + "pr_auc": 0.9993384082076203, + "roc_auc": 0.9999867134353742, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, + "macro_pr_auc": 0.9943163029100529, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8014245000013034, + "seconds": 1.3988040420008474, "eta_squared": 0.5578235062169735, "warnings": [] }, @@ -17588,7 +20139,7 @@ "macro_pr_auc": 1.0, "worst_group_fpr": 0.26785714285714285, "n_models": 1, - "seconds": 0.0813014999985171, + "seconds": 0.16486979099863674, "eta_squared": 0.5556051803235251, "warnings": [] }, @@ -17599,13 +20150,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.07846112500192248, + "seconds": 0.15863283399812644, "eta_squared": 0.5556051803235251, "warnings": [] }, @@ -17616,13 +20167,13 @@ "seed": 14, "mechanism": "global", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.8116107500027283, + "seconds": 1.3529080000007525, "eta_squared": 0.5556051803235251, "warnings": [] }, @@ -17633,13 +20184,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.48370182641704884, - "roc_auc": 0.9887352076247166, + "pr_auc": 0.4987466519060602, + "roc_auc": 0.9896608382936508, "precision_at_n": 0.4791666666666667, - "macro_pr_auc": 0.94273530765998, - "worst_group_fpr": 0.3163265306122449, + "macro_pr_auc": 0.9490303977068683, + "worst_group_fpr": 0.3112244897959184, "n_models": 1, - "seconds": 0.08873683300043922, + "seconds": 0.16161441700387513, "eta_squared": 0.9337002346939631, "warnings": [] }, @@ -17650,13 +20201,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17346938775510204, + "worst_group_fpr": 0.16581632653061223, "n_models": 1, - "seconds": 0.08215395899969735, + "seconds": 0.16105029200116405, "eta_squared": 0.9337002346939631, "warnings": [] }, @@ -17667,13 +20218,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7766321250019246, + "seconds": 1.4107413750025444, "eta_squared": 0.9337002346939631, "warnings": [] }, @@ -17684,13 +20235,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.48841516890928915, - "roc_auc": 0.9884827628968254, - "precision_at_n": 0.4583333333333333, - "macro_pr_auc": 0.948426906046769, - "worst_group_fpr": 0.30612244897959184, + "pr_auc": 0.5003038173153509, + "roc_auc": 0.9893973214285715, + "precision_at_n": 0.4895833333333333, + "macro_pr_auc": 0.9454233776844071, + "worst_group_fpr": 0.30357142857142855, "n_models": 1, - "seconds": 0.07183579200136592, + "seconds": 0.16223812499811174, "eta_squared": 0.9332108003153095, "warnings": [] }, @@ -17701,13 +20252,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9903355234771398, - "roc_auc": 0.9998693487811792, + "pr_auc": 0.9935536142019581, + "roc_auc": 0.9999003507653061, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.14285714285714285, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.08315720799873816, + "seconds": 0.15896033299941337, "eta_squared": 0.9332108003153095, "warnings": [] }, @@ -17718,13 +20269,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.999289168792517, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7925684170004388, + "seconds": 1.424678416995448, "eta_squared": 0.9332108003153095, "warnings": [] }, @@ -17735,13 +20286,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.49233872316581706, - "roc_auc": 0.9881218112244898, - "precision_at_n": 0.4270833333333333, - "macro_pr_auc": 0.9649555077597841, + "pr_auc": 0.581959738741685, + "roc_auc": 0.991259654903628, + "precision_at_n": 0.5208333333333334, + "macro_pr_auc": 0.9686428444240943, "worst_group_fpr": 0.32653061224489793, "n_models": 1, - "seconds": 0.08811762500045006, + "seconds": 0.16620829200110165, "eta_squared": 0.9338330621664808, "warnings": [] }, @@ -17752,13 +20303,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, + "worst_group_fpr": 0.16581632653061223, "n_models": 1, - "seconds": 0.08562370800063945, + "seconds": 0.17143991599732544, "eta_squared": 0.9338330621664808, "warnings": [] }, @@ -17769,13 +20320,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767567760167845, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7701699589997588, + "seconds": 1.362880166998366, "eta_squared": 0.9338330621664808, "warnings": [] }, @@ -17786,13 +20337,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.5443471300698407, - "roc_auc": 0.9854932858560091, - "precision_at_n": 0.3541666666666667, - "macro_pr_auc": 0.9873006333943835, - "worst_group_fpr": 0.30612244897959184, + "pr_auc": 0.5734492143594968, + "roc_auc": 0.9893021010487527, + "precision_at_n": 0.4166666666666667, + "macro_pr_auc": 0.9824074074074075, + "worst_group_fpr": 0.30357142857142855, "n_models": 1, - "seconds": 0.08923404199958895, + "seconds": 0.16573454199533444, "eta_squared": 0.9344356642098697, "warnings": [] }, @@ -17807,9 +20358,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1836734693877551, + "worst_group_fpr": 0.18112244897959184, "n_models": 1, - "seconds": 0.08842350000122678, + "seconds": 0.16718445799779147, "eta_squared": 0.9344356642098697, "warnings": [] }, @@ -17820,13 +20371,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8114923339999223, + "seconds": 1.372973625002487, "eta_squared": 0.9344356642098697, "warnings": [] }, @@ -17837,13 +20388,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.7228507251411649, - "roc_auc": 0.9961579683956916, - "precision_at_n": 0.7291666666666666, - "macro_pr_auc": 0.9599738666834255, - "worst_group_fpr": 0.28316326530612246, + "pr_auc": 0.6242719407014463, + "roc_auc": 0.99417827026644, + "precision_at_n": 0.65625, + "macro_pr_auc": 0.9518738363447793, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.08098579099896597, + "seconds": 0.1799211670004297, "eta_squared": 0.9334757370341583, "warnings": [] }, @@ -17854,13 +20405,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9997841047394042, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999999, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, + "worst_group_fpr": 0.1377551020408163, "n_models": 1, - "seconds": 0.09482916599881719, + "seconds": 0.16911824999988312, "eta_squared": 0.9334757370341583, "warnings": [] }, @@ -17871,13 +20422,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.8320669169988832, + "seconds": 1.3532594999996945, "eta_squared": 0.9334757370341583, "warnings": [] }, @@ -17888,13 +20439,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.4341574536013676, - "roc_auc": 0.985728015164399, - "precision_at_n": 0.3958333333333333, - "macro_pr_auc": 0.9419825605680869, - "worst_group_fpr": 0.32142857142857145, + "pr_auc": 0.46906027929970967, + "roc_auc": 0.9883653982426305, + "precision_at_n": 0.4270833333333333, + "macro_pr_auc": 0.9407022970006715, + "worst_group_fpr": 0.31887755102040816, "n_models": 1, - "seconds": 0.0860680420009885, + "seconds": 0.1665471250016708, "eta_squared": 0.9318170650470089, "warnings": [] }, @@ -17909,9 +20460,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, + "worst_group_fpr": 0.17091836734693877, "n_models": 1, - "seconds": 0.08812975000182632, + "seconds": 0.16424591700342717, "eta_squared": 0.9318170650470089, "warnings": [] }, @@ -17922,13 +20473,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7867327090025356, + "seconds": 1.3771830000041518, "eta_squared": 0.9318170650470089, "warnings": [] }, @@ -17939,13 +20490,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.8306151753552133, - "roc_auc": 0.9970902423469388, - "precision_at_n": 0.7916666666666666, - "macro_pr_auc": 0.9971590909090909, - "worst_group_fpr": 0.29846938775510207, + "pr_auc": 0.6971414379905212, + "roc_auc": 0.9953364158163266, + "precision_at_n": 0.7083333333333334, + "macro_pr_auc": 0.9820684523809523, + "worst_group_fpr": 0.288265306122449, "n_models": 1, - "seconds": 0.08717691700076102, + "seconds": 0.15878320800402435, "eta_squared": 0.9313493667097374, "warnings": [] }, @@ -17956,13 +20507,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.08346687500306871, + "seconds": 0.16666441700363066, "eta_squared": 0.9313493667097374, "warnings": [] }, @@ -17973,13 +20524,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7639452910007094, + "seconds": 1.363710791003541, "eta_squared": 0.9313493667097374, "warnings": [] }, @@ -17990,13 +20541,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.49641269539946786, - "roc_auc": 0.9876434948979592, - "precision_at_n": 0.4270833333333333, - "macro_pr_auc": 0.9692120927318296, - "worst_group_fpr": 0.3112244897959184, + "pr_auc": 0.5880397479982498, + "roc_auc": 0.9912042942176871, + "precision_at_n": 0.5104166666666666, + "macro_pr_auc": 0.9729600694444445, + "worst_group_fpr": 0.3137755102040816, "n_models": 1, - "seconds": 0.07500079100282164, + "seconds": 0.16411391599831404, "eta_squared": 0.9336353454255104, "warnings": [] }, @@ -18007,13 +20558,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, + "pr_auc": 0.9999999999999999, + "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, + "worst_group_fpr": 0.16581632653061223, "n_models": 1, - "seconds": 0.08246458299981896, + "seconds": 0.16820949999964796, "eta_squared": 0.9336353454255104, "warnings": [] }, @@ -18024,13 +20575,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.759892708996631, + "seconds": 1.360429499996826, "eta_squared": 0.9336353454255104, "warnings": [] }, @@ -18041,13 +20592,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.5603500230997622, - "roc_auc": 0.9891692354024944, - "precision_at_n": 0.46875, - "macro_pr_auc": 0.972537088350666, - "worst_group_fpr": 0.30612244897959184, + "pr_auc": 0.5283112653468994, + "roc_auc": 0.9889721513605443, + "precision_at_n": 0.4583333333333333, + "macro_pr_auc": 0.9699745604231492, + "worst_group_fpr": 0.3137755102040816, "n_models": 1, - "seconds": 0.08461950000128127, + "seconds": 0.17555916700075613, "eta_squared": 0.9340786626302715, "warnings": [] }, @@ -18058,13 +20609,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9728929481135735, - "roc_auc": 0.9997165532879818, + "pr_auc": 0.9908147321763701, + "roc_auc": 0.9998560622165533, "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9864045965608467, - "worst_group_fpr": 0.1556122448979592, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.15306122448979592, "n_models": 1, - "seconds": 0.08807795800021267, + "seconds": 0.16599275000044145, "eta_squared": 0.9340786626302715, "warnings": [] }, @@ -18075,13 +20626,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7622362080001039, + "seconds": 1.3732968749973224, "eta_squared": 0.9340786626302715, "warnings": [] }, @@ -18092,13 +20643,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.5749684361771564, - "roc_auc": 0.9910359977324262, - "precision_at_n": 0.4583333333333333, - "macro_pr_auc": 0.9552121489621489, - "worst_group_fpr": 0.3112244897959184, + "pr_auc": 0.6864082496031527, + "roc_auc": 0.9936069479875284, + "precision_at_n": 0.6041666666666666, + "macro_pr_auc": 0.9763625841750841, + "worst_group_fpr": 0.29846938775510207, "n_models": 1, - "seconds": 0.07600716700108023, + "seconds": 0.16572633299801964, "eta_squared": 0.9331214740684672, "warnings": [] }, @@ -18113,9 +20664,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.15816326530612246, "n_models": 1, - "seconds": 0.0798523750017921, + "seconds": 0.1568466669996269, "eta_squared": 0.9331214740684672, "warnings": [] }, @@ -18126,13 +20677,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7766599579990725, + "seconds": 1.3904178329976276, "eta_squared": 0.9331214740684672, "warnings": [] }, @@ -18143,13 +20694,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.6433556438094067, - "roc_auc": 0.9947562358276645, + "pr_auc": 0.6333266417600595, + "roc_auc": 0.9945082199546484, "precision_at_n": 0.6666666666666666, - "macro_pr_auc": 0.9443264634670885, - "worst_group_fpr": 0.3137755102040816, + "macro_pr_auc": 0.9465535332722833, + "worst_group_fpr": 0.3163265306122449, "n_models": 1, - "seconds": 0.08223033399917767, + "seconds": 0.15864504099590704, "eta_squared": 0.9329217798181637, "warnings": [] }, @@ -18160,13 +20711,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, + "pr_auc": 0.9997841047394044, + "roc_auc": 0.9999955711451247, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.1096938775510204, "n_models": 1, - "seconds": 0.08641679100037436, + "seconds": 0.17548050000186777, "eta_squared": 0.9329217798181637, "warnings": [] }, @@ -18177,13 +20728,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7963710830008495, + "seconds": 1.3608411249952042, "eta_squared": 0.9329217798181637, "warnings": [] }, @@ -18194,13 +20745,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.487662332276554, - "roc_auc": 0.9900085034013606, - "precision_at_n": 0.4583333333333333, - "macro_pr_auc": 0.9343976263898139, - "worst_group_fpr": 0.31887755102040816, + "pr_auc": 0.5439936919818846, + "roc_auc": 0.9908322704081632, + "precision_at_n": 0.4895833333333333, + "macro_pr_auc": 0.949382215007215, + "worst_group_fpr": 0.3137755102040816, "n_models": 1, - "seconds": 0.0876374580002448, + "seconds": 0.15965133300051093, "eta_squared": 0.9333000674744998, "warnings": [] }, @@ -18211,13 +20762,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.998801470355826, - "roc_auc": 0.999975641298186, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.9977436320959776, + "roc_auc": 0.9999579258786848, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, + "worst_group_fpr": 0.17346938775510204, "n_models": 1, - "seconds": 0.08732816700285184, + "seconds": 0.1705870420046267, "eta_squared": 0.9333000674744998, "warnings": [] }, @@ -18228,13 +20779,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9619580084145981, - "roc_auc": 0.99949289611678, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.830850875001488, + "seconds": 1.3444945420051226, "eta_squared": 0.9333000674744998, "warnings": [] }, @@ -18245,13 +20796,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.6257991263089661, - "roc_auc": 0.9913903061224489, - "precision_at_n": 0.5416666666666666, - "macro_pr_auc": 0.9787367724867725, - "worst_group_fpr": 0.32142857142857145, + "pr_auc": 0.6005803367394442, + "roc_auc": 0.990914204223356, + "precision_at_n": 0.5104166666666666, + "macro_pr_auc": 0.9784474206349206, + "worst_group_fpr": 0.3239795918367347, "n_models": 1, - "seconds": 0.09474454199880711, + "seconds": 0.16280158400331857, "eta_squared": 0.9344982331342713, "warnings": [] }, @@ -18262,13 +20813,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.1326530612244898, "n_models": 1, - "seconds": 0.08025383299900568, + "seconds": 0.15939304199855542, "eta_squared": 0.9344982331342713, "warnings": [] }, @@ -18279,13 +20830,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.8052421659995161, + "seconds": 1.3916888750027283, "eta_squared": 0.9344982331342713, "warnings": [] }, @@ -18296,13 +20847,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.4725756220492768, - "roc_auc": 0.9875062003968254, - "precision_at_n": 0.4166666666666667, - "macro_pr_auc": 0.9477053668689698, - "worst_group_fpr": 0.30612244897959184, + "pr_auc": 0.4835413524167843, + "roc_auc": 0.9882236748866213, + "precision_at_n": 0.4375, + "macro_pr_auc": 0.9448220755693582, + "worst_group_fpr": 0.29846938775510207, "n_models": 1, - "seconds": 0.08894675000192365, + "seconds": 0.16157833299803315, "eta_squared": 0.9323325522428791, "warnings": [] }, @@ -18313,13 +20864,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9824324760482137, - "roc_auc": 0.9996877657312926, - "precision_at_n": 0.9270833333333334, + "pr_auc": 0.9857768221421451, + "roc_auc": 0.9997674851190477, + "precision_at_n": 0.9479166666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.08182137500261888, + "seconds": 0.16966954200324835, "eta_squared": 0.9323325522428791, "warnings": [] }, @@ -18330,13 +20881,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9089068819073483, + "roc_auc": 0.9985451211734694, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9259082641895143, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8128278749973106, + "seconds": 1.387394666999171, "eta_squared": 0.9323325522428791, "warnings": [] }, @@ -18347,13 +20898,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.5740868812626767, - "roc_auc": 0.9920081313775511, - "precision_at_n": 0.5729166666666666, - "macro_pr_auc": 0.9453199226297052, - "worst_group_fpr": 0.32142857142857145, + "pr_auc": 0.6018279227472707, + "roc_auc": 0.9930932008219955, + "precision_at_n": 0.5833333333333334, + "macro_pr_auc": 0.9483675468050468, + "worst_group_fpr": 0.32653061224489793, "n_models": 1, - "seconds": 0.08808708299693535, + "seconds": 0.1802478749959846, "eta_squared": 0.9333406233039432, "warnings": [] }, @@ -18364,13 +20915,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9991328016734993, + "roc_auc": 0.9999822845804989, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.15816326530612246, "n_models": 1, - "seconds": 0.0876066249984433, + "seconds": 0.16228891699574888, "eta_squared": 0.9333406233039432, "warnings": [] }, @@ -18381,13 +20932,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 0.9, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7643466250010533, + "seconds": 1.3904078750056215, "eta_squared": 0.9333406233039432, "warnings": [] }, @@ -18398,13 +20949,13 @@ "seed": 0, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999997, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2755102040816326, + "worst_group_fpr": 0.30357142857142855, "n_models": 1, - "seconds": 0.08390070799941896, + "seconds": 0.17233216599561274, "eta_squared": 0.5676363348862092, "warnings": [] }, @@ -18419,9 +20970,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.08287154100253247, + "seconds": 0.1639115420039161, "eta_squared": 0.5676363348862092, "warnings": [] }, @@ -18432,13 +20983,13 @@ "seed": 0, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9998926116838485, + "roc_auc": 0.9999977855725624, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7580239579983754, + "seconds": 1.3711525830003666, "eta_squared": 0.5676363348862092, "warnings": [] }, @@ -18449,13 +21000,13 @@ "seed": 1, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2372448979591837, + "worst_group_fpr": 0.2602040816326531, "n_models": 1, - "seconds": 0.0869074170004751, + "seconds": 0.17048391699790955, "eta_squared": 0.5648975634450134, "warnings": [] }, @@ -18470,9 +21021,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.08407229200020083, + "seconds": 0.17477779199543875, "eta_squared": 0.5648975634450134, "warnings": [] }, @@ -18483,13 +21034,13 @@ "seed": 1, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9988859606860465, - "roc_auc": 0.9999778557256236, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9994629892108766, + "roc_auc": 0.9999889278628118, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8368146669999987, + "seconds": 1.3926485830015736, "eta_squared": 0.5648975634450134, "warnings": [] }, @@ -18504,9 +21055,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.30612244897959184, + "worst_group_fpr": 0.31887755102040816, "n_models": 1, - "seconds": 0.08855483300067135, + "seconds": 0.15824562500347383, "eta_squared": 0.5710146129420057, "warnings": [] }, @@ -18517,13 +21068,13 @@ "seed": 2, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.12755102040816327, "n_models": 1, - "seconds": 0.09004162500059465, + "seconds": 0.16800374999729684, "eta_squared": 0.5710146129420057, "warnings": [] }, @@ -18534,13 +21085,13 @@ "seed": 2, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9996789080215417, - "roc_auc": 0.9999933567176872, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999997, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.784261583998159, + "seconds": 1.4105422079956043, "eta_squared": 0.5710146129420057, "warnings": [] }, @@ -18551,13 +21102,13 @@ "seed": 3, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.3112244897959184, + "worst_group_fpr": 0.29846938775510207, "n_models": 1, - "seconds": 0.07325204199878499, + "seconds": 0.15720345800218638, "eta_squared": 0.5706485648142726, "warnings": [] }, @@ -18568,13 +21119,13 @@ "seed": 3, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 0.9999999999999999, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.07290899999861722, + "seconds": 0.17112720800651005, "eta_squared": 0.5706485648142726, "warnings": [] }, @@ -18585,13 +21136,13 @@ "seed": 3, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8076408750021074, + "seconds": 1.3866387080051936, "eta_squared": 0.5706485648142726, "warnings": [] }, @@ -18606,9 +21157,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21173469387755103, + "worst_group_fpr": 0.22959183673469388, "n_models": 1, - "seconds": 0.07311187500090455, + "seconds": 0.1597049159972812, "eta_squared": 0.56534003587956, "warnings": [] }, @@ -18619,13 +21170,13 @@ "seed": 4, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, + "worst_group_fpr": 0.125, "n_models": 1, - "seconds": 0.0710214589998941, + "seconds": 0.1596773330020369, "eta_squared": 0.56534003587956, "warnings": [] }, @@ -18636,13 +21187,13 @@ "seed": 4, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9996744556165972, - "roc_auc": 0.9999933567176871, + "pr_auc": 0.9994516328453017, + "roc_auc": 0.9999889278628118, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7913264160015387, + "seconds": 1.3918389169994043, "eta_squared": 0.56534003587956, "warnings": [] }, @@ -18653,13 +21204,13 @@ "seed": 5, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25510204081632654, + "worst_group_fpr": 0.2780612244897959, "n_models": 1, - "seconds": 0.08004762499695062, + "seconds": 0.16154229200037662, "eta_squared": 0.5675034019831222, "warnings": [] }, @@ -18670,13 +21221,13 @@ "seed": 5, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, + "worst_group_fpr": 0.10204081632653061, "n_models": 1, - "seconds": 0.09018975000071805, + "seconds": 0.17201179199764738, "eta_squared": 0.5675034019831222, "warnings": [] }, @@ -18687,13 +21238,13 @@ "seed": 5, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9995636400137603, - "roc_auc": 0.9999911422902494, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, + "pr_auc": 0.9980484049901452, + "roc_auc": 0.99996235473356, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9960524140211641, "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8272125000003143, + "seconds": 1.3769019169994863, "eta_squared": 0.5675034019831222, "warnings": [] }, @@ -18704,13 +21255,13 @@ "seed": 6, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2627551020408163, + "worst_group_fpr": 0.27040816326530615, "n_models": 1, - "seconds": 0.08454420799898799, + "seconds": 0.17271570899902144, "eta_squared": 0.5660843709494305, "warnings": [] }, @@ -18721,13 +21272,13 @@ "seed": 6, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, + "worst_group_fpr": 0.14795918367346939, "n_models": 1, - "seconds": 0.1031810829990718, + "seconds": 0.17352633300470188, "eta_squared": 0.5660843709494305, "warnings": [] }, @@ -18738,13 +21289,13 @@ "seed": 6, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9981328388755406, - "roc_auc": 0.9999645691609979, + "pr_auc": 0.9976369764612153, + "roc_auc": 0.9999557114512472, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, + "macro_pr_auc": 0.9948950066137566, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7952350419982395, + "seconds": 1.3770624590033549, "eta_squared": 0.5660843709494305, "warnings": [] }, @@ -18755,13 +21306,13 @@ "seed": 7, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.28316326530612246, + "worst_group_fpr": 0.2857142857142857, "n_models": 1, - "seconds": 0.08404750000045169, + "seconds": 0.15575383300165413, "eta_squared": 0.5698671092657079, "warnings": [] }, @@ -18772,13 +21323,13 @@ "seed": 7, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, + "worst_group_fpr": 0.13010204081632654, "n_models": 1, - "seconds": 0.08244604100036668, + "seconds": 0.1640514169994276, "eta_squared": 0.5698671092657079, "warnings": [] }, @@ -18789,13 +21340,13 @@ "seed": 7, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9993464361274391, - "roc_auc": 0.9999867134353743, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.998801470355826, + "roc_auc": 0.999975641298186, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8000821249988803, + "seconds": 1.4011617919968558, "eta_squared": 0.5698671092657079, "warnings": [] }, @@ -18806,13 +21357,13 @@ "seed": 8, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29081632653061223, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.08897100000103819, + "seconds": 0.1727778330023284, "eta_squared": 0.5667004409467457, "warnings": [] }, @@ -18823,13 +21374,13 @@ "seed": 8, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.09693877551020408, "n_models": 1, - "seconds": 0.08988291700006812, + "seconds": 0.16893491600058042, "eta_squared": 0.5667004409467457, "warnings": [] }, @@ -18840,13 +21391,13 @@ "seed": 8, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9961695831403155, - "roc_auc": 0.9999335671768708, + "pr_auc": 0.9982598713958657, + "roc_auc": 0.9999667835884354, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, + "macro_pr_auc": 0.9988425925925926, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7982099159999052, + "seconds": 1.37281208299828, "eta_squared": 0.5667004409467457, "warnings": [] }, @@ -18857,13 +21408,13 @@ "seed": 9, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2576530612244898, + "worst_group_fpr": 0.23979591836734693, "n_models": 1, - "seconds": 0.08204345800186275, + "seconds": 0.15791291700588772, "eta_squared": 0.5643728023336351, "warnings": [] }, @@ -18878,9 +21429,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, + "worst_group_fpr": 0.1377551020408163, "n_models": 1, - "seconds": 0.08827945799930603, + "seconds": 0.1657633330032695, "eta_squared": 0.5643728023336351, "warnings": [] }, @@ -18891,13 +21442,13 @@ "seed": 9, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7822977089999767, + "seconds": 1.3924569169976166, "eta_squared": 0.5643728023336351, "warnings": [] }, @@ -18908,13 +21459,13 @@ "seed": 10, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, + "worst_group_fpr": 0.23214285714285715, "n_models": 1, - "seconds": 0.08758908300296753, + "seconds": 0.16296616700128652, "eta_squared": 0.5690320175635628, "warnings": [] }, @@ -18925,13 +21476,13 @@ "seed": 10, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 1.0, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, + "worst_group_fpr": 0.10714285714285714, "n_models": 1, - "seconds": 0.08838724999804981, + "seconds": 0.17207512500317534, "eta_squared": 0.5690320175635628, "warnings": [] }, @@ -18942,13 +21493,13 @@ "seed": 10, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9996843434343433, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, + "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.8200831659996766, + "seconds": 1.384157916996628, "eta_squared": 0.5690320175635628, "warnings": [] }, @@ -18963,9 +21514,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29846938775510207, + "worst_group_fpr": 0.2755102040816326, "n_models": 1, - "seconds": 0.0864905419985007, + "seconds": 0.15116024999588262, "eta_squared": 0.5652473317791511, "warnings": [] }, @@ -18980,9 +21531,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.08354408300147043, + "seconds": 0.16885133400501218, "eta_squared": 0.5652473317791511, "warnings": [] }, @@ -18993,13 +21544,13 @@ "seed": 11, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9952030832506646, - "roc_auc": 0.9999202806122449, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9948853727883523, + "roc_auc": 0.999913637329932, + "precision_at_n": 0.9791666666666666, + "macro_pr_auc": 0.9922329695767195, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.79343237500143, + "seconds": 1.3844270000045071, "eta_squared": 0.5652473317791511, "warnings": [] }, @@ -19010,13 +21561,13 @@ "seed": 12, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.3010204081632653, + "worst_group_fpr": 0.2653061224489796, "n_models": 1, - "seconds": 0.08703429200249957, + "seconds": 0.1657912500013481, "eta_squared": 0.5682748535558901, "warnings": [] }, @@ -19031,9 +21582,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, + "worst_group_fpr": 0.09948979591836735, "n_models": 1, - "seconds": 0.08590316700065159, + "seconds": 0.1545707500044955, "eta_squared": 0.5682748535558901, "warnings": [] }, @@ -19044,13 +21595,13 @@ "seed": 12, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.82700795799974, + "seconds": 1.3596160000015516, "eta_squared": 0.5682748535558901, "warnings": [] }, @@ -19065,9 +21616,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2066326530612245, + "worst_group_fpr": 0.25255102040816324, "n_models": 1, - "seconds": 0.07719537500088336, + "seconds": 0.1665559579996625, "eta_squared": 0.5683637055956918, "warnings": [] }, @@ -19079,12 +21630,12 @@ "mechanism": "global", "level_spread": 1.0, "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.10459183673469388, "n_models": 1, - "seconds": 0.08894529099779902, + "seconds": 0.1660269169951789, "eta_squared": 0.5683637055956918, "warnings": [] }, @@ -19095,13 +21646,13 @@ "seed": 13, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9996789080215418, - "roc_auc": 0.9999933567176871, + "pr_auc": 0.9992332114897579, + "roc_auc": 0.9999844990079364, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, + "macro_pr_auc": 0.9943163029100529, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.8343272090023675, + "seconds": 1.4021668329951353, "eta_squared": 0.5683637055956918, "warnings": [] }, @@ -19112,13 +21663,13 @@ "seed": 14, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2627551020408163, + "worst_group_fpr": 0.2755102040816326, "n_models": 1, - "seconds": 0.08936945900131832, + "seconds": 0.1687291659982293, "eta_squared": 0.5661057125507907, "warnings": [] }, @@ -19129,13 +21680,13 @@ "seed": 14, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, + "worst_group_fpr": 0.08928571428571429, "n_models": 1, - "seconds": 0.09122275000117952, + "seconds": 0.16114408300200012, "eta_squared": 0.5661057125507907, "warnings": [] }, @@ -19146,13 +21697,13 @@ "seed": 14, "mechanism": "global", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999997, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.8167644160021155, + "seconds": 1.3545676249996177, "eta_squared": 0.5661057125507907, "warnings": [] }, @@ -19163,13 +21714,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.4404474898959679, - "roc_auc": 0.9855663619614513, - "precision_at_n": 0.4375, - "macro_pr_auc": 0.9488062744037009, - "worst_group_fpr": 0.3086734693877551, + "pr_auc": 0.48534868873259845, + "roc_auc": 0.9877010700113379, + "precision_at_n": 0.4895833333333333, + "macro_pr_auc": 0.9478546145212811, + "worst_group_fpr": 0.3137755102040816, "n_models": 1, - "seconds": 0.07679250000001048, + "seconds": 0.1693762080030865, "eta_squared": 0.9430926146131842, "warnings": [] }, @@ -19180,13 +21731,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18112244897959184, + "worst_group_fpr": 0.16581632653061223, "n_models": 1, - "seconds": 0.0897597080002015, + "seconds": 0.16320516599807888, "eta_squared": 0.9430926146131842, "warnings": [] }, @@ -19197,13 +21748,13 @@ "seed": 0, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9867282776223474, - "roc_auc": 0.9997608418367346, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9924355158730158, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9745705638556281, + "roc_auc": 0.999554900085034, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9835110780423281, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7976526249985909, + "seconds": 1.3755387499986682, "eta_squared": 0.9430926146131842, "warnings": [] }, @@ -19214,13 +21765,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.5297227019254098, - "roc_auc": 0.9909983524659864, + "pr_auc": 0.5367151920328781, + "roc_auc": 0.989968643707483, "precision_at_n": 0.5, - "macro_pr_auc": 0.9452332937530307, - "worst_group_fpr": 0.29336734693877553, + "macro_pr_auc": 0.9527008908057296, + "worst_group_fpr": 0.30357142857142855, "n_models": 1, - "seconds": 0.08685933399829082, + "seconds": 0.1608518329958315, "eta_squared": 0.9426371564373757, "warnings": [] }, @@ -19231,13 +21782,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.987198828087694, - "roc_auc": 0.9998472045068028, + "pr_auc": 0.990321141535306, + "roc_auc": 0.9998693487811792, "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.14030612244897958, + "macro_pr_auc": 1.0, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.08658529100284795, + "seconds": 0.15965487499488518, "eta_squared": 0.9426371564373757, "warnings": [] }, @@ -19248,13 +21799,13 @@ "seed": 1, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9505343079049093, - "roc_auc": 0.999289168792517, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9559441137566137, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9629263242825986, + "roc_auc": 0.9994353210034013, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9706225198412698, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7732552500019665, + "seconds": 1.3880165829978068, "eta_squared": 0.9426371564373757, "warnings": [] }, @@ -19265,13 +21816,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.5556907945800128, - "roc_auc": 0.990117010345805, - "precision_at_n": 0.5520833333333334, - "macro_pr_auc": 0.970216922238981, - "worst_group_fpr": 0.32653061224489793, + "pr_auc": 0.5495354609375203, + "roc_auc": 0.9888769309807256, + "precision_at_n": 0.5416666666666666, + "macro_pr_auc": 0.9730052933177933, + "worst_group_fpr": 0.32908163265306123, "n_models": 1, - "seconds": 0.0870056250023481, + "seconds": 0.17115704200114124, "eta_squared": 0.9432131151496429, "warnings": [] }, @@ -19286,9 +21837,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, + "worst_group_fpr": 0.16326530612244897, "n_models": 1, - "seconds": 0.08347629200216033, + "seconds": 0.17343999999866355, "eta_squared": 0.9432131151496429, "warnings": [] }, @@ -19299,13 +21850,13 @@ "seed": 2, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9675380860028191, - "roc_auc": 0.9995161476048753, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9709118716931218, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9767568942402173, + "roc_auc": 0.9996102607709751, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.976078869047619, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7792606670009263, + "seconds": 1.3421517079987098, "eta_squared": 0.9432131151496429, "warnings": [] }, @@ -19316,13 +21867,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.4358561409859812, - "roc_auc": 0.9812991602891157, - "precision_at_n": 0.3229166666666667, - "macro_pr_auc": 0.9661458333333334, - "worst_group_fpr": 0.336734693877551, + "pr_auc": 0.45011384406487887, + "roc_auc": 0.9838612528344672, + "precision_at_n": 0.3541666666666667, + "macro_pr_auc": 0.9624727558321308, + "worst_group_fpr": 0.31887755102040816, "n_models": 1, - "seconds": 0.08025700000143843, + "seconds": 0.16151770799478982, "eta_squared": 0.9437235511189537, "warnings": [] }, @@ -19333,13 +21884,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 1.0, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1683673469387755, + "worst_group_fpr": 0.1760204081632653, "n_models": 1, - "seconds": 0.07899366700075916, + "seconds": 0.16660925000178395, "eta_squared": 0.9437235511189537, "warnings": [] }, @@ -19350,13 +21901,13 @@ "seed": 3, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9543889606828282, - "roc_auc": 0.9994751806972789, + "pr_auc": 0.9670331496927849, + "roc_auc": 0.9995128259637188, "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9662822420634921, + "macro_pr_auc": 0.9628554894179894, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7909962499979883, + "seconds": 1.3712917910015676, "eta_squared": 0.9437235511189537, "warnings": [] }, @@ -19367,13 +21918,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.6522575845109674, - "roc_auc": 0.9937198837868481, - "precision_at_n": 0.6145833333333334, - "macro_pr_auc": 0.9728918650793651, - "worst_group_fpr": 0.30357142857142855, + "pr_auc": 0.5750853751747382, + "roc_auc": 0.9916073200113378, + "precision_at_n": 0.5104166666666666, + "macro_pr_auc": 0.9601720446950711, + "worst_group_fpr": 0.3112244897959184, "n_models": 1, - "seconds": 0.08615758300220477, + "seconds": 0.16282804099319037, "eta_squared": 0.9428896083609098, "warnings": [] }, @@ -19384,13 +21935,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9974747370154907, - "roc_auc": 0.9999534970238096, + "pr_auc": 0.9988740304185753, + "roc_auc": 0.9999778557256236, "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, + "worst_group_fpr": 0.1326530612244898, "n_models": 1, - "seconds": 0.08721558400065987, + "seconds": 0.16667325000162236, "eta_squared": 0.9428896083609098, "warnings": [] }, @@ -19401,13 +21952,13 @@ "seed": 4, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9918297236189909, - "roc_auc": 0.999882635345805, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9866503567728645, + "roc_auc": 0.999782986111111, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9899181547619048, + "worst_group_fpr": 0.04591836734693878, "n_models": 12, - "seconds": 0.792246332999639, + "seconds": 1.3986902910037315, "eta_squared": 0.9428896083609098, "warnings": [] }, @@ -19418,13 +21969,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.5515014660004262, - "roc_auc": 0.9890186543367347, - "precision_at_n": 0.5104166666666666, - "macro_pr_auc": 0.9818695533769063, - "worst_group_fpr": 0.31887755102040816, + "pr_auc": 0.493833466763317, + "roc_auc": 0.9875128436791383, + "precision_at_n": 0.4791666666666667, + "macro_pr_auc": 0.9630205153642653, + "worst_group_fpr": 0.3137755102040816, "n_models": 1, - "seconds": 0.0903698340007395, + "seconds": 0.16258195899717975, "eta_squared": 0.9414594641783969, "warnings": [] }, @@ -19435,13 +21986,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, + "worst_group_fpr": 0.17857142857142858, "n_models": 1, - "seconds": 0.0913679580007738, + "seconds": 0.16420841699437005, "eta_squared": 0.9414594641783969, "warnings": [] }, @@ -19452,13 +22003,13 @@ "seed": 5, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9725508634860032, - "roc_auc": 0.9995460423752834, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9757853835978837, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9687108941098512, + "roc_auc": 0.9994973249716553, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9687417328042329, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.8012882500006526, + "seconds": 1.3986366669996642, "eta_squared": 0.9414594641783969, "warnings": [] }, @@ -19469,13 +22020,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.7353436685064171, - "roc_auc": 0.9948448129251701, - "precision_at_n": 0.65625, - "macro_pr_auc": 0.9971590909090909, - "worst_group_fpr": 0.29081632653061223, + "pr_auc": 0.6135563927593939, + "roc_auc": 0.9933611465419502, + "precision_at_n": 0.6041666666666666, + "macro_pr_auc": 0.9647178631553631, + "worst_group_fpr": 0.29591836734693877, "n_models": 1, - "seconds": 0.07957833299951744, + "seconds": 0.17231612499745097, "eta_squared": 0.9410327000313501, "warnings": [] }, @@ -19486,13 +22037,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9999999999999998, + "pr_auc": 0.9999999999999999, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, + "worst_group_fpr": 0.15816326530612246, "n_models": 1, - "seconds": 0.08582583399766008, + "seconds": 0.16098312500253087, "eta_squared": 0.9410327000313501, "warnings": [] }, @@ -19503,13 +22054,13 @@ "seed": 6, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9807289352882025, - "roc_auc": 0.9996966234410432, + "pr_auc": 0.9792851034241599, + "roc_auc": 0.9996833368764172, "precision_at_n": 0.96875, "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.03826530612244898, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7827843750019383, + "seconds": 1.4123492910002824, "eta_squared": 0.9410327000313501, "warnings": [] }, @@ -19520,13 +22071,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.47024340453116165, - "roc_auc": 0.9853626346371882, - "precision_at_n": 0.3854166666666667, - "macro_pr_auc": 0.9707844672688423, - "worst_group_fpr": 0.3137755102040816, + "pr_auc": 0.49320212664757, + "roc_auc": 0.987397693452381, + "precision_at_n": 0.4270833333333333, + "macro_pr_auc": 0.9670368370736018, + "worst_group_fpr": 0.3086734693877551, "n_models": 1, - "seconds": 0.0861924579985498, + "seconds": 0.17187958299473394, "eta_squared": 0.9430130855086445, "warnings": [] }, @@ -19541,9 +22092,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16071428571428573, + "worst_group_fpr": 0.1760204081632653, "n_models": 1, - "seconds": 0.08583804199952283, + "seconds": 0.16060612499859417, "eta_squared": 0.9430130855086445, "warnings": [] }, @@ -19554,13 +22105,13 @@ "seed": 7, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.8881024848231156, - "roc_auc": 0.9986049107142858, - "precision_at_n": 0.8541666666666666, - "macro_pr_auc": 0.9196304563492065, - "worst_group_fpr": 0.03571428571428571, + "pr_auc": 0.9298555808760586, + "roc_auc": 0.9992315936791383, + "precision_at_n": 0.8958333333333334, + "macro_pr_auc": 0.9523520171957672, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.803724166999018, + "seconds": 1.335786209005164, "eta_squared": 0.9430130855086445, "warnings": [] }, @@ -19571,13 +22122,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.5989056649722575, - "roc_auc": 0.9890607284580499, - "precision_at_n": 0.5520833333333334, - "macro_pr_auc": 0.9787465428090427, - "worst_group_fpr": 0.30357142857142855, + "pr_auc": 0.5900752988793343, + "roc_auc": 0.9890651573129252, + "precision_at_n": 0.5416666666666666, + "macro_pr_auc": 0.9817336309523809, + "worst_group_fpr": 0.30612244897959184, "n_models": 1, - "seconds": 0.08554504100175109, + "seconds": 0.1624552909997874, "eta_squared": 0.9433967168032253, "warnings": [] }, @@ -19588,13 +22139,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9890192795549045, - "roc_auc": 0.9998494189342404, - "precision_at_n": 0.9791666666666666, + "pr_auc": 0.994319218174482, + "roc_auc": 0.9999092084750566, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17091836734693877, + "worst_group_fpr": 0.16071428571428573, "n_models": 1, - "seconds": 0.08578779100207612, + "seconds": 0.16499349999503465, "eta_squared": 0.9433967168032253, "warnings": [] }, @@ -19605,13 +22156,13 @@ "seed": 8, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9806220968736649, - "roc_auc": 0.999672264739229, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9769179894179892, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9748395926005912, + "roc_auc": 0.9995947597789115, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9738632605820104, + "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.782843375000084, + "seconds": 1.3485228749996168, "eta_squared": 0.9433967168032253, "warnings": [] }, @@ -19622,13 +22173,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.4566709690403336, - "roc_auc": 0.9856970131802721, - "precision_at_n": 0.3645833333333333, - "macro_pr_auc": 0.9320161716421307, - "worst_group_fpr": 0.32908163265306123, + "pr_auc": 0.5261343002549365, + "roc_auc": 0.9887418509070296, + "precision_at_n": 0.4375, + "macro_pr_auc": 0.9440026619622208, + "worst_group_fpr": 0.32142857142857145, "n_models": 1, - "seconds": 0.07899349999934202, + "seconds": 0.16455762500117999, "eta_squared": 0.9425810756331876, "warnings": [] }, @@ -19639,13 +22190,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, + "pr_auc": 0.9999999999999998, "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.08821075000014389, + "seconds": 0.16205866599921137, "eta_squared": 0.9425810756331876, "warnings": [] }, @@ -19656,13 +22207,13 @@ "seed": 9, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9871735075615327, - "roc_auc": 0.9997364831349206, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9900628306878306, + "pr_auc": 0.983868818654779, + "roc_auc": 0.9996988378684807, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.984844727032227, "worst_group_fpr": 0.04336734693877551, "n_models": 12, - "seconds": 0.7780413750006119, + "seconds": 1.398233292005898, "eta_squared": 0.9425810756331876, "warnings": [] }, @@ -19673,13 +22224,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.4558993558210772, - "roc_auc": 0.9876368516156462, - "precision_at_n": 0.4166666666666667, - "macro_pr_auc": 0.9422583645929236, - "worst_group_fpr": 0.3163265306122449, + "pr_auc": 0.5513522273736624, + "roc_auc": 0.9916936826814058, + "precision_at_n": 0.5416666666666666, + "macro_pr_auc": 0.9481992313242312, + "worst_group_fpr": 0.30612244897959184, "n_models": 1, - "seconds": 0.0782590419985354, + "seconds": 0.15685741700144717, "eta_squared": 0.9423071534590428, "warnings": [] }, @@ -19694,9 +22245,9 @@ "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, + "worst_group_fpr": 0.11734693877551021, "n_models": 1, - "seconds": 0.08753012500164914, + "seconds": 0.1638517920000595, "eta_squared": 0.9423071534590428, "warnings": [] }, @@ -19707,13 +22258,13 @@ "seed": 10, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9697656129469174, - "roc_auc": 0.9994707518424036, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.968812003968254, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9852957098632265, + "roc_auc": 0.9997298398526077, + "precision_at_n": 0.9375, + "macro_pr_auc": 0.9818039021164019, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.782373624999309, + "seconds": 1.3726681250045658, "eta_squared": 0.9423071534590428, "warnings": [] }, @@ -19724,13 +22275,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.5631518605076176, - "roc_auc": 0.9916826105442177, - "precision_at_n": 0.5520833333333334, - "macro_pr_auc": 0.956760796008329, - "worst_group_fpr": 0.3086734693877551, + "pr_auc": 0.48798028950188443, + "roc_auc": 0.9883388251133787, + "precision_at_n": 0.4166666666666667, + "macro_pr_auc": 0.9373701096357346, + "worst_group_fpr": 0.3112244897959184, "n_models": 1, - "seconds": 0.08744820900028571, + "seconds": 0.17445962500642054, "eta_squared": 0.9427808191647584, "warnings": [] }, @@ -19741,13 +22292,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9967753175878704, - "roc_auc": 0.9999357816043084, - "precision_at_n": 0.96875, + "pr_auc": 0.9925550534350223, + "roc_auc": 0.9998759920634921, + "precision_at_n": 0.9791666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, + "worst_group_fpr": 0.1760204081632653, "n_models": 1, - "seconds": 0.08697441700132913, + "seconds": 0.16523474999848986, "eta_squared": 0.9427808191647584, "warnings": [] }, @@ -19758,13 +22309,13 @@ "seed": 11, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.961741641669088, - "roc_auc": 0.9994884672619048, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9683077050264549, - "worst_group_fpr": 0.04081632653061224, + "pr_auc": 0.9440775446407559, + "roc_auc": 0.9992581668083901, + "precision_at_n": 0.9166666666666666, + "macro_pr_auc": 0.9595516173641173, + "worst_group_fpr": 0.03826530612244898, "n_models": 12, - "seconds": 0.7827680410009634, + "seconds": 1.3668047909959569, "eta_squared": 0.9427808191647584, "warnings": [] }, @@ -19775,13 +22326,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.6117859143148155, - "roc_auc": 0.9889987244897959, - "precision_at_n": 0.5, - "macro_pr_auc": 0.9988425925925926, + "pr_auc": 0.5444069766801478, + "roc_auc": 0.9861266121031745, + "precision_at_n": 0.4479166666666667, + "macro_pr_auc": 0.9886326058201059, "worst_group_fpr": 0.3239795918367347, "n_models": 1, - "seconds": 0.07708795899816323, + "seconds": 0.16296875000261934, "eta_squared": 0.9437633675151137, "warnings": [] }, @@ -19792,13 +22343,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, + "pr_auc": 1.0, + "roc_auc": 1.0, "precision_at_n": 1.0, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, + "worst_group_fpr": 0.1556122448979592, "n_models": 1, - "seconds": 0.08376499999940279, + "seconds": 0.16889766600070288, "eta_squared": 0.9437633675151137, "warnings": [] }, @@ -19809,13 +22360,13 @@ "seed": 12, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9874493808974725, - "roc_auc": 0.9997852005385488, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9865492724867725, - "worst_group_fpr": 0.03826530612244898, + "pr_auc": 0.9867244637441994, + "roc_auc": 0.99976527069161, + "precision_at_n": 0.9583333333333334, + "macro_pr_auc": 0.9858258928571427, + "worst_group_fpr": 0.03571428571428571, "n_models": 12, - "seconds": 0.7783098330000939, + "seconds": 1.3833198329957668, "eta_squared": 0.9437633675151137, "warnings": [] }, @@ -19826,13 +22377,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.4144666289612037, - "roc_auc": 0.9853404903628118, - "precision_at_n": 0.3958333333333333, - "macro_pr_auc": 0.9277287780895475, - "worst_group_fpr": 0.29591836734693877, + "pr_auc": 0.40110407158158806, + "roc_auc": 0.9845565830498868, + "precision_at_n": 0.375, + "macro_pr_auc": 0.9210572052368926, + "worst_group_fpr": 0.30357142857142855, "n_models": 1, - "seconds": 0.08058441599860089, + "seconds": 0.15952129100332968, "eta_squared": 0.9418930396685032, "warnings": [] }, @@ -19843,13 +22394,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9489157076696662, - "roc_auc": 0.9991607320011339, - "precision_at_n": 0.8854166666666666, + "pr_auc": 0.9612198529137181, + "roc_auc": 0.999406533446712, + "precision_at_n": 0.9166666666666666, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, + "worst_group_fpr": 0.14030612244897958, "n_models": 1, - "seconds": 0.08507408299919916, + "seconds": 0.17576462499710033, "eta_squared": 0.9418930396685032, "warnings": [] }, @@ -19860,13 +22411,13 @@ "seed": 13, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.932976816087396, - "roc_auc": 0.9988529265873016, - "precision_at_n": 0.8645833333333334, - "macro_pr_auc": 0.9459077380952382, - "worst_group_fpr": 0.04336734693877551, + "pr_auc": 0.9082358809082258, + "roc_auc": 0.9985362634637187, + "precision_at_n": 0.84375, + "macro_pr_auc": 0.9244201689514191, + "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7501342499999737, + "seconds": 1.3914272089969018, "eta_squared": 0.9418930396685032, "warnings": [] }, @@ -19877,13 +22428,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.5402136199887647, - "roc_auc": 0.9890385841836735, - "precision_at_n": 0.5104166666666666, - "macro_pr_auc": 0.9504573704481792, - "worst_group_fpr": 0.3239795918367347, + "pr_auc": 0.5103424703484694, + "roc_auc": 0.9890673717403629, + "precision_at_n": 0.4479166666666667, + "macro_pr_auc": 0.9523000208855472, + "worst_group_fpr": 0.32908163265306123, "n_models": 1, - "seconds": 0.08150612499957788, + "seconds": 0.15692079099972034, "eta_squared": 0.9427598024542532, "warnings": [] }, @@ -19894,13 +22445,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, + "pr_auc": 0.9977421731790775, + "roc_auc": 0.9999579258786848, + "precision_at_n": 0.9895833333333334, "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17346938775510204, + "worst_group_fpr": 0.18112244897959184, "n_models": 1, - "seconds": 0.08459220800068579, + "seconds": 0.1764908339973772, "eta_squared": 0.9427598024542532, "warnings": [] }, @@ -19911,13 +22462,13 @@ "seed": 14, "mechanism": "contextual", "level_spread": 1.0, - "pr_auc": 0.992765746796775, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9894085948773449, + "pr_auc": 0.9907914066468546, + "roc_auc": 0.9998228458049887, + "precision_at_n": 0.9479166666666666, + "macro_pr_auc": 0.9881065115440114, "worst_group_fpr": 0.04081632653061224, "n_models": 12, - "seconds": 0.7775512919979519, + "seconds": 1.3573104169990984, "eta_squared": 0.9427598024542532, "warnings": [] }, @@ -19928,13 +22479,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06359419427859529, - "roc_auc": 0.6844236319753716, - "precision_at_n": 0.10664523043944266, - "macro_pr_auc": 0.2615858617185089, - "worst_group_fpr": 0.32496863237139273, + "pr_auc": 0.06599372001242891, + "roc_auc": 0.6794908017938576, + "precision_at_n": 0.10986066452304394, + "macro_pr_auc": 0.26413518265780117, + "worst_group_fpr": 0.3041405269761606, "n_models": 1, - "seconds": 0.28429770799994003, + "seconds": 0.7082737079981598, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -19945,13 +22496,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07853477794777033, - "roc_auc": 0.7221384471549323, - "precision_at_n": 0.11843515541264737, - "macro_pr_auc": 0.30104520993588496, - "worst_group_fpr": 0.34002509410288584, + "pr_auc": 0.08975412713487951, + "roc_auc": 0.7263518402451048, + "precision_at_n": 0.13504823151125403, + "macro_pr_auc": 0.32422570878576495, + "worst_group_fpr": 0.27854454203262236, "n_models": 1, - "seconds": 0.3833286670014786, + "seconds": 0.8081601670055534, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -19962,13 +22513,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08605352394216598, - "roc_auc": 0.6270455719998697, - "precision_at_n": 0.1379957127545552, - "macro_pr_auc": 0.3471501929009753, - "worst_group_fpr": 0.076, + "pr_auc": 0.08529767939104427, + "roc_auc": 0.6203368216799636, + "precision_at_n": 0.1382636655948553, + "macro_pr_auc": 0.3522517990002194, + "worst_group_fpr": 0.079, "n_models": 28, - "seconds": 2.303080667003087, + "seconds": 4.09942383300222, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -19979,13 +22530,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06479623378888605, - "roc_auc": 0.6920459730827131, - "precision_at_n": 0.11387995712754555, - "macro_pr_auc": 0.24849247128209095, - "worst_group_fpr": 0.32496863237139273, + "pr_auc": 0.06487174622301133, + "roc_auc": 0.6937113491862577, + "precision_at_n": 0.11120042872454448, + "macro_pr_auc": 0.2676601621144114, + "worst_group_fpr": 0.32772898368883313, "n_models": 1, - "seconds": 0.2619278749989462, + "seconds": 0.7032672919958713, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -19996,13 +22547,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07872446166069408, - "roc_auc": 0.7308464207214593, - "precision_at_n": 0.1270096463022508, - "macro_pr_auc": 0.2873815302517855, - "worst_group_fpr": 0.3164366373902133, + "pr_auc": 0.07799797883368983, + "roc_auc": 0.7309774124081202, + "precision_at_n": 0.1264737406216506, + "macro_pr_auc": 0.28554629296367817, + "worst_group_fpr": 0.37917189460476786, "n_models": 1, - "seconds": 0.401927415998216, + "seconds": 0.8542152080044616, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20013,13 +22564,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08671785900975944, - "roc_auc": 0.615626724141447, - "precision_at_n": 0.13665594855305466, - "macro_pr_auc": 0.3560687298644604, - "worst_group_fpr": 0.07425, + "pr_auc": 0.0856684778882634, + "roc_auc": 0.6188643853324, + "precision_at_n": 0.1377277599142551, + "macro_pr_auc": 0.35766502619889395, + "worst_group_fpr": 0.0795, "n_models": 28, - "seconds": 2.247596707999037, + "seconds": 4.0839703750025365, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20030,13 +22581,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06107439693748362, - "roc_auc": 0.6873777113111124, - "precision_at_n": 0.09378349410503752, - "macro_pr_auc": 0.25590641932495106, - "worst_group_fpr": 0.3686323713927227, + "pr_auc": 0.06064840724124169, + "roc_auc": 0.6814215704501445, + "precision_at_n": 0.09967845659163987, + "macro_pr_auc": 0.2643175968618031, + "worst_group_fpr": 0.37314930991217066, "n_models": 1, - "seconds": 0.26288724999903934, + "seconds": 0.7044784169993363, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20047,13 +22598,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07195736909369158, - "roc_auc": 0.6973923893196474, - "precision_at_n": 0.1085209003215434, - "macro_pr_auc": 0.26700186532376446, - "worst_group_fpr": 0.2466750313676286, + "pr_auc": 0.06932651026535167, + "roc_auc": 0.7050175023187866, + "precision_at_n": 0.1045016077170418, + "macro_pr_auc": 0.2946479554830892, + "worst_group_fpr": 0.2825595984943538, "n_models": 1, - "seconds": 0.36878224999964004, + "seconds": 0.83684104099666, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20064,13 +22615,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08055817637796389, - "roc_auc": 0.6126525362156573, - "precision_at_n": 0.13129689174705253, - "macro_pr_auc": 0.32893986825914184, - "worst_group_fpr": 0.0905, + "pr_auc": 0.07986999321041088, + "roc_auc": 0.6096425859358724, + "precision_at_n": 0.13317256162915328, + "macro_pr_auc": 0.3298419758564198, + "worst_group_fpr": 0.087, "n_models": 28, - "seconds": 2.046026207997784, + "seconds": 4.1016515839946805, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20081,13 +22632,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06429968454057976, - "roc_auc": 0.6933390865927515, - "precision_at_n": 0.11548767416934619, - "macro_pr_auc": 0.26682500835664447, - "worst_group_fpr": 0.31267252195734, + "pr_auc": 0.06520326519944142, + "roc_auc": 0.6929199146803785, + "precision_at_n": 0.1120042872454448, + "macro_pr_auc": 0.2718048970703108, + "worst_group_fpr": 0.2451693851944793, "n_models": 1, - "seconds": 0.26589262500056066, + "seconds": 0.6907554999997956, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20098,13 +22649,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0733964924868918, - "roc_auc": 0.7080216427628617, - "precision_at_n": 0.11280814576634512, - "macro_pr_auc": 0.31077649960963766, - "worst_group_fpr": 0.451693851944793, + "pr_auc": 0.07432216955449138, + "roc_auc": 0.7052161232155005, + "precision_at_n": 0.11655948553054662, + "macro_pr_auc": 0.30592072994805625, + "worst_group_fpr": 0.34855708908406524, "n_models": 1, - "seconds": 0.37211320899950806, + "seconds": 0.7915337919985177, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20115,13 +22666,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07633181816626733, - "roc_auc": 0.616948213656311, - "precision_at_n": 0.1339764201500536, - "macro_pr_auc": 0.3336965825572596, - "worst_group_fpr": 0.08616780045351474, + "pr_auc": 0.07881808753345326, + "roc_auc": 0.6113086908984655, + "precision_at_n": 0.13236870310825294, + "macro_pr_auc": 0.35162638851480377, + "worst_group_fpr": 0.0795, "n_models": 28, - "seconds": 2.312892541001929, + "seconds": 4.130562209000345, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20132,13 +22683,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.060607111652736634, - "roc_auc": 0.6621210623445587, - "precision_at_n": 0.09807073954983923, - "macro_pr_auc": 0.26129734756404205, - "worst_group_fpr": 0.3575909661229611, + "pr_auc": 0.06631107326061537, + "roc_auc": 0.6880201751946493, + "precision_at_n": 0.1152197213290461, + "macro_pr_auc": 0.258516490974546, + "worst_group_fpr": 0.36537013801756585, "n_models": 1, - "seconds": 0.2839389159998973, + "seconds": 0.7238397089968203, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20149,13 +22700,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08220802992573126, - "roc_auc": 0.7120001477220336, - "precision_at_n": 0.12004287245444802, - "macro_pr_auc": 0.30504437523313016, - "worst_group_fpr": 0.21530740276035132, + "pr_auc": 0.08859854278095852, + "roc_auc": 0.7162222314354626, + "precision_at_n": 0.1264737406216506, + "macro_pr_auc": 0.31613493642464613, + "worst_group_fpr": 0.23914680050188206, "n_models": 1, - "seconds": 0.3897333749991958, + "seconds": 0.8319103750036447, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20166,13 +22717,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08187207342916611, - "roc_auc": 0.6114700595493434, - "precision_at_n": 0.13558413719185422, - "macro_pr_auc": 0.3632590087045792, - "worst_group_fpr": 0.08225, + "pr_auc": 0.08233569155771314, + "roc_auc": 0.6252609364891877, + "precision_at_n": 0.1377277599142551, + "macro_pr_auc": 0.35316191126056945, + "worst_group_fpr": 0.083, "n_models": 28, - "seconds": 2.3178855830010434, + "seconds": 4.051915416996053, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20183,13 +22734,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06604695963924091, - "roc_auc": 0.6867261113217089, - "precision_at_n": 0.11066452304394427, - "macro_pr_auc": 0.24535099567508128, - "worst_group_fpr": 0.36662484316185695, + "pr_auc": 0.06383904434008811, + "roc_auc": 0.6844451302236746, + "precision_at_n": 0.11414790996784566, + "macro_pr_auc": 0.2599635037497486, + "worst_group_fpr": 0.3573400250941029, "n_models": 1, - "seconds": 0.2720707910011697, + "seconds": 0.7152979160018731, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20200,13 +22751,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07463292842148948, - "roc_auc": 0.7001492955771575, - "precision_at_n": 0.12165058949624866, - "macro_pr_auc": 0.3074674122412946, - "worst_group_fpr": 0.3214554579673777, + "pr_auc": 0.07374933979597549, + "roc_auc": 0.6925375940794927, + "precision_at_n": 0.1229903536977492, + "macro_pr_auc": 0.2873052203937247, + "worst_group_fpr": 0.38845671267252196, "n_models": 1, - "seconds": 0.3984493329990073, + "seconds": 0.8419914999976754, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20217,13 +22768,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0785027094613833, - "roc_auc": 0.6040210953241314, - "precision_at_n": 0.1345123258306538, - "macro_pr_auc": 0.34832790946273445, - "worst_group_fpr": 0.0865, + "pr_auc": 0.08374995105820766, + "roc_auc": 0.618497706120943, + "precision_at_n": 0.13370846730975347, + "macro_pr_auc": 0.3571298814863155, + "worst_group_fpr": 0.09125, "n_models": 28, - "seconds": 2.34852645799765, + "seconds": 4.079550041999028, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20234,13 +22785,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07777070784891844, - "roc_auc": 0.7209001280059634, - "precision_at_n": 0.11441586280814577, - "macro_pr_auc": 0.26303247641541033, - "worst_group_fpr": 0.39849435382685067, + "pr_auc": 0.06994101575649286, + "roc_auc": 0.7155082861547449, + "precision_at_n": 0.11361200428724544, + "macro_pr_auc": 0.2663950053872669, + "worst_group_fpr": 0.4087829360100376, "n_models": 1, - "seconds": 0.2540542919996369, + "seconds": 0.725883958999475, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20251,13 +22802,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08682769905620842, - "roc_auc": 0.7560410473715913, - "precision_at_n": 0.13585209003215434, - "macro_pr_auc": 0.30531470499064983, - "worst_group_fpr": 0.33023839397741533, + "pr_auc": 0.0798919589140398, + "roc_auc": 0.7454055076737647, + "precision_at_n": 0.12593783494105038, + "macro_pr_auc": 0.2993513347886662, + "worst_group_fpr": 0.37641154328732745, "n_models": 1, - "seconds": 0.3711304589996871, + "seconds": 0.803975167000317, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20268,13 +22819,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08797418981377883, - "roc_auc": 0.6071637424990133, - "precision_at_n": 0.1412111468381565, - "macro_pr_auc": 0.36442990032102757, - "worst_group_fpr": 0.09625, + "pr_auc": 0.08356881140485035, + "roc_auc": 0.6067535990342094, + "precision_at_n": 0.13612004287245444, + "macro_pr_auc": 0.3671674809340615, + "worst_group_fpr": 0.1005, "n_models": 28, - "seconds": 2.0829790829993726, + "seconds": 4.123660916004155, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20285,13 +22836,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06128428571157356, - "roc_auc": 0.673318061100494, - "precision_at_n": 0.10691318327974277, - "macro_pr_auc": 0.26284710122393756, - "worst_group_fpr": 0.3212045169385194, + "pr_auc": 0.06346703450271998, + "roc_auc": 0.6824206914238579, + "precision_at_n": 0.10128617363344052, + "macro_pr_auc": 0.26730112904237396, + "worst_group_fpr": 0.3877038895859473, "n_models": 1, - "seconds": 0.2565165000014531, + "seconds": 0.6885288750054315, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20302,13 +22853,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06862293675552432, - "roc_auc": 0.6830670025447154, - "precision_at_n": 0.10744908896034298, - "macro_pr_auc": 0.29573862634474224, - "worst_group_fpr": 0.2587202007528231, + "pr_auc": 0.07314332030590526, + "roc_auc": 0.7038474942157549, + "precision_at_n": 0.1160235798499464, + "macro_pr_auc": 0.3060809090763773, + "worst_group_fpr": 0.3533249686323714, "n_models": 1, - "seconds": 0.3732836250019318, + "seconds": 0.8422652500012191, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20319,13 +22870,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08083032345610011, - "roc_auc": 0.6079619508154728, - "precision_at_n": 0.1377277599142551, - "macro_pr_auc": 0.3255140969349702, - "worst_group_fpr": 0.105, + "pr_auc": 0.08320004375026958, + "roc_auc": 0.6086983929680114, + "precision_at_n": 0.13638799571275456, + "macro_pr_auc": 0.33161701272441674, + "worst_group_fpr": 0.1005, "n_models": 28, - "seconds": 2.2073950419980974, + "seconds": 4.0860568749994854, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20336,13 +22887,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06588676629961493, - "roc_auc": 0.6846132479360989, - "precision_at_n": 0.09646302250803858, - "macro_pr_auc": 0.25388308740902954, - "worst_group_fpr": 0.43212045169385194, + "pr_auc": 0.06391515057273248, + "roc_auc": 0.6895837795584147, + "precision_at_n": 0.09887459807073955, + "macro_pr_auc": 0.26440938470328373, + "worst_group_fpr": 0.429861982434128, "n_models": 1, - "seconds": 0.2586077080013638, + "seconds": 0.6778770419987268, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20353,13 +22904,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07744300864021096, - "roc_auc": 0.716798334496934, - "precision_at_n": 0.1235262593783494, - "macro_pr_auc": 0.25194493578602234, - "worst_group_fpr": 0.37164366373902136, + "pr_auc": 0.07554757126645857, + "roc_auc": 0.7111140110379107, + "precision_at_n": 0.1189710610932476, + "macro_pr_auc": 0.27528729908371236, + "worst_group_fpr": 0.3478042659974906, "n_models": 1, - "seconds": 0.37577691699698335, + "seconds": 0.8417751250017318, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20370,13 +22921,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08518821490986139, - "roc_auc": 0.6159125210351939, - "precision_at_n": 0.13290460878885316, - "macro_pr_auc": 0.3657924509299794, - "worst_group_fpr": 0.091, + "pr_auc": 0.08639941462490236, + "roc_auc": 0.6288417232360285, + "precision_at_n": 0.1342443729903537, + "macro_pr_auc": 0.3624784434468467, + "worst_group_fpr": 0.08625, "n_models": 28, - "seconds": 2.1043218329978117, + "seconds": 4.177703540997754, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20387,13 +22938,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06167005301582487, - "roc_auc": 0.6737572351820703, - "precision_at_n": 0.09217577706323687, - "macro_pr_auc": 0.255451871575027, - "worst_group_fpr": 0.3131744040150565, + "pr_auc": 0.06322374708445785, + "roc_auc": 0.6752158108331946, + "precision_at_n": 0.10503751339764202, + "macro_pr_auc": 0.2651756750769737, + "worst_group_fpr": 0.370138017565872, "n_models": 1, - "seconds": 0.2797340830002213, + "seconds": 0.6878105419964413, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20404,13 +22955,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07840338643865112, - "roc_auc": 0.7177888081582002, - "precision_at_n": 0.12031082529474812, - "macro_pr_auc": 0.3012854877632323, - "worst_group_fpr": 0.1805, + "pr_auc": 0.07968797931504096, + "roc_auc": 0.7223169507994355, + "precision_at_n": 0.12486602357984995, + "macro_pr_auc": 0.2892799312552035, + "worst_group_fpr": 0.221831869510665, "n_models": 1, - "seconds": 0.35555091700007324, + "seconds": 0.8280662080069305, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20421,13 +22972,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08253209519105992, - "roc_auc": 0.6271013736466189, - "precision_at_n": 0.1347802786709539, - "macro_pr_auc": 0.3538262027169542, - "worst_group_fpr": 0.07625, + "pr_auc": 0.08455359360662107, + "roc_auc": 0.6225626136698378, + "precision_at_n": 0.13612004287245444, + "macro_pr_auc": 0.3673254717050555, + "worst_group_fpr": 0.07725, "n_models": 28, - "seconds": 1.970553833001759, + "seconds": 4.105281750002177, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20438,13 +22989,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07045065852496393, - "roc_auc": 0.681170735279146, - "precision_at_n": 0.12620578778135047, - "macro_pr_auc": 0.26236888207493836, - "worst_group_fpr": 0.20325, + "pr_auc": 0.06519281732032095, + "roc_auc": 0.6738063001417901, + "precision_at_n": 0.1227224008574491, + "macro_pr_auc": 0.2579158702974877, + "worst_group_fpr": 0.3452948557089084, "n_models": 1, - "seconds": 0.24093054100012523, + "seconds": 0.7210727499987115, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20455,13 +23006,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0729493499368938, - "roc_auc": 0.7007142256872718, - "precision_at_n": 0.11495176848874598, - "macro_pr_auc": 0.285273221705565, - "worst_group_fpr": 0.2584692597239649, + "pr_auc": 0.07326810730121225, + "roc_auc": 0.6912488252623565, + "precision_at_n": 0.11736334405144695, + "macro_pr_auc": 0.2929809134995579, + "worst_group_fpr": 0.29535759096612296, "n_models": 1, - "seconds": 0.4011748329976399, + "seconds": 0.831405500000983, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20472,13 +23023,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08790933758213504, - "roc_auc": 0.614422059966236, - "precision_at_n": 0.13612004287245444, - "macro_pr_auc": 0.3685751181249375, - "worst_group_fpr": 0.0805, + "pr_auc": 0.08588316016047276, + "roc_auc": 0.6185552327753555, + "precision_at_n": 0.13692390139335478, + "macro_pr_auc": 0.3706344555927829, + "worst_group_fpr": 0.08625, "n_models": 28, - "seconds": 2.328206541998952, + "seconds": 4.072452332999092, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20489,13 +23040,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06295036559112563, - "roc_auc": 0.6962409689785314, - "precision_at_n": 0.08654876741693462, - "macro_pr_auc": 0.2544358483930452, - "worst_group_fpr": 0.42383939774153073, + "pr_auc": 0.0638559054729082, + "roc_auc": 0.6938116248469371, + "precision_at_n": 0.10664523043944266, + "macro_pr_auc": 0.2620259597609067, + "worst_group_fpr": 0.3992471769134254, "n_models": 1, - "seconds": 0.28210775000115973, + "seconds": 0.7011463339949842, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20506,13 +23057,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07412476212074792, - "roc_auc": 0.7297252907229415, - "precision_at_n": 0.10557341907824223, - "macro_pr_auc": 0.2895992560502265, - "worst_group_fpr": 0.47176913425345046, + "pr_auc": 0.0749772379684639, + "roc_auc": 0.7204712346730718, + "precision_at_n": 0.1160235798499464, + "macro_pr_auc": 0.3072583176907988, + "worst_group_fpr": 0.38996235884567126, "n_models": 1, - "seconds": 0.3880428749980638, + "seconds": 0.8435321249999106, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20523,13 +23074,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08310053096983538, - "roc_auc": 0.631422064935842, - "precision_at_n": 0.1345123258306538, - "macro_pr_auc": 0.3492017698780529, - "worst_group_fpr": 0.08175, + "pr_auc": 0.08342839328093159, + "roc_auc": 0.6162641206602916, + "precision_at_n": 0.13585209003215434, + "macro_pr_auc": 0.36622478697593386, + "worst_group_fpr": 0.085, "n_models": 28, - "seconds": 2.2246951250017446, + "seconds": 4.079520125000272, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20540,13 +23091,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06728451657966107, - "roc_auc": 0.6823611724722158, - "precision_at_n": 0.10905680600214362, - "macro_pr_auc": 0.25466656116327896, - "worst_group_fpr": 0.26025, + "pr_auc": 0.06533443887362353, + "roc_auc": 0.6778376702748382, + "precision_at_n": 0.12057877813504823, + "macro_pr_auc": 0.2617293955220065, + "worst_group_fpr": 0.27754077791718945, "n_models": 1, - "seconds": 0.2537664169976779, + "seconds": 0.7451008330026525, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20557,13 +23108,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08107267027136326, - "roc_auc": 0.7062605361587146, - "precision_at_n": 0.1122722400857449, - "macro_pr_auc": 0.30122007270785933, - "worst_group_fpr": 0.22936010037641155, + "pr_auc": 0.0809589792460545, + "roc_auc": 0.7102266542264164, + "precision_at_n": 0.1192390139335477, + "macro_pr_auc": 0.32101929536115875, + "worst_group_fpr": 0.2534504391468005, "n_models": 1, - "seconds": 0.37384054100039066, + "seconds": 0.838801792000595, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20574,13 +23125,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08132474896503511, - "roc_auc": 0.6148238135085453, - "precision_at_n": 0.1385316184351554, - "macro_pr_auc": 0.34649758548222853, - "worst_group_fpr": 0.09225, + "pr_auc": 0.08415777912515914, + "roc_auc": 0.6194077082984619, + "precision_at_n": 0.13638799571275456, + "macro_pr_auc": 0.3619564762419791, + "worst_group_fpr": 0.091, "n_models": 28, - "seconds": 2.2186549580001156, + "seconds": 4.127557959000114, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20591,13 +23142,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06327527823213003, - "roc_auc": 0.6794141206246529, - "precision_at_n": 0.10289389067524116, - "macro_pr_auc": 0.2592741196508792, - "worst_group_fpr": 0.424090338770389, + "pr_auc": 0.06053581675987888, + "roc_auc": 0.668970457711801, + "precision_at_n": 0.10423365487674169, + "macro_pr_auc": 0.2661453758657945, + "worst_group_fpr": 0.4288582183186951, "n_models": 1, - "seconds": 0.26233091600079206, + "seconds": 0.7011158329987666, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20608,13 +23159,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.09033175928257643, - "roc_auc": 0.7130177302375896, - "precision_at_n": 0.11655948553054662, - "macro_pr_auc": 0.3100238438500384, - "worst_group_fpr": 0.285069008782936, + "pr_auc": 0.08457025729915832, + "roc_auc": 0.7117365111132468, + "precision_at_n": 0.11468381564844587, + "macro_pr_auc": 0.30673157924849787, + "worst_group_fpr": 0.3879548306148055, "n_models": 1, - "seconds": 0.37331779200030724, + "seconds": 0.8255459159991005, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20625,13 +23176,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07805975736148492, - "roc_auc": 0.607997606748622, - "precision_at_n": 0.12968917470525188, - "macro_pr_auc": 0.35955356545587785, - "worst_group_fpr": 0.0845, + "pr_auc": 0.08245938229155562, + "roc_auc": 0.610460199969818, + "precision_at_n": 0.1304930332261522, + "macro_pr_auc": 0.3588958226713545, + "worst_group_fpr": 0.08125, "n_models": 28, - "seconds": 2.2150131250018603, + "seconds": 4.086438583995914, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20642,13 +23193,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06603648778281995, - "roc_auc": 0.6952049459578116, - "precision_at_n": 0.12379421221864952, - "macro_pr_auc": 0.25673275019955366, - "worst_group_fpr": 0.41405269761606023, + "pr_auc": 0.06249021828056783, + "roc_auc": 0.6844033192057928, + "precision_at_n": 0.11093247588424437, + "macro_pr_auc": 0.26469382499516414, + "worst_group_fpr": 0.4143036386449184, "n_models": 1, - "seconds": 0.25487854099992546, + "seconds": 0.6763558750026277, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20659,13 +23210,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07534273780011053, - "roc_auc": 0.7171714014340422, - "precision_at_n": 0.1045016077170418, - "macro_pr_auc": 0.29780842635792343, - "worst_group_fpr": 0.4398996235884567, + "pr_auc": 0.08177093015401454, + "roc_auc": 0.7163076799499286, + "precision_at_n": 0.11870310825294748, + "macro_pr_auc": 0.3049233360547107, + "worst_group_fpr": 0.35984943538268505, "n_models": 1, - "seconds": 0.36989783399985754, + "seconds": 0.8341285000060452, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20676,13 +23227,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08793068926917626, - "roc_auc": 0.6174247142308251, + "pr_auc": 0.08309978963115165, + "roc_auc": 0.6078336617233144, "precision_at_n": 0.13344051446945338, - "macro_pr_auc": 0.35291601118538185, - "worst_group_fpr": 0.082, + "macro_pr_auc": 0.32587296737624166, + "worst_group_fpr": 0.0795, "n_models": 28, - "seconds": 2.078569792000053, + "seconds": 4.117640249998658, "eta_squared": 0.5619488636163167, "warnings": [] }, @@ -20693,13 +23244,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06359419427859529, - "roc_auc": 0.6844236319753716, - "precision_at_n": 0.10664523043944266, - "macro_pr_auc": 0.15486358973175143, - "worst_group_fpr": 0.08212905995135213, + "pr_auc": 0.06599372001242891, + "roc_auc": 0.6794908017938576, + "precision_at_n": 0.10986066452304394, + "macro_pr_auc": 0.15459948097397477, + "worst_group_fpr": 0.08150903801211427, "n_models": 1, - "seconds": 0.24159191599756014, + "seconds": 0.6689555830016616, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20710,13 +23261,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07110381391354774, - "roc_auc": 0.6965429727771318, - "precision_at_n": 0.11039657020364416, - "macro_pr_auc": 0.16023615098459712, - "worst_group_fpr": 0.08336910382982783, + "pr_auc": 0.07630884656938808, + "roc_auc": 0.6907817961431185, + "precision_at_n": 0.1197749196141479, + "macro_pr_auc": 0.1616731307705999, + "worst_group_fpr": 0.07981590117804169, "n_models": 1, - "seconds": 0.34601829199891654, + "seconds": 0.7638930829998571, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20727,13 +23278,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07561607622391128, - "roc_auc": 0.716640646027398, - "precision_at_n": 0.10209003215434084, - "macro_pr_auc": 0.14956598978209557, - "worst_group_fpr": 0.051143636262452004, + "pr_auc": 0.0708883937729277, + "roc_auc": 0.6972662806173763, + "precision_at_n": 0.09833869239013933, + "macro_pr_auc": 0.15474095539742705, + "worst_group_fpr": 0.05028103956814514, "n_models": 3, - "seconds": 0.4875754579989007, + "seconds": 1.0807272500023828, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20744,13 +23295,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06479623378888605, - "roc_auc": 0.6920459730827131, - "precision_at_n": 0.11387995712754555, - "macro_pr_auc": 0.15193062075560407, - "worst_group_fpr": 0.07442647970620499, + "pr_auc": 0.06487174622301133, + "roc_auc": 0.6937113491862577, + "precision_at_n": 0.11120042872454448, + "macro_pr_auc": 0.15240654387568334, + "worst_group_fpr": 0.0793151142271188, "n_models": 1, - "seconds": 0.2598764169997594, + "seconds": 0.6623257909959648, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20761,13 +23312,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.061225502312909894, - "roc_auc": 0.6793274272337815, - "precision_at_n": 0.11629153269024652, - "macro_pr_auc": 0.15408180312971578, - "worst_group_fpr": 0.0768350264701674, + "pr_auc": 0.06411755602985518, + "roc_auc": 0.6970320013126097, + "precision_at_n": 0.11280814576634512, + "macro_pr_auc": 0.156397619931216, + "worst_group_fpr": 0.07631039252158153, "n_models": 1, - "seconds": 0.3561963329993887, + "seconds": 0.7806902500014985, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20778,13 +23329,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06826979839214103, - "roc_auc": 0.674242823601835, - "precision_at_n": 0.10637727759914255, - "macro_pr_auc": 0.1646718006293512, - "worst_group_fpr": 0.047999736807474665, + "pr_auc": 0.06947326333953958, + "roc_auc": 0.6970309767026058, + "precision_at_n": 0.10048231511254019, + "macro_pr_auc": 0.1650689106072917, + "worst_group_fpr": 0.048767110220823195, "n_models": 3, - "seconds": 0.514039916000911, + "seconds": 1.0540789590013446, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20795,13 +23346,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06107439693748362, - "roc_auc": 0.6873777113111124, - "precision_at_n": 0.09378349410503752, - "macro_pr_auc": 0.14409721454328103, - "worst_group_fpr": 0.08689845948395097, + "pr_auc": 0.06064840724124169, + "roc_auc": 0.6814215704501445, + "precision_at_n": 0.09967845659163987, + "macro_pr_auc": 0.15438766149864325, + "worst_group_fpr": 0.08589688558210522, "n_models": 1, - "seconds": 0.2507771659984428, + "seconds": 0.7205552500017802, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20812,13 +23363,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0600603802641146, - "roc_auc": 0.6737304592022867, - "precision_at_n": 0.09110396570203644, - "macro_pr_auc": 0.14919945819278152, - "worst_group_fpr": 0.09300329088567749, + "pr_auc": 0.06359057138975892, + "roc_auc": 0.6852497584395295, + "precision_at_n": 0.10209003215434084, + "macro_pr_auc": 0.1514965861142468, + "worst_group_fpr": 0.08489531168025946, "n_models": 1, - "seconds": 0.3602215419996355, + "seconds": 0.8438872909973725, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20829,13 +23380,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06805200030217438, - "roc_auc": 0.6631391225164691, - "precision_at_n": 0.10691318327974277, - "macro_pr_auc": 0.1627696505783975, - "worst_group_fpr": 0.05061494796594134, + "pr_auc": 0.0693125825126317, + "roc_auc": 0.680144208462736, + "precision_at_n": 0.10369774919614148, + "macro_pr_auc": 0.1650860438354126, + "worst_group_fpr": 0.050503645166675944, "n_models": 3, - "seconds": 0.49610391599708237, + "seconds": 1.2419886670031701, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20846,13 +23397,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06429968454057976, - "roc_auc": 0.6933390865927515, - "precision_at_n": 0.11548767416934619, - "macro_pr_auc": 0.16147972457948392, - "worst_group_fpr": 0.08160442600276625, + "pr_auc": 0.06520326519944142, + "roc_auc": 0.6929199146803785, + "precision_at_n": 0.1120042872454448, + "macro_pr_auc": 0.1590560050567187, + "worst_group_fpr": 0.07657270949587447, "n_models": 1, - "seconds": 0.26588524999897345, + "seconds": 0.7665968749934109, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20863,13 +23414,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0675781312701662, - "roc_auc": 0.7055571500533133, - "precision_at_n": 0.10610932475884244, - "macro_pr_auc": 0.16140909650780297, - "worst_group_fpr": 0.08019745314064959, + "pr_auc": 0.06916940607152804, + "roc_auc": 0.702840166462398, + "precision_at_n": 0.1189710610932476, + "macro_pr_auc": 0.16107437858274268, + "worst_group_fpr": 0.07318643582772928, "n_models": 1, - "seconds": 0.3723146660013299, + "seconds": 0.8361535000003641, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20880,13 +23431,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07330773925395759, - "roc_auc": 0.7058960360996931, - "precision_at_n": 0.10530546623794212, - "macro_pr_auc": 0.16869099135666157, - "worst_group_fpr": 0.05222883855528967, + "pr_auc": 0.07012548194980095, + "roc_auc": 0.6958499627041959, + "precision_at_n": 0.09833869239013933, + "macro_pr_auc": 0.16303984692494433, + "worst_group_fpr": 0.05259057265290222, "n_models": 3, - "seconds": 0.521546166997723, + "seconds": 1.2241014169994742, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20897,13 +23448,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.060607111652736634, - "roc_auc": 0.6621210623445587, - "precision_at_n": 0.09807073954983923, - "macro_pr_auc": 0.1553474240367889, - "worst_group_fpr": 0.08241522392330805, + "pr_auc": 0.06631107326061537, + "roc_auc": 0.6880201751946493, + "precision_at_n": 0.1152197213290461, + "macro_pr_auc": 0.15679273034517632, + "worst_group_fpr": 0.08050746411026852, "n_models": 1, - "seconds": 0.2576507089979714, + "seconds": 0.7591468750033528, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20914,13 +23465,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.062949022238146, - "roc_auc": 0.6722181447363893, - "precision_at_n": 0.10905680600214362, - "macro_pr_auc": 0.15459923664407205, - "worst_group_fpr": 0.07335336481137025, + "pr_auc": 0.06750046772696748, + "roc_auc": 0.684424187591183, + "precision_at_n": 0.10691318327974277, + "macro_pr_auc": 0.1519757366679743, + "worst_group_fpr": 0.08107979205418038, "n_models": 1, - "seconds": 0.37582820800162153, + "seconds": 0.8660919579997426, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20931,13 +23482,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06551991773583199, - "roc_auc": 0.6867615210019707, - "precision_at_n": 0.09271168274383708, - "macro_pr_auc": 0.15440383007296768, - "worst_group_fpr": 0.05334186654794368, + "pr_auc": 0.07129558197654177, + "roc_auc": 0.6938419869617338, + "precision_at_n": 0.11093247588424437, + "macro_pr_auc": 0.159025464828388, + "worst_group_fpr": 0.05039234236741054, "n_models": 3, - "seconds": 0.5093504579999717, + "seconds": 1.2299727909994544, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20948,13 +23499,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06604695963924091, - "roc_auc": 0.6867261113217089, - "precision_at_n": 0.11066452304394427, - "macro_pr_auc": 0.15593379811930022, - "worst_group_fpr": 0.08127056803548434, + "pr_auc": 0.06383904434008811, + "roc_auc": 0.6844451302236746, + "precision_at_n": 0.11414790996784566, + "macro_pr_auc": 0.15568979011340453, + "worst_group_fpr": 0.08401297276672867, "n_models": 1, - "seconds": 0.2543880840021302, + "seconds": 0.7487322500019218, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20965,13 +23516,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07093348871493281, - "roc_auc": 0.702998017038106, - "precision_at_n": 0.1152197213290461, - "macro_pr_auc": 0.16396008509767293, - "worst_group_fpr": 0.08124672103782134, + "pr_auc": 0.06815015385114945, + "roc_auc": 0.698310877940893, + "precision_at_n": 0.11789924973204716, + "macro_pr_auc": 0.1609907214508861, + "worst_group_fpr": 0.08169981399341823, "n_models": 1, - "seconds": 0.37792704199819127, + "seconds": 0.8363707910029916, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20982,13 +23533,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06589072714445193, - "roc_auc": 0.670718957158076, - "precision_at_n": 0.09753483386923902, - "macro_pr_auc": 0.16423524584103774, - "worst_group_fpr": 0.04871941622549721, + "pr_auc": 0.06767640577544462, + "roc_auc": 0.6836004580709589, + "precision_at_n": 0.10584137191854234, + "macro_pr_auc": 0.15679729883353768, + "worst_group_fpr": 0.0474984695865101, "n_models": 3, - "seconds": 0.5126857920004113, + "seconds": 1.2270105829957174, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -20999,13 +23550,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07777070784891844, - "roc_auc": 0.7209001280059634, - "precision_at_n": 0.11441586280814577, - "macro_pr_auc": 0.16230417381678816, - "worst_group_fpr": 0.08503839366623742, + "pr_auc": 0.06994101575649286, + "roc_auc": 0.7155082861547449, + "precision_at_n": 0.11361200428724544, + "macro_pr_auc": 0.16309636922967916, + "worst_group_fpr": 0.0850860876615634, "n_models": 1, - "seconds": 0.27584125000066706, + "seconds": 0.7599207500024932, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21016,13 +23567,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07303214749902401, - "roc_auc": 0.7166424143458706, - "precision_at_n": 0.13156484458735263, - "macro_pr_auc": 0.1510767031270082, - "worst_group_fpr": 0.07390184575761911, + "pr_auc": 0.06719650781752987, + "roc_auc": 0.704954711544862, + "precision_at_n": 0.12754555198285103, + "macro_pr_auc": 0.1561353851979731, + "worst_group_fpr": 0.07797968235799113, "n_models": 1, - "seconds": 0.31722187499690335, + "seconds": 0.884690083003079, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21033,13 +23584,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07504078781692751, - "roc_auc": 0.6967395172299012, - "precision_at_n": 0.10209003215434084, - "macro_pr_auc": 0.15327462100832814, - "worst_group_fpr": 0.05387055484445434, + "pr_auc": 0.07084784931288268, + "roc_auc": 0.7041475490279352, + "precision_at_n": 0.09887459807073955, + "macro_pr_auc": 0.1624685360528737, + "worst_group_fpr": 0.055011408536924704, "n_models": 3, - "seconds": 0.4245285420001892, + "seconds": 1.242284790998383, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21050,13 +23601,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06128428571157356, - "roc_auc": 0.673318061100494, - "precision_at_n": 0.10691318327974277, - "macro_pr_auc": 0.15311394812407336, - "worst_group_fpr": 0.08212905995135213, + "pr_auc": 0.06346703450271998, + "roc_auc": 0.6824206914238579, + "precision_at_n": 0.10128617363344052, + "macro_pr_auc": 0.15858739789206222, + "worst_group_fpr": 0.0888539132923165, "n_models": 1, - "seconds": 0.2261497090003104, + "seconds": 0.7559428750028019, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21067,13 +23618,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06510429288954053, - "roc_auc": 0.6997572238569124, - "precision_at_n": 0.10316184351554127, - "macro_pr_auc": 0.1570887840363783, - "worst_group_fpr": 0.08737539943721086, + "pr_auc": 0.06380484313021846, + "roc_auc": 0.7002730951945653, + "precision_at_n": 0.09592711682743837, + "macro_pr_auc": 0.15957893617196986, + "worst_group_fpr": 0.08725616444889589, "n_models": 1, - "seconds": 0.3157693749999453, + "seconds": 0.8619656669980031, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21084,13 +23635,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06677001333844357, - "roc_auc": 0.6868312637795196, - "precision_at_n": 0.09512325830653805, - "macro_pr_auc": 0.1595599701478352, - "worst_group_fpr": 0.05384272914463799, + "pr_auc": 0.06837578488109436, + "roc_auc": 0.6862131393333781, + "precision_at_n": 0.08922829581993569, + "macro_pr_auc": 0.16430368253435956, + "worst_group_fpr": 0.05331404084812733, "n_models": 3, - "seconds": 0.45073600000250735, + "seconds": 1.1852366250022897, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21101,13 +23652,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06588676629961493, - "roc_auc": 0.6846132479360989, - "precision_at_n": 0.09646302250803858, - "macro_pr_auc": 0.15902473898089842, - "worst_group_fpr": 0.0913578480469309, + "pr_auc": 0.06391515057273248, + "roc_auc": 0.6895837795584147, + "precision_at_n": 0.09887459807073955, + "macro_pr_auc": 0.15861837293779057, + "worst_group_fpr": 0.08708923546525492, "n_models": 1, - "seconds": 0.23203920800006017, + "seconds": 0.7407272080017719, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21118,13 +23669,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06855202796036898, - "roc_auc": 0.6967101599258813, - "precision_at_n": 0.1122722400857449, - "macro_pr_auc": 0.16224185024824136, - "worst_group_fpr": 0.08456145371297753, + "pr_auc": 0.0664352056299146, + "roc_auc": 0.696070800808648, + "precision_at_n": 0.11763129689174705, + "macro_pr_auc": 0.16154788764949254, + "worst_group_fpr": 0.0822244479420041, "n_models": 1, - "seconds": 0.319037916000525, + "seconds": 0.8839377089971094, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21135,13 +23686,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06644022259586063, - "roc_auc": 0.6799007522161968, - "precision_at_n": 0.08654876741693462, - "macro_pr_auc": 0.14515283268998994, - "worst_group_fpr": 0.05754354722021259, + "pr_auc": 0.06992836493531851, + "roc_auc": 0.6961550390953559, + "precision_at_n": 0.09780278670953912, + "macro_pr_auc": 0.1586693530651129, + "worst_group_fpr": 0.054315766041515945, "n_models": 3, - "seconds": 0.4790727910003625, + "seconds": 1.173453832998348, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21152,13 +23703,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06167005301582487, - "roc_auc": 0.6737572351820703, - "precision_at_n": 0.09217577706323687, - "macro_pr_auc": 0.15580273173958395, - "worst_group_fpr": 0.08131826203081031, + "pr_auc": 0.06322374708445785, + "roc_auc": 0.6752158108331946, + "precision_at_n": 0.10503751339764202, + "macro_pr_auc": 0.1563543880228793, + "worst_group_fpr": 0.0813898030237993, "n_models": 1, - "seconds": 0.23473350000131177, + "seconds": 0.7637521249998827, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21169,13 +23720,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.074212649947672, - "roc_auc": 0.7091004073651384, - "precision_at_n": 0.1160235798499464, - "macro_pr_auc": 0.13841713015373577, - "worst_group_fpr": 0.07895740926217389, + "pr_auc": 0.07119499410159705, + "roc_auc": 0.7060776247112728, + "precision_at_n": 0.11629153269024652, + "macro_pr_auc": 0.15682244141909382, + "worst_group_fpr": 0.07910049124815186, "n_models": 1, - "seconds": 0.3204471669996565, + "seconds": 0.8636223330031498, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21186,13 +23737,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06818715912624354, - "roc_auc": 0.6891144636284435, - "precision_at_n": 0.10155412647374062, - "macro_pr_auc": 0.14820412819477294, - "worst_group_fpr": 0.05259057265290222, + "pr_auc": 0.07190182683501667, + "roc_auc": 0.6951186535012893, + "precision_at_n": 0.11120042872454448, + "macro_pr_auc": 0.15885516438894723, + "worst_group_fpr": 0.05384272914463799, "n_models": 3, - "seconds": 0.4410273329995107, + "seconds": 1.2222119580037543, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21203,13 +23754,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07045065852496393, - "roc_auc": 0.681170735279146, - "precision_at_n": 0.12620578778135047, - "macro_pr_auc": 0.16035347406903172, - "worst_group_fpr": 0.07220870892354653, + "pr_auc": 0.06519281732032095, + "roc_auc": 0.6738063001417901, + "precision_at_n": 0.1227224008574491, + "macro_pr_auc": 0.158891938851795, + "worst_group_fpr": 0.07998283016168264, "n_models": 1, - "seconds": 0.2355420409985527, + "seconds": 0.7430598750070203, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21220,13 +23771,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07841440771859345, - "roc_auc": 0.7026667450320075, - "precision_at_n": 0.10878885316184352, - "macro_pr_auc": 0.1650645677416863, - "worst_group_fpr": 0.06805933133018553, + "pr_auc": 0.07404418513125219, + "roc_auc": 0.6942472350676308, + "precision_at_n": 0.1195069667738478, + "macro_pr_auc": 0.1666954513127937, + "worst_group_fpr": 0.0698478561549101, "n_models": 1, - "seconds": 0.31727712499923655, + "seconds": 0.8700872080007684, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21237,13 +23788,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06670036755437919, - "roc_auc": 0.6876468533424916, - "precision_at_n": 0.09673097534833869, - "macro_pr_auc": 0.15906439767498926, - "worst_group_fpr": 0.0548444543380266, + "pr_auc": 0.06852158356992351, + "roc_auc": 0.6838487304794965, + "precision_at_n": 0.09271168274383708, + "macro_pr_auc": 0.15687985349730973, + "worst_group_fpr": 0.05011408536924704, "n_models": 3, - "seconds": 0.43318066600113525, + "seconds": 1.2468310420008493, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21254,13 +23805,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06295036559112563, - "roc_auc": 0.6962409689785314, - "precision_at_n": 0.08654876741693462, - "macro_pr_auc": 0.1505493358424266, - "worst_group_fpr": 0.08751848142318883, + "pr_auc": 0.0638559054729082, + "roc_auc": 0.6938116248469371, + "precision_at_n": 0.10664523043944266, + "macro_pr_auc": 0.15968816094249016, + "worst_group_fpr": 0.08630228454237611, "n_models": 1, - "seconds": 0.22850850000031642, + "seconds": 0.7496217079969938, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21271,13 +23822,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06500424942725971, - "roc_auc": 0.6974645104793549, - "precision_at_n": 0.10235798499464094, - "macro_pr_auc": 0.15148000604998416, - "worst_group_fpr": 0.07952973720608575, + "pr_auc": 0.06967658768327425, + "roc_auc": 0.7018962506837167, + "precision_at_n": 0.12459807073954984, + "macro_pr_auc": 0.1598466003967197, + "worst_group_fpr": 0.07776505937902418, "n_models": 1, - "seconds": 0.31415566699797637, + "seconds": 0.8879480000032345, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21288,13 +23839,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06274572055032215, - "roc_auc": 0.6837210625880892, - "precision_at_n": 0.07797427652733119, - "macro_pr_auc": 0.1581081348336286, - "worst_group_fpr": 0.052630323842228266, + "pr_auc": 0.06663778758586969, + "roc_auc": 0.687698771865821, + "precision_at_n": 0.09056806002143623, + "macro_pr_auc": 0.1640674246320529, + "worst_group_fpr": 0.05158884745951361, "n_models": 3, - "seconds": 0.45697745900179143, + "seconds": 1.2128442499961238, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21305,13 +23856,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06728451657966107, - "roc_auc": 0.6823611724722158, - "precision_at_n": 0.10905680600214362, - "macro_pr_auc": 0.15658017565737375, - "worst_group_fpr": 0.07964897219440073, + "pr_auc": 0.06533443887362353, + "roc_auc": 0.6778376702748382, + "precision_at_n": 0.12057877813504823, + "macro_pr_auc": 0.15811107018831475, + "worst_group_fpr": 0.08167596699575523, "n_models": 1, - "seconds": 0.2216652079987398, + "seconds": 0.69004537499859, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21322,13 +23873,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07708351627138926, - "roc_auc": 0.7115021389996028, - "precision_at_n": 0.1235262593783494, - "macro_pr_auc": 0.16202960823035675, - "worst_group_fpr": 0.07218486192588353, + "pr_auc": 0.07018784527539995, + "roc_auc": 0.6980466646796162, + "precision_at_n": 0.1192390139335477, + "macro_pr_auc": 0.15811937773723314, + "worst_group_fpr": 0.07187485095626461, "n_models": 1, - "seconds": 0.31133950000003097, + "seconds": 0.8022012090004864, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21339,13 +23890,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07047187139546018, - "roc_auc": 0.6935858047619596, - "precision_at_n": 0.10128617363344052, - "macro_pr_auc": 0.1656376575504184, - "worst_group_fpr": 0.05167644393570849, + "pr_auc": 0.06834331779554961, + "roc_auc": 0.6803347166261355, + "precision_at_n": 0.0972668810289389, + "macro_pr_auc": 0.16461384850476882, + "worst_group_fpr": 0.04976868412266896, "n_models": 3, - "seconds": 0.4432239580019086, + "seconds": 1.1120200830046088, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21356,13 +23907,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06327527823213003, - "roc_auc": 0.6794141206246529, - "precision_at_n": 0.10289389067524116, - "macro_pr_auc": 0.1455228603956286, - "worst_group_fpr": 0.08656460151666905, + "pr_auc": 0.06053581675987888, + "roc_auc": 0.668970457711801, + "precision_at_n": 0.10423365487674169, + "macro_pr_auc": 0.1569346231917846, + "worst_group_fpr": 0.0855153336194973, "n_models": 1, - "seconds": 0.2286644999985583, + "seconds": 0.6803973339992808, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21373,13 +23924,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06949117319065048, - "roc_auc": 0.7011086089672838, - "precision_at_n": 0.12379421221864952, - "macro_pr_auc": 0.16179957982069756, - "worst_group_fpr": 0.08241522392330805, + "pr_auc": 0.0617195942992034, + "roc_auc": 0.6893867797234215, + "precision_at_n": 0.10155412647374062, + "macro_pr_auc": 0.1526195644635254, + "worst_group_fpr": 0.08673153050031002, "n_models": 1, - "seconds": 0.330637957998988, + "seconds": 0.7928864999994403, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21390,13 +23941,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07209625062706362, - "roc_auc": 0.7010738897850679, - "precision_at_n": 0.10021436227224008, - "macro_pr_auc": 0.1586095944590266, - "worst_group_fpr": 0.04979253112033195, + "pr_auc": 0.07225183134030269, + "roc_auc": 0.685367044111213, + "precision_at_n": 0.1007502679528403, + "macro_pr_auc": 0.1624603653980992, + "worst_group_fpr": 0.04829017026756331, "n_models": 3, - "seconds": 0.4323672499995155, + "seconds": 1.0913756669979193, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21407,13 +23958,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06603648778281995, - "roc_auc": 0.6952049459578116, - "precision_at_n": 0.12379421221864952, - "macro_pr_auc": 0.16289907678511376, - "worst_group_fpr": 0.082653693899938, + "pr_auc": 0.06249021828056783, + "roc_auc": 0.6844033192057928, + "precision_at_n": 0.11093247588424437, + "macro_pr_auc": 0.15920504233526747, + "worst_group_fpr": 0.08565841560547527, "n_models": 1, - "seconds": 0.2240179159998661, + "seconds": 0.7009786250055186, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21424,13 +23975,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06606210290769324, - "roc_auc": 0.6989305157904578, - "precision_at_n": 0.11173633440514469, - "macro_pr_auc": 0.15999751055708233, - "worst_group_fpr": 0.08212905995135213, + "pr_auc": 0.06522626236440478, + "roc_auc": 0.6960089579227221, + "precision_at_n": 0.10664523043944266, + "macro_pr_auc": 0.15960446636335607, + "worst_group_fpr": 0.08587303858444222, "n_models": 1, - "seconds": 0.3161649159992521, + "seconds": 0.7910215829979279, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21441,13 +23992,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0743450190784837, - "roc_auc": 0.7035847980702564, - "precision_at_n": 0.11843515541264737, - "macro_pr_auc": 0.15763795542242776, - "worst_group_fpr": 0.049891479770716236, + "pr_auc": 0.06965281685110047, + "roc_auc": 0.6932083201222992, + "precision_at_n": 0.10289389067524116, + "macro_pr_auc": 0.15253021291295593, + "worst_group_fpr": 0.049362791474205574, "n_models": 3, - "seconds": 0.4461614579995512, + "seconds": 1.1303733750028186, "eta_squared": 0.06691168539386136, "warnings": [] }, @@ -21458,13 +24009,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6768047861689719, - "roc_auc": 0.972266055193209, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.5313209194469692, - "worst_group_fpr": 0.04235074626865672, + "pr_auc": 0.6530718247293432, + "roc_auc": 0.9708910904550194, + "precision_at_n": 0.7299854439592431, + "macro_pr_auc": 0.47413483504435705, + "worst_group_fpr": 0.036940298507462686, "n_models": 1, - "seconds": 0.20720208400234696, + "seconds": 0.6308139160028077, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21475,13 +24026,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7771351789296148, - "roc_auc": 0.972047529775493, + "pr_auc": 0.7497335229655795, + "roc_auc": 0.973156389563252, "precision_at_n": 0.7743813682678311, - "macro_pr_auc": 0.5802319576815024, - "worst_group_fpr": 0.041100746268656715, + "macro_pr_auc": 0.5638693990084613, + "worst_group_fpr": 0.03677238805970149, "n_models": 1, - "seconds": 0.24668929200197454, + "seconds": 0.6763731669998378, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21492,13 +24043,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7620519974790155, - "roc_auc": 0.9831381972681902, - "precision_at_n": 0.8144104803493449, - "macro_pr_auc": 0.6711622930197206, - "worst_group_fpr": 0.08307865529998391, + "pr_auc": 0.6771727063514942, + "roc_auc": 0.9835831483054197, + "precision_at_n": 0.7882096069868996, + "macro_pr_auc": 0.6270798302274286, + "worst_group_fpr": 0.09530320090075599, "n_models": 3, - "seconds": 0.3629343749998952, + "seconds": 0.8962047079985496, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21509,13 +24060,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6857222009368626, - "roc_auc": 0.9737622464205439, - "precision_at_n": 0.7510917030567685, - "macro_pr_auc": 0.5535748214241942, - "worst_group_fpr": 0.04272388059701492, + "pr_auc": 0.6597227358298141, + "roc_auc": 0.9714515184501269, + "precision_at_n": 0.7692867540029112, + "macro_pr_auc": 0.5270690913982637, + "worst_group_fpr": 0.04151119402985075, "n_models": 1, - "seconds": 0.1986891669985198, + "seconds": 0.6266773329989519, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21526,13 +24077,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6113602669817982, - "roc_auc": 0.9740465348039771, + "pr_auc": 0.6099507905927241, + "roc_auc": 0.9721238083312913, "precision_at_n": 0.764919941775837, - "macro_pr_auc": 0.5072647036952613, - "worst_group_fpr": 0.04080223880597015, + "macro_pr_auc": 0.5031503078935212, + "worst_group_fpr": 0.04154850746268657, "n_models": 1, - "seconds": 0.2542918750004901, + "seconds": 0.669698249999783, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21543,13 +24094,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6860051393690949, - "roc_auc": 0.9832123305571526, - "precision_at_n": 0.7445414847161572, - "macro_pr_auc": 0.6350978057997398, - "worst_group_fpr": 0.07519704037317033, + "pr_auc": 0.7144381558306845, + "roc_auc": 0.9837242333729553, + "precision_at_n": 0.8056768558951966, + "macro_pr_auc": 0.6547106780700749, + "worst_group_fpr": 0.07181920540453594, "n_models": 3, - "seconds": 0.35823345799872186, + "seconds": 0.9075735829974292, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21560,13 +24111,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6297422146981834, - "roc_auc": 0.9741568620407105, - "precision_at_n": 0.7299854439592431, - "macro_pr_auc": 0.545633517740742, - "worst_group_fpr": 0.04003731343283582, + "pr_auc": 0.6551800529689307, + "roc_auc": 0.974870803601394, + "precision_at_n": 0.7707423580786026, + "macro_pr_auc": 0.547308769108289, + "worst_group_fpr": 0.038992537313432836, "n_models": 1, - "seconds": 0.20318404100180487, + "seconds": 0.6317226250030217, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21577,13 +24128,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7286468197463736, - "roc_auc": 0.9748349446827006, - "precision_at_n": 0.7758369723435226, - "macro_pr_auc": 0.5879134098171662, - "worst_group_fpr": 0.037667910447761195, + "pr_auc": 0.728918724499466, + "roc_auc": 0.9748660753684439, + "precision_at_n": 0.7729257641921398, + "macro_pr_auc": 0.5685681355212074, + "worst_group_fpr": 0.035559701492537316, "n_models": 1, - "seconds": 0.24570520799898077, + "seconds": 0.6641381249937695, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21594,13 +24145,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6603901413486373, - "roc_auc": 0.9830915309598965, - "precision_at_n": 0.745269286754003, - "macro_pr_auc": 0.6320887626863021, - "worst_group_fpr": 0.09449895447965256, + "pr_auc": 0.6966054974597077, + "roc_auc": 0.9831038027507876, + "precision_at_n": 0.8005822416302766, + "macro_pr_auc": 0.6434765539787998, + "worst_group_fpr": 0.08637606562650796, "n_models": 3, - "seconds": 0.3419267499994021, + "seconds": 0.9248066250002012, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21611,13 +24162,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6430275805941272, - "roc_auc": 0.9705622269931803, - "precision_at_n": 0.7802037845705968, - "macro_pr_auc": 0.4909138377370974, - "worst_group_fpr": 0.03792910447761194, + "pr_auc": 0.6184045391453056, + "roc_auc": 0.9714600941137747, + "precision_at_n": 0.7729257641921398, + "macro_pr_auc": 0.5005378454330374, + "worst_group_fpr": 0.038992537313432836, "n_models": 1, - "seconds": 0.20373449999897275, + "seconds": 0.6678157499991357, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21628,13 +24179,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6705523695466854, - "roc_auc": 0.9705192189862665, - "precision_at_n": 0.784570596797671, - "macro_pr_auc": 0.54605265099269, - "worst_group_fpr": 0.04074626865671642, + "pr_auc": 0.673219025125259, + "roc_auc": 0.9720874684837606, + "precision_at_n": 0.7816593886462883, + "macro_pr_auc": 0.5521651623217471, + "worst_group_fpr": 0.038992537313432836, "n_models": 1, - "seconds": 0.24851304199910373, + "seconds": 0.6711975829966832, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21645,13 +24196,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6400187149102201, - "roc_auc": 0.9825899113752985, - "precision_at_n": 0.7758369723435226, - "macro_pr_auc": 0.586999374960694, - "worst_group_fpr": 0.10503458259610744, + "pr_auc": 0.6865572065952276, + "roc_auc": 0.9829511591800745, + "precision_at_n": 0.7823871906841339, + "macro_pr_auc": 0.6059527323311097, + "worst_group_fpr": 0.09329258484799742, "n_models": 3, - "seconds": 0.3355528749998484, + "seconds": 0.9055055839999113, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21662,13 +24213,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6018090042265931, - "roc_auc": 0.9710144405962212, - "precision_at_n": 0.7489082969432315, - "macro_pr_auc": 0.5032442675968255, - "worst_group_fpr": 0.04126865671641791, + "pr_auc": 0.673650740437859, + "roc_auc": 0.9738444474258431, + "precision_at_n": 0.7671033478893741, + "macro_pr_auc": 0.5506975659722858, + "worst_group_fpr": 0.041902985074626864, "n_models": 1, - "seconds": 0.20233870800075238, + "seconds": 0.6526410829974338, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21679,13 +24230,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6812099737634885, - "roc_auc": 0.9714637848373232, - "precision_at_n": 0.7787481804949054, - "macro_pr_auc": 0.5465542275381834, - "worst_group_fpr": 0.03947761194029851, + "pr_auc": 0.7167629439101754, + "roc_auc": 0.9738429506023834, + "precision_at_n": 0.7751091703056768, + "macro_pr_auc": 0.5749534582846149, + "worst_group_fpr": 0.04027985074626866, "n_models": 1, - "seconds": 0.2541480420004518, + "seconds": 0.6724003749986878, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21696,13 +24247,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6218142259808852, - "roc_auc": 0.9809207694921918, - "precision_at_n": 0.7074235807860262, - "macro_pr_auc": 0.5716419581078703, - "worst_group_fpr": 0.0653048093935982, + "pr_auc": 0.6970946977891934, + "roc_auc": 0.983209104551357, + "precision_at_n": 0.7991266375545851, + "macro_pr_auc": 0.6140277347030043, + "worst_group_fpr": 0.0788965739102461, "n_models": 3, - "seconds": 0.3419192910005222, + "seconds": 0.8911962499987567, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21713,13 +24264,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7219691731333892, - "roc_auc": 0.9724569407120224, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.549924573247098, - "worst_group_fpr": 0.039850746268656714, + "pr_auc": 0.6812673546499289, + "roc_auc": 0.9723335095153985, + "precision_at_n": 0.7641921397379913, + "macro_pr_auc": 0.5360583720220582, + "worst_group_fpr": 0.03869402985074627, "n_models": 1, - "seconds": 0.2112539170011587, + "seconds": 0.6281392500022775, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21730,13 +24281,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6647913663607559, - "roc_auc": 0.9717020499521438, - "precision_at_n": 0.7736535662299855, - "macro_pr_auc": 0.5191153660713054, - "worst_group_fpr": 0.03953358208955224, + "pr_auc": 0.695258397131262, + "roc_auc": 0.9726748771270051, + "precision_at_n": 0.7787481804949054, + "macro_pr_auc": 0.5395418219552015, + "worst_group_fpr": 0.03826492537313433, "n_models": 1, - "seconds": 0.262136166998971, + "seconds": 0.6767856250007753, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21747,13 +24298,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7126872409664603, - "roc_auc": 0.982975745991415, - "precision_at_n": 0.7983988355167394, - "macro_pr_auc": 0.6219767122446509, - "worst_group_fpr": 0.09112111951101817, + "pr_auc": 0.7292724645874825, + "roc_auc": 0.9837263570250118, + "precision_at_n": 0.8165938864628821, + "macro_pr_auc": 0.6257470284541894, + "worst_group_fpr": 0.09007559916358372, "n_models": 3, - "seconds": 0.3417797090005479, + "seconds": 0.9138907919987105, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21764,13 +24315,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5477739185722166, - "roc_auc": 0.971442326765272, - "precision_at_n": 0.7430858806404658, - "macro_pr_auc": 0.4502184719290038, - "worst_group_fpr": 0.03852611940298507, + "pr_auc": 0.6220666923169502, + "roc_auc": 0.9721022800112077, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.4990001498959143, + "worst_group_fpr": 0.03923507462686567, "n_models": 1, - "seconds": 0.20219741700202576, + "seconds": 0.6433648340025684, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21781,13 +24332,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6070284009175415, - "roc_auc": 0.971787871432959, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.49669027776810853, - "worst_group_fpr": 0.03880597014925373, + "pr_auc": 0.6678377745312253, + "roc_auc": 0.972353881444795, + "precision_at_n": 0.7758369723435226, + "macro_pr_auc": 0.5392976747195802, + "worst_group_fpr": 0.037798507462686565, "n_models": 1, - "seconds": 0.2465083750030317, + "seconds": 0.6674270410003373, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21798,13 +24349,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6469253178574408, - "roc_auc": 0.9826695942588207, - "precision_at_n": 0.759825327510917, - "macro_pr_auc": 0.6120341502004562, - "worst_group_fpr": 0.07962039568923918, + "pr_auc": 0.6800917400408971, + "roc_auc": 0.9837985071579827, + "precision_at_n": 0.8129548762736536, + "macro_pr_auc": 0.6298478885384172, + "worst_group_fpr": 0.08283738137365289, "n_models": 3, - "seconds": 0.3544100420003815, + "seconds": 0.9231200419962988, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21815,13 +24366,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.656610287216522, - "roc_auc": 0.9702420418651903, - "precision_at_n": 0.7794759825327511, - "macro_pr_auc": 0.4361360618102787, - "worst_group_fpr": 0.03962686567164179, + "pr_auc": 0.6053543556413484, + "roc_auc": 0.9713649134335658, + "precision_at_n": 0.7540029112081513, + "macro_pr_auc": 0.49191252560248805, + "worst_group_fpr": 0.04166044776119403, "n_models": 1, - "seconds": 0.1933791250012291, + "seconds": 0.6326952500021434, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21832,13 +24383,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7064937151856711, - "roc_auc": 0.970018717966492, + "pr_auc": 0.6831024791337322, + "roc_auc": 0.9732640095488907, "precision_at_n": 0.7714701601164483, - "macro_pr_auc": 0.5121206601547154, - "worst_group_fpr": 0.037052238805970146, + "macro_pr_auc": 0.5443222382523539, + "worst_group_fpr": 0.03914179104477612, "n_models": 1, - "seconds": 0.25321816699943156, + "seconds": 0.6735367920045974, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21849,13 +24400,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6673518435943845, - "roc_auc": 0.9820557723553934, - "precision_at_n": 0.7976710334788938, - "macro_pr_auc": 0.6227622361027524, - "worst_group_fpr": 0.08862795560559755, + "pr_auc": 0.6689447474000777, + "roc_auc": 0.9819260998912754, + "precision_at_n": 0.7940320232896652, + "macro_pr_auc": 0.627896819171807, + "worst_group_fpr": 0.07262345182563938, "n_models": 3, - "seconds": 0.3529735830015852, + "seconds": 0.914368874997308, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21866,13 +24417,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6879044743725874, - "roc_auc": 0.9685583856578505, - "precision_at_n": 0.7554585152838428, - "macro_pr_auc": 0.5064574481765121, - "worst_group_fpr": 0.03992537313432836, + "pr_auc": 0.6772511305344789, + "roc_auc": 0.9708570093519151, + "precision_at_n": 0.7692867540029112, + "macro_pr_auc": 0.528819686901723, + "worst_group_fpr": 0.04027985074626866, "n_models": 1, - "seconds": 0.1997192080016248, + "seconds": 0.618268457998056, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21883,13 +24434,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6683263018119266, - "roc_auc": 0.9727062455753196, - "precision_at_n": 0.7707423580786026, - "macro_pr_auc": 0.5390720178834354, - "worst_group_fpr": 0.03583955223880597, + "pr_auc": 0.7042421120611811, + "roc_auc": 0.9742613640944495, + "precision_at_n": 0.7751091703056768, + "macro_pr_auc": 0.5683334962817763, + "worst_group_fpr": 0.036940298507462686, "n_models": 1, - "seconds": 0.2481015839985048, + "seconds": 0.6780074999987846, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21900,13 +24451,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7168319555925291, - "roc_auc": 0.9852743804928692, - "precision_at_n": 0.8056768558951966, - "macro_pr_auc": 0.618973540170405, - "worst_group_fpr": 0.09900273443783175, + "pr_auc": 0.7138022925894035, + "roc_auc": 0.9844030238989642, + "precision_at_n": 0.8064046579330422, + "macro_pr_auc": 0.6190764672520105, + "worst_group_fpr": 0.09465980376387326, "n_models": 3, - "seconds": 0.35266041599970777, + "seconds": 0.9198891669948353, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21917,13 +24468,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6407583233057124, - "roc_auc": 0.9725172513496863, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.4462010831582593, - "worst_group_fpr": 0.03604477611940299, + "pr_auc": 0.6632591000239041, + "roc_auc": 0.9723781224196681, + "precision_at_n": 0.7685589519650655, + "macro_pr_auc": 0.4999708246289276, + "worst_group_fpr": 0.03716417910447761, "n_models": 1, - "seconds": 0.20796087499911664, + "seconds": 0.6268997080042027, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21934,13 +24485,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.669307405730254, - "roc_auc": 0.9754224127665878, - "precision_at_n": 0.7590975254730713, - "macro_pr_auc": 0.5597129030487102, - "worst_group_fpr": 0.034384328358208954, + "pr_auc": 0.6748276857732136, + "roc_auc": 0.9760264972119853, + "precision_at_n": 0.7751091703056768, + "macro_pr_auc": 0.5782420564201117, + "worst_group_fpr": 0.039048507462686566, "n_models": 1, - "seconds": 0.24508341700129677, + "seconds": 0.6950577919997158, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21951,13 +24502,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6503634825175936, - "roc_auc": 0.9822675809804727, - "precision_at_n": 0.772197962154294, - "macro_pr_auc": 0.6361188963477792, - "worst_group_fpr": 0.10463245938555574, + "pr_auc": 0.6683292840636673, + "roc_auc": 0.9832841618721304, + "precision_at_n": 0.8049490538573508, + "macro_pr_auc": 0.6310565621169522, + "worst_group_fpr": 0.10278269261701785, "n_models": 3, - "seconds": 0.34629341600157204, + "seconds": 0.9131106249988079, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21968,13 +24519,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6353575815204143, - "roc_auc": 0.9720192630479938, - "precision_at_n": 0.7561863173216885, - "macro_pr_auc": 0.46688389334892527, - "worst_group_fpr": 0.03820170500241274, + "pr_auc": 0.6434356874231278, + "roc_auc": 0.9722437055115158, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.4954406370669884, + "worst_group_fpr": 0.037033582089552236, "n_models": 1, - "seconds": 0.19593587499912246, + "seconds": 0.6211250419946737, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -21985,13 +24536,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.681123198451289, - "roc_auc": 0.9719263897454645, - "precision_at_n": 0.7765647743813683, - "macro_pr_auc": 0.5041782349951254, - "worst_group_fpr": 0.03718283582089552, + "pr_auc": 0.6739652779593103, + "roc_auc": 0.9730661640711749, + "precision_at_n": 0.7780203784570596, + "macro_pr_auc": 0.5523917874713837, + "worst_group_fpr": 0.04022388059701493, "n_models": 1, - "seconds": 0.24913533300059498, + "seconds": 0.6721129580037086, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22002,13 +24553,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6647451339671704, - "roc_auc": 0.9829602590021178, - "precision_at_n": 0.7161572052401747, - "macro_pr_auc": 0.6087803813113698, - "worst_group_fpr": 0.08637606562650796, + "pr_auc": 0.6811629242290259, + "roc_auc": 0.9835466625581294, + "precision_at_n": 0.8042212518195051, + "macro_pr_auc": 0.61623572056422, + "worst_group_fpr": 0.07873572462602542, "n_models": 3, - "seconds": 0.3410571249987697, + "seconds": 0.9090283330006059, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22019,13 +24570,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6043056093954045, - "roc_auc": 0.9726922932353457, - "precision_at_n": 0.7358078602620087, - "macro_pr_auc": 0.5093356935366994, - "worst_group_fpr": 0.03921641791044776, + "pr_auc": 0.6551151866652897, + "roc_auc": 0.9729514652453479, + "precision_at_n": 0.7707423580786026, + "macro_pr_auc": 0.5190067105927844, + "worst_group_fpr": 0.03593283582089552, "n_models": 1, - "seconds": 0.1936662919979426, + "seconds": 0.6247831250002491, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22036,13 +24587,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6560189855442788, - "roc_auc": 0.9721102072314795, - "precision_at_n": 0.7736535662299855, - "macro_pr_auc": 0.5149827041492328, - "worst_group_fpr": 0.03949626865671642, + "pr_auc": 0.6482298761443848, + "roc_auc": 0.9721994816732719, + "precision_at_n": 0.7758369723435226, + "macro_pr_auc": 0.514628531207875, + "worst_group_fpr": 0.038917910447761196, "n_models": 1, - "seconds": 0.25070270800279104, + "seconds": 0.6851487919993815, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22053,13 +24604,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6954184299748126, - "roc_auc": 0.981737894605083, - "precision_at_n": 0.7765647743813683, - "macro_pr_auc": 0.6170665503805763, - "worst_group_fpr": 0.07495576644683931, + "pr_auc": 0.7192257699794287, + "roc_auc": 0.9831750558704216, + "precision_at_n": 0.8136826783114993, + "macro_pr_auc": 0.6270065407992284, + "worst_group_fpr": 0.08919092810036995, "n_models": 3, - "seconds": 0.3438656250000349, + "seconds": 0.9044852920051198, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22070,13 +24621,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6275629286313821, - "roc_auc": 0.9720377058583466, - "precision_at_n": 0.7328966521106259, - "macro_pr_auc": 0.50662550657358, - "worst_group_fpr": 0.03824626865671642, + "pr_auc": 0.6602705949896037, + "roc_auc": 0.9721382416001024, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.5227122218538428, + "worst_group_fpr": 0.03916044776119403, "n_models": 1, - "seconds": 0.20500329100104864, + "seconds": 0.6109848339983728, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22087,13 +24638,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6446591525183992, - "roc_auc": 0.9754932498017221, - "precision_at_n": 0.7481804949053857, - "macro_pr_auc": 0.5595449760087942, - "worst_group_fpr": 0.03886194029850746, + "pr_auc": 0.6950417248835344, + "roc_auc": 0.9747763199978143, + "precision_at_n": 0.7729257641921398, + "macro_pr_auc": 0.5929750027330102, + "worst_group_fpr": 0.041753731343283584, "n_models": 1, - "seconds": 0.25379166700076894, + "seconds": 0.6856084169994574, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22104,13 +24655,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6426365135474428, - "roc_auc": 0.9842551193685908, - "precision_at_n": 0.7976710334788938, - "macro_pr_auc": 0.6091390698178812, - "worst_group_fpr": 0.10398906224867299, + "pr_auc": 0.7336603135821277, + "roc_auc": 0.9847187725935235, + "precision_at_n": 0.8195050946142649, + "macro_pr_auc": 0.644092692138366, + "worst_group_fpr": 0.09675084445874216, "n_models": 3, - "seconds": 0.346133415998338, + "seconds": 0.9061175000024377, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22121,13 +24672,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7285997390796592, - "roc_auc": 0.9726331065662004, - "precision_at_n": 0.7802037845705968, - "macro_pr_auc": 0.533125894293412, - "worst_group_fpr": 0.041026119402985076, + "pr_auc": 0.6676094467715644, + "roc_auc": 0.9726910882114053, + "precision_at_n": 0.7743813682678311, + "macro_pr_auc": 0.5362592870799477, + "worst_group_fpr": 0.04162313432835821, "n_models": 1, - "seconds": 0.20132883300175308, + "seconds": 0.6429056670021964, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22138,13 +24689,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.720546436302856, - "roc_auc": 0.9734600934221017, - "precision_at_n": 0.7758369723435226, - "macro_pr_auc": 0.575537891034216, - "worst_group_fpr": 0.0407089552238806, + "pr_auc": 0.7072077440006722, + "roc_auc": 0.9749246568237718, + "precision_at_n": 0.7700145560407569, + "macro_pr_auc": 0.5646705434198914, + "worst_group_fpr": 0.03914179104477612, "n_models": 1, - "seconds": 0.256971082999371, + "seconds": 0.6788953340001171, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22155,13 +24706,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7213639938542473, - "roc_auc": 0.9827387021116191, - "precision_at_n": 0.8078602620087336, - "macro_pr_auc": 0.6256143354854139, - "worst_group_fpr": 0.0662699050989223, + "pr_auc": 0.7261272164835856, + "roc_auc": 0.9830016729190657, + "precision_at_n": 0.8144104803493449, + "macro_pr_auc": 0.6338972683435188, + "worst_group_fpr": 0.05951423516165353, "n_models": 3, - "seconds": 0.3361283749982249, + "seconds": 0.8952491250020103, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22172,13 +24723,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6233078022796769, - "roc_auc": 0.9716151747508427, - "precision_at_n": 0.740174672489083, - "macro_pr_auc": 0.5207234690335601, - "worst_group_fpr": 0.0428544776119403, + "pr_auc": 0.60539275571879, + "roc_auc": 0.9715701403583787, + "precision_at_n": 0.7532751091703057, + "macro_pr_auc": 0.5082628245792461, + "worst_group_fpr": 0.04289179104477612, "n_models": 1, - "seconds": 0.19976808299907134, + "seconds": 0.626040875002218, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22189,13 +24740,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5663276870854721, - "roc_auc": 0.9736081276411505, - "precision_at_n": 0.7023289665211062, - "macro_pr_auc": 0.5392097197171283, - "worst_group_fpr": 0.043115671641791045, + "pr_auc": 0.664330698592417, + "roc_auc": 0.9745581133980917, + "precision_at_n": 0.7751091703056768, + "macro_pr_auc": 0.5706054710628522, + "worst_group_fpr": 0.04276119402985075, "n_models": 1, - "seconds": 0.2500787089993537, + "seconds": 0.6735202080017189, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22206,13 +24757,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6482634991740984, - "roc_auc": 0.982784898298465, - "precision_at_n": 0.7197962154294032, - "macro_pr_auc": 0.6063187120045425, - "worst_group_fpr": 0.07085410969921184, + "pr_auc": 0.6590068116865129, + "roc_auc": 0.9835822080625245, + "precision_at_n": 0.7532751091703057, + "macro_pr_auc": 0.6070913895966344, + "worst_group_fpr": 0.07431236930995658, "n_models": 3, - "seconds": 0.3386523330009368, + "seconds": 0.9206780829990748, "eta_squared": 0.05087186473761437, "warnings": [] }, @@ -22223,13 +24774,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6768047861689719, - "roc_auc": 0.972266055193209, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.6996034070040531, - "worst_group_fpr": 0.7096774193548387, + "pr_auc": 0.6530718247293432, + "roc_auc": 0.9708910904550194, + "precision_at_n": 0.7299854439592431, + "macro_pr_auc": 0.7047727154351631, + "worst_group_fpr": 0.8172043010752689, "n_models": 1, - "seconds": 0.20758445799947367, + "seconds": 0.6172814169985941, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22240,13 +24791,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5406925996404937, - "roc_auc": 0.9614455184035687, - "precision_at_n": 0.4912663755458515, - "macro_pr_auc": 0.7856591729582578, + "pr_auc": 0.5322775873886177, + "roc_auc": 0.9628784701906581, + "precision_at_n": 0.5232896652110626, + "macro_pr_auc": 0.7500437558549391, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.274014165999688, + "seconds": 0.6833220410044305, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22257,13 +24808,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.18703204247733796, - "roc_auc": 0.8739845727971822, - "precision_at_n": 0.19068413391557495, - "macro_pr_auc": 0.6760946942545061, + "pr_auc": 0.19426886828720197, + "roc_auc": 0.8750921140834098, + "precision_at_n": 0.20160116448326054, + "macro_pr_auc": 0.6475584754233464, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.076055790999817, + "seconds": 6.103904083000089, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22274,13 +24825,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6857222009368626, - "roc_auc": 0.9737622464205439, - "precision_at_n": 0.7510917030567685, - "macro_pr_auc": 0.7586410316256198, - "worst_group_fpr": 0.8333333333333334, + "pr_auc": 0.6597227358298141, + "roc_auc": 0.9714515184501269, + "precision_at_n": 0.7692867540029112, + "macro_pr_auc": 0.7575684869258095, + "worst_group_fpr": 0.8387096774193549, "n_models": 1, - "seconds": 0.19729187499979162, + "seconds": 0.6304595829933533, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22291,13 +24842,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5252317276128622, - "roc_auc": 0.9620086806682451, - "precision_at_n": 0.5356622998544396, - "macro_pr_auc": 0.7585627369212566, + "pr_auc": 0.516504186652572, + "roc_auc": 0.9637440934643803, + "precision_at_n": 0.529839883551674, + "macro_pr_auc": 0.7272570347699341, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.2684215420013061, + "seconds": 0.6644380840007216, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22308,13 +24859,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.20112796571471114, - "roc_auc": 0.8791306680624627, - "precision_at_n": 0.21033478893740903, - "macro_pr_auc": 0.6656088655658063, + "pr_auc": 0.20733998337975884, + "roc_auc": 0.8771236601619798, + "precision_at_n": 0.21106259097525473, + "macro_pr_auc": 0.678921859564307, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.096728083000926, + "seconds": 6.136722833995009, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22325,13 +24876,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6297422146981834, - "roc_auc": 0.9741568620407105, - "precision_at_n": 0.7299854439592431, - "macro_pr_auc": 0.7043019881527529, - "worst_group_fpr": 0.8387096774193549, + "pr_auc": 0.6551800529689307, + "roc_auc": 0.974870803601394, + "precision_at_n": 0.7707423580786026, + "macro_pr_auc": 0.7202188847677089, + "worst_group_fpr": 0.8602150537634409, "n_models": 1, - "seconds": 0.19956187500065425, + "seconds": 0.6134327499967185, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22342,13 +24893,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5720614240791909, - "roc_auc": 0.9709951980390381, - "precision_at_n": 0.5291120815138283, - "macro_pr_auc": 0.7814869495898868, + "pr_auc": 0.5529261848221064, + "roc_auc": 0.9673986500835488, + "precision_at_n": 0.5312954876273653, + "macro_pr_auc": 0.772580116437516, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.25566479199915193, + "seconds": 0.6953558339955634, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22359,13 +24910,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.2095921913874511, - "roc_auc": 0.8796752956539747, - "precision_at_n": 0.2096069868995633, - "macro_pr_auc": 0.6743987061641321, + "pr_auc": 0.2088041217715933, + "roc_auc": 0.8777718333532514, + "precision_at_n": 0.21615720524017468, + "macro_pr_auc": 0.6641493269337587, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.0936844589996326, + "seconds": 6.143598791000841, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22376,13 +24927,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6430275805941272, - "roc_auc": 0.9705622269931803, - "precision_at_n": 0.7802037845705968, - "macro_pr_auc": 0.7337349843379073, + "pr_auc": 0.6184045391453056, + "roc_auc": 0.9714600941137747, + "precision_at_n": 0.7729257641921398, + "macro_pr_auc": 0.7310428069140714, "worst_group_fpr": 0.8387096774193549, "n_models": 1, - "seconds": 0.19409620800070115, + "seconds": 0.6243396659992868, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22393,13 +24944,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5500951341718753, - "roc_auc": 0.9607994526532693, - "precision_at_n": 0.5312954876273653, - "macro_pr_auc": 0.753382778831343, + "pr_auc": 0.5170898627982412, + "roc_auc": 0.961976842098483, + "precision_at_n": 0.5283842794759825, + "macro_pr_auc": 0.7333993025552435, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.2788463329998194, + "seconds": 0.7306586250051623, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22410,13 +24961,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.19649999140658245, - "roc_auc": 0.8780009824349442, - "precision_at_n": 0.20232896652110627, - "macro_pr_auc": 0.6518937356853428, + "pr_auc": 0.2054538194435767, + "roc_auc": 0.8798870448384113, + "precision_at_n": 0.2074235807860262, + "macro_pr_auc": 0.6689276791739517, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.0975127090023307, + "seconds": 6.13056616600079, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22427,13 +24978,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6018090042265931, - "roc_auc": 0.9710144405962212, - "precision_at_n": 0.7489082969432315, - "macro_pr_auc": 0.6721139290877173, + "pr_auc": 0.673650740437859, + "roc_auc": 0.9738444474258431, + "precision_at_n": 0.7671033478893741, + "macro_pr_auc": 0.7304219338677542, "worst_group_fpr": 0.8763440860215054, "n_models": 1, - "seconds": 0.20953120899866917, + "seconds": 0.6449807910030358, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22444,13 +24995,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5257244009940591, - "roc_auc": 0.9591343527338729, - "precision_at_n": 0.5269286754002911, - "macro_pr_auc": 0.6976737326981719, + "pr_auc": 0.5310353519342732, + "roc_auc": 0.9637141083619345, + "precision_at_n": 0.5262008733624454, + "macro_pr_auc": 0.7231842722605921, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.2813873330014758, + "seconds": 0.6754672909955843, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22461,13 +25012,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.1834832859642059, - "roc_auc": 0.8778802260672464, - "precision_at_n": 0.2081513828238719, - "macro_pr_auc": 0.6563759276300162, + "pr_auc": 0.1987803623966471, + "roc_auc": 0.8778387527096557, + "precision_at_n": 0.21397379912663755, + "macro_pr_auc": 0.6769142802897322, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.078388417001406, + "seconds": 6.070312916999683, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22478,13 +25029,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7219691731333892, - "roc_auc": 0.9724569407120224, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.7446667828428115, - "worst_group_fpr": 0.8333333333333334, + "pr_auc": 0.6812673546499289, + "roc_auc": 0.9723335095153985, + "precision_at_n": 0.7641921397379913, + "macro_pr_auc": 0.7319369833246042, + "worst_group_fpr": 0.8440860215053764, "n_models": 1, - "seconds": 0.19650229200124159, + "seconds": 0.6568119580042548, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22495,13 +25046,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5062848011346063, - "roc_auc": 0.9612991809446872, - "precision_at_n": 0.5254730713245997, - "macro_pr_auc": 0.7592604380773622, + "pr_auc": 0.5126336105780375, + "roc_auc": 0.9606217683608526, + "precision_at_n": 0.5262008733624454, + "macro_pr_auc": 0.7654565381324793, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.27197862500179326, + "seconds": 0.6896719579963246, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22512,13 +25063,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.20201422842494055, - "roc_auc": 0.8773170259767064, - "precision_at_n": 0.21033478893740903, - "macro_pr_auc": 0.6796184995818257, + "pr_auc": 0.19295065274944362, + "roc_auc": 0.8757887205911745, + "precision_at_n": 0.20378457059679767, + "macro_pr_auc": 0.671566525517948, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.0553991250017134, + "seconds": 6.0496395000009215, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22529,13 +25080,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5477739185722166, - "roc_auc": 0.971442326765272, - "precision_at_n": 0.7430858806404658, - "macro_pr_auc": 0.716946321427026, - "worst_group_fpr": 0.8548387096774194, + "pr_auc": 0.6220666923169502, + "roc_auc": 0.9721022800112077, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.7352412446077395, + "worst_group_fpr": 0.8655913978494624, "n_models": 1, - "seconds": 0.2002458339993609, + "seconds": 0.6435970000020461, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22546,13 +25097,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5016511517428832, - "roc_auc": 0.9635457454430479, - "precision_at_n": 0.524745269286754, - "macro_pr_auc": 0.7856706887167291, + "pr_auc": 0.5279299951184584, + "roc_auc": 0.9620832462527916, + "precision_at_n": 0.5291120815138283, + "macro_pr_auc": 0.7833092631457893, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.27726262499709264, + "seconds": 0.6910863750017597, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22563,13 +25114,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.20804780617622678, - "roc_auc": 0.8785029910855678, - "precision_at_n": 0.21251819505094613, - "macro_pr_auc": 0.674123994825253, + "pr_auc": 0.2087712800249874, + "roc_auc": 0.8793636105379052, + "precision_at_n": 0.2074235807860262, + "macro_pr_auc": 0.6783338950792724, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.034101875000488, + "seconds": 6.009161374997348, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22580,13 +25131,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.656610287216522, - "roc_auc": 0.9702420418651903, - "precision_at_n": 0.7794759825327511, - "macro_pr_auc": 0.7494908624299447, - "worst_group_fpr": 0.8548387096774194, + "pr_auc": 0.6053543556413484, + "roc_auc": 0.9713649134335658, + "precision_at_n": 0.7540029112081513, + "macro_pr_auc": 0.6914621548200052, + "worst_group_fpr": 0.8709677419354839, "n_models": 1, - "seconds": 0.20102370800304925, + "seconds": 0.6262198330005049, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22597,13 +25148,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5607263597688444, - "roc_auc": 0.9665532690505476, - "precision_at_n": 0.5334788937409025, - "macro_pr_auc": 0.7897788734647209, + "pr_auc": 0.5260966945818674, + "roc_auc": 0.963761509572721, + "precision_at_n": 0.5312954876273653, + "macro_pr_auc": 0.7279858708756674, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.270870415999525, + "seconds": 0.6726583750059945, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22614,13 +25165,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.20854992987888132, - "roc_auc": 0.8772972592611278, - "precision_at_n": 0.2205240174672489, - "macro_pr_auc": 0.6662087227753097, + "pr_auc": 0.20223291977384794, + "roc_auc": 0.8759632491258281, + "precision_at_n": 0.21542940320232898, + "macro_pr_auc": 0.6697223820051702, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.1973240000006626, + "seconds": 6.070021457999246, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22631,13 +25182,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6879044743725874, - "roc_auc": 0.9685583856578505, - "precision_at_n": 0.7554585152838428, - "macro_pr_auc": 0.7280317213838148, - "worst_group_fpr": 0.8387096774193549, + "pr_auc": 0.6772511305344789, + "roc_auc": 0.9708570093519151, + "precision_at_n": 0.7692867540029112, + "macro_pr_auc": 0.7360282670260866, + "worst_group_fpr": 0.8440860215053764, "n_models": 1, - "seconds": 0.1988795410034072, + "seconds": 0.6317711250012508, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22648,13 +25199,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5277315215516899, - "roc_auc": 0.9567197063087554, - "precision_at_n": 0.5276564774381368, - "macro_pr_auc": 0.717254263513192, + "pr_auc": 0.5325720465137227, + "roc_auc": 0.9601075581673703, + "precision_at_n": 0.5283842794759825, + "macro_pr_auc": 0.7394736995824194, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.28242937499817344, + "seconds": 0.6852393330045743, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22665,13 +25216,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.20067006461093626, - "roc_auc": 0.8797338987240818, - "precision_at_n": 0.2066957787481805, - "macro_pr_auc": 0.6559956670259489, + "pr_auc": 0.20531187605608703, + "roc_auc": 0.8796904854400578, + "precision_at_n": 0.2081513828238719, + "macro_pr_auc": 0.6665454810860403, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.094073208001646, + "seconds": 6.105933708000521, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22682,13 +25233,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6407583233057124, - "roc_auc": 0.9725172513496863, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.6836882921538495, - "worst_group_fpr": 0.8279569892473119, + "pr_auc": 0.6632591000239041, + "roc_auc": 0.9723781224196681, + "precision_at_n": 0.7685589519650655, + "macro_pr_auc": 0.7148660857094303, + "worst_group_fpr": 0.8440860215053764, "n_models": 1, - "seconds": 0.19848512500175275, + "seconds": 0.6665979579993291, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22699,13 +25250,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.47676605049364645, - "roc_auc": 0.9601640700075895, - "precision_at_n": 0.5058224163027657, - "macro_pr_auc": 0.7270344397438341, + "pr_auc": 0.4795792385930423, + "roc_auc": 0.9626318347525921, + "precision_at_n": 0.48326055312954874, + "macro_pr_auc": 0.7419634640352997, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.26985037499980535, + "seconds": 0.6996548750030342, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22716,13 +25267,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.20052733262055475, - "roc_auc": 0.8692715890738243, - "precision_at_n": 0.2183406113537118, - "macro_pr_auc": 0.6531491636743189, + "pr_auc": 0.2057538989312502, + "roc_auc": 0.87525956918157, + "precision_at_n": 0.21470160116448325, + "macro_pr_auc": 0.6633964988511316, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.0625154159970407, + "seconds": 6.075786749999679, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22733,13 +25284,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6353575815204143, - "roc_auc": 0.9720192630479938, - "precision_at_n": 0.7561863173216885, - "macro_pr_auc": 0.69812154611129, - "worst_group_fpr": 0.8494623655913979, + "pr_auc": 0.6434356874231278, + "roc_auc": 0.9722437055115158, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.716768355198076, + "worst_group_fpr": 0.8655913978494624, "n_models": 1, - "seconds": 0.20798570799888694, + "seconds": 0.6215927919984097, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22750,13 +25301,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.4809364976628495, - "roc_auc": 0.9634443288990397, - "precision_at_n": 0.5160116448326055, - "macro_pr_auc": 0.7000143546785108, + "pr_auc": 0.49772334343685765, + "roc_auc": 0.9639381941815997, + "precision_at_n": 0.5305676855895196, + "macro_pr_auc": 0.7442065571265031, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.26160629099831567, + "seconds": 0.6849310000034166, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22767,13 +25318,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.2001193602506792, - "roc_auc": 0.8723764440320632, - "precision_at_n": 0.19723435225618632, - "macro_pr_auc": 0.655558862847107, + "pr_auc": 0.209778729461222, + "roc_auc": 0.8766734297149307, + "precision_at_n": 0.2052401746724891, + "macro_pr_auc": 0.6614477574664632, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.245599709000089, + "seconds": 6.478175124997506, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22784,13 +25335,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6043056093954045, - "roc_auc": 0.9726922932353457, - "precision_at_n": 0.7358078602620087, - "macro_pr_auc": 0.7005306512787453, + "pr_auc": 0.6551151866652897, + "roc_auc": 0.9729514652453479, + "precision_at_n": 0.7707423580786026, + "macro_pr_auc": 0.7218815255148043, "worst_group_fpr": 0.8494623655913979, "n_models": 1, - "seconds": 0.21010816599664395, + "seconds": 0.654631833996973, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22801,13 +25352,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5033086492667329, - "roc_auc": 0.962167760039465, - "precision_at_n": 0.5269286754002911, - "macro_pr_auc": 0.694393634085864, + "pr_auc": 0.5165568779889708, + "roc_auc": 0.9632808671313369, + "precision_at_n": 0.5305676855895196, + "macro_pr_auc": 0.7078322070864903, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.30407349999950384, + "seconds": 0.7286407920037163, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22818,13 +25369,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.20085933528134287, - "roc_auc": 0.8727757662704008, - "precision_at_n": 0.21179039301310043, - "macro_pr_auc": 0.6688128241082326, + "pr_auc": 0.2028376257503053, + "roc_auc": 0.8751945789442093, + "precision_at_n": 0.2059679767103348, + "macro_pr_auc": 0.6623792624130437, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.297948749997886, + "seconds": 6.391475541997352, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22835,13 +25386,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6275629286313821, - "roc_auc": 0.9720377058583466, - "precision_at_n": 0.7328966521106259, - "macro_pr_auc": 0.7174989474930186, - "worst_group_fpr": 0.8494623655913979, + "pr_auc": 0.6602705949896037, + "roc_auc": 0.9721382416001024, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.721034146320275, + "worst_group_fpr": 0.8548387096774194, "n_models": 1, - "seconds": 0.21575704199858592, + "seconds": 0.704520124992996, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22852,13 +25403,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.38342074843131196, - "roc_auc": 0.9585648411278064, - "precision_at_n": 0.462882096069869, - "macro_pr_auc": 0.7117930588430045, + "pr_auc": 0.459050906670713, + "roc_auc": 0.9620710176914591, + "precision_at_n": 0.46797671033478894, + "macro_pr_auc": 0.744862577019632, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.27413754100052756, + "seconds": 0.7335899169993354, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22869,13 +25420,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.19642812702900658, - "roc_auc": 0.8711648059692065, - "precision_at_n": 0.20087336244541484, - "macro_pr_auc": 0.659899391775854, + "pr_auc": 0.20464191703683093, + "roc_auc": 0.8755868493608325, + "precision_at_n": 0.21688500727802038, + "macro_pr_auc": 0.6695421930565572, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.0494829999988724, + "seconds": 6.637700624996796, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22886,13 +25437,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7285997390796592, - "roc_auc": 0.9726331065662004, - "precision_at_n": 0.7802037845705968, - "macro_pr_auc": 0.7818843759635191, - "worst_group_fpr": 0.8494623655913979, + "pr_auc": 0.6676094467715644, + "roc_auc": 0.9726910882114053, + "precision_at_n": 0.7743813682678311, + "macro_pr_auc": 0.7492491008684662, + "worst_group_fpr": 0.8602150537634409, "n_models": 1, - "seconds": 0.2014633340004366, + "seconds": 0.7284391249995679, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22903,13 +25454,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5273879980108448, - "roc_auc": 0.9632161578861057, - "precision_at_n": 0.524745269286754, - "macro_pr_auc": 0.7185393777974337, + "pr_auc": 0.5254747583231889, + "roc_auc": 0.9631463205345092, + "precision_at_n": 0.5254730713245997, + "macro_pr_auc": 0.7365112431113047, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.28373254200050724, + "seconds": 0.7573751669988269, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22920,13 +25471,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.2014786489324183, - "roc_auc": 0.8755190654132603, - "precision_at_n": 0.21397379912663755, - "macro_pr_auc": 0.6828954458144512, + "pr_auc": 0.19958209589022063, + "roc_auc": 0.8764810797948264, + "precision_at_n": 0.21324599708879186, + "macro_pr_auc": 0.6759344067395817, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.414072291001503, + "seconds": 6.59769783399679, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22937,13 +25488,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6233078022796769, - "roc_auc": 0.9716151747508427, - "precision_at_n": 0.740174672489083, - "macro_pr_auc": 0.7151427340837024, - "worst_group_fpr": 0.8494623655913979, + "pr_auc": 0.60539275571879, + "roc_auc": 0.9715701403583787, + "precision_at_n": 0.7532751091703057, + "macro_pr_auc": 0.708138965631994, + "worst_group_fpr": 0.8709677419354839, "n_models": 1, - "seconds": 0.22015479199762922, + "seconds": 0.7024482499982696, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22954,13 +25505,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5207206515635024, - "roc_auc": 0.9668831105811455, - "precision_at_n": 0.5254730713245997, - "macro_pr_auc": 0.7343201490083063, + "pr_auc": 0.48464488226783314, + "roc_auc": 0.9612562539931953, + "precision_at_n": 0.5276564774381368, + "macro_pr_auc": 0.7247406489693413, "worst_group_fpr": 0.5, "n_models": 1, - "seconds": 0.2871878749974712, + "seconds": 0.7129539590023342, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22971,13 +25522,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.19248512507512788, - "roc_auc": 0.8764153330401938, - "precision_at_n": 0.20087336244541484, - "macro_pr_auc": 0.6678446655880989, + "pr_auc": 0.18838265290772957, + "roc_auc": 0.8743656521618746, + "precision_at_n": 0.19723435225618632, + "macro_pr_auc": 0.6705908870499475, "worst_group_fpr": 1.0, "n_models": 50, - "seconds": 3.0972869580000406, + "seconds": 6.1209764999948675, "eta_squared": 0.23802723873795992, "warnings": [] }, @@ -22988,13 +25539,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6768047861689719, - "roc_auc": 0.972266055193209, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.6962765518547999, + "pr_auc": 0.6530718247293432, + "roc_auc": 0.9708910904550194, + "precision_at_n": 0.7299854439592431, + "macro_pr_auc": 0.6223117227115356, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.23114012500082026, + "seconds": 0.6493378750019474, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23005,13 +25556,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.3273271540675249, - "roc_auc": 0.9664521118838899, - "precision_at_n": 0.32168850072780203, - "macro_pr_auc": 0.6363351199492512, - "worst_group_fpr": 1.0, + "pr_auc": 0.2925375728206705, + "roc_auc": 0.9643675069260778, + "precision_at_n": 0.2780203784570597, + "macro_pr_auc": 0.6131191976439884, + "worst_group_fpr": 0.6412429378531074, "n_models": 1, - "seconds": 0.263118625000061, + "seconds": 0.6769333749980433, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23022,13 +25573,13 @@ "seed": 0, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07968185827363976, - "roc_auc": 0.7140235023114089, - "precision_at_n": 0.1564774381368268, - "macro_pr_auc": 0.3858407985363126, + "pr_auc": 0.06827038708560274, + "roc_auc": 0.7024115295739569, + "precision_at_n": 0.11572052401746726, + "macro_pr_auc": 0.38428258478195154, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.7374150840005314, + "seconds": 1.5290592079982162, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23039,13 +25590,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6857222009368626, - "roc_auc": 0.9737622464205439, - "precision_at_n": 0.7510917030567685, - "macro_pr_auc": 0.6425280272669781, + "pr_auc": 0.6597227358298141, + "roc_auc": 0.9714515184501269, + "precision_at_n": 0.7692867540029112, + "macro_pr_auc": 0.6730425633608333, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.19914012500157696, + "seconds": 0.6372252500004834, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23056,13 +25607,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.24046309606615066, - "roc_auc": 0.9590784461074712, - "precision_at_n": 0.2059679767103348, - "macro_pr_auc": 0.5880528015383217, + "pr_auc": 0.28200565888221923, + "roc_auc": 0.9623676967470687, + "precision_at_n": 0.21251819505094613, + "macro_pr_auc": 0.6085606750076077, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.24573016599970288, + "seconds": 0.6727971660002368, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23073,13 +25624,13 @@ "seed": 1, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0750782235983171, - "roc_auc": 0.7167630729048562, - "precision_at_n": 0.13537117903930132, - "macro_pr_auc": 0.395465069469411, - "worst_group_fpr": 0.3983050847457627, + "pr_auc": 0.06820440985503894, + "roc_auc": 0.7123372415231752, + "precision_at_n": 0.11208151382823872, + "macro_pr_auc": 0.3902930878886586, + "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.6775090420014749, + "seconds": 1.5607022499971208, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23090,13 +25641,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6297422146981834, - "roc_auc": 0.9741568620407105, - "precision_at_n": 0.7299854439592431, - "macro_pr_auc": 0.6925223121673637, + "pr_auc": 0.6551800529689307, + "roc_auc": 0.974870803601394, + "precision_at_n": 0.7707423580786026, + "macro_pr_auc": 0.7199726852544707, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.19627287500043167, + "seconds": 0.6439404170014313, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23107,13 +25658,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.46016034585866533, - "roc_auc": 0.9745935562322855, - "precision_at_n": 0.574235807860262, - "macro_pr_auc": 0.6241088073386754, - "worst_group_fpr": 0.547945205479452, + "pr_auc": 0.3913764879820988, + "roc_auc": 0.9727182309703861, + "precision_at_n": 0.4556040756914119, + "macro_pr_auc": 0.6532797230546793, + "worst_group_fpr": 0.5684931506849316, "n_models": 1, - "seconds": 0.24542595900129527, + "seconds": 0.6842081249997136, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23124,13 +25675,13 @@ "seed": 2, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07690179233831149, - "roc_auc": 0.7192531008724352, - "precision_at_n": 0.17248908296943233, - "macro_pr_auc": 0.40360496245498223, - "worst_group_fpr": 0.5454545454545454, + "pr_auc": 0.07744738380734927, + "roc_auc": 0.7355907560160252, + "precision_at_n": 0.13755458515283842, + "macro_pr_auc": 0.4034029553432024, + "worst_group_fpr": 0.6363636363636364, "n_models": 9, - "seconds": 0.6885379580016888, + "seconds": 1.5249805839994224, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23141,13 +25692,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6430275805941272, - "roc_auc": 0.9705622269931803, - "precision_at_n": 0.7802037845705968, - "macro_pr_auc": 0.6836070915297823, + "pr_auc": 0.6184045391453056, + "roc_auc": 0.9714600941137747, + "precision_at_n": 0.7729257641921398, + "macro_pr_auc": 0.6627545279891364, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.20553808299882803, + "seconds": 0.6204587500033085, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23158,13 +25709,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.2589765025486777, - "roc_auc": 0.9598066101928685, - "precision_at_n": 0.22780203784570596, - "macro_pr_auc": 0.6008786154330514, - "worst_group_fpr": 0.576271186440678, + "pr_auc": 0.27882098766438645, + "roc_auc": 0.9643165284693337, + "precision_at_n": 0.2612809315866084, + "macro_pr_auc": 0.6168349334626622, + "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.2481297500016808, + "seconds": 0.678875915997196, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23175,13 +25726,13 @@ "seed": 3, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0637751855247328, - "roc_auc": 0.7069178597970749, - "precision_at_n": 0.0982532751091703, - "macro_pr_auc": 0.3945021474398532, - "worst_group_fpr": 0.4011299435028249, + "pr_auc": 0.06988953355746912, + "roc_auc": 0.7285389991462379, + "precision_at_n": 0.10116448326055313, + "macro_pr_auc": 0.39571710183865605, + "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.6691163750001579, + "seconds": 1.50486500000261, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23192,13 +25743,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6018090042265931, - "roc_auc": 0.9710144405962212, - "precision_at_n": 0.7489082969432315, - "macro_pr_auc": 0.630740432852108, - "worst_group_fpr": 1.0, + "pr_auc": 0.673650740437859, + "roc_auc": 0.9738444474258431, + "precision_at_n": 0.7671033478893741, + "macro_pr_auc": 0.6703092351092165, + "worst_group_fpr": 0.9794520547945206, "n_models": 1, - "seconds": 0.1993304169991461, + "seconds": 0.6531753750023199, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23209,13 +25760,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.3234510522653249, - "roc_auc": 0.9669947833378844, - "precision_at_n": 0.3158660844250364, - "macro_pr_auc": 0.6259982222975145, - "worst_group_fpr": 0.5342465753424658, + "pr_auc": 0.4333450080079327, + "roc_auc": 0.9762862420136362, + "precision_at_n": 0.5327510917030568, + "macro_pr_auc": 0.6493967498853038, + "worst_group_fpr": 0.5547945205479452, "n_models": 1, - "seconds": 0.26843716700022924, + "seconds": 0.7706693750005797, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23226,13 +25777,13 @@ "seed": 4, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08640878122573586, - "roc_auc": 0.7189830836469692, - "precision_at_n": 0.1433770014556041, - "macro_pr_auc": 0.40852179948277323, + "pr_auc": 0.08561459107619543, + "roc_auc": 0.7184035968203016, + "precision_at_n": 0.17321688500727803, + "macro_pr_auc": 0.4086851369556044, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.9446155420009745, + "seconds": 1.535262249999505, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23243,13 +25794,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7219691731333892, - "roc_auc": 0.9724569407120224, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.6724135119096966, - "worst_group_fpr": 0.952054794520548, + "pr_auc": 0.6812673546499289, + "roc_auc": 0.9723335095153985, + "precision_at_n": 0.7641921397379913, + "macro_pr_auc": 0.6779446797938639, + "worst_group_fpr": 0.958904109589041, "n_models": 1, - "seconds": 0.22371366600054898, + "seconds": 0.6306185420035035, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23260,13 +25811,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.42668957005806235, - "roc_auc": 0.9717321701469596, - "precision_at_n": 0.5203784570596798, - "macro_pr_auc": 0.6252719546383938, - "worst_group_fpr": 1.0, + "pr_auc": 0.3585514128845653, + "roc_auc": 0.9694663252655521, + "precision_at_n": 0.35807860262008734, + "macro_pr_auc": 0.6543872673773167, + "worst_group_fpr": 0.5342465753424658, "n_models": 1, - "seconds": 0.2623475000000326, + "seconds": 0.6698882499986212, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23277,13 +25828,13 @@ "seed": 5, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07996626606127319, - "roc_auc": 0.7514170973465459, - "precision_at_n": 0.11499272197962154, - "macro_pr_auc": 0.4051010495650098, + "pr_auc": 0.07531711806829501, + "roc_auc": 0.7476751305602911, + "precision_at_n": 0.10407569141193596, + "macro_pr_auc": 0.3993828703937818, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.6614609999996901, + "seconds": 1.5176797910025925, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23294,13 +25845,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.5477739185722166, - "roc_auc": 0.971442326765272, - "precision_at_n": 0.7430858806404658, - "macro_pr_auc": 0.621669004706941, + "pr_auc": 0.6220666923169502, + "roc_auc": 0.9721022800112077, + "precision_at_n": 0.7656477438136827, + "macro_pr_auc": 0.6444243052139519, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.20584945900191087, + "seconds": 0.6436851670005126, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23311,13 +25862,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.20737521034393283, - "roc_auc": 0.9453792151980602, - "precision_at_n": 0.21251819505094613, - "macro_pr_auc": 0.6019847338794133, - "worst_group_fpr": 0.8698630136986302, + "pr_auc": 0.3116587794206096, + "roc_auc": 0.9665341021450918, + "precision_at_n": 0.22634643377001457, + "macro_pr_auc": 0.6211672285158993, + "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.25065941700086114, + "seconds": 0.687488666997524, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23328,13 +25879,13 @@ "seed": 6, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08017149867796952, - "roc_auc": 0.7134330081584336, - "precision_at_n": 0.1462882096069869, - "macro_pr_auc": 0.40062922236528836, + "pr_auc": 0.07848036388509816, + "roc_auc": 0.7032876414192861, + "precision_at_n": 0.12299854439592431, + "macro_pr_auc": 0.4012301923957859, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.6690729170004488, + "seconds": 1.5172687500016764, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23345,13 +25896,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.656610287216522, - "roc_auc": 0.9702420418651903, - "precision_at_n": 0.7794759825327511, - "macro_pr_auc": 0.6754417961969063, - "worst_group_fpr": 1.0, + "pr_auc": 0.6053543556413484, + "roc_auc": 0.9713649134335658, + "precision_at_n": 0.7540029112081513, + "macro_pr_auc": 0.6766219848505614, + "worst_group_fpr": 0.958904109589041, "n_models": 1, - "seconds": 0.20678316699923016, + "seconds": 0.6216738750008517, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23362,13 +25913,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.395278085800411, - "roc_auc": 0.9732555797850024, - "precision_at_n": 0.5094614264919942, - "macro_pr_auc": 0.5920150302290529, - "worst_group_fpr": 0.5753424657534246, + "pr_auc": 0.36309104554360055, + "roc_auc": 0.9709513578631249, + "precision_at_n": 0.4243085880640466, + "macro_pr_auc": 0.5981743246147424, + "worst_group_fpr": 0.7191780821917808, "n_models": 1, - "seconds": 0.24940754199997173, + "seconds": 0.6808708329990623, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23379,13 +25930,13 @@ "seed": 7, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07631598617840837, - "roc_auc": 0.7293813595138456, - "precision_at_n": 0.09243085880640466, - "macro_pr_auc": 0.39603941873799964, + "pr_auc": 0.07690028702241986, + "roc_auc": 0.7066107408031114, + "precision_at_n": 0.10116448326055313, + "macro_pr_auc": 0.39231441855606347, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.6708718749978289, + "seconds": 1.5898154169990448, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23396,13 +25947,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6879044743725874, - "roc_auc": 0.9685583856578505, - "precision_at_n": 0.7554585152838428, - "macro_pr_auc": 0.6771777353932684, + "pr_auc": 0.6772511305344789, + "roc_auc": 0.9708570093519151, + "precision_at_n": 0.7692867540029112, + "macro_pr_auc": 0.6722284388637143, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.1932590420001361, + "seconds": 0.6161034579999978, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23413,13 +25964,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.3130157461431265, - "roc_auc": 0.9644009179710267, - "precision_at_n": 0.2685589519650655, - "macro_pr_auc": 0.6222887020900015, - "worst_group_fpr": 0.8531073446327684, + "pr_auc": 0.33804833606603873, + "roc_auc": 0.9686204524963242, + "precision_at_n": 0.3042212518195051, + "macro_pr_auc": 0.643267830351372, + "worst_group_fpr": 0.6384180790960452, "n_models": 1, - "seconds": 0.26523350000206847, + "seconds": 0.6863054170025862, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23430,13 +25981,13 @@ "seed": 8, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07639087290321192, - "roc_auc": 0.7220214353333034, - "precision_at_n": 0.12518195050946143, - "macro_pr_auc": 0.40693402893363, + "pr_auc": 0.08181791184607733, + "roc_auc": 0.722418974352357, + "precision_at_n": 0.13537117903930132, + "macro_pr_auc": 0.40322326447032014, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.6679120419976243, + "seconds": 1.491453749993525, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23447,13 +25998,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6407583233057124, - "roc_auc": 0.9725172513496863, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.7003670689921986, - "worst_group_fpr": 0.9383561643835616, + "pr_auc": 0.6632591000239041, + "roc_auc": 0.9723781224196681, + "precision_at_n": 0.7685589519650655, + "macro_pr_auc": 0.6651138433157199, + "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.19455912500052364, + "seconds": 0.6094639580041985, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23464,13 +26015,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.3364051211985619, - "roc_auc": 0.9655589621888561, - "precision_at_n": 0.31077147016011647, - "macro_pr_auc": 0.5965132042321021, - "worst_group_fpr": 0.6610169491525424, + "pr_auc": 0.35174263797174954, + "roc_auc": 0.9691429681687144, + "precision_at_n": 0.36681222707423583, + "macro_pr_auc": 0.58867333788018, + "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.24625333299991325, + "seconds": 0.6934875830047531, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23481,13 +26032,13 @@ "seed": 9, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.0760674281761912, - "roc_auc": 0.7293366061135113, - "precision_at_n": 0.13901018922852984, - "macro_pr_auc": 0.4172472929121744, + "pr_auc": 0.06856972617361667, + "roc_auc": 0.7178113356591268, + "precision_at_n": 0.12008733624454149, + "macro_pr_auc": 0.40376882066536895, "worst_group_fpr": 0.45454545454545453, "n_models": 9, - "seconds": 0.6731629589994554, + "seconds": 1.498864124994725, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23498,13 +26049,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6353575815204143, - "roc_auc": 0.9720192630479938, - "precision_at_n": 0.7561863173216885, - "macro_pr_auc": 0.650372407487826, - "worst_group_fpr": 1.0, + "pr_auc": 0.6434356874231278, + "roc_auc": 0.9722437055115158, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.6728144483691012, + "worst_group_fpr": 0.952054794520548, "n_models": 1, - "seconds": 0.20414445899950806, + "seconds": 0.6182659159967443, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23515,13 +26066,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.3480830797503195, - "roc_auc": 0.9703641329455036, - "precision_at_n": 0.38573508005822416, - "macro_pr_auc": 0.5814514554227593, - "worst_group_fpr": 0.6301369863013698, + "pr_auc": 0.31794713082169135, + "roc_auc": 0.9669186939114042, + "precision_at_n": 0.2867540029112082, + "macro_pr_auc": 0.5600754112405154, + "worst_group_fpr": 0.9322033898305084, "n_models": 1, - "seconds": 0.26166016700153705, + "seconds": 0.6796225420039264, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23532,13 +26083,13 @@ "seed": 10, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06438051583272147, - "roc_auc": 0.7163338196010209, - "precision_at_n": 0.07496360989810771, - "macro_pr_auc": 0.3859154054264402, - "worst_group_fpr": 0.45454545454545453, + "pr_auc": 0.06836061567799057, + "roc_auc": 0.7435770548830153, + "precision_at_n": 0.09097525473071325, + "macro_pr_auc": 0.38933945894079874, + "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.706516541999008, + "seconds": 1.4779979160011862, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23549,13 +26100,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6043056093954045, - "roc_auc": 0.9726922932353457, - "precision_at_n": 0.7358078602620087, - "macro_pr_auc": 0.634287625877879, - "worst_group_fpr": 0.9315068493150684, + "pr_auc": 0.6551151866652897, + "roc_auc": 0.9729514652453479, + "precision_at_n": 0.7707423580786026, + "macro_pr_auc": 0.6727633798882601, + "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.3409611249990121, + "seconds": 0.6319372079960885, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23566,13 +26117,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.38277031030813824, - "roc_auc": 0.9698135396749323, - "precision_at_n": 0.38355167394468703, - "macro_pr_auc": 0.6796146639100208, - "worst_group_fpr": 0.6027397260273972, + "pr_auc": 0.3740197046131641, + "roc_auc": 0.9709152719892499, + "precision_at_n": 0.3922852983988355, + "macro_pr_auc": 0.6510074992250879, + "worst_group_fpr": 0.636986301369863, "n_models": 1, - "seconds": 0.2751555830000143, + "seconds": 0.7073003330006031, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23583,13 +26134,13 @@ "seed": 11, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.07489059141273685, - "roc_auc": 0.7470951303826177, - "precision_at_n": 0.09970887918486172, - "macro_pr_auc": 0.4024024556919111, - "worst_group_fpr": 0.423728813559322, + "pr_auc": 0.07391365513712772, + "roc_auc": 0.7449540297956705, + "precision_at_n": 0.09461426491994178, + "macro_pr_auc": 0.3997538857745237, + "worst_group_fpr": 0.45454545454545453, "n_models": 9, - "seconds": 0.6938686250032333, + "seconds": 1.4909879580009147, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23600,13 +26151,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6275629286313821, - "roc_auc": 0.9720377058583466, - "precision_at_n": 0.7328966521106259, - "macro_pr_auc": 0.6401234203460929, + "pr_auc": 0.6602705949896037, + "roc_auc": 0.9721382416001024, + "precision_at_n": 0.7663755458515283, + "macro_pr_auc": 0.6526067810644735, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.23530379200019524, + "seconds": 0.6282843750013853, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23617,13 +26168,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.26900659319698456, - "roc_auc": 0.9645175188974232, - "precision_at_n": 0.21033478893740903, - "macro_pr_auc": 0.6039773125607502, - "worst_group_fpr": 0.5, + "pr_auc": 0.3108883708145589, + "roc_auc": 0.9681155420615929, + "precision_at_n": 0.26564774381368267, + "macro_pr_auc": 0.6079535655714939, + "worst_group_fpr": 0.6242937853107344, "n_models": 1, - "seconds": 0.27582854199863505, + "seconds": 0.6849255420020199, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23634,13 +26185,13 @@ "seed": 12, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.08429731402335623, - "roc_auc": 0.7179036199589228, - "precision_at_n": 0.1586608442503639, - "macro_pr_auc": 0.4099720782816015, + "pr_auc": 0.078883980951485, + "roc_auc": 0.7297432395509132, + "precision_at_n": 0.11644832605531295, + "macro_pr_auc": 0.4047703027308915, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.7225801250024233, + "seconds": 1.4905037909993553, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23651,13 +26202,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.7285997390796592, - "roc_auc": 0.9726331065662004, - "precision_at_n": 0.7802037845705968, - "macro_pr_auc": 0.7060445397113666, - "worst_group_fpr": 0.9315068493150684, + "pr_auc": 0.6676094467715644, + "roc_auc": 0.9726910882114053, + "precision_at_n": 0.7743813682678311, + "macro_pr_auc": 0.6760766023815652, + "worst_group_fpr": 0.952054794520548, "n_models": 1, - "seconds": 0.22078720900026383, + "seconds": 0.650708916997246, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23668,13 +26219,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.39602434244127666, - "roc_auc": 0.9715310932597532, - "precision_at_n": 0.4104803493449782, - "macro_pr_auc": 0.6008872050536501, - "worst_group_fpr": 0.9915254237288136, + "pr_auc": 0.3857369689812618, + "roc_auc": 0.9723167634652131, + "precision_at_n": 0.3937409024745269, + "macro_pr_auc": 0.609113254613314, + "worst_group_fpr": 0.7465753424657534, "n_models": 1, - "seconds": 0.27268545900005847, + "seconds": 0.6833039580014884, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23685,13 +26236,13 @@ "seed": 13, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.06710228407045235, - "roc_auc": 0.6961358945809176, - "precision_at_n": 0.10116448326055313, - "macro_pr_auc": 0.3926832918461665, + "pr_auc": 0.07273420453606227, + "roc_auc": 0.7159277373404886, + "precision_at_n": 0.11935953420669577, + "macro_pr_auc": 0.3943276800611047, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.6953132090020517, + "seconds": 1.517019207996782, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23702,13 +26253,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.6233078022796769, - "roc_auc": 0.9716151747508427, - "precision_at_n": 0.740174672489083, - "macro_pr_auc": 0.6710047505870849, - "worst_group_fpr": 0.9383561643835616, + "pr_auc": 0.60539275571879, + "roc_auc": 0.9715701403583787, + "precision_at_n": 0.7532751091703057, + "macro_pr_auc": 0.6665123742922626, + "worst_group_fpr": 0.9452054794520548, "n_models": 1, - "seconds": 0.21069662499940023, + "seconds": 0.630134041995916, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23719,13 +26270,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.3534844492103426, - "roc_auc": 0.9712877162496516, - "precision_at_n": 0.31368267831149926, - "macro_pr_auc": 0.5719889990063187, + "pr_auc": 0.30897573098961106, + "roc_auc": 0.9648347590117471, + "precision_at_n": 0.22634643377001457, + "macro_pr_auc": 0.5950681046342272, "worst_group_fpr": 1.0, "n_models": 1, - "seconds": 0.2774734589984291, + "seconds": 0.697411250002915, "eta_squared": 0.18908304175391513, "warnings": [] }, @@ -23736,13 +26287,13 @@ "seed": 14, "mechanism": "real", "level_spread": NaN, - "pr_auc": 0.09778967463280377, - "roc_auc": 0.7424727504099728, - "precision_at_n": 0.19359534206695778, - "macro_pr_auc": 0.407532612033249, + "pr_auc": 0.09274576542190417, + "roc_auc": 0.7378824791918303, + "precision_at_n": 0.17467248908296942, + "macro_pr_auc": 0.404764037047564, "worst_group_fpr": 0.5454545454545454, "n_models": 9, - "seconds": 0.8698225000007369, + "seconds": 1.5153749590026564, "eta_squared": 0.18908304175391513, "warnings": [] } diff --git a/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.md b/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.md new file mode 100644 index 000000000..7eff5b654 --- /dev/null +++ b/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.md @@ -0,0 +1,99 @@ +# Anomaly conditioning experiment — 2026-08-25 + +DQX `f9c703a1` · datasets: synthetic, smd, nslkdd, tabular · seeds per cell: 15 · cells: 1545 + +PR-AUC is the primary metric. No point-adjusted F1 appears anywhere; see `metrics.py`. +DQX scores rows independently, so figures on time-series data are not comparable with +published sequence-model results — that is a different task, not a worse implementation. + +## Does removing the heterogeneity gate cost anything? + +Rules fixed before running: **no gate** if the worst delta below eta-squared 0.1 exceeds -0.01; **gate needed** if any such cell reaches -0.02. + +- **Pre-registered rule (worst single cell): gate needed; refit the threshold from this sweep** +- **Variance-robust companion (worst per-grouping median): no gate needed** + +The pre-registered rule takes a minimum over individual cells, so it is maximally sensitive to estimator variance. It is reported unchanged, alongside the median form of the same question, so the criterion set in advance and the answer it gave are both visible. Where the two disagree, the per-grouping deltas below show why. + +| statistic | value | +|---|---| +| n_low_eta_cells | 90 | +| n_low_eta_groupings | 4 | +| worst_delta_single_cell | -0.0498 | +| n_harmful_cells | 1 | +| worst_median_delta_per_grouping | +0.0000 | +| n_harmful_groupings | 0 | +| median_delta_below_threshold | +0.0000 | + +### Does eta-squared predict the benefit at all? + +Spearman rho(eta-squared, delta) = **+0.571** (bootstrap 95% CI +0.490 to +0.645). + +Least squares `delta ~ 1 + eta_squared + is_contextual`, which asks whether eta-squared still carries signal once the anomaly mechanism is controlled for: + +| term | coefficient | +|---|---| +| intercept | -0.1015 | +| eta_squared | +0.2808 | +| is_contextual | +0.0849 | +| R-squared | 0.578 (n=390) | + +## Plain tabular benchmarks (no grouping) + +No grouping exists in these datasets, so conditioning is not applicable and only the pooled +configuration runs. This characterises the mechanism on the benchmarks the field reports, and +is the regression check that adding conditioning did not disturb the ungrouped path. + +PR-AUC is **not comparable across rows** — it moves with the base rate — so each dataset is +read against its own random floor. `lift` is DQX PR-AUC divided by the random floor. + +| dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random | beats max-abs-z | +|---|---|---|---|---|---|---|---|---| +| campaign | 30000 | 62 | 11.27% | 0.2867 | 0.1160 | 0.2398 | 2.5x | yes | +| cardio | 1831 | 21 | 9.61% | 0.5811 | 0.1012 | 0.5534 | 5.7x | yes | +| covertype | 30000 | 10 | 0.96% | 0.0534 | 0.0094 | 0.1122 | 5.7x | **no** | +| fraud | 30000 | 29 | 0.17% | 0.2240 | 0.0019 | 0.1365 | 115.4x | yes | +| mammography | 11183 | 6 | 2.32% | 0.2193 | 0.0236 | 0.1779 | 9.3x | yes | +| mnist | 7603 | 100 | 9.21% | 0.2766 | 0.0999 | 0.3367 | 2.8x | **no** | +| satellite | 6435 | 36 | 31.64% | 0.6668 | 0.3131 | 0.5946 | 2.1x | yes | +| shuttle | 30000 | 9 | 7.15% | 0.9788 | 0.0731 | 0.8983 | 13.4x | yes | +| spambase | 4207 | 57 | 39.91% | 0.4758 | 0.4033 | 0.4039 | 1.2x | yes | +| thyroid | 3772 | 6 | 2.47% | 0.5389 | 0.0251 | 0.3007 | 21.5x | yes | + +### Every grouping below eta-squared 0.1, seed by seed + +| dataset | grouping | eta-squared | median delta | min | max | seeds agree? | +|---|---|---|---|---|---|---| +| nslkdd | protocol_type | 0.0509 | +0.0396 | -0.0498 | +0.0967 | **no** | +| smd | machine_family | 0.0669 | +0.0029 | -0.0027 | +0.0103 | **no** | +| synthetic | spread=0.000 | 0.0008 | +0.0000 | -0.0027 | +0.0299 | **no** | +| synthetic | spread=0.050 | 0.0593 | +0.0000 | -0.0002 | +0.0222 | **no** | + +## Paired comparisons + +### baseline-relative minus pooled (PR-AUC) + +| mechanism | n | median delta | IQR | Wilcoxon p | +|---|---|---|---|---| +| contextual | 195 | +0.0742 | +0.0076 to +0.3070 | 1.31e-31 | +| global | 195 | +0.0000 | -0.0000 to +0.0000 | 2.26e-01 | +| real | 75 | +0.0025 | -0.1452 to +0.0124 | 1.75e-02 | +| *all* | 465 | +0.0001 | +0.0000 to +0.0431 | 1.50e-25 | + +### baseline-relative minus per-group (PR-AUC) + +| mechanism | n | median delta | IQR | Wilcoxon p | +|---|---|---|---|---| +| contextual | 195 | +0.0252 | +0.0147 to +0.0356 | 9.41e-34 | +| global | 195 | +0.0005 | +0.0000 to +0.0019 | 6.12e-29 | +| real | 75 | +0.0054 | -0.0051 to +0.2832 | 2.74e-04 | +| *all* | 465 | +0.0065 | +0.0002 to +0.0254 | 3.42e-56 | + +## Cost + +| config | median models | median seconds | +|---|---|---| +| pooled | 1 | 0.17 | +| relative | 1 | 0.17 | +| per_group | 12 | 1.38 | + diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 027317781..c5c2c6cb0 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -343,11 +343,13 @@ unit suite): The second row is why conditioning is on by default when a grouping is available: it costs nothing measurable when the anomaly was already visible. -Across a wider sweep — 1,395 configurations over synthetic data, the Server Machine Dataset and -NSL-KDD — conditioning is worth about **+0.07 PR-AUC** where anomalies are contextual, and about -**−0.001** where they are not. That asymmetry, not a hunch, is why a discovered grouping is used -rather than ignored. See [Anomaly detection quality](/docs/reference/anomaly_detection_quality) for -the full results, the datasets, and what these numbers do *not* mean. +Across a wider sweep — 1,545 configurations over synthetic data, the Server Machine Dataset, NSL-KDD +and ten classical tabular benchmarks — conditioning is worth about **+0.07 PR-AUC** where anomalies are +contextual, and **nothing measurable** where they are not. Against the previous release, on the same +tables through the real pipeline, it moves a contextual collapse from 0.0376 to 0.5703. That +asymmetry, not a hunch, is why a discovered grouping is used rather than ignored. See +[Anomaly detection quality](/docs/reference/anomaly_detection_quality) for the full results, the +datasets, and what these numbers do *not* mean. ### Baseline columns are not features diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index a55c7cad7..e173f7efc 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -79,16 +79,16 @@ model identity rather than scores. ## Results -1,395 cells: a synthetic two-factor sweep plus the Server Machine Dataset (28 entities) and NSL-KDD, -15 seeds each. +1,545 cells: a synthetic two-factor sweep, the Server Machine Dataset (28 entities), NSL-KDD, and ten +classical tabular benchmarks, 15 seeds each. ### `baseline_by` versus comparing against the whole table | anomaly mechanism | n | median ΔPR-AUC | IQR | Wilcoxon p | |---|---|---|---|---| -| contextual (ordinary globally, wrong for its group) | 195 | **+0.0734** | +0.0068 to +0.2649 | 5.3e-32 | -| global (extreme for the whole table) | 195 | +0.0000 | −0.0000 to +0.0000 | 2.0e-01 | -| real datasets (mechanism unknown) | 75 | −0.0010 | −0.1622 to +0.0110 | 3.1e-03 | +| contextual (ordinary globally, wrong for its group) | 195 | **+0.0742** | +0.0076 to +0.3070 | 1.3e-31 | +| global (extreme for the whole table) | 195 | +0.0000 | −0.0000 to +0.0000 | 2.3e-01 | +| real datasets (mechanism unknown) | 75 | +0.0025 | −0.1452 to +0.0124 | 1.8e-02 | Read these three rows together, because they are the whole argument: @@ -101,31 +101,39 @@ Read these three rows together, because they are the whole argument: the median difference is zero and the test does not reject (p = 0.20). At low heterogeneity every group median approaches the global median, so the relative feature degenerates into a monotone transform of the raw metric: a near-duplicate of an informative column rather than noise. -- **On real datasets, conditioning is marginally worse, and detectably so.** Median −0.0010 at - p = 0.003. The magnitude is negligible; the sign is real. SMD and NSL-KDD anomalies are largely - globally extreme or sequence-dependent rather than contextual with respect to a categorical group, - so the extra feature dilutes slightly without adding signal. - -The practical reading: `baseline_by` buys roughly +0.07 PR-AUC where it applies and costs roughly -0.001 where it does not. That asymmetry is why auto-discovery enables it rather than requiring you to -opt in. +- **On real datasets, conditioning is marginally better.** Median +0.0025 at p = 0.018, with a wide + spread either side. SMD and NSL-KDD anomalies are largely globally extreme or sequence-dependent + rather than contextual with respect to a categorical group, so there is little for conditioning to + find — and it does not get in the way. + + An earlier revision of this page reported **−0.0010 at p = 0.003** here and described conditioning as + marginally *worse* on real data. That was an artefact of the harness: it fitted 100 trees where DQX + ships 200, which understated the forest on exactly the datasets where the forest does the work. + Correcting the harness moved this row from marginally negative to marginally positive. The + contextual and global rows barely moved, because a paired comparison holds the configuration + constant on both sides — it was the real-data row, where the two mechanisms are closest, that the + fidelity gap could flip. + +The practical reading: `baseline_by` buys roughly +0.07 PR-AUC where it applies and costs nothing +measurable where it does not. That asymmetry is why auto-discovery enables it rather than requiring +you to opt in. ### `baseline_by` versus one model per group | anomaly mechanism | n | median ΔPR-AUC | IQR | Wilcoxon p | |---|---|---|---|---| -| contextual | 195 | **+0.0274** | +0.0128 to +0.0413 | 1.2e-33 | -| global | 195 | +0.0006 | +0.0000 to +0.0017 | 3.4e-26 | -| real datasets | 75 | **+0.0123** | −0.0043 to +0.2785 | 3.2e-05 | +| contextual | 195 | **+0.0252** | +0.0147 to +0.0356 | 9.4e-34 | +| global | 195 | +0.0005 | +0.0000 to +0.0019 | 6.1e-29 | +| real datasets | 75 | **+0.0054** | −0.0051 to +0.2832 | 2.7e-04 | One conditioned model beats one model per group in **every** mechanism, including on real data, while training a single model instead of one per group: | configuration | median models trained | median fit seconds | |---|---|---| -| pooled | 1 | 0.09 | -| relative | 1 | 0.09 | -| per-group | 12 | 0.79 | +| pooled | 1 | 0.17 | +| relative | 1 | 0.17 | +| per-group | 12 | 1.38 | Per-group models also fail in a particular way that averages hide. On the Server Machine Dataset one entity produced 15,963 false positives across 28,392 normal rows — a 56% false-alarm rate — while the @@ -134,6 +142,58 @@ own rows, so a group containing nothing unusual still has its most-unusual few p extreme. This is why DQX reports the **worst** group's false-positive rate rather than the average, and why `segment_by` is retained for compatibility rather than recommended. +## Plain tabular benchmarks + +Ten classical benchmarks with no grouping at all, from [ADBench](https://github.com/Minqi824/ADBench) +(BSD-2-Clause), which redistributes the ODDS / UCI / Kaggle collections. Conditioning does not apply +here — these characterise the ungrouped path, which this work does not change, and act as its +regression baseline. + +PR-AUC **moves with the base rate**, so these rows are not comparable with each other. Each is read +against its own random floor, and against `max-abs-z` — the largest absolute z-score across features, +which is the cheapest defensible detector and a genuinely competitive one. + +| dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random | beats max-abs-z | +|---|---|---|---|---|---|---|---|---| +| shuttle | 30000 | 9 | 7.15% | **0.9788** | 0.0731 | 0.8983 | 13.4x | yes | +| satellite | 6435 | 36 | 31.64% | 0.6668 | 0.3131 | 0.5946 | 2.1x | yes | +| cardio | 1831 | 21 | 9.61% | 0.5811 | 0.1012 | 0.5534 | 5.7x | yes | +| thyroid | 3772 | 6 | 2.47% | 0.5389 | 0.0251 | 0.3007 | 21.5x | yes | +| spambase | 4207 | 57 | 39.91% | 0.4758 | 0.4033 | 0.4039 | 1.2x | yes | +| campaign | 30000 | 62 | 11.27% | 0.2867 | 0.1160 | 0.2398 | 2.5x | yes | +| mnist | 7603 | 100 | 9.21% | 0.2766 | 0.0999 | 0.3367 | 2.8x | **no** | +| fraud | 30000 | 29 | 0.17% | 0.2240 | 0.0019 | 0.1365 | **115.4x** | yes | +| mammography | 11183 | 6 | 2.32% | 0.2193 | 0.0236 | 0.1779 | 9.3x | yes | +| covertype | 30000 | 10 | 0.96% | 0.0534 | 0.0094 | 0.1122 | 5.7x | **no** | + +DQX beats the random floor on all ten and `max-abs-z` on eight. `fraud` — the Kaggle credit-card set, +at a 0.17% base rate — is the standout at 115x the floor. + +### Where it underperforms, and why + +Two rows say **no**, and they say it for the same reason. + +- **`covertype`** (10 features, 0.96% base rate): 0.0534 against a z-score's 0.1122, so roughly half. +- **`mnist`** (100 features): 0.2766 against 0.3367. + +This is Isolation Forest's inductive bias, not a tuning gap. It splits on one randomly chosen feature +at a time, so when an anomaly is a single extreme value among few dimensions the signal is diluted +across the other axes, and in 100 dimensions the random choice rarely lands on the informative one. A +global max-abs-z captures "any feature is extreme" directly. + +Tuning does not recover it, and the obvious knob makes things worse elsewhere: raising `max_samples` +from 256 to 4096 lifts `fraud` (0.1926 → 0.2924) while dropping `shuttle` (0.9789 → 0.8546) and +`cardio` (0.5766 → 0.4835). sklearn's default is well chosen — subsampling is *why* Isolation Forest +works. + +Adding a complementary z-score detector to the ensemble was measured and **rejected**: it wins on 5 of +10 datasets with a median of −0.0028 and a −0.1409 worst case. And it cannot be fixed by choosing per +dataset, because choosing needs labels and DQX is unsupervised. See +`benchmarks/anomaly_conditioning/complementary_detector.py` for the full result. + +One caveat on `spambase`: a 39.9% base rate is not anomaly detection, it is classification with an +unusual framing, and its 1.2x lift should be read that way. + ## Why there is no heterogeneity threshold DQX briefly gated conditioning on eta-squared — the share of variance the grouping explains — skipping @@ -153,10 +213,10 @@ The seed-by-seed breakdown shows why: | dataset | grouping | eta-squared | median Δ | min | max | seeds agree? | |---|---|---|---|---|---|---| -| NSL-KDD | `protocol_type` | 0.0509 | +0.0285 | −0.0744 | +0.1003 | no | -| SMD | machine family | 0.0669 | +0.0033 | −0.0047 | +0.0125 | no | -| synthetic | spread 0.000 | 0.0008 | +0.0000 | −0.0019 | +0.0283 | no | -| synthetic | spread 0.050 | 0.0593 | +0.0000 | −0.0019 | +0.0169 | no | +| NSL-KDD | `protocol_type` | 0.0509 | +0.0396 | −0.0498 | +0.0967 | no | +| SMD | machine family | 0.0669 | +0.0029 | −0.0027 | +0.0103 | no | +| synthetic | spread 0.000 | 0.0008 | +0.0000 | −0.0027 | +0.0299 | no | +| synthetic | spread 0.050 | 0.0593 | +0.0000 | −0.0002 | +0.0222 | no | Every low-heterogeneity grouping straddles zero. The pre-registered rule takes a minimum over individual cells, which makes it maximally sensitive to estimator variance, and it fired on single From 7375f9b748262cb793420cb0527492e12187a02f Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 20:11:44 +0100 Subject: [PATCH 024/107] Write the quality page for users, not for the team The page had accumulated material that existed to convince us the work was an improvement, not to help anyone use the feature: a build-versus-build comparison against v0.16.0, the story of getting that measurement wrong the first time, a note about which harness revision reported which number, and the design rationale for a heterogeneity gate users never saw. That belongs in the changelog, the dev docs and the PR, and it is already in those places. 265 lines down to 151. What a reader actually needs is now the shape of the page: whether anomaly detection suits their data, whether to give it a grouping, and what it is bad at. The conditioning tables are reframed from "mechanism A versus mechanism B" to "your anomalies are like this, so do that", and the underperformance section ends with what to do instead -- a range check or an outlier rule serves single-column extremes better, and both can run together. Kept, because they are the reader's protection rather than our defence: rows are scored independently so time-series comparisons are a category error, PR-AUC moves with the base rate so the columns are not comparable, no point-adjusted F1, and every figure measures separability rather than generalisation. --- .../reference/anomaly_detection_quality.mdx | 294 ++++++------------ 1 file changed, 90 insertions(+), 204 deletions(-) diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index e173f7efc..bf1ecab4b 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -8,150 +8,78 @@ sidebar_position: 509 # Anomaly Detection Quality -How well does row anomaly detection actually detect? This page reports measured detection quality -rather than timing — [Benchmarks](/docs/reference/benchmarks) covers performance. +How well does row anomaly detection actually detect? This page reports measured detection quality; +[Benchmarks](/docs/reference/benchmarks) covers timing. -The numbers come from `benchmarks/anomaly_conditioning/` in the DQX repository, which is run manually -rather than nightly: it downloads third-party datasets and produces a correlation rather than a -pass/fail. Its README documents how to reproduce everything below. +Use it to decide two things: whether anomaly detection suits your data at all, and whether to give it +a grouping via `baseline_by`. ## Read this first -**DQX scores rows independently.** It is not a sequence model, and it does not consume a window of -history to make a prediction. Published results on time-series anomaly benchmarks routinely exceed -0.80 PR-AUC using models that do. A DQX figure of 0.15 on the same data is **a different task, not a -worse implementation**. Comparing the two directly is a category error. +**DQX scores rows independently.** It is not a sequence model and does not consume a window of history +to make a prediction. Published results on time-series anomaly benchmarks routinely exceed 0.80 PR-AUC +using models that do. A DQX figure of 0.15 on the same data is **a different task, not a worse +implementation**. -**PR-AUC is the primary metric.** These datasets are heavily imbalanced, and ROC-AUC flatters a -detector that merely ranks the majority class well. +**PR-AUC moves with the base rate.** A score of 0.22 at a 0.17% anomaly rate is strong; 0.48 at 40% is +weak. Never compare PR-AUC across datasets — compare each against its own floor, which is what the +tables below do. -**No point-adjusted F1 appears anywhere.** Under the point-adjust protocol — crediting an entire -labelled anomaly segment when any single point inside it is detected — a *random* anomaly score -achieves state-of-the-art F1 ([Kim et al., AAAI 2022](https://arxiv.org/abs/2109.05257)). Numbers -produced that way are uninterpretable, so DQX does not compute them. +**No point-adjusted F1 appears anywhere.** Under that protocol — crediting a whole labelled anomaly +segment when any single point inside it is detected — a *random* score achieves state-of-the-art F1 +([Kim et al., AAAI 2022](https://arxiv.org/abs/2109.05257)). DQX does not compute it. -## What was compared +## When conditioning helps -Three ways of relating a model to a group, each fitted and scored on identical rows: +`baseline_by` judges each metric against its own group's baseline rather than against the whole table. +Whether that helps depends on what makes your anomalies anomalous. -| configuration | what it does | DQX equivalent | +| your anomalies are... | median ΔPR-AUC with `baseline_by` | what to do | |---|---|---| -| pooled | one model over the raw metrics; no notion of a group | no `baseline_by` | -| relative | one model over the raw metrics **plus** each metric's deviation from its own group's baseline | `baseline_by` | -| per-group | one model per group, trained only on that group's rows | the legacy `segment_by` | - -Comparisons are **paired by seed** and tested with Wilcoxon signed-rank. The seed drives both the -data draw and the forest, and dominates the variance between configurations, so unpaired means would -largely measure the seed. - -## Against the previous release - -The comparison that matters most: the released v0.16.0 build and this one, run through the **real DQX -pipeline** — Spark, MLflow, Unity Catalog — over the same Delta tables, so neither side can differ by -anything except the code. - -| scenario | how it was called | v0.16.0 | current | change | -|---|---|---|---|---| -| contextual anomaly | zero config | 0.1885 | **0.5703** | **+0.3818** | -| contextual anomaly | explicit `columns` | 0.0376 | **0.5703** | **+0.5327** | -| globally extreme anomaly | zero config | 0.3743 | **1.0000** | **+0.6257** | -| globally extreme anomaly | explicit `columns` | 1.0000 | 1.0000 | +0.0000 | - -Better on three, identical on the fourth, worse on none. Two rows are worth reading closely: - -- **`contextual` / explicit columns, 0.0376 → 0.5703.** v0.16.0 has no conditioning mechanism at all, - so this is the cleanest statement of what conditioning buys: a fifteenfold improvement on an anomaly - that is invisible to a whole-table comparison. -- **`global` / explicit columns, both exactly 1.0000.** This path is untouched by the change, and the - two builds agree exactly. That is the regression check: nothing was traded away to get the rows - above. - -Getting this measurement right took two attempts, and the reason is worth recording because it applies -to anyone repeating it. The first run showed the *new* build losing on three of four cells. It was -measuring the sampler: training samples 30% of rows by default via `.sample`, which under Spark -Connect depends on partition ordering, so the two builds trained on different rows. With -`sample_fraction=1.0` every cell is reproducible to four decimal places across repeats, and the -picture inverts. **A single-point comparison of this pipeline is not evidence.** - -Both `RobustScaler` removal and the configuration-hash change landed after these figures were -measured. Neither can move them: the scaler is provably a no-op (see below), and the hash affects -model identity rather than scores. - -## Results - -1,545 cells: a synthetic two-factor sweep, the Server Machine Dataset (28 entities), NSL-KDD, and ten -classical tabular benchmarks, 15 seeds each. - -### `baseline_by` versus comparing against the whole table - -| anomaly mechanism | n | median ΔPR-AUC | IQR | Wilcoxon p | -|---|---|---|---|---| -| contextual (ordinary globally, wrong for its group) | 195 | **+0.0742** | +0.0076 to +0.3070 | 1.3e-31 | -| global (extreme for the whole table) | 195 | +0.0000 | −0.0000 to +0.0000 | 2.3e-01 | -| real datasets (mechanism unknown) | 75 | +0.0025 | −0.1452 to +0.0124 | 1.8e-02 | - -Read these three rows together, because they are the whole argument: - -- **When anomalies are contextual, conditioning is transformative.** This is the case - [#1484](https://github.com/databrickslabs/dqx/issues/1484) exists for: a group whose volume - collapses while the daily total stays flat is invisible to any whole-table comparison. Measured - offline on such a collapse, a pooled model scored PR-AUC 0.0028 against a 0.0026 base rate — that - is chance — while conditioning reached 0.6962. -- **When anomalies are globally extreme, conditioning costs nothing.** Not "costs little" — - the median difference is zero and the test does not reject (p = 0.20). At low heterogeneity every - group median approaches the global median, so the relative feature degenerates into a monotone - transform of the raw metric: a near-duplicate of an informative column rather than noise. -- **On real datasets, conditioning is marginally better.** Median +0.0025 at p = 0.018, with a wide - spread either side. SMD and NSL-KDD anomalies are largely globally extreme or sequence-dependent - rather than contextual with respect to a categorical group, so there is little for conditioning to - find — and it does not get in the way. - - An earlier revision of this page reported **−0.0010 at p = 0.003** here and described conditioning as - marginally *worse* on real data. That was an artefact of the harness: it fitted 100 trees where DQX - ships 200, which understated the forest on exactly the datasets where the forest does the work. - Correcting the harness moved this row from marginally negative to marginally positive. The - contextual and global rows barely moved, because a paired comparison holds the configuration - constant on both sides — it was the real-data row, where the two mechanisms are closest, that the - fidelity gap could flip. - -The practical reading: `baseline_by` buys roughly +0.07 PR-AUC where it applies and costs nothing -measurable where it does not. That asymmetry is why auto-discovery enables it rather than requiring -you to opt in. - -### `baseline_by` versus one model per group - -| anomaly mechanism | n | median ΔPR-AUC | IQR | Wilcoxon p | -|---|---|---|---|---| -| contextual | 195 | **+0.0252** | +0.0147 to +0.0356 | 9.4e-34 | -| global | 195 | +0.0005 | +0.0000 to +0.0019 | 6.1e-29 | -| real datasets | 75 | **+0.0054** | −0.0051 to +0.2832 | 2.7e-04 | - -One conditioned model beats one model per group in **every** mechanism, including on real data, while -training a single model instead of one per group: - -| configuration | median models trained | median fit seconds | +| **contextual** — ordinary for the table, wrong for their group | **+0.0742** | use `baseline_by` | +| **globally extreme** — unusual against the whole table | +0.0000 | costs nothing; leave it on | +| mixed or unknown (real-world datasets) | +0.0025 | leave it on | + +Read that as one asymmetry rather than three results: conditioning is worth a great deal where it +applies and nothing measurable where it does not. That is why DQX enables it automatically when it +finds a usable grouping, rather than asking you to opt in. + +The contextual case deserves a concrete example, because whole-table models and rules both miss it. One +group's daily volume collapses by 80% while the total across all groups stays flat. Measured on exactly +that shape, a model comparing against the whole table scored PR-AUC 0.0028 against a 0.0026 base rate — +chance. Conditioned on the group, 0.6962. + +To compare against the whole table anyway, pass `baseline_by=[]`. + +## One model, not one per group + +The legacy `segment_by` trains a separate model per group. Conditioning expresses the grouping as +features on a single model instead, and wins on both axes. + +| your anomalies are... | median ΔPR-AUC vs one-model-per-group | +|---|---| +| contextual | **+0.0252** | +| globally extreme | +0.0005 | +| mixed or unknown | +0.0054 | + +| approach | models trained | median fit time | |---|---|---| -| pooled | 1 | 0.17 | -| relative | 1 | 0.17 | -| per-group | 12 | 1.38 | +| `baseline_by` | 1 | 0.17s | +| `segment_by` | one per group (12 in this sweep) | 1.38s | -Per-group models also fail in a particular way that averages hide. On the Server Machine Dataset one -entity produced 15,963 false positives across 28,392 normal rows — a 56% false-alarm rate — while the -aggregate metric looked merely mediocre. Each per-group model calibrates its own contamination on its -own rows, so a group containing nothing unusual still has its most-unusual few percent scored as -extreme. This is why DQX reports the **worst** group's false-positive rate rather than the average, -and why `segment_by` is retained for compatibility rather than recommended. +Per-group models also fail in a way that averages conceal. On the Server Machine Dataset one entity +produced 15,963 false positives across 28,392 normal rows — a 56% false-alarm rate — while the +aggregate metric looked merely mediocre. Each per-group model calibrates its threshold on its own rows, +so a group containing nothing unusual still has its most-unusual few percent flagged. ## Plain tabular benchmarks -Ten classical benchmarks with no grouping at all, from [ADBench](https://github.com/Minqi824/ADBench) -(BSD-2-Clause), which redistributes the ODDS / UCI / Kaggle collections. Conditioning does not apply -here — these characterise the ungrouped path, which this work does not change, and act as its -regression baseline. +Ten classical benchmarks with no grouping, from [ADBench](https://github.com/Minqi824/ADBench) +(BSD-2-Clause), which redistributes the ODDS / UCI / Kaggle collections. These characterise detection +without any conditioning. -PR-AUC **moves with the base rate**, so these rows are not comparable with each other. Each is read -against its own random floor, and against `max-abs-z` — the largest absolute z-score across features, -which is the cheapest defensible detector and a genuinely competitive one. +Each row is read against its own random floor, and against `max-abs-z` — the largest absolute z-score +across features, which is the cheapest defensible detector and a genuinely competitive one. | dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random | beats max-abs-z | |---|---|---|---|---|---|---|---|---| @@ -166,70 +94,35 @@ which is the cheapest defensible detector and a genuinely competitive one. | mammography | 11183 | 6 | 2.32% | 0.2193 | 0.0236 | 0.1779 | 9.3x | yes | | covertype | 30000 | 10 | 0.96% | 0.0534 | 0.0094 | 0.1122 | 5.7x | **no** | -DQX beats the random floor on all ten and `max-abs-z` on eight. `fraud` — the Kaggle credit-card set, -at a 0.17% base rate — is the standout at 115x the floor. - -### Where it underperforms, and why +DQX beats the random floor on all ten and `max-abs-z` on eight. `fraud` — the Kaggle credit-card set at +a 0.17% base rate — is the standout at 115x the floor. -Two rows say **no**, and they say it for the same reason. +One row to read carefully rather than celebrate: `spambase` at a 39.9% base rate is not really anomaly +detection, and its 1.2x lift should be read that way. -- **`covertype`** (10 features, 0.96% base rate): 0.0534 against a z-score's 0.1122, so roughly half. -- **`mnist`** (100 features): 0.2766 against 0.3367. +### Where it underperforms -This is Isolation Forest's inductive bias, not a tuning gap. It splits on one randomly chosen feature -at a time, so when an anomaly is a single extreme value among few dimensions the signal is diluted -across the other axes, and in 100 dimensions the random choice rarely lands on the informative one. A -global max-abs-z captures "any feature is extreme" directly. +Two datasets say **no**, for the same reason. -Tuning does not recover it, and the obvious knob makes things worse elsewhere: raising `max_samples` -from 256 to 4096 lifts `fraud` (0.1926 → 0.2924) while dropping `shuttle` (0.9789 → 0.8546) and -`cardio` (0.5766 → 0.4835). sklearn's default is well chosen — subsampling is *why* Isolation Forest -works. - -Adding a complementary z-score detector to the ensemble was measured and **rejected**: it wins on 5 of -10 datasets with a median of −0.0028 and a −0.1409 worst case. And it cannot be fixed by choosing per -dataset, because choosing needs labels and DQX is unsupervised. See -`benchmarks/anomaly_conditioning/complementary_detector.py` for the full result. - -One caveat on `spambase`: a 39.9% base rate is not anomaly detection, it is classification with an -unusual framing, and its 1.2x lift should be read that way. - -## Why there is no heterogeneity threshold - -DQX briefly gated conditioning on eta-squared — the share of variance the grouping explains — skipping -it below 0.10, on the theory that a grouping which explains little contributes noise. +| dataset | shape | DQX | max-abs-z | +|---|---|---|---| +| `covertype` | 10 features, 0.96% base rate | 0.0534 | 0.1122 | +| `mnist` | 100 features | 0.2766 | 0.3367 | -The decision rules were fixed before the experiment ran: **no gate** if the worst delta below -eta-squared 0.10 stayed above −0.01; **gate needed** if any such cell reached −0.02. +Isolation Forest splits on one randomly chosen feature at a time. When an anomaly is a single extreme +value among few dimensions the signal is diluted across the other axes, and in 100 dimensions the +random choice rarely lands on the informative one. A global max-abs-z captures "any feature is extreme" +directly. -The two forms of that criterion disagreed, and both are published: +**What this means for you.** If your anomalies are single extreme values in a handful of numeric +columns, a range check or an outlier rule will serve you better — and you can run both. Anomaly +detection earns its place on unusual *combinations* across columns, which is what rules struggle to +express. -| criterion | verdict | -|---|---| -| pre-registered (worst single cell) | gate needed — 3 harmful cells of 90 | -| variance-robust (worst per-grouping median) | **no gate needed** — 0 harmful groupings | - -The seed-by-seed breakdown shows why: - -| dataset | grouping | eta-squared | median Δ | min | max | seeds agree? | -|---|---|---|---|---|---|---| -| NSL-KDD | `protocol_type` | 0.0509 | +0.0396 | −0.0498 | +0.0967 | no | -| SMD | machine family | 0.0669 | +0.0029 | −0.0027 | +0.0103 | no | -| synthetic | spread 0.000 | 0.0008 | +0.0000 | −0.0027 | +0.0299 | no | -| synthetic | spread 0.050 | 0.0593 | +0.0000 | −0.0002 | +0.0222 | no | - -Every low-heterogeneity grouping straddles zero. The pre-registered rule takes a minimum over -individual cells, which makes it maximally sensitive to estimator variance, and it fired on single -unlucky seeds of groupings whose medians are positive. Raising the seed count fivefold moved no -grouping's median below zero. **No grouping is systematically harmed at low heterogeneity**, so the -gate has nothing to gate on and was removed. - -Eta-squared is not useless — it correlates with the *size* of the benefit (Spearman ρ = +0.597, -bootstrap 95% CI +0.523 to +0.668, and a coefficient of +0.276 in -`Δ ~ eta_squared + is_contextual`, so it carries signal even after controlling for the anomaly -mechanism). But it predicts **how much you gain, never whether you lose**, and a gate needs to -identify harm. Computing it cost a full Spark aggregation per training run to answer a question that -never changes the decision. +Tuning does not close this gap, and the obvious knob makes matters worse elsewhere: raising the rows +sampled per tree lifts `fraud` while dropping `shuttle` and `cardio` substantially. Adding a +complementary z-score detector to the ensemble was measured and rejected — it wins on half the datasets +and loses badly on the others. ## Datasets @@ -239,27 +132,20 @@ Downloaded at run time and cached; DQX redistributes none of them. |---|---|---|---| | Server Machine Dataset | MIT, via [`NetManAIOps/OmniAnomaly`](https://github.com/NetManAIOps/OmniAnomaly) | server entity (28), machine family (3) | Su et al., *Robust Anomaly Detection for Multivariate Time Series through Stochastic Recurrent Neural Networks*, KDD 2019 | | NSL-KDD | redistributable with citation | `service` (65), `flag` (11), `protocol_type` (3) | Tavallaee et al., *A detailed analysis of the KDD CUP 99 data set*, CISDA 2009 | -| synthetic | n/a | 13 heterogeneity levels × 2 anomaly mechanisms | `benchmarks/anomaly_conditioning/datasets/synthetic.py` | +| ADBench (10 tabular sets) | BSD-2-Clause | none — ungrouped | Han et al., *ADBench: Anomaly Detection Benchmark*, NeurIPS 2022 | +| synthetic | n/a | 13 heterogeneity levels x 2 anomaly mechanisms | generated by the harness | SMAP and MSL are excluded: their data files carry "© Original Authors" with no permissive licence. -ADBench is not used because it ships pre-processed numeric matrices, so categorical column identity — -the grouping this experiment measures — is gone. - -SMD is capped at 4,000 rows per entity so a run takes minutes rather than hours, and NSL-KDD's attacks -are downsampled from roughly 46% to 2% so the task is anomaly detection rather than classification. -Every configuration sees identical rows, so neither affects the comparison. ## Caveats -- The harness reimplements the baseline-relative transform in numpy rather than calling DQX, so a - sweep of a few thousand fits needs no Spark session. It therefore measures the **mechanism**, not - DQX's implementation of it. Pipeline fidelity is asserted separately in - `tests/integration_anomaly/test_anomaly_quality.py`, which runs the real Spark and MLflow path, and - the release comparison above runs the pipeline end to end. -- The harness fits the same estimator configuration DQX ships (200 trees, contamination from - `expected_anomaly_rate`). An earlier version used 100 trees, which understated the product; paired - comparisons were unaffected, since the configuration is held constant on both sides, but absolute - per-dataset figures moved. -- Each configuration is fitted and scored on the same rows. This measures separability, not - generalisation to unseen data. -- Results are indicative. Always benchmark against your own data and environment. +- **1,545 measurements**, 15 seeds per configuration, compared pairwise by seed and tested with + Wilcoxon signed-rank. Medians are reported, not means. +- Each configuration is fitted and scored on the same rows, so these figures measure separability + rather than generalisation to unseen data. +- SMD is capped at 4,000 rows per entity, and NSL-KDD's attacks are downsampled to a 2% rate so the + task is anomaly detection rather than classification. Every configuration sees identical rows. +- Results are indicative. Benchmark against your own data before relying on a number here. + +Everything above is reproducible from `benchmarks/anomaly_conditioning/` in the DQX repository, which +documents how to run it. From 8df14ab627559d0c78b7bb0c57047143b1de7750 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 20:18:37 +0100 Subject: [PATCH 025/107] Refresh the quality benchmarks in the nightly The detection-quality numbers sat in a page nobody regenerated, so they would go stale the first time anyone changed the model. They now refresh with the timing benchmarks and ride the same PR. Not routed through pytest-benchmark, for a specific reason. The nightly merges baseline.json with if ($old[.] != null) then $old[.] else $new[.] end so an entry that already exists keeps its previous value entirely -- which is why no existing benchmark mean has changed across the last four baseline PRs. Quality published through extra_info would be frozen at first observation and republished as fresh for ever. emit_docs.py writes the numbers into the page directly instead. Generated tables sit between HTML comment markers, so regeneration replaces the numbers and leaves the prose. That is what makes this page different from benchmarks.mdx, which generate_md_report.py rewrites wholesale and where hand-written guidance therefore cannot survive -- the reason the two are separate pages rather than one. Scoped for CI at 5 seeds rather than 15, and marked continue-on-error: the real datasets are ~250MB fetched from GitHub on a cold cache, and a third-party download failing should not fail the timing benchmarks that already ran. The trade to be aware of is that the nightly now depends on an external host it did not before. emit_docs warns rather than stays quiet when handed a results file with no tabular data, since a stale table that looks current is worse than an obvious gap. --- .github/workflows/nightly.yml | 23 ++- benchmarks/anomaly_conditioning/emit_docs.py | 174 ++++++++++++++++++ .../anomaly_conditioning/run_experiment.py | 3 + .../reference/anomaly_detection_quality.mdx | 8 + 4 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 benchmarks/anomaly_conditioning/emit_docs.py diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 16c0b5c47..e639e810e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -555,6 +555,23 @@ jobs: ${{ env.NEW_BASELINE }} ${{ env.UPDATED_BASELINE }} + - name: Refresh anomaly detection quality benchmarks + timeout-minutes: 90 + continue-on-error: true + run: | + # Detection quality, not timing, so it does not go through pytest-benchmark. The baseline + # merge below is keep-old-on-conflict, which would freeze an extra_info payload at its first + # observation and publish a fossil for ever; this writes the numbers straight into the page + # instead, between markers, so the hand-written guidance around them survives. + # + # Scoped for CI: 5 seeds rather than 15, and the real datasets are ~250MB fetched from + # GitHub on a cold cache. continue-on-error because a third-party download failing should + # not fail the timing benchmarks that ran before it. + UV_FROZEN=1 uv run --all-extras python benchmarks/anomaly_conditioning/run_experiment.py \ + --seeds 5 --datasets synthetic smd nslkdd tabular + LATEST=$(ls -t benchmarks/anomaly_conditioning/results/*.json | head -1) + UV_FROZEN=1 uv run --all-extras python benchmarks/anomaly_conditioning/emit_docs.py "$LATEST" + - name: Create PR with updated baseline if changed env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -565,6 +582,8 @@ jobs: # Stage baseline and report git add $FINAL_BASELINE || true git add $BENCHMARK_REPORT || true + git add docs/dqx/docs/reference/anomaly_detection_quality.mdx || true + git add benchmarks/anomaly_conditioning/results || true # Check if there are actual changes if git diff --cached --quiet; then @@ -585,11 +604,13 @@ jobs: EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --state open --json number --jq '.[0].number') if [ -z "$EXISTING_PR" ]; then gh pr create \ - --title "Update performance benchmark baseline" \ + --title "Update performance and detection quality benchmarks" \ --body "$(cat <<'EOF' ## Summary - Updated `tests/perf/.benchmarks/baseline.json` with latest nightly benchmark results - Regenerated `docs/dqx/docs/reference/benchmarks.mdx` report + - Refreshed the generated tables in `docs/dqx/docs/reference/anomaly_detection_quality.mdx` + and added a dated results file under `benchmarks/anomaly_conditioning/results/` ## Action required This commit is **not GPG-signed** (created by GitHub Actions). diff --git a/benchmarks/anomaly_conditioning/emit_docs.py b/benchmarks/anomaly_conditioning/emit_docs.py new file mode 100644 index 000000000..5b61164d4 --- /dev/null +++ b/benchmarks/anomaly_conditioning/emit_docs.py @@ -0,0 +1,174 @@ +"""Refresh the generated tables in the user-facing quality page from a results JSON. + + python benchmarks/anomaly_conditioning/emit_docs.py results/2026-08-25-abc1234.json + +Only the regions between marker comments are replaced, so the hand-written prose around them -- +which is most of the page, and the part that tells a reader what to do about a number -- survives +regeneration. That is the difference from ``docs/dqx/docs/reference/benchmarks.mdx``, which +``tests/perf/generate_md_report.py`` rewrites wholesale and where narrative therefore cannot live. + +Deliberately not routed through pytest-benchmark's ``extra_info``. The nightly merges baseline.json +with keep-old-on-conflict semantics, so a benchmark that already exists keeps its previous entry and +its extra_info is frozen at first observation. Quality published that way would never refresh. +""" + +import json +import pathlib +import sys + +MARKERS = { + "conditioning": ("", ""), + "per_group": ("", ""), + "tabular": ("", ""), + "cost": ("", ""), +} + +PAGE = ( + pathlib.Path(__file__).resolve().parents[2] + / "docs" + / "dqx" + / "docs" + / "reference" + / "anomaly_detection_quality.mdx" +) + +MECHANISM_LABELS = { + "contextual": "**contextual** — ordinary for the table, wrong for their group", + "global": "**globally extreme** — unusual against the whole table", + "real": "mixed or unknown (real-world datasets)", +} +ADVICE = { + "contextual": "use `baseline_by`", + "global": "costs nothing; leave it on", + "real": "leave it on", +} + + +def _median(values: list[float]) -> float: + ordered = sorted(values) + if not ordered: + return float("nan") + mid = len(ordered) // 2 + return ordered[mid] if len(ordered) % 2 else (ordered[mid - 1] + ordered[mid]) / 2 + + +def _paired_medians(cells: list[dict], left: str, right: str) -> dict[str, float]: + """Median per-mechanism delta between two configurations, paired on the cell they share.""" + index: dict[tuple, dict[str, dict]] = {} + for cell in cells: + key = (cell["dataset"], cell["grouping"], cell["mechanism"], cell["seed"]) + index.setdefault(key, {})[cell["config"]] = cell + + by_mechanism: dict[str, list[float]] = {} + for (_dataset, _grouping, mechanism, _seed), configs in index.items(): + if left not in configs or right not in configs: + continue + delta = configs[left]["pr_auc"] - configs[right]["pr_auc"] + if delta == delta: # skip NaN + by_mechanism.setdefault(mechanism, []).append(delta) + return {m: _median(v) for m, v in by_mechanism.items()} + + +def conditioning_table(cells: list[dict]) -> str: + medians = _paired_medians(cells, "relative", "pooled") + rows = [ + "| your anomalies are... | median ΔPR-AUC with `baseline_by` | what to do |", + "|---|---|---|", + ] + for mechanism in ("contextual", "global", "real"): + if mechanism not in medians: + continue + value = medians[mechanism] + emphasis = f"**{value:+.4f}**" if value > 0.01 else f"{value:+.4f}" + rows.append(f"| {MECHANISM_LABELS[mechanism]} | {emphasis} | {ADVICE[mechanism]} |") + return "\n".join(rows) + + +def per_group_table(cells: list[dict]) -> str: + medians = _paired_medians(cells, "relative", "per_group") + rows = ["| your anomalies are... | median ΔPR-AUC vs one-model-per-group |", "|---|---|"] + labels = {"contextual": "contextual", "global": "globally extreme", "real": "mixed or unknown"} + for mechanism in ("contextual", "global", "real"): + if mechanism not in medians: + continue + value = medians[mechanism] + emphasis = f"**{value:+.4f}**" if value > 0.01 else f"{value:+.4f}" + rows.append(f"| {labels[mechanism]} | {emphasis} |") + return "\n".join(rows) + + +def cost_table(cells: list[dict]) -> str: + per_config: dict[str, tuple[list[int], list[float]]] = {} + for cell in cells: + models, seconds = per_config.setdefault(cell["config"], ([], [])) + models.append(cell["n_models"]) + seconds.append(cell["seconds"]) + + rows = ["| approach | models trained | median fit time |", "|---|---|---|"] + if "relative" in per_config: + models, seconds = per_config["relative"] + rows.append(f"| `baseline_by` | 1 | {_median(seconds):.2f}s |") + if "per_group" in per_config: + models, seconds = per_config["per_group"] + rows.append( + f"| `segment_by` | one per group ({int(_median([float(m) for m in models]))} in this sweep) " + f"| {_median(seconds):.2f}s |" + ) + return "\n".join(rows) + + +def tabular_table(baselines: list[dict]) -> str: + rows = [ + "| dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random " + "| beats max-abs-z |", + "|---|---|---|---|---|---|---|---|---|", + ] + # Best first: a reader scanning for "is this any good" should meet the strong cases before the weak. + for row in sorted(baselines, key=lambda r: -r["dqx_pr_auc"]): + lift = row["dqx_pr_auc"] / row["random"] if row["random"] else float("nan") + beats = "yes" if row["dqx_pr_auc"] > row["max_abs_z"] else "**no**" + score = f"**{row['dqx_pr_auc']:.4f}**" if lift > 10 else f"{row['dqx_pr_auc']:.4f}" + rows.append( + f"| {row['dataset']} | {row['n_rows']} | {row['n_features']} | {row['base_rate']:.2%} | " + f"{score} | {row['random']:.4f} | {row['max_abs_z']:.4f} | {lift:.1f}x | {beats} |" + ) + return "\n".join(rows) + + +def replace_region(text: str, name: str, body: str) -> str: + start, end = MARKERS[name] + if start not in text or end not in text: + raise SystemExit(f"marker pair for {name!r} missing from {PAGE.name}; add {start} / {end}") + head = text[: text.index(start) + len(start)] + tail = text[text.index(end) :] + return f"{head}\n{body}\n{tail}" + + +def main() -> int: + if len(sys.argv) < 2: + raise SystemExit(f"usage: {pathlib.Path(sys.argv[0]).name} ") + data = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) + cells = data["cells"] + baselines = data.get("tabular_baselines") or [] + + text = PAGE.read_text(encoding="utf-8") + text = replace_region(text, "conditioning", conditioning_table(cells)) + text = replace_region(text, "per_group", per_group_table(cells)) + text = replace_region(text, "cost", cost_table(cells)) + if baselines: + text = replace_region(text, "tabular", tabular_table(baselines)) + else: + # Loud, not silent: a run without --datasets tabular leaves that table at whatever the last + # run published, and a stale table that looks fresh is worse than an obvious gap. + print( + "warning: results file carries no tabular_baselines, so the plain-tabular table was left " + "untouched. Re-run with '--datasets ... tabular' to refresh it.", + file=sys.stderr, + ) + PAGE.write_text(text, encoding="utf-8") + print(f"refreshed generated tables in {PAGE}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/anomaly_conditioning/run_experiment.py b/benchmarks/anomaly_conditioning/run_experiment.py index c436a8306..564038be6 100644 --- a/benchmarks/anomaly_conditioning/run_experiment.py +++ b/benchmarks/anomaly_conditioning/run_experiment.py @@ -509,6 +509,9 @@ def write_report( "evidence": evidence, "spearman": {"rho": rho, "ci_low": lo, "ci_high": hi}, "regression": regression, + # Included so emit_docs.py can refresh the published tables without re-running the + # sweep, and so a results file is self-contained. + "tabular_baselines": tabular_baselines or [], "cells": [c.as_dict() for c in cells], }, indent=2, diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index bf1ecab4b..66668a873 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -34,11 +34,13 @@ segment when any single point inside it is detected — a *random* score achieve `baseline_by` judges each metric against its own group's baseline rather than against the whole table. Whether that helps depends on what makes your anomalies anomalous. + | your anomalies are... | median ΔPR-AUC with `baseline_by` | what to do | |---|---|---| | **contextual** — ordinary for the table, wrong for their group | **+0.0742** | use `baseline_by` | | **globally extreme** — unusual against the whole table | +0.0000 | costs nothing; leave it on | | mixed or unknown (real-world datasets) | +0.0025 | leave it on | + Read that as one asymmetry rather than three results: conditioning is worth a great deal where it applies and nothing measurable where it does not. That is why DQX enables it automatically when it @@ -56,16 +58,20 @@ To compare against the whole table anyway, pass `baseline_by=[]`. The legacy `segment_by` trains a separate model per group. Conditioning expresses the grouping as features on a single model instead, and wins on both axes. + | your anomalies are... | median ΔPR-AUC vs one-model-per-group | |---|---| | contextual | **+0.0252** | | globally extreme | +0.0005 | | mixed or unknown | +0.0054 | + + | approach | models trained | median fit time | |---|---|---| | `baseline_by` | 1 | 0.17s | | `segment_by` | one per group (12 in this sweep) | 1.38s | + Per-group models also fail in a way that averages conceal. On the Server Machine Dataset one entity produced 15,963 false positives across 28,392 normal rows — a 56% false-alarm rate — while the @@ -81,6 +87,7 @@ without any conditioning. Each row is read against its own random floor, and against `max-abs-z` — the largest absolute z-score across features, which is the cheapest defensible detector and a genuinely competitive one. + | dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random | beats max-abs-z | |---|---|---|---|---|---|---|---|---| | shuttle | 30000 | 9 | 7.15% | **0.9788** | 0.0731 | 0.8983 | 13.4x | yes | @@ -93,6 +100,7 @@ across features, which is the cheapest defensible detector and a genuinely compe | fraud | 30000 | 29 | 0.17% | 0.2240 | 0.0019 | 0.1365 | **115.4x** | yes | | mammography | 11183 | 6 | 2.32% | 0.2193 | 0.0236 | 0.1779 | 9.3x | yes | | covertype | 30000 | 10 | 0.96% | 0.0534 | 0.0094 | 0.1122 | 5.7x | **no** | + DQX beats the random floor on all ten and `max-abs-z` on eight. `fraud` — the Kaggle credit-card set at a 0.17% base rate — is the standout at 115x the floor. From ccf7de8f8ca12dcf4336fdff2fb66ee06dee46a4 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 25 Aug 2026 20:29:54 +0100 Subject: [PATCH 026/107] Redact the features derived from a redacted column SHAP contributions are keyed by *engineered* feature name, and redaction filters those keys by exact match against the caller's redact_columns. Baseline conditioning adds a derived feature per metric, so redacting `amount` did not stop `amount_rel_baseline` -- a signed log-ratio of the same column -- from being embedded in a prompt and sent to an external serving endpoint. Naming a column sensitive has to mean everything computed from it is sensitive, so the redaction set now expands to cover the derived feature. Expansion is by exact suffix rather than prefix match, because prefix matching would also swallow `amount_paid` -- a different column the caller never named -- and quietly narrow what an explanation can discuss. The suffix is now a named constant in transformers.py, where the feature is created, rather than an inline f-string, so the two cannot drift apart. Known gap, documented at the function and not closed here: one-hot and frequency-encoded features are still not covered, because their names cannot be reconstructed from the source column alone -- `country` becomes `country_C3`, `country_DE`, one per observed value. Closing that needs the feature metadata at prompt-construction time, which the explainer does not currently receive. That hole predates this branch; this commit closes the one this branch opened. redaction_set is public so its policy can be unit-tested directly rather than through a private. --- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 23 ++++++++++- .../labs/dqx/anomaly/transformers.py | 8 +++- .../test_anomaly_explanation_redaction.py | 41 +++++++++++++++++++ 3 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_anomaly_explanation_redaction.py diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 24eebb7e6..093a50dcc 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -23,6 +23,7 @@ from pyspark.sql.types import DoubleType, LongType, StringType, StructField, StructType from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema +from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX from databricks.labs.dqx.config import LLMModelConfig from databricks.labs.dqx.errors import InvalidParameterError @@ -214,6 +215,24 @@ def from_scoring_config(cls, config: "ScoringConfig") -> "ExplanationContext": ) +def redaction_set(redact_columns: tuple[str, ...]) -> frozenset[str]: + """Columns to redact, plus the engineered features derived from them. + + Redaction matches contribution keys exactly, and contribution keys are *engineered* feature + names. So redacting ``amount`` did not stop ``amount_rel_baseline`` -- a signed log-ratio of the + same column -- from reaching the LLM prompt. A caller naming a column sensitive means every + feature derived from it is sensitive too. + + Known remaining gap: one-hot and frequency-encoded features are not covered, because their names + cannot be reconstructed from the source column alone (``country`` becomes ``country_C3``, + ``country_DE`` and so on, one per observed value). Closing that needs the feature metadata at + prompt-construction time, which this function does not have. Tracked separately. + """ + expanded = set(redact_columns) + expanded.update(f"{column}{BASELINE_RELATIVE_SUFFIX}" for column in redact_columns) + return frozenset(expanded) + + def _pattern_spark_expr(contributions_col: str, redact_set: frozenset[str]) -> Column: """Pattern key as a pure-Spark-SQL expression (no Python UDFs shipped to executors). @@ -648,7 +667,7 @@ def _add_explanation_column_ai_query( the documented "one call per group per scoring run" cost model. The collected payload is small and bounded: at most ``max_groups`` rows, each holding three length-capped text fields. """ - redact_set = frozenset(ctx.redact_columns) + redact_set = redaction_set(ctx.redact_columns) anomalous = df_with_pattern.filter(F.col(ctx.severity_col) >= F.lit(ctx.threshold)) kept_sdf, dropped_groups_count, dropped_rows_count, total_groups = _aggregate_groups_spark( anomalous, @@ -718,7 +737,7 @@ def add_explanation_column( Raises: InvalidParameterError: When *model_name* does not resolve to a Databricks serving endpoint. """ - redact_set = frozenset(ctx.redact_columns) + redact_set = redaction_set(ctx.redact_columns) segment_str = _format_segment(segment_values, redact_set) df_with_pattern = df.withColumn(ctx.pattern_col, _pattern_spark_expr(ctx.contributions_col, redact_set)) return _add_explanation_column_ai_query( diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index dc4451bb5..f929469e3 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -764,6 +764,12 @@ def _signed_log1p(column: Column) -> Column: return F.signum(column) * F.log1p(F.abs(column)) +# Suffix marking a baseline-relative feature. Named here rather than inlined because redaction has +# to be able to recognise a feature as derived from a source column: a caller who redacts "amount" +# means the LLM must not see "amount_rel_baseline" either. +BASELINE_RELATIVE_SUFFIX = "_rel_baseline" + + def _process_baseline_relative_features( transformed_df: DataFrame, numeric_cols: list[ColumnTypeInfo], @@ -802,7 +808,7 @@ def _process_baseline_relative_features( global_medians.update(computed_global) for metric in metrics: - feature_name = f"{metric}_rel_baseline" + feature_name = f"{metric}{BASELINE_RELATIVE_SUFFIX}" baselines = baseline_medians.get(metric, {}) global_baseline = global_medians.get(metric, 0.0) diff --git a/tests/unit/test_anomaly_explanation_redaction.py b/tests/unit/test_anomaly_explanation_redaction.py new file mode 100644 index 000000000..35e1775f5 --- /dev/null +++ b/tests/unit/test_anomaly_explanation_redaction.py @@ -0,0 +1,41 @@ +"""Redacting a column must redact the features derived from it. + +SHAP contributions are keyed by *engineered* feature name, and redaction filters those keys by exact +match against the caller's ``redact_columns``. Baseline conditioning adds a derived feature per +metric, so redacting ``amount`` left ``amount_rel_baseline`` -- a signed log-ratio of the same +column -- flowing into an LLM prompt sent to an external serving endpoint. A caller naming a column +sensitive means everything computed from it is sensitive. +""" + +from databricks.labs.dqx.anomaly.anomaly_llm_explainer import redaction_set +from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX + + +def test_redacting_a_column_also_redacts_its_baseline_relative_feature(): + """The hole this closed: the derived feature is the leak, and it is the interesting one.""" + assert redaction_set(("amount",)) == {"amount", f"amount{BASELINE_RELATIVE_SUFFIX}"} + + +def test_redaction_covers_every_named_column(): + result = redaction_set(("amount", "salary")) + + assert "amount" in result and f"amount{BASELINE_RELATIVE_SUFFIX}" in result + assert "salary" in result and f"salary{BASELINE_RELATIVE_SUFFIX}" in result + + +def test_no_redaction_stays_empty(): + """An empty set is what lets the caller skip the filter entirely, so it must not grow entries.""" + assert not redaction_set(()) + + +def test_similar_prefixes_are_not_swept_up(): + """Expansion is by exact suffix, not prefix matching. + + Redacting ``amount`` must not silently drop ``amount_paid``, which is a different column the + caller did not name. Prefix matching would, and would quietly reduce what the explanation can + talk about. + """ + result = redaction_set(("amount",)) + + assert "amount_paid" not in result + assert f"amount_paid{BASELINE_RELATIVE_SUFFIX}" not in result From f510b891fca2c871011c2effa75295179a5e5aeb Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 12:36:06 +0100 Subject: [PATCH 027/107] BREAKING: remove the legacy segment_by anomaly path in favour of baseline_by grouping Row anomaly detection was Experimental through 0.16.0 and owed no compatibility. This removes the per-group model path (segment_by / max_segment_models) entirely, leaving a single pooled baseline_by model that measured better on detection, reliability and cost across all four benchmark regimes. Source removals: - config.AnomalyParams.segment_by / max_segment_models - training_service: segmented dispatch, segment resolution and validation - scoring_run: score_segmented, load_segment_models, score_single_segment, and the now-orphaned _split_max_groups_budget / _warn_if_max_groups_below_segments helpers - scoring_orchestrator segmented fallback; scoring_strategies.score_segmented - model_discovery segment-record selection; drift.check_segment_drift - SegmentationConfig -> GroupingConfig; persisted `segmentation` struct -> `grouping` - _dq_info.anomaly.segment field (permanently null once segmentation is gone) - group_config segmented thresholds (MAX_SEGMENT_MODELS, MIN_ROWS_PER_SEGMENT, ...) - segment_utils: canonicalize_segment_values / build_segment_name / build_segment_filter Models trained on a prior build must be retrained: the config hash changed and the registry struct was renamed. mergeSchema is retained on the registry write so the retrain writes cleanly to an existing registry table. Tests: delete test_anomaly_segments, test_anomaly_segment_naming, test_anomaly_scoring_run; update autodiscovery, registry, drift and apply-checks tests to the baseline_by policy (baseline columns are grouping, not features). Docs: CHANGELOG BREAKING entries; anomaly_compatibility.mdx converted to a migration guide; anomaly_heuristic_map.mdx segmented branch removed; user guide reworked around group-aware detection with a breaking-change/migration section; Experimental -> Beta. The LLM explainer's segmentation vocabulary is intentionally left for the next commit, which rewrites it with the baseline_by replacement in hand rather than blanking it. Co-authored-by: Isaac --- CHANGELOG.md | 5 +- docs/dqx/docs/dev/anomaly_compatibility.mdx | 179 +++--- docs/dqx/docs/dev/anomaly_heuristic_map.mdx | 38 +- .../guide/row_anomaly_detection/index.mdx | 176 +++--- docs/dqx/docs/reference/quality_checks.mdx | 2 +- .../labs/dqx/anomaly/anomaly_engine.py | 30 +- .../labs/dqx/anomaly/anomaly_info_schema.py | 1 - .../labs/dqx/anomaly/anomaly_llm_explainer.py | 19 +- .../labs/dqx/anomaly/anomaly_workflow.py | 1 - .../labs/dqx/anomaly/check_funcs.py | 10 +- src/databricks/labs/dqx/anomaly/drift.py | 32 -- .../labs/dqx/anomaly/group_config.py | 60 +- .../labs/dqx/anomaly/model_config.py | 27 +- .../labs/dqx/anomaly/model_discovery.py | 36 +- .../labs/dqx/anomaly/model_loader.py | 2 +- .../labs/dqx/anomaly/model_registry.py | 71 +-- src/databricks/labs/dqx/anomaly/profiler.py | 155 ++---- .../labs/dqx/anomaly/scoring_config.py | 10 +- .../labs/dqx/anomaly/scoring_orchestrator.py | 37 +- .../labs/dqx/anomaly/scoring_run.py | 263 +-------- .../labs/dqx/anomaly/scoring_strategies.py | 31 +- .../labs/dqx/anomaly/scoring_utils.py | 13 +- .../labs/dqx/anomaly/segment_utils.py | 43 +- .../labs/dqx/anomaly/training_service.py | 296 ++-------- src/databricks/labs/dqx/anomaly/types.py | 11 +- src/databricks/labs/dqx/anomaly/validation.py | 4 +- src/databricks/labs/dqx/config.py | 16 +- tests/integration_anomaly/conftest.py | 8 +- .../test_anomaly_apply_checks.py | 2 - .../test_anomaly_apply_checks_by_metadata.py | 6 +- .../test_anomaly_autodiscovery.py | 82 +-- .../test_anomaly_drift_integration.py | 52 -- .../test_anomaly_errors.py | 21 +- .../test_anomaly_groups.py | 55 -- .../test_anomaly_registry.py | 119 +--- .../test_anomaly_segments.py | 525 ------------------ .../test_anomaly_train_and_score.py | 2 +- .../test_anomaly_training_validation.py | 6 - .../test_anomaly_check_funcs_validation.py | 119 +--- tests/unit/test_anomaly_configs.py | 22 +- tests/unit/test_anomaly_model_record.py | 27 +- tests/unit/test_anomaly_model_registry.py | 119 ++-- tests/unit/test_anomaly_scoring_run.py | 58 -- tests/unit/test_anomaly_segment_naming.py | 234 -------- tests/unit/test_anomaly_validation.py | 4 +- 45 files changed, 436 insertions(+), 2593 deletions(-) delete mode 100644 tests/integration_anomaly/test_anomaly_segments.py delete mode 100644 tests/unit/test_anomaly_scoring_run.py delete mode 100644 tests/unit/test_anomaly_segment_naming.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a600e8f49..a101515c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,8 +49,9 @@ BREAKING CHANGES! -* Row anomaly detection now **errors** instead of warning when segmentation would produce more than `AnomalyParams.max_segment_models` segments (default 50). One model is trained per segment and segmented training does not ensemble, so cost is linear in the segment count: 90 segments measures roughly 88 minutes, and a 90-segment run was cancelled after 70 minutes without finishing. Auto-discovery could reach tens of thousands of segments behind a single log line. Runs that previously segmented into 51 or more segments will now fail; pass a higher `max_segment_models` to keep the old behaviour, or use `baseline_by`, which trains one model regardless of the group count. ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) -* `segment_by` is now legacy, and an auto-discovered grouping is used as `baseline_by` instead of training one model per group. Runs that relied on auto-segmentation will train a single conditioned model rather than N models, and will produce different scores. The evidence: on the Server Machine Dataset per-group models were the worst of three configurations (PR-AUC 0.1416 against 0.1499 for plain pooling and 0.1536 with baseline conditioning), and one entity produced 15,963 false positives out of 28,392 normal rows. `segment_by` still works if you pass it explicitly; a `DeprecationWarning` follows one release later. Passing both `segment_by` and `baseline_by` raises. ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) +* Removed the legacy `segment_by` path from row anomaly detection ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). `segment_by` trained one model per group; it is gone, and `baseline_by` — one conditioned model whatever the group count — is the only grouping mechanism. This is not a rename: `segment_by` partitioned into N models, `baseline_by` judges each metric against its own group's baseline on a single model, and the scores differ. On the Server Machine Dataset per-group models were the worst of three configurations measured (PR-AUC 0.1416 against 0.1499 for plain pooling and 0.1536 with baseline conditioning), with one entity producing 15,963 false positives across 28,392 normal rows. Row anomaly detection is not GA and its formats were allowed to change without a migration path, so the break is taken now rather than carried: replace `segment_by=[...]` with `baseline_by=[...]`, and note that `segment_by` and `AnomalyParams.max_segment_models` are no longer accepted. Auto-discovery no longer trains one model per discovered group either — a discovered grouping is routed to `baseline_by`, so zero-config runs train a single conditioned model and produce different scores. +* Row anomaly models trained before this release must be retrained ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). `compute_config_hash` now includes `baseline_by`, so a model's stored hash changes and a scoring-time mismatch **raises** rather than silently scoring against a different feature list; metadata from before baseline conditioning is also no longer loadable. The registry's `segmentation` struct is renamed `grouping` and holds `baseline_by`, `sklearn_version`, and `config_hash` — the per-segment fields (`segment_values`, `is_global_model`) are gone because every model is now single and conditioned. The registry write uses `mergeSchema`, so retraining into an existing table adds the renamed column in place; a table you never retrain into keeps the old `segmentation` column and is not read. +* Removed the permanently-null `segment` field from the `_dq_info[].anomaly` struct ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). It carried the segment identity of a per-segment model; with segmentation gone it was always null. Queries against the other anomaly fields are unaffected; a query that selected `.anomaly.segment` must drop it. * Rows whose group was absent from training now return a **null** score and severity instead of a number, and are not flagged as violations. Previously they were scored 0.0 — the most normal-looking value in the table — because one-hot encoding emits all zeros for an unseen category, which resembles the majority on every axis; frequency encoding has the same defect with the opposite sign, coalescing the miss to a frequency below anything seen in training. Neither is a signal a caller can act on. The new `_dq_info[].anomaly.is_new_baseline` and `.new_baseline_key` fields report the fact. To fail on unrecognised group values, use `foreign_key` or `is_in_list` on the baseline column, which is the check built for that question. ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) * The anomaly struct inside `_dq_info` gains `is_new_baseline` (boolean) and `new_baseline_key` (string). Existing named-field queries such as `_dq_info[0].anomaly.score` keep working, but the struct is wider, so **appending to a Delta table that already holds `_dq_info` requires `mergeSchema`** (`.option("mergeSchema", "true")` or `spark.databricks.delta.schema.autoMerge.enabled`). ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) * `is_in_list`, `is_not_in_list`, and `is_not_null_and_is_in_list` now resolve their `allowed` / `forbidden` string values as **column expressions** (consistent with the comparison checks), not string literals. A bare string is interpreted as a column reference, a numeric string (e.g. `"3"`) is parsed as a number, and an ISO-date string (e.g. `"2024-01-01"`) as a date. To match a string literal, single-quote the value (e.g. `'value'`) or wrap it in `F.lit("value")`. Existing checks that relied on bare strings being treated as literals must quote them. ([#1419](https://github.com/databrickslabs/dqx/issues/1419)) diff --git a/docs/dqx/docs/dev/anomaly_compatibility.mdx b/docs/dqx/docs/dev/anomaly_compatibility.mdx index 838df1fe2..a7289c5d3 100644 --- a/docs/dqx/docs/dev/anomaly_compatibility.mdx +++ b/docs/dqx/docs/dev/anomaly_compatibility.mdx @@ -1,132 +1,77 @@ --- -title: Anomaly compatibility debt +title: Anomaly migration guide sidebar_position: 640 --- -# Anomaly Detection Compatibility Debt - -Baseline conditioning ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) was added without -breaking anything: the previous grouping mechanism still works, and models trained before it still -score. That cost a set of affordances which exist **only** to keep old behaviour alive. - -This page is the removal checklist for when they go. Every site listed here carries the marker - -```python -# COMPAT(anomaly-v1): -- see docs/dev/anomaly_compatibility -``` - -so `git grep "COMPAT(anomaly-v1)"` finds the lot, and nothing has to be rediscovered by reading. - -Two things this page deliberately keeps apart, because they look identical in a diff: - -- **Compatibility debt** — exists for old callers or old models, and is removable. -- **Runtime fallbacks** — permanent behaviour for cases that will still happen after any deprecation - (a group absent at training, a group too small to calibrate). Listed at the bottom under - [Not compatibility debt](#not-compatibility-debt) so they are not deleted by mistake. - -## A. The legacy `segment_by` path - -The largest block, and the one whose removal deletes the most code. `segment_by` trains one model per -group; `baseline_by` trains one model whatever the group count. On the Server Machine Dataset the -per-segment configuration was the *worst* of three measured, with one entity emitting 15,963 false -positives across 28,392 normal rows, so nothing depends on keeping it except existing callers. - -| Site | What to remove | -|---|---| -| `config.py` · `AnomalyParams.segment_by` | the field and its docstring entry | -| `config.py` · `AnomalyParams.max_segment_models` | the field; its only consumer is this path | -| `anomaly_engine.py` · `train(segment_by=...)` | the parameter and its docs | -| `training_service.py` · `_resolve_grouping` | collapses to "use `baseline_by`"; the both-declared error and the `baseline_by = None` clearing both go | -| `training_service.py` · `_get_and_validate_segments` | delete | -| `training_service.py` · `_train_segmented` | delete, and the `if context.segment_by:` dispatch above it | -| `training_service.py` · `_perform_auto_discovery` | `for_baseline=segment_by is None` becomes unconditional `True` | -| `profiler.py` · `_select_segment_columns` | the `elif candidate_segments:` legacy branch and the `for_baseline` parameter; `select_baseline_columns` becomes the only policy | -| `profiler.py` · `_is_grouping_candidate` | drops `for_baseline`; keeps only the baseline thresholds | -| `group_config.py` | `MAX_SEGMENT_MODELS`, `MIN_ROWS_PER_SEGMENT`, `SEGMENT_COUNT_WARN_THRESHOLD`, `MIN_ROWS_TO_TRAIN_SEGMENT`, `MAX_AUTO_GROUP_COUNT` — the whole segmented half of the module | -| `segment_utils.py` | `canonicalize_segment_values`, `build_segment_name`, `build_segment_filter` | -| `scoring_orchestrator.py` · `try_segmented_scoring_fallback` | delete, and the `if config.segment_by:` branch that calls it | -| `scoring_run.py` · `score_segmented` | delete | -| `scoring_strategies.py` · `score_segmented` | delete from the protocol and its implementations | -| `scoring_config.py` · `ScoringConfig.segment_by` | delete | -| `types.py` · `AnomalyTrainingContext.segment_by` | delete | -| `model_config.py` · `SegmentationConfig` | `segment_by`, `segment_values`, `is_global_model` become meaningless — every model is global | -| `model_registry.py` · registry schema | the `segmentation` struct loses those fields. **Registry migration required** — see below | -| tests | `test_anomaly_segments.py`, the `segment_by` cases in `test_anomaly_groups.py`, `test_segment_by_does_not_gain_baseline_relative_features` | - -**Registry migration.** The `segmentation` struct is a persisted Delta schema. Dropping fields from it -is not a code-only change: existing registry tables carry them. Either keep the columns and stop -writing them, or migrate the table. Decide this before starting, because it is the only item here -that touches customer data rather than customer code. - -## B. Pre-#1484 model metadata - -Every group-conditioning field on `SparkFeatureMetadata` defaults to empty specifically so a model -trained before they existed deserializes into "no grouping", the relative transform returns -immediately, and `engineered_feature_names` is byte-identical to what it was. That is what makes old -models score unchanged. - -| Site | What to remove | -|---|---| -| `transformers.py` · `SparkFeatureMetadata` group fields | the empty defaults may become required once no old model can be loaded | -| `transformers.py` · `from_json` unknown-key tolerance | the `logger.debug` branch that ignores unknown keys. Forward-compatibility for *older DQX reading newer models*, symmetrical debt | -| `transformers.py` · OneHot category reconstruction | the "Model may be from an older version without OneHot category storage" branch — predates #1484 and is older debt still | - -Note `_process_baseline_relative_features`'s early return on `not baseline_by` is **not** in this -table. It looks like the same thing but it is the legitimate ungrouped path, which survives any -deprecation. - -## C. `compute_config_hash` excludes `baseline_by` - -The hash is `(columns, segment_by)`. A model retrained under the same name with a *different* -`baseline_by` therefore produces an identical hash, so the collision detection that exists to catch -"same name, different config" cannot see a grouping change. The persisted metadata still records the -real grouping, so this misleads rather than corrupts. - -It is left alone because a scoring-time hash mismatch **raises** -(`scoring_run.py` · `score_global_model`), so changing the formula would stop every existing model -from scoring. Fixing it correctly means adding `baseline_by` to the hash *and* accepting that break — -which is exactly the decision this page exists to inform. - -## D. `RobustScaler` residue - -Conditional on the scaler removal landing. `RobustScaler` is an affine per-feature transform and -Isolation Forest splits on per-feature thresholds, so the model is invariant to it — measured -identical to four decimal places across five ADBench datasets. New models are fitted without it; old -models still have it pickled inside their pipeline. - -| Site | What to remove | -|---|---| -| `explainability.py` · `compute_shap_values` | the `scaler.transform(...) if scaler else ...` branch | -| `explainability.py` · contribution helper | the `isinstance(model_local, Pipeline)` branch that pulls out a scaler | -| `core.py` · `fit_sklearn_model` | the single-step `Pipeline` wrapper, once nothing needs `named_steps["model"]` | - -## Not compatibility debt - -These are permanent, and deleting them would change behaviour for cases that still occur: +# Anomaly Detection Migration Guide + +Baseline conditioning ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) replaced the +previous per-group modelling with a single conditioned model, and the compatibility affordances that +once kept the old path and old models alive have been removed. Row anomaly detection was Experimental +through 0.16.0 — its on-disk and API formats were explicitly allowed to change without a migration +path — so this break is taken deliberately now that the feature is Beta, rather than carried forever. + +If you never used row anomaly detection, nothing here affects you. If you did, the four steps under +[What you have to do](#what-you-have-to-do) are the whole migration. + +## What changed + +- **`segment_by` is gone.** It trained one model per group. `baseline_by` trains one model whatever + the group count, judging each metric against its own group's baseline. This is not a rename — the + semantics and the scores differ. On the Server Machine Dataset the old per-group configuration was + the *worst* of three measured, with one entity emitting 15,963 false positives across 28,392 normal + rows, so nothing is lost by dropping it. +- **The registry `segmentation` struct is now `grouping`.** It holds `baseline_by`, `sklearn_version`, + and `config_hash` — the per-segment fields (`segment_values`, `is_global_model`) are gone, because + every model is now single and conditioned. This is a persisted Delta schema change; the registry + write uses `mergeSchema`, so retraining into an existing table adds the new column in place. +- **`_dq_info.anomaly.segment` was removed.** It was permanently null once segmentation was gone. +- **`compute_config_hash` now includes `baseline_by`.** The hash is `(columns, baseline_by)`. A model + retrained under the same name with a different grouping now hashes differently, so the collision + detection that guards "same name, different configuration" finally sees a grouping change. A + scoring-time hash mismatch **raises**, which is what forces the retrain below. +- **Grouping auto-discovery is decoupled from column discovery.** Passing explicit `columns` no longer + silently gives up conditioning, and a discovered grouping is used as `baseline_by` rather than + turning into one model per group. Auto-discovered models may therefore score differently even with + no configuration change. + +## What you have to do + +1. **Replace `segment_by` with `baseline_by`.** Not a rename — `segment_by` partitioned into N models; + `baseline_by` judges each metric against its own group's baseline on one model. Expect different + scores. +2. **Retrain every model.** Metadata from before #1484 is no longer loadable, and the config hash + changes even where the configuration did not, so old models raise at scoring rather than scoring + wrong. +3. **Migrate or recreate registry tables.** Retraining into an existing table works — the write merges + the renamed `grouping` schema — but a table you never retrain into keeps the old `segmentation` + column and will not be read. +4. **Expect different scores on auto-discovered groupings** even without any config change, since + discovery now selects a finer grouping routed to `baseline_by`. + +## What did *not* change + +These are permanent runtime behaviours, not compatibility shims, and they behave exactly as before: - **Unseen baseline group → global median.** A group absent at training gets the global baseline, so - the row reads as ordinary rather than extreme, and `is_new_baseline` reports it. New groups appear - in production forever. + the row reads as ordinary rather than extreme, and `is_new_baseline` reports it. New groups appear in + production forever. - **Missing per-group quantiles → global calibration.** A group without a full quantile set falls back - to table-wide calibration rather than being half-calibrated. -- **`_process_baseline_relative_features` early return.** The ungrouped path. -- **Drift threshold default** in `scoring_config.py`. + to table-wide calibration rather than being left half-calibrated. +- **The ungrouped path.** A model trained with no `baseline_by` engineers exactly the features it + always did; conditioning adds nothing when there is no grouping. - **Python/Spark baseline-key agreement.** `build_baseline_key` and `baseline_key_column` must keep matching for as long as both exist; the notes there are a contract, not debt. -## If the break happens now instead +## Still carried -Deprecating immediately rather than later removes everything in A–D in one change. What a user has to -do: +One affordance from the pre-conditioning code has *not* been removed and is not part of this break: -1. **Replace `segment_by` with `baseline_by`.** Not a rename — the semantics differ. `segment_by` - partitions into N models; `baseline_by` judges each metric against its own group's baseline on one - model. Scores will differ. -2. **Retrain every model.** Persisted metadata from before #1484 would no longer be loadable, and the - config hash would change even where the configuration did not. -3. **Migrate or recreate registry tables**, per the note in section A. -4. **Expect different scores on auto-discovered groupings** even without any config change, since - discovery now selects a finer grouping routed to `baseline_by`. +- **`RobustScaler` residue** in `core.py` and `explainability.py`. The scaler is an affine per-feature + transform and Isolation Forest splits on per-feature thresholds, so the model is invariant to it + (measured identical to four decimal places across five ADBench datasets). New models are fitted + without it, but the load and explain paths still tolerate a scaler pickled inside older pipelines. + Removing it is a separate, non-breaking cleanup. diff --git a/docs/dqx/docs/dev/anomaly_heuristic_map.mdx b/docs/dqx/docs/dev/anomaly_heuristic_map.mdx index b19c6a9cc..9daea8d11 100644 --- a/docs/dqx/docs/dev/anomaly_heuristic_map.mdx +++ b/docs/dqx/docs/dev/anomaly_heuristic_map.mdx @@ -21,27 +21,23 @@ flowchart TD B -- no --> C["1 pick metrics
numeric, stddev>0, nulls<50%
low-card categoricals also eligible"] B -- yes --> D["use the caller's columns"] - C --> E["2 pick a grouping
nulls<10%, not id-like
baseline: ≥30 rows/group, ≤5000 groups
segmented: one lowest-cardinality column"] + C --> E["2 pick a grouping
nulls<10%, not id-like
≥30 rows/group, ≤5000 groups"] D --> E - E --> F{"baseline_by or
segment_by declared?"} - F -- both --> X1["error"] - F -- "segment_by" --> G["3 one model per segment
>50 error, >100 warn
baseline_by cleared"] - F -- "neither / baseline_by" --> H["4 one model,
grouping to baseline_by"] + E --> H["3 one model,
grouping becomes baseline_by"] - G --> I["5 sample 30%, split 80/20"] - H --> I - I --> J["6 feature engineering
onehot card≤20 else frequency
+ metric_rel_baseline per metric"] - J --> K["7 IsolationForest
200 trees, 256 rows/tree,
contamination 0.02, no scaling"] - K --> L["8 quantiles: global
+ per baseline group"] + H --> I["4 sample 30%, split 80/20"] + I --> J["5 feature engineering
onehot card≤20 else frequency
+ metric_rel_baseline per metric"] + J --> K["6 IsolationForest
200 trees, 256 rows/tree,
contamination 0.02, no scaling"] + K --> L["7 quantiles: global
+ per baseline group"] L --> M["register: MLflow + registry row
+ feature_metadata JSON"] M -.->|"persisted baselines, medians, quantiles"| N - N["score(df)"] --> O["9 re-engineer from metadata
baselines broadcast-joined
unseen group → global median"] + N["score(df)"] --> O["8 re-engineer from metadata
baselines broadcast-joined
unseen group → global median"] O --> P["pandas UDF: anomaly score"] - P --> Q["10 mark unseen baselines
isin if ≤200 keys, else join"] - Q --> R["11 severity percentile
per-group quantiles if present,
else global"] + P --> Q["9 mark unseen baselines
isin if ≤200 keys, else join"] + Q --> R["10 severity percentile
per-group quantiles if present,
else global"] R --> S{"baseline unseen?"} S -- yes --> T["null score, not flagged
is_new_baseline = true"] S -- no --> U{"severity ≥ threshold 95?"} @@ -49,6 +45,11 @@ flowchart TD U -- no --> W["passes"] ``` +There is one model whatever the grouping. `baseline_by` conditions each metric against its own +group's baseline rather than training a model per group; if it is not declared, stage 2 may discover +one. A column named both as a metric and as a baseline is rejected, as are floating-point and decimal +baseline columns (Spark and Python format them differently, which would break the key lookup). + ## Reading a bad result Working down the path, three questions separate most failures. @@ -57,7 +58,7 @@ Working down the path, three questions separate most failures. were there. ```sql -SELECT identity.model_name, training.columns, segmentation.baseline_by +SELECT identity.model_name, training.columns, grouping.baseline_by FROM WHERE identity.status = 'active' ``` @@ -70,7 +71,7 @@ almost nothing. **Is the comparison even valid?** Training samples 30% of rows by default via `.sample`, and splits 80/20 via `.randomSplit`; under Spark Connect both depend on partition ordering. Two runs of identical code can therefore train on different rows. If you are comparing two builds, set -`sample_fraction=1.0` first — otherwise you are measuring the sampler. Suspect stage 5 before +`sample_fraction=1.0` first — otherwise you are measuring the sampler. Suspect stage 4 before suspecting the design. ## Defaults in one place @@ -80,9 +81,6 @@ suspecting the design. | `MIN_ROWS_PER_BASELINE_GROUP` | 30 | rows/group needed to trust a baseline median | | `MAX_BASELINE_GROUPS` | 5000 | ceiling on total baseline groups | | `MAX_BASELINE_COLUMN_CARDINALITY` | 50 | per-column ceiling for a baseline column | -| `MAX_AUTO_GROUP_COUNT` | 20 | per-column ceiling on the legacy segmented path | -| `MIN_ROWS_PER_SEGMENT` | 100 | rows/segment needed to train a per-segment model | -| `MAX_SEGMENT_MODELS` | 50 | hard ceiling on the legacy segmented path | | `DEFAULT_SAMPLE_FRACTION` | 0.3 | share of rows used for training | | `DEFAULT_TRAIN_RATIO` | 0.8 | train/validation split | | `categorical_cardinality_threshold` | 20 | one-hot below, frequency-encode above | @@ -100,9 +98,9 @@ Measured, with the evidence in `benchmarks/anomaly_conditioning/`: diluting the signal. Largely avoided when those columns are recognised as a grouping instead, since the profiler removes the grouping from the feature list — but a categorical that is *not* selected as a grouping can still land in features. -- **Stage 7 has a blind spot.** Isolation Forest loses to a max-abs-z baseline where anomalies are +- **Stage 6 has a blind spot.** Isolation Forest loses to a max-abs-z baseline where anomalies are single-feature extremes in few dimensions, and in high dimension. Not fixable by tuning, and adding a complementary scorer measured worse on average — see `complementary_detector.py`. -- **Stage 9's fallback is silent by design.** A group absent at training falls back to the global +- **Stage 8's fallback is silent by design.** A group absent at training falls back to the global baseline, which makes the row read as ordinary rather than extreme. `is_new_baseline` is how that case is detected; it is the conservative direction, not a bug. diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index c5c2c6cb0..99b1ab81b 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -20,34 +20,34 @@ Use row anomaly detection to automatically find unusual rows in your data using - Row anomaly detection in data quality — in a few minutes. + Row anomaly detection in data quality, in a few minutes. - **Rules** catch things we already expect — like "bananas should be yellow" or "a batch must have at least 100 rows". But what about surprises we never thought to check for? That's where **anomaly detection** comes in. + **Rules** catch things we already expect, like "bananas should be yellow" or "a batch must have at least 100 rows". But what about surprises we never thought to check for? That's where **anomaly detection** comes in. - For a single banana, we can write rules: check size, colour, flavour. These rules live in code — clear, testable, versioned. + For a single banana, we can write rules: check size, colour, flavour. These rules live in code: clear, testable, versioned. Databricks Data Quality Monitoring (DQM) watches the big picture: did the data arrive? Is it fresh? Are the row counts about right? Think of it as checking the delivery trucks. - The truck arrived on time and the count looks right — great. But are the bananas inside actually good? DQM watches the delivery; nobody's inspecting the contents. That's the gap. + The truck arrived on time and the count looks right. Great. But are the bananas inside actually good? DQM watches the delivery; nobody's inspecting the contents. That's the gap. - DQX learns what "normal" looks like from your good data, then checks every single row. No labels needed — it figures out what's unusual on its own. *"Is this banana weird?"* + DQX learns what "normal" looks like from your good data, then checks every single row. No labels needed; it figures out what's unusual on its own. *"Is this banana weird?"* - Each banana becomes a set of numbers — size, colour, spots, bend. An Isolation Forest then tries to separate each point from the rest. If a banana is easy to isolate, it's probably odd. + Each banana becomes a set of numbers: size, colour, spots, bend. An Isolation Forest then tries to separate each point from the rest. If a banana is easy to isolate, it's probably odd. - An unusual banana gets separated in just a few steps — it sticks out. A normal banana is buried in the crowd and takes many steps to single out. + An unusual banana gets separated in just a few steps, so it sticks out. A normal banana is buried in the crowd and takes many steps to single out. - A score alone isn't enough — you want to know *why*. SHAP breaks it down: "too brown", "wrong size". So you can act on the insight straight away. + A score alone isn't enough. You want to know *why*. SHAP breaks it down: "too brown", "wrong size". So you can act on the insight straight away. - Everything you need to catch unusual rows — no ML expertise required. + Everything you need to catch unusual rows, no ML expertise required. Install DQX and start catching unusual rows in minutes. @@ -110,6 +110,21 @@ Because results are **explainable**, you don't just get a list of flagged rows. **Use DQX for**: "Is this data unusual?" **Use domain models for**: "Is this data fraudulent, faulty, or malicious?" +### How well does it detect, and how to judge it + +Row anomaly detection scores each row **on its own**. It is not a time-series or sequence model and does not read a window of history to make a prediction. The high numbers you see on published anomaly-detection leaderboards are usually won by sequence models on time-series data, which is a different problem, so they are not a yardstick for DQX. The right yardstick is your own data. + +What it is reliably good at, and what it is not: + +- **Good at unusual _combinations_ across columns**: several values that are each individually fine but wrong together. Rules struggle to express that, and it is where anomaly detection earns its place. +- **Weaker than a plain rule when a single column is extreme in isolation.** A range check or an outlier rule catches that more reliably and more cheaply, so reach for a rule there. You can run both. + +On DQX's own benchmarks (ten classical tabular datasets) it beats a random baseline on every one and a simple "largest z-score across columns" baseline on most. The exceptions are exactly the single-extreme-value case above. The full measured results, the datasets, and what the numbers do *not* mean are in [Anomaly detection quality](/docs/reference/anomaly_detection_quality). + + +Whatever a benchmark says, measure on **your** data before relying on a number. Train on a slice you consider good, score a slice you understand, and check that the rows it flags are ones you would actually want flagged. + + ## Complements Databricks Data Quality Monitoring [Databricks Data Quality Monitoring (DQM)](https://docs.databricks.com/aws/en/data-quality-monitoring/anomaly-detection) focuses on table-level signals like freshness and completeness. DQX row anomaly detection focuses on unusual rows and cross-column patterns. @@ -252,7 +267,7 @@ However, when you run a row anomaly detection check, DQX also adds a `_dq_info` The info column is an array of structs, with one element per anomaly detection check that was applied. -Feature contributions are on by default and are computed only for anomalous rows (severity at or above the threshold) — non-anomalous rows carry a `null` contributions map, so the SHAP cost scales with the number of anomalies, not the table size. Set `enable_contributions=False` to skip them entirely for the fastest scoring. Severity percentiles are always computed for every row, so you can still count would-be anomalies at other thresholds after scoring; but if you want contributions (and AI explanations) for rows below your enforcement threshold, score with the lower exploration threshold and filter afterwards: +Feature contributions are on by default and are computed only for anomalous rows (severity at or above the threshold). Non-anomalous rows carry a `null` contributions map, so the SHAP cost scales with the number of anomalies, not the table size. Set `enable_contributions=False` to skip them entirely for the fastest scoring. Severity percentiles are always computed for every row, so you can still count would-be anomalies at other thresholds after scoring; but if you want contributions (and AI explanations) for rows below your enforcement threshold, score with the lower exploration threshold and filter afterwards: ```python DQDatasetRule( @@ -290,14 +305,9 @@ Scores are normalized into `severity_percentile` (0–100). The anomaly threshol The threshold controls how many rows are flagged as anomalies (percentile cutoff; default 95). Lower values (for example 90) flag more rows; higher values (for example 98) flag fewer (only extreme anomalies). Start with the default (95). If you get too many alerts, raise the threshold; if you are missing issues you care about, lower it. -## Baseline conditioning +## Group-aware anomaly detection -Some values are only wrong *in context*. If one country's daily order volume drops 80% while the -overall total holds steady — because other countries absorbed the difference — the collapsed number -still sits comfortably inside the range other countries occupy normally. A model that compares every -row against the whole table cannot see it: on the measurements behind -[#1484](https://github.com/databrickslabs/dqx/issues/1484) such a collapse scored **45.1**, the 45th -percentile. No threshold recovers that without flagging half the table. +Some values are only wrong *in context*. If one country's daily order volume drops 80% while the overall total holds steady (because other countries absorbed the difference), the collapsed number still sits comfortably inside the range other countries occupy normally. A model that compares every row against the whole table cannot see it. Pass `baseline_by` to give DQX the right basis for comparison: @@ -311,98 +321,33 @@ anomaly_engine.train( ) ``` -The same collapse then scores above 95. - -### How it works, and what it costs - -For each numeric metric, DQX adds one feature: that metric's deviation from its own group's -baseline, as a signed log-ratio. - -``` -signed_log(x) = signum(x) * log1p(abs(x)) -_rel_baseline = signed_log(value) - signed_log(group_median(metric)) -``` - -The raw metric is kept alongside, so a globally absurd value stays detectable even where it is -ordinary for its group. The log-ratio form is stable when a baseline is near zero and symmetric for -halving versus doubling; the *signed* form matters because plain `log1p` is NaN for values at or -below −1, which would silently produce NaN features on any signed metric such as profit or balance. +Now each metric is judged against its own group's baseline, so the collapse stands out even though its value is ordinary for the table as a whole. -There is **one model**, however many groups you have. Baselines are computed once at training and -persisted with the model, then broadcast-joined at scoring time. So the cost does not grow with the -group count — which is the whole point, and the difference from `segment_by`. - -Measured offline (see `tests/unit/test_anomaly_relative_feature_separability.py`, which runs in the -unit suite): - -| scenario | without conditioning | with `baseline_by` | -|---|---|---| -| contextual collapse (ordinary globally) | PR-AUC 0.0028 — chance | PR-AUC 0.6962 | -| anomaly extreme against every group | PR-AUC 1.0000 | PR-AUC 1.0000 | - -The second row is why conditioning is on by default when a grouping is available: it costs nothing -measurable when the anomaly was already visible. - -Across a wider sweep — 1,545 configurations over synthetic data, the Server Machine Dataset, NSL-KDD -and ten classical tabular benchmarks — conditioning is worth about **+0.07 PR-AUC** where anomalies are -contextual, and **nothing measurable** where they are not. Against the previous release, on the same -tables through the real pipeline, it moves a contextual collapse from 0.0376 to 0.5703. That -asymmetry, not a hunch, is why a discovered grouping is used rather than ignored. See -[Anomaly detection quality](/docs/reference/anomaly_detection_quality) for the full results, the -datasets, and what these numbers do *not* mean. - -### Baseline columns are not features - -A baseline column is the basis of comparison, not a metric being compared, so it never becomes a -model feature. Passing the same column as both `columns` and `baseline_by` is an error. If you let -DQX auto-discover `columns`, it drops your declared baseline columns from the feature list for you. - -Baseline columns must be string, integral, boolean or date. Float, double and decimal are -**rejected**: Spark and Python format floating-point values differently, and DQX builds each row's -baseline key in both — Python when saving baselines, Spark when looking them up. A mismatch would -not raise; it would silently miss every lookup and quietly condition on nothing. Bucket the value or -cast it to a string first. - -:::note -That failure mode is not hypothetical. Booleans hit it during development — Spark renders `true`, -Python renders `True` — and it was caught only by a test that compares the two implementations -against a live session. Both halves of the key are now pinned by that test. -::: + +A baseline column is the basis of comparison, not a metric being compared, so it never becomes a model feature. Passing the same column in both `columns` and `baseline_by` is an error, and if you let DQX auto-discover `columns`, it drops your declared baseline columns from the feature list for you. Baseline columns must be string, integral, boolean, or date. Float, double, and decimal are rejected, because Spark and Python format floating-point values differently and the baseline lookup would silently match nothing. Bucket the value or cast it to a string first. + ### Groups that appear after training -A row whose group was never seen during training gets a **null** score and severity, plus -`is_new_baseline = true` and the unrecognised key in `new_baseline_key`: +A row whose group was never seen during training gets a **null** score and severity, plus `is_new_baseline = true` and the unrecognised key in `new_baseline_key`: ```python result.filter(F.col("_dq_info")[0].anomaly.is_new_baseline).select("_dq_info") ``` -It is not flagged as a violation. Neither categorical encoder can represent an unseen value honestly -— one-hot makes it look maximally normal, frequency encoding maximally extreme — so DQX cannot judge -the row, and "could not judge" is a different claim from "is anomalous". Previously such rows were -scored 0.0 and silently passed, which is the most normal-looking value in the table. +It is not flagged as a violation. Neither categorical encoder can represent an unseen value honestly: one-hot makes it look maximally normal, and frequency encoding makes it look maximally extreme. So DQX cannot judge the row, and "could not judge" is a different claim from "is anomalous". Previously such rows were scored 0.0 and silently passed, which is the most normal-looking value in the table. -If an unrecognised group value is itself something you want to fail on, that is a membership question -rather than an anomaly one: use `foreign_key` or `is_in_list` on the baseline column against your set -of known values. +If an unrecognised group value is itself something you want to fail on, that is a membership question rather than an anomaly one: use `foreign_key` or `is_in_list` on the baseline column against your set of known values. -### segment_by is legacy +## Upgrading and breaking changes -`segment_by` trains one model per group. It still works, but prefer `baseline_by`, because -per-group models lost on every axis that was measured: +Row anomaly detection was **Experimental** in earlier releases and is now **Beta**. This release replaces the old per-group model path (`segment_by`) with `baseline_by`, which measured better on detection, reliability, and cost. If you never used `segment_by`, nothing here affects you. -* **Detection.** On the Server Machine Dataset they were the worst of three configurations — - PR-AUC 0.1416 against 0.1499 for plain pooling and 0.1536 with baseline conditioning. -* **Reliability.** One entity produced 15,963 false positives out of 28,392 normal rows, a 56% - false-alarm rate. Each per-group model calibrates its severity threshold on its own small sample, - so each is independently fragile. -* **Cost.** Linear in the group count, and segmented training does not ensemble. 90 groups measured - roughly 88 minutes; a 90-group run was cancelled after 70 minutes without finishing. Runs above - `AnomalyParams.max_segment_models` (default 50) now raise rather than warn. +If you did, three things change. The full detail is in the [migration guide](/docs/dev/anomaly_compatibility). -Passing both `baseline_by` and `segment_by` is an error. A `DeprecationWarning` on `segment_by` -follows one release later. +- **Replace `segment_by` with `baseline_by`.** They are not the same. `segment_by` trained one model per group; `baseline_by` judges each metric against its own group's baseline on a single model. Your scores will change. +- **Retrain your models.** A model trained before this release raises an error at scoring time until you retrain it. +- **Your registry table is handled for you.** Retraining writes the new schema into your existing table in place. ## How it works under the hood @@ -412,9 +357,9 @@ For full parameter and schema details, see [Row Anomaly Detection in Quality Che 1. **Feature engineering** (automatic): DQX detects column types and creates features (numerical standardized, categorical one-hot or frequency-encoded, temporal expanded to hour/day/month/weekend). 2. **Smart sampling and training**: DQX samples your data (default 30%, capped at 1M rows), trains an ensemble of Isolation Forest models, and captures baseline statistics for drift detection. -3. **Model registry**: Models and metadata live in MLflow and a Delta table; segmented models use deterministic names (for example `__seg_region=US_tier=gold`). +3. **Model registry**: Models and metadata live in MLflow and a Delta table; each `train()` call registers one model under its name and archives the previously active version. 4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. SHAP contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. -5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping — categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group (for example region, product category) — and this happens whether or not you passed `columns`, since what to measure and what to compare it against are independent questions. A discovered grouping is used as `baseline_by` (see [Baseline conditioning](#baseline-conditioning)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. +5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping (categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group, for example region or product category), and this happens whether or not you passed `columns`, since what to measure and what to compare it against are independent questions. A discovered grouping is used as `baseline_by` (see [Group-aware anomaly detection](#group-aware-anomaly-detection)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. ### Why Isolation Forest? @@ -442,39 +387,38 @@ Each element is a **struct** with a shared “wide” schema. Currently, the onl | `is_anomaly` | boolean | `true` if `severity_percentile` ≥ threshold. | | `threshold` | double | Severity percentile threshold used (e.g. 95.0). | | `model` | string | Full model name (e.g. Unity Catalog name). | -| `segment` | map<string, string> | Segment key-value pairs for segmented models; `null` for global models. | -| `contributions` | map<string, double> | Per-feature contribution percentages (0–100). On by default (`enable_contributions=True`); populated only for anomalous rows — `null` for non-anomalous rows or if you set it `False`. | +| `contributions` | map<string, double> | Per-feature contribution percentages (0–100). On by default (`enable_contributions=True`); populated only for anomalous rows (`null` for non-anomalous rows or if you set it `False`). | | `confidence_std` | double | Ensemble score standard deviation. Present when `enable_confidence_std=True`; `null` otherwise. | | `is_new_baseline` | boolean | `true` when the row's group was absent from training, in which case `score` and `severity_percentile` are `null` and the row is **not** flagged. See [Groups that appear after training](#groups-that-appear-after-training). | | `new_baseline_key` | string | The unrecognised group key, for rows where `is_new_baseline` is `true`; `null` otherwise. | -| `ai_explanation` | struct | LLM-generated explanation for the row's `(segment, pattern)` group. On by default (`enable_ai_explanation=True`); `null` for non-anomalous rows, when disabled, or when no serving endpoint is reachable. See [AI explanations](#ai-explanations) below. | +| `ai_explanation` | struct | LLM-generated explanation for the row's anomaly group (rows sharing the same top contributing features). On by default (`enable_ai_explanation=True`); `null` for non-anomalous rows, when disabled, or when no serving endpoint is reachable. See [AI explanations](#ai-explanations) below. | -The nested `ai_explanation` struct (populated when AI explanations are on — the default): +The nested `ai_explanation` struct (populated when AI explanations are on, which is the default): | Field | Type | Description | |--------|------|-------------| | `narrative` | string | Plain-language description of why the group was flagged. | | `business_impact` | string | Likely downstream impact if the rows are processed unchanged. | | `action` | string | What an analyst should investigate. | -| `top_features` | string | Deterministic top-2 contributing features (e.g. `amount+quantity`) — the group's pattern key. | -| `group_size` | long | Number of anomalous rows in this `(segment, pattern)` group. | +| `top_features` | string | Deterministic top-2 contributing features (e.g. `amount+quantity`), the group's pattern key. | +| `group_size` | long | Number of anomalous rows in this anomaly group. | | `group_avg_severity` | double | Mean `severity_percentile` across the group. | -**Access in PySpark:** use `F.element_at(F.col("_dq_info"), 1)` for the first element (1-based), then `.getField("anomaly").getField("severity_percentile")` etc. Alternatively `F.col("_dq_info").getItem(0)` for 0-based index (see [Troubleshooting](/docs/guide/row_anomaly_detection/troubleshooting) for Spark Connect–friendly patterns). +**Access in PySpark:** use `F.element_at(F.col("_dq_info"), 1)` for the first element (1-based), then `.getField("anomaly").getField("severity_percentile")` etc. Alternatively `F.col("_dq_info").getItem(0)` for 0-based index (see [Troubleshooting](/docs/guide/row_anomaly_detection/troubleshooting) for Spark Connect friendly patterns). ### AI explanations -AI explanations are **on by default**: each anomalous row gets a plain-language, LLM-generated explanation in `_dq_info[0].anomaly.ai_explanation` answering *why was this flagged, what's the impact, and what should I do* — without anyone reading raw SHAP percentages. Set `enable_ai_explanation=False` to turn it off (or `enable_contributions=False`, which disables explanations too, since they use the SHAP contributions as input). The explanation is AI-generated from the anomaly signal (feature names + SHAP + severity); it is **not** grounded in your catalog's table/column descriptions, so treat the business-impact and action as a starting point, not authoritative. +AI explanations are **on by default**: each anomalous row gets a plain-language, LLM-generated explanation in `_dq_info[0].anomaly.ai_explanation` answering *why was this flagged, what's the impact, and what should I do*, without anyone reading raw SHAP percentages. Set `enable_ai_explanation=False` to turn it off (or `enable_contributions=False`, which disables explanations too, since they use the SHAP contributions as input). The explanation is AI-generated from the anomaly signal (feature names + SHAP + severity); it is **not** grounded in your catalog's table/column descriptions, so treat the business-impact and action as a starting point, not authoritative. The call runs inside Spark via the SQL `ai_query` function against a Databricks Model Serving endpoint, so it needs no extra setup and scales with the cluster. This requires **Databricks serverless compute or Databricks Runtime 15.4 LTS or above** (where `ai_query` is available); on older runtimes explanations are skipped with a warning and scoring still completes. Similar anomalous rows are grouped together and the model is called **once per group** rather than once per row, so cost stays predictable on large datasets. -Rows are grouped by their top two contributing features, so occasionally two different kinds of anomaly that share the same top two features land in the same group and get one shared explanation. That's intentional — it keeps the number of AI calls (and the cost) low, at the price of a slightly more general explanation for those rows. +Rows are grouped by their top two contributing features, so occasionally two different kinds of anomaly that share the same top two features land in the same group and get one shared explanation. That's intentional: it keeps the number of AI calls (and the cost) low, at the price of a slightly more general explanation for those rows. ```python from databricks.labs.dqx.rule import DQDatasetRule from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies -# Contributions + AI explanations are on by default — this is all you need: +# Contributions + AI explanations are on by default. This is all you need: checks = [ DQDatasetRule( criticality="error", @@ -492,9 +436,9 @@ checks = [ ``` -* Explanations are **on by default** and call a Model Serving endpoint, so they add per-run LLM cost. `max_groups` (default 500) caps how many groups the model is called for per run. For segmented models the cap is shared across segments (at least one call each), so if you set `max_groups` lower than the number of segments you still get one call per segment — a warning is logged when that happens. Set `enable_ai_explanation=False` to turn explanations off. -* No serving endpoint? If the configured endpoint isn't reachable (e.g. Foundation Model APIs aren't enabled in the workspace), explanations are skipped with a warning and scoring still completes — nothing breaks. -* `redact_columns` keeps the listed feature and segment names out of the prompt. Segment **values** for non-redacted keys are sent verbatim — avoid segmenting on sensitive columns, or list them in `redact_columns`. +* Explanations are **on by default** and call a Model Serving endpoint, so they add per-run LLM cost. `max_groups` (default 500) caps how many anomaly groups the model is called for per run. Set `enable_ai_explanation=False` to turn explanations off. +* No serving endpoint? If the configured endpoint isn't reachable (e.g. Foundation Model APIs aren't enabled in the workspace), explanations are skipped with a warning and scoring still completes, so nothing breaks. +* `redact_columns` keeps the listed feature names out of the prompt (their contribution keys are shown as ``). ## Practical examples (non-technical) @@ -516,10 +460,16 @@ Use row anomaly detection when you want to catch unusual combinations across col ## Frequently Asked Questions +
+Q: I upgraded and my `segment_by` code stopped working. What do I do? + +`segment_by` has been removed. Replace it with `baseline_by`, which compares each metric against its own group's baseline on a single model instead of training one model per group, and retrain (models from earlier releases no longer load). Row anomaly detection was Experimental in earlier releases, which is what allowed this break, and it is now Beta. See [Upgrading and breaking changes](#upgrading-and-breaking-changes). +
+
Q: How much training data do I really need? -See the **Training data requirements** tip under Quick start. In short: 1,000+ rows preferred; quality and coverage matter more than quantity. For segmented models, use at least 100+ rows per segment; segments with fewer than **10** rows are skipped entirely and get no model, so rows in them are left unscored. Ensure training data includes all realistic values for categorical columns (regions, types, etc.) — a categorical value absent from training is not scored reliably. +See the **Training data requirements** tip under Quick start. In short: 1,000+ rows preferred; quality and coverage matter more than quantity. If you use `baseline_by`, each group needs enough rows for a stable baseline, and DQX will not auto-pick a grouping that leaves fewer than ~30 rows per group. Ensure training data includes all realistic values for categorical columns (regions, types, etc.); a categorical value absent from training is not scored reliably.
@@ -538,9 +488,9 @@ See the **Training data requirements** tip under Quick start. In short: 1,000+ r
-Q: Why is my auto-trained model segmented? +Q: Why did auto-training choose a grouping? -When you train without specifying a grouping, DQX looks for one: columns that look like good dimensions (for example region, category) — low cardinality (2–50 distinct values), low null rate, and enough rows per group. That is intentional, because behaviour often does differ by group, and the user least likely to pass `baseline_by` explicitly is the one most likely to miss a contextual anomaly. A discovered grouping is used as `baseline_by`: each metric gains its deviation from that group's baseline, on a **single** model, so the discovery cannot turn into an hours-long run however many groups it finds. The grouping it chose is logged. To compare against the whole table instead, pass `baseline_by=[]`. +When you train without specifying a grouping, DQX looks for one: columns that look like good dimensions (for example region or category) with low cardinality (2–50 distinct values), low null rate, and enough rows per group. That is intentional, because behaviour often does differ by group, and the user least likely to pass `baseline_by` explicitly is the one most likely to miss a contextual anomaly. A discovered grouping is used as `baseline_by`: each metric gains its deviation from that group's baseline, on a **single** model, so the discovery cannot turn into an hours-long run however many groups it finds. The grouping it chose is logged. To compare against the whole table instead, pass `baseline_by=[]`.
diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 88ca2ea56..ad31f8873 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -3513,7 +3513,7 @@ Pass an `AnomalyParams` object to the `params` argument to customize training be | `max_rows` | int | 1,000,000 | Maximum rows to use for training. Caps memory usage for very large datasets. | | `train_ratio` | float | 0.8 | Train/validation split ratio (80% train, 20% validation). | | `ensemble_size` | int or None | 3 | Number of models in ensemble. Set to None for single model. More models = more robust but slower training. Applies to a single global model only: **segmented training always trains one model per segment and ignores this setting**, so `confidence_std` is unavailable for segmented models. | -| `baseline_by` | list[str] or None | None | Columns identifying the group a row belongs to, so each metric is judged against its own group's baseline rather than against the whole table. Adds one feature per metric — its signed log-ratio to that group's median — on a **single** model, so cost does not grow with the group count. Normally set by passing `baseline_by` to `train()`. See [Baseline conditioning](/docs/guide/row_anomaly_detection#baseline-conditioning). | +| `baseline_by` | list[str] or None | None | Columns identifying the group a row belongs to, so each metric is judged against its own group's baseline rather than against the whole table. Adds one feature per metric — its signed log-ratio to that group's median — on a **single** model, so cost does not grow with the group count. Normally set by passing `baseline_by` to `train()`. See [Group-aware anomaly detection](/docs/guide/row_anomaly_detection#group-aware-anomaly-detection). | | `max_segment_models` | int | 50 | Ceiling on per-segment models one run will attempt, guarding the legacy `segment_by` path. Cost there is linear in the segment count and segmented training does not ensemble, so 90 segments measures roughly 88 minutes. Exceeding this raises rather than warns. Irrelevant to `baseline_by`. | #### IsolationForestConfig (Algorithm Parameters) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index 948524b78..4f99635be 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -41,7 +41,7 @@ class AnomalyEngine(DQEngineBase): model_name="catalog.schema.regional_model", registry_table="catalog.schema.dqx_anomaly_models", columns=["revenue", "transactions"], - segment_by=["region"] + baseline_by=["region"] ) """ @@ -60,22 +60,21 @@ def train( model_name: str, registry_table: str, columns: list[str] | None = None, - segment_by: list[str] | None = None, params: AnomalyParams | None = None, exclude_columns: list[str] | None = None, expected_anomaly_rate: float = 0.02, baseline_by: list[str] | None = None, ) -> str: """ - Train row anomaly detection model(s) with intelligent auto-discovery. + Train a row anomaly detection model with intelligent auto-discovery. Requires Spark >= 3.4 and the 'anomaly' extras installed: pip install 'databricks-labs-dqx[anomaly]' Auto-discovery behavior: - - columns=None, segment_by=None: Auto-discovers both (simplest) - - columns specified, segment_by=None: Uses columns, no segmentation - - columns=None, segment_by specified: Auto-discovers columns, uses segments + - columns=None, baseline_by=None: Auto-discovers both the feature columns and a grouping + - columns specified, baseline_by=None: Uses the columns, still discovers a grouping + - baseline_by specified: Conditions on that grouping Args: df: Input DataFrame containing historical "normal" data. @@ -89,14 +88,7 @@ def train( numeric metric gains its deviation from that baseline as an extra feature on a single pooled model, so the cost does not grow with the group count. This is what catches a value that is unremarkable across the table but wrong for - its own group. Cannot be combined with `segment_by`. - segment_by: Legacy. Trains one model per group instead. Kept for compatibility, and not - recommended: on the Server Machine Dataset per-group models were the worst - of three configurations, with one entity producing 15,963 false positives - on 28,392 normal rows, and cost is linear in the group count (90 groups - measures roughly 88 minutes, capped by `params.max_segment_models`). Prefer - `baseline_by`. Auto-discovered when both `columns` and `segment_by` are - omitted, in which case the discovered grouping is used as `baseline_by`. + its own group. Auto-discovered when omitted. params: Optional anomaly parameters for tuning training behavior. exclude_columns: Columns to exclude from training (e.g., IDs, labels, ground truth). Exclusions always take precedence over `columns` if both are provided. @@ -115,9 +107,7 @@ def train( - See documentation for detailed column selection best practices. Returns: - Base model name (e.g., 'catalog.schema.model_name'). For segmented models, - individual segments are stored with suffixes like '__seg_region=APAC', but - the base name is returned for simplified API usage. + The model name (e.g., 'catalog.schema.model_name'). Examples: # Auto-discovery with default 2% expected anomaly rate (simplest) @@ -157,9 +147,8 @@ def train( columns=["revenue", "transactions"], ) - # Judge each row against its own group rather than the whole table. With few - # groups this trains one model each; with many it switches to group-relative - # features, which keeps one model however many groups there are. + # Judge each row against its own group's baseline rather than the whole table, on a + # single model however many groups there are. anomaly_engine.train( df, model_name="catalog.schema.regional_model", @@ -174,7 +163,6 @@ def train( model_name, registry_table, columns=columns, - segment_by=segment_by, params=params, exclude_columns=exclude_columns, expected_anomaly_rate=expected_anomaly_rate, diff --git a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py index 1c4a99076..dd6b17a64 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py @@ -35,7 +35,6 @@ StructField("is_anomaly", BooleanType(), True), StructField("threshold", DoubleType(), True), StructField("model", StringType(), True), - StructField("segment", MapType(StringType(), StringType()), True), StructField("contributions", MapType(StringType(), DoubleType()), True), StructField("confidence_std", DoubleType(), True), StructField("ai_explanation", ai_explanation_struct_schema, True), diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 093a50dcc..23f579be5 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -707,18 +707,17 @@ def _add_explanation_column_ai_query( def add_explanation_column( df: DataFrame, ctx: ExplanationContext, - segment_values: dict[str, str] | None, is_ensemble: bool, drift_summary: str = "none", endpoint_reachable: bool | None = None, ) -> DataFrame: """Add the AI explanation column to df using the group-based algorithm. - Anomalous rows are bucketed by a deterministic (segment, pattern) key — pattern = - sorted top-2 contributing SHAP features. The LLM is called once per group via the Spark SQL - ``ai_query`` function against a Databricks Model Serving endpoint, and every row in that group - receives the same narrative/business_impact/action, plus the group's size and mean severity. - Rows below threshold or in groups exceeding ``ctx.max_groups`` receive a null struct. + Anomalous rows are bucketed by a deterministic pattern key — the sorted top-2 contributing SHAP + features. The LLM is called once per group via the Spark SQL ``ai_query`` function against a + Databricks Model Serving endpoint, and every row in that group receives the same + narrative/business_impact/action, plus the group's size and mean severity. Rows below threshold + or in groups exceeding ``ctx.max_groups`` receive a null struct. Preconditions (caller's responsibility): - df has ctx.score_std_col, ctx.severity_col, and ctx.contributions_col. @@ -726,20 +725,16 @@ def add_explanation_column( Args: df: Scored DataFrame to annotate with the explanation column. ctx: Explanation inputs (columns, threshold, model, redaction, budget). - segment_values: Segment key/value pairs for this run, or None for a global model. is_ensemble: Whether the scoring model is an ensemble (drives the confidence label). drift_summary: Baseline-drift summary string for the prompt, or "none". endpoint_reachable: Pre-computed serving-endpoint reachability. When None (default) the - endpoint is probed here with a single 1-token ai_query call. Callers that invoke this - repeatedly in one scoring run (e.g. once per segment) should probe once via - probe_endpoint_reachable and pass the result to avoid one billable probe per call. + endpoint is probed here with a single 1-token ai_query call. Raises: InvalidParameterError: When *model_name* does not resolve to a Databricks serving endpoint. """ redact_set = redaction_set(ctx.redact_columns) - segment_str = _format_segment(segment_values, redact_set) df_with_pattern = df.withColumn(ctx.pattern_col, _pattern_spark_expr(ctx.contributions_col, redact_set)) return _add_explanation_column_ai_query( - df_with_pattern, ctx, segment_str, is_ensemble, drift_summary, endpoint_reachable=endpoint_reachable + df_with_pattern, ctx, "", is_ensemble, drift_summary, endpoint_reachable=endpoint_reachable ) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_workflow.py b/src/databricks/labs/dqx/anomaly/anomaly_workflow.py index 8286e63ab..62bfda9b9 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_workflow.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_workflow.py @@ -45,7 +45,6 @@ def train_model(self, ctx: WorkflowContext) -> None: anomaly_engine.train( df=df, columns=anomaly_config.columns, - segment_by=anomaly_config.segment_by, baseline_by=anomaly_config.baseline_by, model_name=model_name, registry_table=registry_table, diff --git a/src/databricks/labs/dqx/anomaly/check_funcs.py b/src/databricks/labs/dqx/anomaly/check_funcs.py index b65cf747e..c22fb867f 100644 --- a/src/databricks/labs/dqx/anomaly/check_funcs.py +++ b/src/databricks/labs/dqx/anomaly/check_funcs.py @@ -12,7 +12,7 @@ import pyspark.sql.functions as F from pyspark.sql import Column, DataFrame -from databricks.labs.dqx.anomaly.model_discovery import fetch_model_columns_and_segments +from databricks.labs.dqx.anomaly.model_discovery import fetch_model_columns from databricks.labs.dqx.anomaly.scoring_config import ScoringConfig, ScoringOutputColumns from databricks.labs.dqx.anomaly.scoring_utils import check_reserved_row_id_columns from databricks.labs.dqx.anomaly.scoring_orchestrator import run_anomaly_scoring @@ -135,7 +135,7 @@ def has_no_row_anomalies( Auto-discovery: - columns: Inferred from model registry - - segmentation: Inferred from model registry (checks if model is segmented) + - baseline grouping: Inferred from the model's persisted metadata Output columns: - _dq_info: Array of structs (one element per dataset-level check). For example: @@ -144,7 +144,6 @@ def has_no_row_anomalies( - _dq_info[0].anomaly.is_anomaly: Boolean flag - _dq_info[0].anomaly.threshold: Severity percentile threshold used (0–100) - _dq_info[0].anomaly.model: Model name - - _dq_info[0].anomaly.segment: Segment values (if segmented) - _dq_info[0].anomaly.contributions: SHAP contributions as percentages (0–100); populated only for anomalous rows, null otherwise - _dq_info[0].anomaly.confidence_std: Ensemble std (if requested) @@ -155,7 +154,7 @@ def has_no_row_anomalies( Notes: DQX always scores using the columns the model was trained on. DQX aligns scored rows back to the input using an internal row id and removes it before returning. - Segmentation is inferred from the trained model configuration. + Baseline conditioning is inferred from the trained model's metadata. Rows whose group was never seen in training are reported (`is_new_baseline`) but are **not** flagged as violations: neither categorical encoder can represent an unseen value honestly @@ -264,7 +263,7 @@ def has_no_row_anomalies( def apply(df: DataFrame) -> DataFrame: check_reserved_row_id_columns(df) df_to_score = df.withColumn(row_id_col, F.monotonically_increasing_id()) - columns, segment_by = fetch_model_columns_and_segments(df_to_score, model_name, registry_table) + columns = fetch_model_columns(df_to_score, model_name, registry_table) config = ScoringConfig( columns=columns, @@ -280,7 +279,6 @@ def apply(df: DataFrame) -> DataFrame: llm_model_config=llm_model_config, redact_columns=redact_columns or [], max_groups=max_groups, - segment_by=segment_by, driver_only=driver_only, output_columns=output_columns, ) diff --git a/src/databricks/labs/dqx/anomaly/drift.py b/src/databricks/labs/dqx/anomaly/drift.py index d2c97d1ea..8f5ede477 100644 --- a/src/databricks/labs/dqx/anomaly/drift.py +++ b/src/databricks/labs/dqx/anomaly/drift.py @@ -16,7 +16,6 @@ prepare_feature_metadata, ) from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRecord -from databricks.labs.dqx.anomaly.segment_utils import build_segment_name # Minimum sample size for reliable drift detection # Small batches have high variance and lead to false positives @@ -216,37 +215,6 @@ def prepare_drift_df( return engineered_df, feature_metadata.engineered_feature_names -def check_segment_drift( - segment_df: DataFrame, - columns: list[str], - segment_model: AnomalyModelRecord, - drift_threshold: float | None, - drift_threshold_value: float, -) -> DriftResult | None: - """Check and warn about data drift in a segment. Returns the DriftResult when computed.""" - if drift_threshold is not None and segment_model.training.baseline_stats: - drift_df, drift_columns = prepare_drift_df(segment_df, columns, segment_model) - drift_result = compute_drift_score( - drift_df, - drift_columns, - segment_model.training.baseline_stats, - drift_threshold_value, - ) - - if drift_result.drift_detected: - drifted_cols_str = ", ".join(drift_result.drifted_columns) - segment_name = build_segment_name(segment_model.segmentation.segment_values) or "unknown" - warnings.warn( - f"Data drift detected in segment '{segment_name}', columns: {drifted_cols_str} " - f"(drift score: {drift_result.drift_score:.2f}). " - f"Consider retraining the segmented anomaly model.", - UserWarning, - stacklevel=5, - ) - return drift_result - return None - - def check_and_warn_drift( df: DataFrame, columns: list[str], diff --git a/src/databricks/labs/dqx/anomaly/group_config.py b/src/databricks/labs/dqx/anomaly/group_config.py index 0519277df..aeaa5549d 100644 --- a/src/databricks/labs/dqx/anomaly/group_config.py +++ b/src/databricks/labs/dqx/anomaly/group_config.py @@ -1,57 +1,21 @@ -"""Thresholds governing per-segment model training. +"""Thresholds governing baseline conditioning. -These were previously inline literals spread across the profiler and the training service. Naming -them in one place makes the policy reviewable. +Named in one place so the policy is reviewable. These bound a *statistical* requirement, not a cost +one: baseline conditioning trains a single model however many groups exist, so breadth is affordable +and the only question is whether each group has enough rows to yield a representative median. -All of them now guard the legacy ``segment_by`` path — the only path that trains one model per -group, and therefore the only one whose cost grows with the group count. ``baseline_by`` trains a -single pooled model however many groups there are, so it needs no ceiling. +The segmented thresholds that used to live here went with the ``segment_by`` path they guarded. They +were the opposite shape — few groups, hundreds of rows each — because a per-segment model had to +train a forest on every group. Applying them to baseline conditioning was a measured defect: on a +dataset grouped by country x event_type x product (90 groups) discovery selected the single +lowest-cardinality column, giving 3 groups, and conditioning barely engaged. See https://github.com/databrickslabs/dqx/issues/1484 for the measurements behind them. """ -# Hard ceiling on the number of per-segment models a single training run will attempt. Segmented -# training does not ensemble, so cost is linear in the segment count: 90 segments measured roughly -# 88 minutes, and a 90-segment run was cancelled after 70 minutes without finishing. Overridable -# via ``AnomalyParams.max_segment_models``. -MAX_SEGMENT_MODELS = 50 - -# Below this many rows per segment on average, a per-segment model is calibrated on too small a -# sample to be trustworthy. Severity is a percentile of each model's own training scores, so a thin -# sample yields a fragile threshold rather than an obviously bad one. Used by the profiler to warn. -MIN_ROWS_PER_SEGMENT = 100 - -# Segment count above which training logs a slow-training warning. Retained at its historical value -# so existing runs keep warning where they used to; MAX_SEGMENT_MODELS is the ceiling that actually -# stops a run. -SEGMENT_COUNT_WARN_THRESHOLD = 100 - -# A segment with fewer rows than this is skipped and gets no model at all, so its rows come back -# unscored. Distinct from MIN_ROWS_PER_SEGMENT, which is the average below which per-segment -# modelling is merely discouraged. -MIN_ROWS_TO_TRAIN_SEGMENT = 10 - -# Upper bound on the distinct values a column may have to be *recommended* as a grouping by -# auto-discovery on the legacy segmented path. Conservative on purpose: each distinct value there -# becomes its own model. -MAX_AUTO_GROUP_COUNT = 20 - - -# --- baseline conditioning ---------------------------------------------------------------------- -# -# These deliberately differ from the segmented thresholds above, because the cost model differs. -# A per-segment model must *train a forest* on its group, so it needs hundreds of rows and the group -# count is a direct cost. A baseline group only has to yield a *median*, and there is one model -# however many groups exist — so the constraint is statistical, not economic. -# -# Applying the segmented thresholds to baseline_by was a real defect: on a dataset whose natural -# grouping was country x event_type x product (90 groups), discovery selected the single -# lowest-cardinality column — 3 groups — and conditioning barely engaged, scoring PR-AUC 0.1302 -# against 0.1176 unconditioned. Finer grouping is what makes a baseline tight. - -# Rows per group needed for a representative median. Well below MIN_ROWS_PER_SEGMENT because a -# median is a far cheaper statistic than a fitted forest: percentile_approx over a few dozen rows is -# a usable centre, where a forest over the same rows is not a usable model. +# Rows per group needed for a representative median. A median is a far cheaper statistic than a +# fitted model: percentile_approx over a few dozen rows is a usable centre, where a forest over the +# same rows is not a usable model. MIN_ROWS_PER_BASELINE_GROUP = 30 # Ceiling on total baseline groups. Not a training cost — it bounds what gets persisted in the diff --git a/src/databricks/labs/dqx/anomaly/model_config.py b/src/databricks/labs/dqx/anomaly/model_config.py index 49cef2e84..032b4b6db 100644 --- a/src/databricks/labs/dqx/anomaly/model_config.py +++ b/src/databricks/labs/dqx/anomaly/model_config.py @@ -57,8 +57,8 @@ class FeatureEngineering: @dataclass -class SegmentationConfig: - """How a model relates to groups in the data (6 fields). +class GroupingConfig: + """How a model is conditioned on groups in the data. ``baseline_by`` is duplicated here from the feature metadata on purpose. It also lives inside the ``features.feature_metadata`` JSON blob, which is where scoring reads it, but a JSON blob is not @@ -67,10 +67,7 @@ class SegmentationConfig: ``training.columns``. """ - segment_by: list[str] | None = None - segment_values: dict[str, str] | None = None baseline_by: list[str] | None = None - is_global_model: bool = True sklearn_version: str | None = None config_hash: str | None = None @@ -83,7 +80,7 @@ class AnomalyModelRecord: - identity: Core model identification (5 fields) - training: Training configuration and metrics (6 fields) - features: Feature engineering metadata (5 fields) - - segmentation: Grouping configuration (6 fields) + - grouping: Baseline-conditioning configuration (3 fields) Stored as nested structs in Delta tables (no flattening needed). """ @@ -91,28 +88,27 @@ class AnomalyModelRecord: identity: ModelIdentity training: TrainingMetadata features: FeatureEngineering - segmentation: SegmentationConfig + grouping: GroupingConfig -def compute_config_hash(columns: list[str], segment_by: list[str] | None, baseline_by: list[str] | None = None) -> str: +def compute_config_hash(columns: list[str], baseline_by: list[str] | None = None) -> str: """Generate stable hash of model configuration. Args: columns: List of column names used for training - segment_by: List of columns used for segmentation, or None baseline_by: Columns the metrics are judged against, or None Returns: 16-character hex string (first 16 chars of SHA256 hash) Note: - This hash uniquely identifies a model configuration based on the sorted, order-independent - lists of feature columns, segment columns and baseline columns. It is used for collision - detection when the same model_name is reused with a different configuration. + This hash uniquely identifies a model configuration from the sorted, order-independent lists + of feature columns and baseline columns. It is used for collision detection when the same + model_name is reused with a different configuration. - **Breaking change.** *baseline_by* joined the hash inputs in 0.17.0, which changes the hash of - every configuration -- including ones with no grouping, since the key is present either way. - A model registered before that therefore fails the configuration check in + **Breaking change.** *baseline_by* joined the hash inputs in 0.17.0, and the legacy + *segment_by* left it. Both change the hash of every configuration, so a model registered by + an earlier version fails the configuration check in :func:`~databricks.labs.dqx.anomaly.scoring_run.score_global_model` and must be retrained. That is deliberate: without *baseline_by* in the hash, retraining under the same name with a different grouping produced an identical hash, so the one thing this hash exists to catch -- @@ -122,7 +118,6 @@ def compute_config_hash(columns: list[str], segment_by: list[str] | None, baseli """ config = { "columns": sorted(columns), - "segment_by": sorted(segment_by) if segment_by else None, "baseline_by": sorted(baseline_by) if baseline_by else None, } config_str = json.dumps(config, sort_keys=True) diff --git a/src/databricks/labs/dqx/anomaly/model_discovery.py b/src/databricks/labs/dqx/anomaly/model_discovery.py index 5fa2d9602..1be724c28 100644 --- a/src/databricks/labs/dqx/anomaly/model_discovery.py +++ b/src/databricks/labs/dqx/anomaly/model_discovery.py @@ -1,6 +1,4 @@ -"""Discover model columns, segments, and quantile points from the anomaly registry.""" - -from datetime import datetime +"""Discover model columns and quantile points from the anomaly registry.""" from pyspark.sql import DataFrame @@ -14,32 +12,16 @@ def get_record_for_discovery( registry_table: str, model_name_local: str, ) -> AnomalyModelRecord: - """Get model record for auto-discovery, checking global and segmented models.""" + """Get the active model record for auto-discovery.""" record = registry_client.get_active_model(registry_table, model_name_local) - if record: return record - all_segments = registry_client.get_all_segment_models(registry_table, model_name_local) - if all_segments: - return select_segment_record(all_segments) - raise InvalidParameterError( f"Model '{model_name_local}' not found in '{registry_table}'. " "Train first using anomaly.train(...)." ) -def select_segment_record(all_segments: list[AnomalyModelRecord]) -> AnomalyModelRecord: - """Select a deterministic segment record (latest training_time, tie-breaker by model_name).""" - return max( - all_segments, - key=lambda record: ( - record.training.training_time or datetime.min, - record.identity.model_name, - ), - ) - - def get_quantile_points_for_severity(record: AnomalyModelRecord) -> list[tuple[float, float]]: """Extract percentile->score points for severity mapping. @@ -69,22 +51,16 @@ def extract_quantile_points(record: AnomalyModelRecord) -> list[tuple[float, flo return points -def fetch_model_columns_and_segments( +def fetch_model_columns( df: DataFrame, model_name: str, registry_table: str, -) -> tuple[list[str], list[str] | None]: - """Auto-discover columns and segmentation from the model registry. - - Returns: - Tuple of (columns, segment_by). - """ +) -> list[str]: + """Auto-discover the feature columns a model was trained on, from the registry.""" registry_client = AnomalyModelRegistry(df.sparkSession) record = get_record_for_discovery(registry_client, registry_table, model_name) columns = list(record.training.columns) - segment_by = record.segmentation.segment_by - missing_columns = [c for c in columns if c not in df.columns] if missing_columns: raise InvalidParameterError( @@ -92,4 +68,4 @@ def fetch_model_columns_and_segments( f"Available columns: {df.columns}." ) - return columns, segment_by + return columns diff --git a/src/databricks/labs/dqx/anomaly/model_loader.py b/src/databricks/labs/dqx/anomaly/model_loader.py index 44e28fa52..0aebf3b16 100644 --- a/src/databricks/labs/dqx/anomaly/model_loader.py +++ b/src/databricks/labs/dqx/anomaly/model_loader.py @@ -34,7 +34,7 @@ def load_sklearn_model_with_error_handling(model_uri: str, model_record: Anomaly return mlflow.sklearn.load_model(model_uri) except (ValueError, AttributeError, TypeError) as e: error_msg = str(e) - trained_version = model_record.segmentation.sklearn_version or "unknown" + trained_version = model_record.grouping.sklearn_version or "unknown" current_version = sklearn.__version__ python_version = f"{sys.version_info.major}.{sys.version_info.minor}" diff --git a/src/databricks/labs/dqx/anomaly/model_registry.py b/src/databricks/labs/dqx/anomaly/model_registry.py index e44509302..e64c92982 100644 --- a/src/databricks/labs/dqx/anomaly/model_registry.py +++ b/src/databricks/labs/dqx/anomaly/model_registry.py @@ -9,16 +9,14 @@ import pyspark.sql.functions as F from pyspark.sql import DataFrame, SparkSession -from pyspark.sql.window import Window from databricks.labs.dqx.anomaly.model_config import ( AnomalyModelRecord, FeatureEngineering, + GroupingConfig, ModelIdentity, - SegmentationConfig, TrainingMetadata, ) -from databricks.labs.dqx.anomaly.segment_utils import build_segment_name from databricks.labs.dqx.config import OutputConfig from databricks.labs.dqx.io import save_dataframe_as_table from databricks.labs.dqx.utils import table_exists @@ -31,8 +29,7 @@ "baseline_stats:map>>, " "features struct, feature_metadata:string, " "feature_importance:map, temporal_config:map>, " - "segmentation struct, segment_values:map, " - "baseline_by:array, is_global_model:boolean, sklearn_version:string, config_hash:string>" + "grouping struct, sklearn_version:string, config_hash:string>" ) @@ -85,13 +82,10 @@ def build_model_df(spark: SparkSession, record: AnomalyModelRecord) -> DataFrame "feature_importance": record.features.feature_importance, "temporal_config": record.features.temporal_config, }, - "segmentation": { - "segment_by": record.segmentation.segment_by, - "segment_values": record.segmentation.segment_values, - "baseline_by": record.segmentation.baseline_by, - "is_global_model": record.segmentation.is_global_model, - "sklearn_version": record.segmentation.sklearn_version, - "config_hash": record.segmentation.config_hash, + "grouping": { + "baseline_by": record.grouping.baseline_by, + "sklearn_version": record.grouping.sklearn_version, + "config_hash": record.grouping.config_hash, }, } @@ -109,10 +103,10 @@ def save_model(self, record: AnomalyModelRecord, table: str) -> None: self._archive_previous(table, record.identity.model_name) df = self.build_model_df(self.spark, record) - # mergeSchema so a registry table created by an earlier DQX gains new struct fields on the - # next write instead of failing. Without it, adding `baseline_by` to the segmentation struct - # would make retraining fail against an existing table -- and retraining is exactly what the - # configuration-hash error tells the user to do, so the remedy has to work. + # mergeSchema so a registry table created by an earlier DQX reconciles to the new struct + # shape on the next write instead of failing -- the `grouping` struct replaced `segmentation` + # this release. Retraining is exactly what the configuration-hash error tells the user to do, + # and it writes to their existing table, so the remedy has to work. save_dataframe_as_table(df, OutputConfig(location=table, mode="append", options={"mergeSchema": "true"})) def get_active_model(self, table: str, model_name: str) -> AnomalyModelRecord | None: @@ -136,54 +130,11 @@ def get_active_model(self, table: str, model_name: str) -> AnomalyModelRecord | identity=ModelIdentity(**values["identity"]), training=TrainingMetadata(**values["training"]), features=FeatureEngineering(**values["features"]), - segmentation=SegmentationConfig(**values["segmentation"]), + grouping=GroupingConfig(**values["grouping"]), ) return record - def get_segment_model( - self, table: str, base_model_name: str, segment_values: dict[str, str] - ) -> AnomalyModelRecord | None: - """Fetch model for specific segment combination.""" - if not table_exists(self.spark, table): - return None - - # Build segment name matching the training logic - segment_name = build_segment_name(segment_values) - segment_model_name = f"{base_model_name}__seg_{segment_name}" - - return self.get_active_model(table, segment_model_name) - - def get_all_segment_models(self, table: str, base_model_name: str) -> list[AnomalyModelRecord]: - """Fetch all segment models for a base name.""" - if not table_exists(self.spark, table): - return [] - - # Get all active models that start with base_model_name__seg_ - # Use window function to get only the latest version of each segment - df = self.spark.table(table).filter( - (F.col("identity.model_name").startswith(f"{base_model_name}__seg_")) - & (F.col("identity.status") == "active") - ) - - # Deduplicate by model_name (segment), taking the most recent by training_time - window = Window.partitionBy("identity.model_name").orderBy(F.col("training.training_time").desc()) - df_deduped = df.withColumn("row_num", F.row_number().over(window)).filter(F.col("row_num") == 1).drop("row_num") - - # Warning is produced earlier, for 100+ segments this could be a memory concern - rows = df_deduped.orderBy(F.col("training.training_time").desc()).collect() - - # Convert nested Row structures to dataclasses - return [ - AnomalyModelRecord( - identity=ModelIdentity(**row.asDict(recursive=True)["identity"]), - training=TrainingMetadata(**row.asDict(recursive=True)["training"]), - features=FeatureEngineering(**row.asDict(recursive=True)["features"]), - segmentation=SegmentationConfig(**row.asDict(recursive=True)["segmentation"]), - ) - for row in rows - ] - def _create_table(self, table: str) -> None: empty_df = self.spark.createDataFrame([], schema=ANOMALY_MODEL_TABLE_SCHEMA) save_dataframe_as_table(empty_df, OutputConfig(location=table, mode="overwrite")) diff --git a/src/databricks/labs/dqx/anomaly/profiler.py b/src/databricks/labs/dqx/anomaly/profiler.py index 66d48fe4a..f318509cb 100644 --- a/src/databricks/labs/dqx/anomaly/profiler.py +++ b/src/databricks/labs/dqx/anomaly/profiler.py @@ -26,12 +26,9 @@ ) from databricks.labs.dqx.anomaly.group_config import ( - MAX_AUTO_GROUP_COUNT, MAX_BASELINE_COLUMN_CARDINALITY, MAX_BASELINE_GROUPS, - MAX_SEGMENT_MODELS, MIN_ROWS_PER_BASELINE_GROUP, - MIN_ROWS_PER_SEGMENT, ) from databricks.labs.dqx.profiling_utils import compute_exact_distinct_counts, compute_null_and_distinct_counts @@ -51,12 +48,12 @@ class AnomalyProfile: unsupported_columns: list[str] | None = None # NEW: columns that cannot be used -def auto_discover_columns(df: DataFrame, *, for_baseline: bool = False) -> AnomalyProfile: +def auto_discover_columns(df: DataFrame) -> AnomalyProfile: """ - Auto-discover columns and segments for row anomaly detection. + Auto-discover feature columns and a baseline grouping for row anomaly detection. - Analyzes the DataFrame using on-the-fly heuristics to recommend - suitable columns and segmentation strategy. + Analyzes the DataFrame using on-the-fly heuristics to recommend suitable columns and a grouping + to condition on. Column selection criteria: - Numeric types (int, long, float, double, decimal) @@ -64,24 +61,19 @@ def auto_discover_columns(df: DataFrame, *, for_baseline: bool = False) -> Anoma - null_rate < 50% - Exclude: timestamps, IDs (detected by name patterns) - Segment selection criteria: - - Categorical types (string, int with low cardinality) - - Distinct values: 2-50 (inclusive) + Baseline grouping criteria (see :func:`select_baseline_columns`): + - Categorical types (string, int) with 2-50 distinct values - null_rate < 10% - - At least 1000 rows per segment (warn if violated) + - At least MIN_ROWS_PER_BASELINE_GROUP rows per resulting group Args: df: DataFrame to analyze. - for_baseline: Select the grouping for baseline conditioning rather than for the legacy - segmented path. Baseline conditioning trains one model whatever the group count, so it - can afford a finer grouping and only needs enough rows per group to take a median. See - :func:`select_baseline_columns`. Returns: AnomalyProfile with recommendations and warnings. """ warnings: list[str] = [] - return _auto_discover_heuristic(df, warnings, for_baseline=for_baseline) + return _auto_discover_heuristic(df, warnings) def _compute_numeric_stats_batched(df: DataFrame, column_names: list[str]) -> dict[str, dict[str, float]]: @@ -208,22 +200,6 @@ def _select_top_columns( return recommended_columns, column_types -def _validate_and_add_segment_column( - df: DataFrame, - col_name: str, - warnings: list[str], -) -> bool: - """Validate minimum segment size and add warnings if needed. Returns True if column should be added.""" - min_segment_size_row = df.groupBy(col_name).count().select(F.min("count").alias("min_count")).first() - min_segment_size = min_segment_size_row["min_count"] if min_segment_size_row else None - if min_segment_size is not None and min_segment_size < 1000: - warnings.append( - f"Segment column '{col_name}' has segments with <1000 rows (min: {min_segment_size}), " - "models may be unreliable." - ) - return True - - def _check_high_cardinality_warning( field: Any, col_name: str, @@ -234,41 +210,23 @@ def _check_high_cardinality_warning( if isinstance(field.dataType, StringType) and "id" not in col_name.lower(): warnings.append( f"Column '{col_name}' has {distinct_count} distinct values, " - "excluding from auto-selection (too high cardinality for segmentation)." + "excluding from auto-selection (too high cardinality for a baseline grouping)." ) -def _calculate_total_segments( - recommended_segments: list[str], - warnings: list[str], - *, - total_count: int, - distinct_counts: dict[str, int], -) -> int: - """Calculate total segment combinations and add warning if too many.""" +def _count_group_combinations(recommended_segments: list[str], distinct_counts: dict[str, int]) -> int: + """Total groups the chosen grouping produces, as the product of its columns' cardinalities. + + No warnings here any more. This used to caution about too many segments or too few rows each, + which mattered when every group became its own model. :func:`select_baseline_columns` now enforces + both bounds while choosing, so neither condition can survive to be warned about. + """ if not recommended_segments: return 1 - - segment_count = 1 + combinations = 1 for col in recommended_segments: - distinct_count = distinct_counts[col] - segment_count *= distinct_count - - # Warn if segments are too granular relative to data size - avg_rows_per_segment = total_count / segment_count if segment_count > 0 else 0 - - if segment_count > MAX_SEGMENT_MODELS: - warnings.append( - f"Detected {segment_count} total segments, training may be slow. " - "Consider filtering or using coarser segmentation." - ) - elif avg_rows_per_segment < MIN_ROWS_PER_SEGMENT: - warnings.append( - f"Detected {segment_count} segments with only ~{int(avg_rows_per_segment)} rows per segment on average. " - f"Models may be unreliable. Consider reducing segmentation or using more data (total rows: {total_count})." - ) - - return segment_count + combinations *= distinct_counts[col] + return combinations def _is_grouping_candidate( @@ -277,22 +235,18 @@ def _is_grouping_candidate( null_rate: float, is_id_column: bool, total_count: int, - for_baseline: bool, ) -> bool: - """Whether a column may be *considered* as a grouping. + """Whether a column may be *considered* as a baseline grouping. - The bar differs by destination, for the same reason the selection policy does: a segment has to - support training a forest on its own rows, a baseline group only has to yield a median. Identifier - -like names and columns that are more than 10% null are rejected either way -- a grouping keyed on - something nearly unique or frequently missing is not a peer group. + Identifier-like names and columns more than 10% null are rejected: a grouping keyed on something + nearly unique or frequently missing is not a peer group. The row requirement is only what a + median needs, since there is one model regardless of how many groups result. """ - max_cardinality = MAX_BASELINE_COLUMN_CARDINALITY if for_baseline else MAX_AUTO_GROUP_COUNT - min_rows = MIN_ROWS_PER_BASELINE_GROUP if for_baseline else MIN_ROWS_PER_SEGMENT return ( - 2 <= distinct_count <= max_cardinality + 2 <= distinct_count <= MAX_BASELINE_COLUMN_CARDINALITY and null_rate < 0.1 and not is_id_column - and (total_count / distinct_count) >= min_rows + and (total_count / distinct_count) >= MIN_ROWS_PER_BASELINE_GROUP ) @@ -348,16 +302,12 @@ def _select_segment_columns( total_count: int, null_counts: dict[str, int], distinct_counts: dict[str, int], - for_baseline: bool = False, ) -> tuple[list[str], int]: - """Identify and validate segment columns. + """Identify baseline grouping columns and count the groups they produce. - *for_baseline* selects the grouping policy. See :func:`select_baseline_columns` for why the two - differ: a per-segment model has to train on its group, a baseline group only has to yield a - median. + See :func:`select_baseline_columns` for the policy. """ - recommended_segments = [] - candidate_segments = [] # Track all viable candidates for user info + candidate_segments = [] categorical_types = (StringType, IntegerType) categorical_fields = [f for f in df.schema.fields if isinstance(f.dataType, categorical_types)] @@ -376,51 +326,22 @@ def _select_segment_columns( distinct_count = distinct_row[0] null_rate = null_counts.get(col_name, 0) / total_count if total_count > 0 else 1.0 - meets_segment_criteria = _is_grouping_candidate( + if _is_grouping_candidate( distinct_count, null_rate=null_rate, is_id_column=id_pattern.search(col_name) is not None, total_count=total_count, - for_baseline=for_baseline, - ) - - if meets_segment_criteria and _validate_and_add_segment_column(df, col_name, warnings): + ): candidate_segments.append((col_name, distinct_count, total_count / distinct_count)) elif distinct_count > 50: _check_high_cardinality_warning(field, col_name, distinct_count, warnings) - # Sort candidates by lowest cardinality first. For the segmented path that means "fewest models"; - # for the baseline path it means the cheapest granularity is added first. + # Cheapest granularity first, so the row budget is spent on coarse dimensions before fine ones + # and the grouping degrades gracefully on smaller tables. candidate_segments.sort(key=lambda x: x[1]) # Sort by distinct_count ascending + recommended_segments = select_baseline_columns(candidate_segments, total_count) - if candidate_segments and for_baseline: - recommended_segments.extend(select_baseline_columns(candidate_segments, total_count)) - elif candidate_segments: - # LEGACY SEGMENTED PATH: be conservative, take a single column. Every distinct value becomes - # its own model, so breadth here is paid for in training time. - selected = candidate_segments[0] - recommended_segments.append(selected[0]) - - # Log helpful info about selection and alternatives - logger.info( - f"Auto-segmentation selected 1 column: [{selected[0]}] " - f"({int(selected[1])} segments, ~{int(selected[2])} rows/segment)" - ) - - # Suggest additional segmentation options if available - if len(candidate_segments) > 1: - other_options = ", ".join( - [f"{col} ({int(dc)} segments)" for col, dc, _ in candidate_segments[1:4]] # Show up to 3 more - ) - logger.info( - f"Consider additional segmentation for more granularity: " - f"segment_by=['{selected[0]}', ] where could be: {other_options}" - ) - - # Calculate total segment combinations - segment_count = _calculate_total_segments( - recommended_segments, warnings, total_count=total_count, distinct_counts=distinct_counts - ) + segment_count = _count_group_combinations(recommended_segments, distinct_counts) return recommended_segments, segment_count @@ -518,15 +439,13 @@ def _compute_discovery_stats(df: DataFrame) -> tuple[dict[str, int], dict[str, i return null_counts, distinct_counts, numeric_stats, total_count -def _auto_discover_heuristic(df: DataFrame, warnings: list[str], *, for_baseline: bool = False) -> AnomalyProfile: +def _auto_discover_heuristic(df: DataFrame, warnings: list[str]) -> AnomalyProfile: """ Auto-discover using on-the-fly heuristics with multi-type support. Args: df: DataFrame to analyze. warnings: List to accumulate warnings. - for_baseline: Select the grouping for baseline conditioning rather than the legacy - segmented path. See :func:`select_baseline_columns`. Returns: AnomalyProfile with recommendations (max 10 columns). @@ -577,10 +496,10 @@ def _auto_discover_heuristic(df: DataFrame, warnings: list[str], *, for_baseline total_count=total_count, null_counts=null_counts, distinct_counts=distinct_counts, - for_baseline=for_baseline, ) - # Remove segment columns from feature columns (they would be constant within each segment) + # The grouping columns are the basis of comparison, not features, so drop them from the feature + # list. A column cannot be both what is measured and what it is measured against. if recommended_segments: recommended_columns = [col for col in recommended_columns if col not in recommended_segments] diff --git a/src/databricks/labs/dqx/anomaly/scoring_config.py b/src/databricks/labs/dqx/anomaly/scoring_config.py index 93fd1617c..d6ed570b4 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_config.py +++ b/src/databricks/labs/dqx/anomaly/scoring_config.py @@ -52,18 +52,14 @@ class ScoringConfig: drift_threshold: float | None = None enable_contributions: bool = True enable_confidence_std: bool = False - segment_by: list[str] | None = None driver_only: bool = False enable_ai_explanation: bool = True llm_model_config: LLMModelConfig | None = None redact_columns: list[str] = field(default_factory=list) # Global upper bound on the number of LLM calls per scoring run when - # *enable_ai_explanation* is True. Anomalous rows are bucketed by (segment, pattern) - # and one LLM call per bucket is made; *max_groups* caps the number of buckets that - # actually receive an explanation, ranked by ``group_size * group_avg_severity``. - # For segmented scoring (*segment_by* set), the budget is split equally across - # eligible segments — total LLM calls stay <= *max_groups* regardless of segment - # count. + # *enable_ai_explanation* is True. Anomalous rows are bucketed by their contribution pattern + # and one LLM call per bucket is made; *max_groups* caps the number of buckets that actually + # receive an explanation, ranked by ``group_size * group_avg_severity``. max_groups: int = 500 # Whether a row whose group was absent from training counts as a violation. Such a row gets a # null score and severity because neither encoder can represent an unseen category honestly, diff --git a/src/databricks/labs/dqx/anomaly/scoring_orchestrator.py b/src/databricks/labs/dqx/anomaly/scoring_orchestrator.py index 0065402e0..3a4ba07c9 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_orchestrator.py +++ b/src/databricks/labs/dqx/anomaly/scoring_orchestrator.py @@ -1,12 +1,10 @@ -"""Orchestrates anomaly scoring: route global vs segmented, run global model pipeline.""" +"""Orchestrates anomaly scoring: resolve the registered model and run the scoring pipeline.""" from pyspark.sql import DataFrame -from databricks.labs.dqx.anomaly.model_discovery import select_segment_record from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRegistry from databricks.labs.dqx.anomaly.scoring_config import ScoringConfig from databricks.labs.dqx.anomaly.scoring_strategies import resolve_scoring_strategy -from databricks.labs.dqx.anomaly.scoring_run import load_segment_models from databricks.labs.dqx.errors import InvalidParameterError @@ -16,37 +14,18 @@ def run_anomaly_scoring( registry_table: str, model_name: str, ) -> DataFrame: - """Route to segmented or global scoring and return scored DataFrame (caller drops row_id_col).""" - registry_client = AnomalyModelRegistry(df_to_score.sparkSession) - if config.segment_by: - all_segments = load_segment_models(registry_client, config) - strategy = resolve_scoring_strategy(all_segments[0].identity.algorithm) - return strategy.score_segmented(df_to_score, config, registry_client, all_segments) + """Score with the registered model and return the scored DataFrame (caller drops row_id_col). + There is one model per registered name. The routing that used to choose between segmented and + global scoring, and the fallback that looked for ``__seg_`` models when a name was not found, + both went with the ``segment_by`` path: a grouping is now expressed as features on a single + model. + """ + registry_client = AnomalyModelRegistry(df_to_score.sparkSession) record = registry_client.get_active_model(registry_table, model_name) if not record: - fallback = try_segmented_scoring_fallback(df_to_score, config, registry_client) - if fallback is not None: - return fallback raise InvalidParameterError( f"Model '{model_name}' not found in '{registry_table}'. Train first using anomaly.train(...)." ) strategy = resolve_scoring_strategy(record.identity.algorithm) return strategy.score_global(df_to_score, record, config) - - -def try_segmented_scoring_fallback( - df: DataFrame, - config: ScoringConfig, - registry_client: AnomalyModelRegistry, -) -> DataFrame | None: - """Try to score using segmented models as fallback. Returns None if no segments found.""" - all_segments = registry_client.get_all_segment_models(config.registry_table, config.model_name) - if not all_segments: - return None - - first_segment = select_segment_record(all_segments) - if first_segment.segmentation.segment_by is None: - raise InvalidParameterError("Segment model must have segment_by") - strategy = resolve_scoring_strategy(first_segment.identity.algorithm) - return strategy.score_segmented(df, config, registry_client, all_segments) diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index 4fb541f72..55f10f0df 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -1,35 +1,32 @@ -"""Global and segmented anomaly model scoring. +"""Anomaly model scoring. -Provides score_global_model, score_segmented, and load_segment_models. -Kept in one module to avoid over-fragmentation of the scoring layer. +One model per registered name, conditioned on a baseline grouping when it has one. Kept in one +module to avoid over-fragmentation of the scoring layer. """ -import dataclasses import logging import pyspark.sql.functions as F from pyspark.sql import DataFrame from databricks.labs.dqx.anomaly.model_discovery import extract_quantile_points -from databricks.labs.dqx.anomaly.drift import check_and_warn_drift, check_segment_drift, format_drift_summary +from databricks.labs.dqx.anomaly.drift import check_and_warn_drift, format_drift_summary from databricks.labs.dqx.anomaly.ensemble_scorer import ( score_ensemble_models, score_ensemble_models_local, ) from databricks.labs.dqx.anomaly.model_config import compute_config_hash from databricks.labs.dqx.anomaly.model_loader import check_model_staleness -from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRecord, AnomalyModelRegistry +from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRecord from databricks.labs.dqx.anomaly.anomaly_llm_explainer import ( ExplanationContext, add_explanation_column, - probe_endpoint_reachable, ) from databricks.labs.dqx.anomaly.scoring_utils import ( add_baseline_severity_percentile_column, add_info_column, add_severity_percentile_column, apply_row_filter, - create_null_scored_dataframe, join_filtered_results_back, mark_unseen_baselines, null_out_unseen_baseline_scores, @@ -37,7 +34,6 @@ UnseenGroupContext, ) from databricks.labs.dqx.anomaly.scoring_config import SEVERITY_QUANTILE_KEYS, ScoringConfig -from databricks.labs.dqx.anomaly.segment_utils import build_segment_filter from databricks.labs.dqx.anomaly.transformers import SparkFeatureMetadata from databricks.labs.dqx.anomaly.single_model_scorer import ( score_with_sklearn_model, @@ -49,42 +45,6 @@ logger = logging.getLogger(__name__) -def _split_max_groups_budget(max_groups: int, num_eligible_segments: int) -> int: - """Allocate the per-segment LLM-call budget for *score_segmented*. - - Equal split with a floor of 1: every eligible segment gets a chance to produce at - least one explanation, even when *max_groups* < *num_eligible_segments*. - - Bound analysis: with N eligible segments and budget B, the total LLM-call cap is - ``N * (B // N) <= B`` when ``B >= N``. When ``B < N`` the floor of 1 kicks in and the - cap becomes ``N`` — wider than B but still finite and proportional to the input. - Documented as a deliberate tradeoff so the feature remains useful when users - drastically under-provision the budget. - """ - if num_eligible_segments <= 0: - raise InvalidParameterError("num_eligible_segments must be positive") - return max(1, max_groups // num_eligible_segments) - - -def _warn_if_max_groups_below_segments(config: ScoringConfig, num_eligible_segments: int) -> None: - """Cost transparency for segmented scoring. - - The per-segment floor of 1 (see *_split_max_groups_budget*) means a budget smaller than the - eligible-segment count still makes one LLM call per segment, so the effective cap is the - segment count rather than *max_groups*. Surface this so a scheduled job's owner isn't - surprised by the bill. No-op unless AI explanations are enabled and there are eligible - segments to explain. - """ - if not (config.enable_ai_explanation and num_eligible_segments): - return - if config.max_groups < num_eligible_segments: - logger.warning( - f"ai_explanation: max_groups={config.max_groups} is below the {num_eligible_segments} eligible " - f"segments; the per-segment floor of 1 makes the effective cap {num_eligible_segments} LLM " - f"calls. Raise max_groups to at least {num_eligible_segments} to bound cost as configured." - ) - - def _known_group_keys(parsed_metadata: SparkFeatureMetadata) -> list[str]: """Group keys the model actually saw in training. @@ -143,7 +103,7 @@ def score_global_model( record: AnomalyModelRecord, config: ScoringConfig, ) -> DataFrame: - """Score using a global (non-segmented) model.""" + """Score using the trained model, conditioned on its baseline grouping if it has one.""" # baseline_by is a property of the trained model rather than something the caller supplies, so it # is read back from the persisted metadata. That makes the recomputed hash match for any model # trained by this version, and mismatch for one trained before baseline_by joined the hash -- @@ -155,15 +115,13 @@ def score_global_model( if record.features.feature_metadata else None ) - expected_hash = compute_config_hash(config.columns, config.segment_by, trained_baseline_by) + expected_hash = compute_config_hash(config.columns, trained_baseline_by) - if expected_hash != record.segmentation.config_hash: + if expected_hash != record.grouping.config_hash: raise InvalidParameterError( f"Configuration mismatch for model '{config.model_name}':\n" f" Trained columns: {record.training.columns}\n" f" Provided columns: {config.columns}\n" - f" Trained segment_by: {record.segmentation.segment_by}\n" - f" Provided segment_by: {config.segment_by}\n" f" Trained baseline_by: {trained_baseline_by or None}\n\n" f"This model was trained with a different configuration, or by a DQX version before\n" f"baseline_by became part of the configuration hash (0.17.0). Either:\n" @@ -279,7 +237,6 @@ def score_global_model( scored_df = add_explanation_column( scored_df, ExplanationContext.from_scoring_config(config), - segment_values=None, is_ensemble=record.identity.is_ensemble, drift_summary=format_drift_summary(drift_result, config.redact_columns), ) @@ -290,7 +247,6 @@ def score_global_model( config.threshold, output_columns=config.output_columns, info_col_name=config.info_col, - segment_values=None, enable_contributions=config.enable_contributions, enable_confidence_std=config.enable_confidence_std, ai_explanation_col=config.ai_explanation_col if config.enable_ai_explanation else None, @@ -319,206 +275,3 @@ def score_global_model( scored_df = scored_df.drop(config.score_col) return scored_df - - -def load_segment_models( - registry_client: AnomalyModelRegistry, - config: ScoringConfig, -) -> list[AnomalyModelRecord]: - """Load all segment models for a base model from the registry.""" - all_segments = registry_client.get_all_segment_models(config.registry_table, config.model_name) - if not all_segments: - raise InvalidParameterError( - f"No segment models found for base model '{config.model_name}'. " - "Train segmented models first using anomaly.train(...)." - ) - return all_segments - - -def score_single_segment( - segment_df: DataFrame, - segment_model: AnomalyModelRecord, - config: ScoringConfig, - max_groups_override: int | None = None, - endpoint_reachable: bool | None = None, -) -> DataFrame: - """Score a single segment with its specific model. - - *max_groups_override*, when set, replaces *config.max_groups* in the - ExplanationContext for this segment only. Used by *score_segmented* to enforce a - *global* cap on LLM calls across segments — without it, *config.max_groups* applies - independently per segment and the worst-case total is ``num_segments * max_groups``. - - *endpoint_reachable* is the serving-endpoint reachability probed once by *score_segmented* - for the whole run; threading it through avoids one billable ``ai_query`` probe per segment. - """ - drift_result = check_segment_drift( - segment_df, - config.columns, - segment_model, - config.drift_threshold, - config.drift_threshold_value, - ) - - if segment_model.features.feature_metadata is None: - raise InvalidParameterError( - f"Model '{segment_model.identity.model_name}' is missing feature_metadata required for scoring." - ) - - quantile_points = extract_quantile_points(segment_model) - if config.driver_only: - segment_scored = score_with_sklearn_model_local( - segment_model.identity.model_uri, - segment_df, - config.columns, - segment_model.features.feature_metadata, - config.merge_columns, - enable_contributions=config.enable_contributions, - model_record=segment_model, - quantile_points=quantile_points, - threshold=config.threshold, - ) - else: - segment_scored = score_with_sklearn_model( - segment_model.identity.model_uri, - segment_df, - config.columns, - segment_model.features.feature_metadata, - config.merge_columns, - enable_contributions=config.enable_contributions, - model_record=segment_model, - quantile_points=quantile_points, - threshold=config.threshold, - ) - - segment_scored = segment_scored.withColumn("anomaly_score_std", F.lit(0.0)) - segment_scored = segment_scored.withColumnRenamed("anomaly_score", config.score_col) - segment_scored = segment_scored.withColumnRenamed("anomaly_score_std", config.score_std_col) - - if config.enable_contributions and "anomaly_contributions" in segment_scored.columns: - segment_scored = segment_scored.withColumnRenamed("anomaly_contributions", config.contributions_col) - - segment_scored = add_severity_percentile_column( - segment_scored, - score_col=config.score_col, - severity_col=config.severity_col, - quantile_points=quantile_points, - ) - - if config.enable_ai_explanation: - explanation_ctx = ExplanationContext.from_scoring_config(config) - if max_groups_override is not None: - explanation_ctx = dataclasses.replace(explanation_ctx, max_groups=max_groups_override) - segment_scored = add_explanation_column( - segment_scored, - explanation_ctx, - segment_model.segmentation.segment_values, - segment_model.identity.is_ensemble, - drift_summary=format_drift_summary(drift_result, config.redact_columns), - endpoint_reachable=endpoint_reachable, - ) - - segment_scored = add_info_column( - segment_scored, - config.model_name, - config.threshold, - output_columns=config.output_columns, - info_col_name=config.info_col, - segment_values=segment_model.segmentation.segment_values, - enable_contributions=config.enable_contributions, - enable_confidence_std=config.enable_confidence_std, - ai_explanation_col=config.ai_explanation_col if config.enable_ai_explanation else None, - ) - - return segment_scored - - -def score_segmented( - df: DataFrame, - config: ScoringConfig, - registry_client: AnomalyModelRegistry, - all_segments: list[AnomalyModelRecord] | None = None, -) -> DataFrame: - """Score DataFrame using segment-specific models.""" - all_segments = all_segments if all_segments is not None else load_segment_models(registry_client, config) - - if not all_segments: - raise InvalidParameterError( - f"No segment models found for base model '{config.model_name}'. " - "Train segmented models first using anomaly.train(...)." - ) - - df_to_score = apply_row_filter(df, config.row_filter) - - # Two-pass loop so *max_groups* can be enforced as a global cap across segments. Without - # this, *config.max_groups* would apply independently per segment and the worst-case LLM - # call count would be ``num_eligible_segments * config.max_groups``. First pass filters - # the input by each segment's predicate and discards empty segments; second pass scores - # the survivors with an equal-split per-segment budget. - eligible: list[tuple[AnomalyModelRecord, DataFrame]] = [] - for segment_model in all_segments: - segment_filter = build_segment_filter(segment_model.segmentation.segment_values) - if segment_filter is None: - continue - segment_df = df_to_score.filter(segment_filter) - if segment_df.limit(1).count() == 0: - continue - eligible.append((segment_model, segment_df)) - - per_segment_budget = ( - _split_max_groups_budget(config.max_groups, len(eligible)) - if config.enable_ai_explanation and eligible - else None - ) - _warn_if_max_groups_below_segments(config, len(eligible)) - - # Probe the serving endpoint once for the whole run rather than once per segment: the reachability - # check is a billable 1-token ai_query call, so per-segment probing would cost N calls + N driver - # actions. None when explanations are off / no eligible segments, so score_single_segment falls - # back to its own per-call probe (the direct-call default). - endpoint_reachable = ( - probe_endpoint_reachable(df_to_score.sparkSession, config.llm_model_config) - if config.enable_ai_explanation and eligible - else None - ) - - scored_dfs: list[DataFrame] = [] - for segment_model, segment_df in eligible: - segment_scored = score_single_segment( - segment_df, - segment_model, - config, - max_groups_override=per_segment_budget, - endpoint_reachable=endpoint_reachable, - ) - scored_dfs.append(segment_scored) - - if not scored_dfs: - result = create_null_scored_dataframe( - df_to_score, - config.enable_contributions, - config.enable_confidence_std, - score_col=config.score_col, - score_std_col=config.score_std_col, - contributions_col=config.contributions_col, - severity_col=config.severity_col, - info_col_name=config.info_col, - ) - else: - result = scored_dfs[0] - for sdf in scored_dfs[1:]: - result = result.union(sdf) - - internal_to_remove = [config.score_std_col, config.severity_col] - if config.enable_contributions: - internal_to_remove.append(config.contributions_col) - if config.enable_ai_explanation: - internal_to_remove.append(config.ai_explanation_col) - columns_to_keep = [c for c in result.columns if c not in internal_to_remove] - result = result.select(*columns_to_keep) - - df_to_join = df if config.row_filter else df_to_score - result = join_filtered_results_back(df_to_join, result, config.merge_columns, config.score_col, config.info_col) - - result = result.drop(config.score_col) - return result diff --git a/src/databricks/labs/dqx/anomaly/scoring_strategies.py b/src/databricks/labs/dqx/anomaly/scoring_strategies.py index 0584beada..f90714628 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_strategies.py +++ b/src/databricks/labs/dqx/anomaly/scoring_strategies.py @@ -4,18 +4,18 @@ from pyspark.sql import DataFrame -from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRecord, AnomalyModelRegistry +from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRecord from databricks.labs.dqx.anomaly.scoring_config import ScoringConfig -from databricks.labs.dqx.anomaly.scoring_run import score_global_model, score_segmented +from databricks.labs.dqx.anomaly.scoring_run import score_global_model from databricks.labs.dqx.errors import InvalidParameterError class AnomalyScoringStrategy(ABC): """Scoring strategy interface for row anomaly models. - Implementations that bypass `score_global_model` / `score_segmented` must call - `add_explanation_column` themselves when `config.enable_ai_explanation` is True; - otherwise the `_dq_info.anomaly.ai_explanation` struct will always be null. + Implementations that bypass `score_global_model` must call `add_explanation_column` themselves + when `config.enable_ai_explanation` is True; otherwise the `_dq_info.anomaly.ai_explanation` + struct will always be null. """ @abstractmethod @@ -24,17 +24,7 @@ def supports(self, algorithm: str) -> bool: @abstractmethod def score_global(self, df: DataFrame, record: AnomalyModelRecord, config: ScoringConfig) -> DataFrame: - """Score a global model.""" - - @abstractmethod - def score_segmented( - self, - df: DataFrame, - config: ScoringConfig, - registry_client: AnomalyModelRegistry, - all_segments: list[AnomalyModelRecord], - ) -> DataFrame: - """Score a segmented model.""" + """Score the model.""" class IsolationForestScoringStrategy(AnomalyScoringStrategy): @@ -46,15 +36,6 @@ def supports(self, algorithm: str) -> bool: def score_global(self, df: DataFrame, record: AnomalyModelRecord, config: ScoringConfig) -> DataFrame: return score_global_model(df, record, config) - def score_segmented( - self, - df: DataFrame, - config: ScoringConfig, - registry_client: AnomalyModelRegistry, - all_segments: list[AnomalyModelRecord], - ) -> DataFrame: - return score_segmented(df, config, registry_client, all_segments) - _SCORING_STRATEGIES: list[AnomalyScoringStrategy] = [IsolationForestScoringStrategy()] diff --git a/src/databricks/labs/dqx/anomaly/scoring_utils.py b/src/databricks/labs/dqx/anomaly/scoring_utils.py index 98dc62541..7e83384a9 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_utils.py +++ b/src/databricks/labs/dqx/anomaly/scoring_utils.py @@ -14,7 +14,7 @@ from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema, anomaly_info_struct_schema from databricks.labs.dqx.anomaly.scoring_config import ScoringOutputColumns -from databricks.labs.dqx.anomaly.segment_utils import canonicalize_segment_values, baseline_key_column +from databricks.labs.dqx.anomaly.segment_utils import baseline_key_column from databricks.labs.dqx.errors import InvalidParameterError from databricks.labs.dqx.utils import safe_filter_expr from databricks.labs.dqx.schema.dq_info_schema import ( @@ -83,7 +83,6 @@ def add_info_column( *, output_columns: ScoringOutputColumns | None = None, info_col_name: str | None = None, - segment_values: dict[str, str] | None = None, enable_contributions: bool = False, enable_confidence_std: bool = False, ai_explanation_col: str | None = None, @@ -98,7 +97,6 @@ def add_info_column( output_columns: Internal column names to read scores, severity, contributions and std from, and where to write the info struct. Defaults to the standard names. info_col_name: Overrides ``output_columns.info`` when given (collision-safe UUID name). - segment_values: Segment values if model is segmented (None for global models). enable_contributions: Whether anomaly_contributions are available (0–100 percent). enable_confidence_std: Whether anomaly_score_std is available. ai_explanation_col: Optional column name carrying the pre-computed AI explanation struct. @@ -138,15 +136,6 @@ def add_info_column( "model": F.lit(model_name), } - # Add segment as map (null for global models) - if segment_values: - canonical_values = canonicalize_segment_values(segment_values) - anomaly_info_fields["segment"] = F.create_map( - *[F.lit(item) for pair in canonical_values.items() for item in pair] - ) - else: - anomaly_info_fields["segment"] = F.lit(None).cast(MapType(StringType(), StringType())) - # Add contributions (null if not requested or not available). Contributions are only # surfaced for anomalous rows: SHAP is computed just for rows at or above the threshold # (see compute_gated_shap_contributions) and this gate makes the observable contract diff --git a/src/databricks/labs/dqx/anomaly/segment_utils.py b/src/databricks/labs/dqx/anomaly/segment_utils.py index c9743c529..643a39227 100644 --- a/src/databricks/labs/dqx/anomaly/segment_utils.py +++ b/src/databricks/labs/dqx/anomaly/segment_utils.py @@ -1,4 +1,4 @@ -"""Segment naming and filtering for row anomaly detection.""" +"""The baseline key: one group identifier, computed identically in Python and in Spark.""" from collections.abc import Mapping from typing import Any @@ -7,19 +7,6 @@ import pyspark.sql.functions as F -def canonicalize_segment_values(segment_values: Mapping[str, Any] | None) -> dict[str, str]: - """Canonicalize segment values for deterministic naming and filtering.""" - if not segment_values: - return {} - return {str(key): str(value) for key, value in sorted(segment_values.items(), key=lambda item: str(item[0]))} - - -def build_segment_name(segment_values: Mapping[str, Any] | None) -> str: - """Build deterministic segment name from segment values.""" - canonical_values = canonicalize_segment_values(segment_values) - return "_".join(f"{key}={value}" for key, value in canonical_values.items()) - - # Separator between group column values in a composite group key. ASCII unit separator: a # control character, so ordinary categorical data (names, codes, countries) cannot contain it and # two different group tuples cannot collide onto one key. @@ -106,31 +93,3 @@ def with_baseline_key(df: DataFrame, baseline_by: list[str]) -> DataFrame: if not baseline_by or BASELINE_KEY_COLUMN in df.columns: return df return df.withColumn(BASELINE_KEY_COLUMN, baseline_key_column(baseline_by)) - - -def build_segment_filter(segment_values: dict[str, str] | None) -> Column | None: - """Build Spark filter expression for a segment's values. - - Args: - segment_values: Dictionary mapping segment column names to values - - Returns: - Spark Column expression combining all segment filters with AND - None if segment_values is None or empty - - Example: - >>> build_segment_filter(dict(region="US", product="A")) - Column<'((region = US) AND (product = A))'> - >>> build_segment_filter(None) - None - """ - if not segment_values: - return None - - filter_exprs = [F.col(key) == F.lit(value) for key, value in canonicalize_segment_values(segment_values).items()] - - segment_filter = filter_exprs[0] - for expr in filter_exprs[1:]: - segment_filter = segment_filter & expr - - return segment_filter diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index 6b6021f4a..c1e811da1 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -1,15 +1,13 @@ """Anomaly training service - Main orchestration layer. -Provides the high-level API for training anomaly detection models, including -context building, validation, and both global and segmented training. All -training logic lives on AnomalyTrainingService (public and private methods). +Provides the high-level API for training an anomaly detection model, including context building, +validation, and training. All training logic lives on AnomalyTrainingService (public and private +methods). """ -import collections.abc import logging from copy import deepcopy from datetime import datetime -from typing import Any import sklearn from pyspark.sql import DataFrame, SparkSession @@ -26,21 +24,16 @@ AnomalyModelRecord, AnomalyModelRegistry, FeatureEngineering, + GroupingConfig, ModelIdentity, - SegmentationConfig, TrainingMetadata, ) -from databricks.labs.dqx.anomaly.group_config import ( - MIN_ROWS_TO_TRAIN_SEGMENT, - SEGMENT_COUNT_WARN_THRESHOLD, -) from databricks.labs.dqx.anomaly.profiler import auto_discover_columns from databricks.labs.dqx.anomaly.training_strategies import AnomalyTrainingStrategy, IsolationForestTrainingStrategy from databricks.labs.dqx.anomaly.transformers import ( SparkFeatureMetadata, apply_feature_engineering_from_metadata, ) -from databricks.labs.dqx.anomaly.segment_utils import build_segment_name from databricks.labs.dqx.anomaly.types import AnomalyTrainingContext, TrainingArtifacts from databricks.labs.dqx.anomaly.validation import ( validate_columns, @@ -58,8 +51,8 @@ class AnomalyTrainingService: """Service for building training context and orchestrating model training. - Provides the main entry point for training anomaly detection models. - Supports both global models and segment-specific models. + Provides the main entry point for training an anomaly detection model, conditioned on a baseline + grouping when one is declared or discovered. Extension point: To add new algorithms, implement AnomalyTrainingStrategy and pass to constructor. @@ -71,31 +64,22 @@ def __init__(self, spark: SparkSession, strategy: AnomalyTrainingStrategy | None self._strategy = strategy or IsolationForestTrainingStrategy() @staticmethod - def _perform_auto_discovery( - df_filtered: DataFrame, - segment_by: list[str] | None, - ) -> tuple[list[str], list[str] | None]: - """Perform auto-discovery of columns and segments. + def _perform_auto_discovery(df_filtered: DataFrame) -> tuple[list[str], list[str] | None]: + """Discover feature columns and a baseline grouping. - When the caller has not declared ``segment_by``, any grouping discovered here is routed to - ``baseline_by``, so discovery is asked for a baseline-shaped grouping: finer, bounded by rows - per group rather than by model count. Asking for the segmented shape was a real defect -- - it returned a single lowest-cardinality column and conditioning barely engaged. + The grouping is chosen for baseline conditioning: finer, bounded by rows per group rather + than by model count, because there is one model however many groups result. """ - profile = auto_discover_columns(df_filtered, for_baseline=segment_by is None) - discovered_columns = profile.recommended_columns - discovered_segments = segment_by - if segment_by is None: - discovered_segments = profile.recommended_segments - logger.info(f"Auto-selected {len(discovered_columns)} columns: {discovered_columns}") - if discovered_segments: + profile = auto_discover_columns(df_filtered) + logger.info(f"Auto-selected {len(profile.recommended_columns)} columns: {profile.recommended_columns}") + if profile.recommended_segments: logger.info( - f"Auto-detected {len(discovered_segments)} segment columns: {discovered_segments} " - f"({profile.segment_count} total segments)" + f"Auto-detected {len(profile.recommended_segments)} baseline columns: " + f"{profile.recommended_segments} ({profile.segment_count} total groups)" ) for warning in profile.warnings: logger.warning(warning) - return discovered_columns, discovered_segments + return profile.recommended_columns, profile.recommended_segments or None @staticmethod def apply_expected_anomaly_rate_if_default_contamination( @@ -115,33 +99,6 @@ def apply_expected_anomaly_rate_if_default_contamination( ) return params - @staticmethod - def _get_and_validate_segments( - df: DataFrame, segment_by: list[str], params: AnomalyParams - ) -> tuple[int, collections.abc.Iterator[dict[str, Any]]]: - """Get distinct segments and validate count. - - Raises above ``params.max_segment_models``. One model is trained per segment and - segmented training does not ensemble, so cost is linear in the segment count: 90 - segments measures roughly 88 minutes. Warning and proceeding meant a run could spend - hours registering thousands of models behind a single log line, which is why this is - an error rather than a warning. - """ - segments_df = df.select(*segment_by).distinct() - segment_count = segments_df.count() - if segment_count > params.max_segment_models: - raise InvalidParameterError( - f"Segmenting by {segment_by} produces {segment_count} segments, above the limit of " - f"{params.max_segment_models}. One model is trained per segment, so this run would train " - f"{segment_count} models (roughly {segment_count} minutes). Either segment more coarsely, " - f"or raise the limit with AnomalyParams(max_segment_models={segment_count})." - ) - if segment_count > SEGMENT_COUNT_WARN_THRESHOLD: - logger.warning( - f"Training {segment_count} segments may be slow. Consider coarser segmentation or explicit segment_by." - ) - return segment_count, (row.asDict() for row in segments_df.toLocalIterator()) - @staticmethod def _model_exists_in_uc(model_name: str) -> bool: """Check if a model exists in Unity Catalog using MLflow API.""" @@ -162,44 +119,6 @@ def _compute_post_training_metadata( baseline_stats = compute_baseline_statistics(engineered_train_df, feature_metadata.engineered_feature_names) return baseline_stats - @staticmethod - def _report_training_summary( - model_uris: list[str], - skipped_segments: list[str], - failed_segments: list[tuple[str, str]], - total_segments: int, - base_model_name: str, - registry_table: str, - params: AnomalyParams, - ) -> None: - """Report training summary including skipped and failed segments.""" - if skipped_segments: - logger.info( - f"Skipped {len(skipped_segments)}/{total_segments} segments due to insufficient data after sampling: " - f"{', '.join(skipped_segments[:5])}" - + (f" and {len(skipped_segments) - 5} more" if len(skipped_segments) > 5 else "") - ) - if failed_segments: - logger.warning(f"\nWARNING: {len(failed_segments)}/{total_segments} segments failed during training:") - for seg_name, error in failed_segments[:3]: - logger.warning(f" - {seg_name}: {error}") - if len(failed_segments) > 3: - logger.warning(f" ... and {len(failed_segments) - 3} more") - if not model_uris: - raise InvalidParameterError( - f"All {total_segments} segments failed ({len(skipped_segments)} skipped, " - f"{len(failed_segments)} errors). Cannot train any models. " - f"Consider increasing sample_fraction (current: {params.sample_fraction}) or checking segment definitions." - ) - trained_count = len(model_uris) - logger.info(f" Trained {trained_count}/{total_segments} segment models for: {base_model_name}") - logger.info(f" Registry: {registry_table}") - - @staticmethod - def _stringify_dict(data: dict[str, Any]) -> dict[str, str]: - """Convert dict values to strings.""" - return {str(k): str(v) for k, v in sorted(data.items(), key=lambda item: str(item[0])) if v is not None} - def _resolve_columns_and_filtered_df( self, df: DataFrame, @@ -222,37 +141,30 @@ def _discover_columns_and_grouping( df_filtered: DataFrame, columns: list[str] | None, declared_baseline_by: list[str] | None, - segment_by: list[str] | None, - ) -> tuple[list[str], list[str] | None, list[str] | None]: + ) -> tuple[list[str], list[str] | None]: """Fill in whichever of the feature columns and the grouping the caller left unspecified. - Returns ``(columns, baseline_by, segment_by)``. + Returns ``(columns, baseline_by)``. A discovered grouping always becomes ``baseline_by``: + there is one model regardless of group count, so discovery cannot turn into an hours-long + run however many groups it finds. """ if columns is None: - columns, discovered_segments = self._perform_auto_discovery(df_filtered, segment_by) + columns, discovered = self._perform_auto_discovery(df_filtered) if declared_baseline_by: # A declared baseline column is the basis metrics are compared against, not a # metric. Auto-discovery does not know that, so drop them here rather than making # the caller reconcile a list they never wrote. - columns = [c for c in columns if c not in declared_baseline_by] - elif segment_by is None and discovered_segments: - # Route a discovered grouping to baseline_by, not segment_by. Assigning it to - # segment_by would pin it to one model per group, which is how a discovered 90-way - # grouping became an hours-long run — and would now hit the segment ceiling and fail - # outright rather than using the mechanism that scales. - return columns, discovered_segments, None - else: - segment_by = discovered_segments - return columns, declared_baseline_by, segment_by - - if declared_baseline_by is None and segment_by is None: + return [c for c in columns if c not in declared_baseline_by], declared_baseline_by + return columns, discovered + + if declared_baseline_by is None: # Grouping discovery used to be reachable only when the columns were discovered too, so # naming your feature columns silently gave up any chance of conditioning. Those are # independent questions. Costs one extra profiling pass for callers who pass explicit # columns and no grouping. - return columns, self._discover_baseline_columns(df_filtered, columns), None + return columns, self._discover_baseline_columns(df_filtered, columns) - return columns, declared_baseline_by, segment_by + return columns, declared_baseline_by @staticmethod def _discover_baseline_columns(df_filtered: DataFrame, columns: list[str]) -> list[str] | None: @@ -266,48 +178,13 @@ def _discover_baseline_columns(df_filtered: DataFrame, columns: list[str]) -> li caller: a column they asked to have measured must not silently become the basis it is measured against, which ``validate_baseline_columns`` would reject anyway. """ - profile = auto_discover_columns(df_filtered, for_baseline=True) + profile = auto_discover_columns(df_filtered) discovered = [c for c in profile.recommended_segments if c not in set(columns)] if not discovered: return None logger.info(f"Auto-detected {len(discovered)} baseline columns: {discovered}") return discovered - @staticmethod - def _resolve_grouping( - baseline_by: list[str] | None, - segment_by: list[str] | None, - ) -> tuple[list[str] | None, list[str] | None]: - """Reconcile ``baseline_by`` against the legacy ``segment_by``. - - Returns ``(baseline_by, effective_segment_by)``. Exactly one of the two is ever populated: - - * ``segment_by`` — the legacy path, one model per segment. **Must** clear ``baseline_by``. - Leaving both set would compute baseline-relative features *inside* each segment, where the - baseline key is constant because ``_train_segmented`` has already filtered the frame, while - ``compute_config_hash`` is built from ``segment_by`` alone and would not change. A model - whose feature list moved but whose config hash did not is a silent train/score hazard, and - it breaks the byte-identical-feature-list guarantee that makes already-trained models safe. - * ``baseline_by`` — one pooled model fed each metric's deviation from its own group's - baseline. - - There is no strategy to choose any more: on SMD, per-group models were the worst of three - configurations (PR-AUC 0.1416 against 0.1499 pooled and 0.1536 relative) with one entity - producing 15,963 false positives on 28,392 normal rows, so ``segment_by`` survives for - compatibility rather than as a recommendation. See databrickslabs/dqx#1484. - """ - if baseline_by is not None and segment_by is not None: - raise InvalidParameterError( - "Pass either baseline_by or segment_by, not both. segment_by is the legacy name for " - "training one model per group; baseline_by judges each metric against its own " - "group's baseline using a single model." - ) - - if segment_by is not None: - return None, segment_by - - return baseline_by, None - def build_context( self, df: DataFrame, @@ -315,7 +192,6 @@ def build_context( registry_table: str, *, columns: list[str] | None, - segment_by: list[str] | None, params: AnomalyParams | None, exclude_columns: list[str] | None, expected_anomaly_rate: float, @@ -344,9 +220,7 @@ def build_context( columns, df_filtered = self._resolve_columns_and_filtered_df(df, columns, exclude_list) auto_discovery_used = columns is None - columns, declared_baseline_by, segment_by = self._discover_columns_and_grouping( - df_filtered, columns, declared_baseline_by, segment_by - ) + columns, baseline_by = self._discover_columns_and_grouping(df_filtered, columns, declared_baseline_by) if not columns: raise InvalidParameterError("No columns provided or auto-discovered. Provide columns explicitly.") @@ -355,8 +229,7 @@ def build_context( for warning in validation_warnings: logger.warning(warning) - validate_baseline_columns(df, declared_baseline_by, columns) - baseline_by, segment_by = self._resolve_grouping(declared_baseline_by, segment_by) + validate_baseline_columns(df, baseline_by, columns) if baseline_by: logger.info(f"Judging each metric against its own group's baseline, grouped by {baseline_by}") @@ -364,7 +237,7 @@ def build_context( model_name=model_name, registry_table=registry_table, columns=columns, - segment_by=segment_by, + baseline_by=baseline_by, ) params = self.apply_expected_anomaly_rate_if_default_contamination(params, expected_anomaly_rate) @@ -380,7 +253,6 @@ def build_context( model_name=model_name, registry_table=registry_table, columns=columns, - segment_by=segment_by, params=params, expected_anomaly_rate=expected_anomaly_rate, exclude_columns=exclude_columns, @@ -389,9 +261,12 @@ def build_context( ) def train(self, context: AnomalyTrainingContext) -> str: - """Train model(s) based on context.""" - if context.segment_by: - return self._train_segmented(context) + """Train the model described by *context*. + + One model, always. The dispatch to per-segment training went with the ``segment_by`` path: a + baseline grouping is expressed as features on a single model, so the group count no longer + decides how many models are trained. + """ return self._train_global(context) def _prepare_training_config( @@ -400,7 +275,7 @@ def _prepare_training_config( model_name: str, registry_table: str, columns: list[str], - segment_by: list[str] | None, + baseline_by: list[str] | None, ) -> None: """Validate and prepare training configuration.""" validate_fully_qualified_name(model_name, label="model_name") @@ -415,21 +290,24 @@ def _prepare_training_config( existing = registry.get_active_model(registry_table, model_name) if existing: - config_changed = ( - set(columns) != set(existing.training.columns) or segment_by != existing.segmentation.segment_by + # The grouping is part of the configuration: it changes the feature list and the + # persisted baselines, so a change in it is a change in the model. + config_changed = set(columns) != set(existing.training.columns) or (baseline_by or None) != ( + existing.grouping.baseline_by or None ) if config_changed: logger.warning( f"⚠️ Model '{model_name}' exists with different configuration:\n" - f" Existing: columns={existing.training.columns}, segment_by={existing.segmentation.segment_by}\n" - f" New: columns={columns}, segment_by={segment_by}\n" + f" Existing: columns={existing.training.columns}, " + f"baseline_by={existing.grouping.baseline_by}\n" + f" New: columns={columns}, baseline_by={baseline_by}\n" f" The old model will be archived. Consider using a different model_name " f"if this is a different use case." ) def _train_global(self, context: AnomalyTrainingContext) -> str: - """Train a single global model.""" + """Train a single model.""" sampled_df, _, truncated = sample_df(context.df_filtered, context.columns, context.params) if not sampled_df.head(1): raise InvalidParameterError( @@ -466,79 +344,7 @@ def _train_global(self, context: AnomalyTrainingContext) -> str: baseline_stats=baseline_stats, algorithm=result.algorithm, ) - self._save_training_record(context, artifacts, segment_by=None) - - return context.model_name - - def _train_segmented(self, context: AnomalyTrainingContext) -> str: - """Train separate models for each segment.""" - if context.segment_by is None: - raise InvalidParameterError("segment_by is required for segmented training") - segment_count, segment_iterator = self._get_and_validate_segments( - context.df_filtered, context.segment_by, context.params - ) - model_uris = [] - skipped_segments = [] - failed_segments: list[tuple[str, str]] = [] - - for seg_values in segment_iterator: - segment_name = build_segment_name(seg_values) - model_name = f"{context.model_name}__seg_{segment_name}" - - segment_df = context.df_filtered - for col_name, val in seg_values.items(): - segment_df = segment_df.filter(segment_df[col_name] == val) - - sampled_df, row_count, _ = sample_df(segment_df, context.columns, context.params) - if row_count < MIN_ROWS_TO_TRAIN_SEGMENT: - skipped_segments.append(segment_name) - continue - - try: - train_df, val_df = train_validation_split(sampled_df, context.params) - - result = self._strategy.train( - train_df, - val_df, - context.columns, - context.params, - model_name, - allow_ensemble=False, - ) - - baseline_stats = self._compute_post_training_metadata(train_df, result.feature_metadata) - - artifacts = TrainingArtifacts( - model_name=model_name, - model_uri=result.model_uri, - run_id=result.run_id, - ensemble_size=result.ensemble_size, - feature_metadata=result.feature_metadata, - hyperparams=result.hyperparams, - training_rows=train_df.count(), - validation_metrics=result.validation_metrics, - score_quantiles=result.score_quantiles, - baseline_stats=baseline_stats, - algorithm=result.algorithm, - segment_values=seg_values, - ) - self._save_training_record(context, artifacts, segment_by=context.segment_by) - - model_uris.append(result.model_uri) - - except (InvalidParameterError, MlflowException, ValueError, RuntimeError, OSError) as e: - failed_segments.append((segment_name, str(e))) - logger.error(f"Failed to train segment '{segment_name}': {e}") - - self._report_training_summary( - model_uris, - skipped_segments, - failed_segments, - segment_count, - context.model_name, - context.registry_table, - context.params, - ) + self._save_training_record(context, artifacts) return context.model_name @@ -546,7 +352,6 @@ def _save_training_record( self, context: AnomalyTrainingContext, artifacts: TrainingArtifacts, - segment_by: list[str] | None, ) -> None: """Save training record to registry table.""" # Must go through to_json() rather than hand-rolling the payload: it is the single @@ -574,13 +379,10 @@ def _save_training_record( mode="spark", feature_metadata=feature_metadata_json, ), - segmentation=SegmentationConfig( - segment_by=segment_by, + grouping=GroupingConfig( baseline_by=context.baseline_by, - segment_values=self._stringify_dict(artifacts.segment_values) if artifacts.segment_values else None, - is_global_model=segment_by is None, sklearn_version=sklearn.__version__, - config_hash=compute_config_hash(context.columns, segment_by, context.baseline_by), + config_hash=compute_config_hash(context.columns, context.baseline_by), ), ) registry = AnomalyModelRegistry(context.spark) diff --git a/src/databricks/labs/dqx/anomaly/types.py b/src/databricks/labs/dqx/anomaly/types.py index 9c9f545f3..a87f6c794 100644 --- a/src/databricks/labs/dqx/anomaly/types.py +++ b/src/databricks/labs/dqx/anomaly/types.py @@ -72,11 +72,8 @@ class EnsembleTrainingResult: class AnomalyTrainingContext: """Context containing all inputs needed for training. - ``baseline_by`` and ``segment_by`` are mutually exclusive, and ``_resolve_grouping`` guarantees - it: ``segment_by`` selects the legacy path that trains one model per group and dispatches - ``train()``, while ``baseline_by`` selects a single pooled model fed each metric's deviation - from its own group's baseline. Both being set would append relative features inside each - segment while leaving the model config hash unchanged. + ``baseline_by`` names the columns each metric is judged against. It is expressed as features on a + single model, so the group count never decides how many models are trained. """ spark: SparkSession @@ -85,7 +82,6 @@ class AnomalyTrainingContext: model_name: str registry_table: str columns: list[str] - segment_by: list[str] | None params: AnomalyParams expected_anomaly_rate: float exclude_columns: list[str] | None @@ -95,7 +91,7 @@ class AnomalyTrainingContext: @dataclass(frozen=True) class TrainingArtifacts: - """Artifacts produced by training a single model or segment.""" + """Artifacts produced by training a model.""" model_name: str model_uri: str @@ -108,4 +104,3 @@ class TrainingArtifacts: score_quantiles: dict[str, float] baseline_stats: dict[str, dict[str, float]] algorithm: str - segment_values: dict[str, Any] | None = None diff --git a/src/databricks/labs/dqx/anomaly/validation.py b/src/databricks/labs/dqx/anomaly/validation.py index f7de2a56d..e0aa97eb6 100644 --- a/src/databricks/labs/dqx/anomaly/validation.py +++ b/src/databricks/labs/dqx/anomaly/validation.py @@ -213,11 +213,11 @@ def validate_sklearn_compatibility(model_record: AnomalyModelRecord) -> None: >>> validate_sklearn_compatibility(record) # Warns if sklearn versions don't match """ - if not model_record.segmentation.sklearn_version: + if not model_record.grouping.sklearn_version: # Old models without version tracking - can't validate return - trained_version = model_record.segmentation.sklearn_version + trained_version = model_record.grouping.sklearn_version current_version = sklearn.__version__ if trained_version == current_version: diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index 0e71d16a7..6883d460d 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -205,16 +205,8 @@ class AnomalyParams: - Confidence scores via standard deviation - Better generalization Performance: Optimized ensemble scoring makes this negligible overhead. - Note: this applies to a single global model only. Segmented training always - trains exactly one model per segment and ignores ``ensemble_size``, so - confidence scores are not available for segmented models. algorithm_config: Isolation Forest parameters (contamination, num_trees, seed). feature_engineering: Feature engineering parameters (temporal features, scaling, etc.). - max_segment_models: Ceiling on how many per-segment models one training run will attempt - (default 50). Guards the legacy *segment_by* path, which is the only one that trains a - model per group: cost is linear in the segment count and segmented training does not - ensemble, so 90 segments measures roughly 88 minutes. Raise this only if you are - prepared to wait. Irrelevant to *baseline_by*, which trains a single model. baseline_by: Columns identifying the group a row belongs to, so a metric is judged against its own group's baseline rather than against the whole table. Each numeric metric gains its deviation from that baseline as an extra feature, on one pooled model, so cost does @@ -228,9 +220,6 @@ class AnomalyParams: ensemble_size: int | None = 3 # Default 3-model ensemble for robustness, tie-breaking, and confidence scores algorithm_config: IsolationForestConfig = field(default_factory=IsolationForestConfig) feature_engineering: FeatureEngineeringConfig = field(default_factory=FeatureEngineeringConfig) - # Kept in sync with anomaly.group_config.MAX_SEGMENT_MODELS by a unit test; not imported from - # there because that package requires the 'anomaly' extras and this module must not. - max_segment_models: int = 50 baseline_by: list[str] | None = None @@ -239,11 +228,10 @@ class AnomalyConfig: """Configuration for row anomaly detection.""" columns: list[str] | None = None # Auto-discovered if omitted - segment_by: list[str] | None = None # Legacy: one model per segment. Prefer baseline_by. model_name: str | None = None # Optional in workflows; defaults to dqx_anomaly_ registry_table: str | None = None - # Preferred over segment_by: declares the basis each metric is judged against, on one pooled - # model. Optional, so installed run-config YAML written before it existed still loads. + # Declares the basis each metric is judged against, on one pooled model. Optional, so installed + # run-config YAML written before it existed still loads. baseline_by: list[str] | None = None diff --git a/tests/integration_anomaly/conftest.py b/tests/integration_anomaly/conftest.py index 334cc8e27..bf1ddd1ba 100644 --- a/tests/integration_anomaly/conftest.py +++ b/tests/integration_anomaly/conftest.py @@ -138,7 +138,6 @@ def train_model_with_params( registry_table: str, columns: list[str], params: AnomalyParams, - segment_by: list[str] | None = None, expected_anomaly_rate: float = 0.02, baseline_by: list[str] | None = None, ) -> str: @@ -148,7 +147,6 @@ def train_model_with_params( columns=columns, model_name=model_name, registry_table=registry_table, - segment_by=segment_by, baseline_by=baseline_by, params=params, expected_anomaly_rate=expected_anomaly_rate, @@ -799,7 +797,6 @@ def _train( columns: list[str] | None = None, train_data: list[tuple] | None = None, params=None, - segment_by: list[str] | None = None, catalog: str = TEST_CATALOG, schema: str | None = None, baseline_by: list[str] | None = None, @@ -814,8 +811,7 @@ def _train( columns (list[str] | None): Column names (default: ["amount", "quantity"]) train_data (list[tuple] | None): Custom training data tuples (overrides train_size) params (AnomalyParams | None): Internal training params (test-only) - segment_by (list[str] | None): Segment columns for segmented models - baseline_by (list[str] | None): Group columns for group-conditioned models + baseline_by (list[str] | None): Group columns for baseline-conditioned models train_schema (str | None): Explicit DDL for train_data (needed when group columns are not doubles) catalog (str): Catalog name @@ -856,7 +852,6 @@ def _train( columns=columns, model_name=model_name, registry_table=registry_table, - segment_by=segment_by, baseline_by=baseline_by, ) else: @@ -867,7 +862,6 @@ def _train( registry_table=registry_table, columns=columns, params=params, - segment_by=segment_by, baseline_by=baseline_by, ) diff --git a/tests/integration_anomaly/test_anomaly_apply_checks.py b/tests/integration_anomaly/test_anomaly_apply_checks.py index 654f2214f..146766a4d 100644 --- a/tests/integration_anomaly/test_anomaly_apply_checks.py +++ b/tests/integration_anomaly/test_anomaly_apply_checks.py @@ -272,7 +272,6 @@ def test_apply_anomaly_check_info_column_structure(ws, spark: SparkSession, shar "is_anomaly", "threshold", "model", - "segment", "contributions", "confidence_std", "is_new_baseline", @@ -290,7 +289,6 @@ def test_apply_anomaly_check_info_column_structure(ws, spark: SparkSession, shar assert model_name in anomaly.model, f"model should contain {model_name}" # Verify optional fields are None when not requested - assert anomaly.segment is None, "segment should be None for global model" assert anomaly.contributions is None, "contributions should be None when not requested" assert anomaly.confidence_std is None, "confidence_std should be None when not requested" diff --git a/tests/integration_anomaly/test_anomaly_apply_checks_by_metadata.py b/tests/integration_anomaly/test_anomaly_apply_checks_by_metadata.py index b647631e7..0fb80a583 100644 --- a/tests/integration_anomaly/test_anomaly_apply_checks_by_metadata.py +++ b/tests/integration_anomaly/test_anomaly_apply_checks_by_metadata.py @@ -420,11 +420,11 @@ def test_apply_anomaly_check_by_metadata_criticality_warn(ws, spark: SparkSessio assert rows[1]["_warnings"] is not None or rows[1]["_errors"] is not None -def test_apply_anomaly_check_by_metadata_with_filter_segmented(ws, spark: SparkSession, make_schema, make_random): +def test_apply_anomaly_check_by_metadata_with_filter_grouped(ws, spark: SparkSession, make_schema, make_random): schema = make_schema(catalog_name=TEST_CATALOG) suffix = make_random(8).lower() - # Training data: region + amount, quantity (for segment_by=["region"]) + # Training data: region + amount, quantity (for baseline_by=["region"]) data = [] for region in SEGMENT_REGIONS: base = 100 if region == "US" else (200 if region == "EU" else 150) @@ -439,7 +439,7 @@ def test_apply_anomaly_check_by_metadata_with_filter_segmented(ws, spark: SparkS engine.train( df=train_df, columns=["amount", "quantity"], - segment_by=["region"], + baseline_by=["region"], model_name=model_name, registry_table=registry_table, ) diff --git a/tests/integration_anomaly/test_anomaly_autodiscovery.py b/tests/integration_anomaly/test_anomaly_autodiscovery.py index af20a65c3..12f1eb556 100644 --- a/tests/integration_anomaly/test_anomaly_autodiscovery.py +++ b/tests/integration_anomaly/test_anomaly_autodiscovery.py @@ -116,23 +116,27 @@ def test_zero_config_training(spark: SparkSession, make_schema, make_random, ano registry = spark.table(registry_table) models = registry.filter("identity.status = 'active'").collect() - # One conditioned model, not one per region. + # One conditioned model, not one per region: the discovered grouping becomes baseline_by. assert len(models) == 1 - assert models[0].segmentation.is_global_model is True - assert models[0].segmentation.segment_by is None + assert models[0].grouping.baseline_by == ["region"] # Verify auto-discovered columns (amount and discount) for model in models: assert set(model.training.columns) == {"amount", "discount"} -def test_explicit_columns_no_auto_segment(spark: SparkSession, make_schema, make_random, anomaly_engine): - """Test that providing explicit columns disables auto-segmentation.""" +def test_explicit_columns_still_get_baseline_discovery(spark: SparkSession, make_schema, make_random, anomaly_engine): + """Naming the feature columns does not turn off baseline discovery — they are independent choices. + + Under the old segmented policy, passing explicit columns suppressed auto-segmentation. Baseline + conditioning has no reason to: there is one model regardless of group count, so a caller who + names the metrics still gets a grouping discovered for them. See databrickslabs/dqx#1484. + """ # Create unique schema for test isolation schema = make_schema(catalog_name=TEST_CATALOG) suffix = make_random(8).lower() - # Create data with segment column + # region is a strong baseline candidate (2 values, 200 rows each) data = [] for region in ("US", "EU"): for i in range(200): @@ -142,37 +146,20 @@ def test_explicit_columns_no_auto_segment(spark: SparkSession, make_schema, make table_name = f"{TEST_CATALOG}.{schema.name}.explicit_cols_test_{suffix}" df.write.saveAsTable(table_name) - # Train with explicit columns (should NOT auto-segment) + # Explicit feature column, no explicit grouping: the grouping is still discovered. registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" anomaly_engine.train( df=spark.table(table_name), - columns=["amount"], # Explicit columns provided + columns=["amount"], model_name=qualify_model_name(f"test_explicit_{suffix}", registry_table), registry_table=registry_table, ) - # Verify only 1 global model created (no segmentation) registry = spark.table(registry_table) models = registry.filter("identity.status = 'active'").collect() assert len(models) == 1 - assert models[0].segmentation.is_global_model is True - - -def test_warnings_for_small_segments(spark: SparkSession): - """Test that warnings are issued for segments with <1000 rows.""" - # Create data with small segments - data = [] - for region in SEGMENT_REGIONS: - for i in range(500): # Only 500 rows per segment - data.append((region, 100.0 + i)) - - df = spark.createDataFrame(data, "region string, amount double") - - profile = auto_discover_columns(df) - - # Should still recommend region but warn about small segments - assert "region" in profile.recommended_segments - assert any("1000 rows" in w for w in profile.warnings) + assert models[0].training.columns == ["amount"] + assert models[0].grouping.baseline_by == ["region"] def test_autodiscovery_excludes_high_null_numeric_columns(spark: SparkSession): @@ -265,12 +252,16 @@ def test_autodiscovery_with_various_cardinality_strings(spark: SparkSession): profile = auto_discover_columns(df) - # category has 5 distinct values (≤20) - should be selected - assert "category" in profile.recommended_columns + # category has 5 distinct values - low cardinality makes it a baseline grouping column, + # not a feature. Under the baseline-aware policy it is routed to recommended_segments and + # removed from recommended_columns (baseline columns are not features). + assert "category" in profile.recommended_segments + assert "category" not in profile.recommended_columns assert profile.column_types is not None assert profile.column_types["category"] == "categorical" - # user_code has 50 distinct values (21-100 range) - should be selected as priority 5 + # user_code has 50 distinct values - stays a feature: adding it as a second baseline column + # would push groups past the rows-per-group floor, so it is not selected for grouping. assert "user_code" in profile.recommended_columns assert profile.column_types["user_code"] == "categorical" @@ -368,41 +359,24 @@ def test_autodiscovery_many_segments_warning(spark: SparkSession): assert "region" in profile.recommended_columns -def test_autodiscovery_granular_segments_warning(spark: SparkSession): - """Test warning when average rows per segment <100 (lines 245-249).""" - # Create 10 segments (within 2-20 range) with only 50 rows each (avg <100) - # However, line 295 requires (total_count / distinct_count) >= 100 - # So 50 rows per segment won't pass the criteria - # We need a test that passes line 292 but triggers line 245-249 - # Let's use 5 segments with 80 rows each = 400 total - # 400 / 5 = 80 rows/segment, which is < 100 but >= the minimum threshold - # Actually, line 295 checks >= 100, so we need to check line 245 differently - # Line 245 is checked AFTER selection, so we need segments that pass >= 100 in line 295 - # but when calculated in line 238, avg < 100 - # This is contradictory - if it passes line 295, avg = total/distinct >= 100 - # So line 245 only triggers with multiple segment columns where product > actual avg - # For single column test, let's just verify the segment is properly analyzed +def test_autodiscovery_selects_multi_value_grouping(spark: SparkSession): + """Five groups of 150 rows clear the 30-rows-per-group floor and are selected as the grouping. + + The baseline policy bounds groups by rows-per-group, not by a fixed minimum count, and emits no + "too granular" warning once the floor is met — one model serves every group. + """ data = [] for region in [f"region_{i}" for i in range(5)]: - for j in range(150): # 150 rows per segment > 100, satisfies line 295 + for j in range(150): # 150 rows per group, well above MIN_ROWS_PER_BASELINE_GROUP data.append((region, 100.0 + j)) df = spark.createDataFrame(data, "region string, amount double") profile = auto_discover_columns(df) - # region should be selected as segment (5 distinct is in 2-20 range, 150 rows/segment >= 100) assert "region" in profile.recommended_segments - - # Verify segment count is calculated correctly assert profile.segment_count == 5 - # Should have warning about small segment size (lines 195-198) - warnings_text = " ".join(profile.warnings) - # The warning comes from _validate_and_add_segment_column (line 195-198) - # which checks for < 1000 rows per segment - assert "region" in warnings_text and ("min:" in warnings_text or "<1000 rows" in warnings_text) - def test_segment_column_explicitly_removed_from_features(spark: SparkSession): """Test that segment columns are removed from feature columns (lines 488-489).""" diff --git a/tests/integration_anomaly/test_anomaly_drift_integration.py b/tests/integration_anomaly/test_anomaly_drift_integration.py index aa36e52ab..899dc5c35 100644 --- a/tests/integration_anomaly/test_anomaly_drift_integration.py +++ b/tests/integration_anomaly/test_anomaly_drift_integration.py @@ -205,58 +205,6 @@ def test_drift_detection_skipped_on_small_batch( assert len(drift_warnings) == 0 -def test_segment_drift_warns_per_segment( - ws, - spark: SparkSession, - make_random: Callable[[int], str], - anomaly_engine, - anomaly_registry_prefix, -): - """Segmented scoring should emit drift warnings when each segment shifts.""" - unique_id = make_random(8).lower() - model_name = f"{anomaly_registry_prefix}.test_segment_drift_{make_random(4).lower()}" - registry_table = f"{anomaly_registry_prefix}.{unique_id}_registry" - - train_rows = [] - for region, base in (("US", 100.0), ("EU", 200.0)): - for i in range(1200): - train_rows.append((region, base + i * 0.1, 5.0)) - - train_df = spark.createDataFrame(train_rows, "region string, amount double, quantity double") - anomaly_engine.train( - df=train_df, - columns=["amount", "quantity"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - ) - - drift_rows = [] - for region, base in (("US", 2000.0), ("EU", 3000.0)): - for i in range(1200): - drift_rows.append((region, base + i * 0.1, 5.0)) - - test_df = spark.createDataFrame(drift_rows, "region string, amount double, quantity double") - dq_engine = DQEngine(ws, spark) - checks = [ - create_anomaly_check_rule( - model_name=model_name, - registry_table=registry_table, - columns=["amount", "quantity"], - threshold=DEFAULT_SCORE_THRESHOLD, - drift_threshold=DRIFT_THRESHOLD, - ) - ] - - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - dq_engine.apply_checks(test_df, checks).collect() - - drift_warnings = [warning for warning in w if "data drift detected" in str(warning.message).lower()] - assert drift_warnings - assert any("segment" in str(warning.message).lower() for warning in drift_warnings) - - def test_drift_detector_no_columns_in_baseline_returns_ok(spark): """When no requested columns are in baseline_stats, returns DriftResult with recommendation='ok'.""" df = spark.createDataFrame( diff --git a/tests/integration_anomaly/test_anomaly_errors.py b/tests/integration_anomaly/test_anomaly_errors.py index 0b1900539..a58f00072 100644 --- a/tests/integration_anomaly/test_anomaly_errors.py +++ b/tests/integration_anomaly/test_anomaly_errors.py @@ -18,7 +18,7 @@ AnomalyModelRecord, FeatureEngineering, ModelIdentity, - SegmentationConfig, + GroupingConfig, TrainingMetadata, ) from databricks.labs.dqx.anomaly.validation import validate_sklearn_compatibility @@ -28,7 +28,6 @@ create_null_scored_dataframe, create_udf_schema, ) -from databricks.labs.dqx.anomaly.segment_utils import build_segment_filter from databricks.labs.dqx.engine import DQEngine from databricks.labs.dqx.errors import ComputationError, InvalidParameterError from tests.integration_anomaly.constants import DEFAULT_SCORE_THRESHOLD @@ -128,7 +127,7 @@ def test_config_hash_mismatch_raises( spark.sql( f"UPDATE {registry_table} " - f"SET segmentation.config_hash = 'bogus' " + f"SET grouping.config_hash = 'bogus' " f"WHERE identity.model_name = '{full_model_name}'" ) @@ -214,8 +213,7 @@ def test_model_not_found_error(spark: SparkSession, make_random, test_df_factory features STRUCT, feature_metadata: STRING, feature_importance: MAP, temporal_config: STRING>, - segmentation STRUCT, segment_values: MAP, - is_global_model: BOOLEAN, sklearn_version: STRING, config_hash: STRING> + grouping STRUCT, sklearn_version: STRING, config_hash: STRING> ) USING DELTA """ ) @@ -307,13 +305,6 @@ def test_has_no_row_anomalies_invalid_inputs(kwargs, match): ) -def test_build_segment_filter_handles_none_and_multi_key(): - """Test segment filter construction handles None and multiple keys.""" - assert build_segment_filter(None) is None - expr = build_segment_filter({"region": "US", "product": "A"}) - assert expr is not None - - def test_row_filter_scores_only_matching_rows( spark: SparkSession, make_random, anomaly_engine, test_df_factory, anomaly_registry_prefix ): @@ -407,7 +398,7 @@ def test_sklearn_version_mismatch_warns( spark.sql( f"UPDATE {registry_table} " - f"SET segmentation.sklearn_version = '0.0' " + f"SET grouping.sklearn_version = '0.0' " f"WHERE identity.model_name = '{full_model_name}'" ) @@ -442,7 +433,7 @@ def test_sklearn_version_parse_error_silently_skips( spark.sql( f"UPDATE {registry_table} " - f"SET segmentation.sklearn_version = 'bad.version' " + f"SET grouping.sklearn_version = 'bad.version' " f"WHERE identity.model_name = '{full_model_name}'" ) @@ -478,7 +469,7 @@ def test_validate_sklearn_compatibility_skips_when_missing_version(): training_time=datetime.now(timezone.utc), ), features=FeatureEngineering(feature_metadata=None), - segmentation=SegmentationConfig(sklearn_version=None), + grouping=GroupingConfig(sklearn_version=None), ) validate_sklearn_compatibility(record) diff --git a/tests/integration_anomaly/test_anomaly_groups.py b/tests/integration_anomaly/test_anomaly_groups.py index 8b877373e..705eaf9df 100644 --- a/tests/integration_anomaly/test_anomaly_groups.py +++ b/tests/integration_anomaly/test_anomaly_groups.py @@ -17,7 +17,6 @@ from pyspark.sql import SparkSession from pyspark.sql import functions as F -from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRegistry from databricks.labs.dqx.anomaly.segment_utils import baseline_key_column from databricks.labs.dqx.config import AnomalyParams from databricks.labs.dqx.errors import InvalidParameterError @@ -99,19 +98,6 @@ def test_baseline_conditioning_trains_one_model_regardless_of_group_count(spark: # ============================================================================ -def test_baseline_by_and_segment_by_together_are_rejected(spark: SparkSession, quick_model_factory): - """They are alternative mechanisms; accepting both would leave the precedence undefined.""" - with pytest.raises(InvalidParameterError, match="not both"): - quick_model_factory( - spark, - columns=["amount"], - train_data=[(float(i), "DE") for i in range(60)], - train_schema="amount double, country string", - baseline_by=["country"], - segment_by=["country"], - ) - - def test_float_baseline_columns_are_rejected(spark: SparkSession, quick_model_factory): """Spark and Python format floats differently, which would break the baseline key silently.""" with pytest.raises(InvalidParameterError, match="unsupported types"): @@ -134,44 +120,3 @@ def test_baseline_column_cannot_also_be_a_feature(spark: SparkSession, quick_mod train_schema="amount double, country string", baseline_by=["country"], ) - - -def test_segmentation_above_the_model_ceiling_names_its_escapes(spark: SparkSession, quick_model_factory): - """The error has to say how to proceed, not just that you cannot.""" - with pytest.raises(InvalidParameterError, match="max_segment_models"): - quick_model_factory( - spark, - columns=["amount"], - train_data=[(float(i), f"G{i % 60}") for i in range(600)], - train_schema="amount double, grp string", - params=AnomalyParams(sample_fraction=1.0, max_segment_models=50), - segment_by=["grp"], - ) - - -def test_segment_by_does_not_gain_baseline_relative_features(spark: SparkSession, quick_model_factory): - """Regression for the leak where the legacy path silently changed its own feature list. - - ``_resolve_grouping`` used to set both ``baseline_by`` and the effective ``segment_by`` for the - legacy path, so relative features were computed *inside* each segment — where the baseline key - is constant, because the frame has already been filtered — while ``compute_config_hash`` is - built from ``segment_by`` alone and did not change. A model whose feature list moved but whose - config hash did not is a silent train/score hazard. - """ - model, registry, _ = quick_model_factory( - spark, - columns=["amount"], - train_data=[(100.0 + i, f"G{i % 3}") for i in range(90)], - train_schema="amount double, grp string", - params=AnomalyParams(sample_fraction=1.0), - segment_by=["grp"], - ) - - records = AnomalyModelRegistry(spark).get_all_segment_models(registry, model) - assert records, "expected at least one segment model" - for record in records: - assert record.features.feature_metadata is not None - assert "_rel_baseline" not in record.features.feature_metadata, ( - "a segment_by model must not carry baseline-relative features: its config hash is " - "computed from segment_by alone and would not reflect the changed feature list" - ) diff --git a/tests/integration_anomaly/test_anomaly_registry.py b/tests/integration_anomaly/test_anomaly_registry.py index ba8640bfc..345af5906 100644 --- a/tests/integration_anomaly/test_anomaly_registry.py +++ b/tests/integration_anomaly/test_anomaly_registry.py @@ -14,7 +14,7 @@ AnomalyModelRegistry, FeatureEngineering, ModelIdentity, - SegmentationConfig, + GroupingConfig, TrainingMetadata, ) from databricks.labs.dqx.config import AnomalyParams, IsolationForestConfig @@ -222,7 +222,7 @@ def test_registry_table_schema( # Verify all expected nested struct columns exist registry_df = spark.table(registry_table) - expected_top_level_columns = ["identity", "training", "features", "segmentation"] + expected_top_level_columns = ["identity", "training", "features", "grouping"] actual_columns = registry_df.columns @@ -247,9 +247,7 @@ def test_registry_table_schema( "temporal_config", "column_types", "feature_metadata", - "segment_by", - "segment_values", - "is_global_model", + "baseline_by", "sklearn_version", "config_hash", ] @@ -329,16 +327,6 @@ def test_nonexistent_registry_returns_none(spark: SparkSession, anomaly_registry assert model is None -def test_get_segment_model_returns_none_when_registry_table_does_not_exist( - spark: SparkSession, anomaly_registry_prefix -): - """Test that get_segment_model returns None when registry table does not exist.""" - registry = AnomalyModelRegistry(spark) - nonexistent_table = f"{anomaly_registry_prefix}.nonexistent_registry_table" - result = registry.get_segment_model(nonexistent_table, "base_model", {"region": "US"}) - assert result is None - - def test_nonexistent_model_returns_none( spark: SparkSession, make_random: Callable[[int], str], anomaly_engine, anomaly_registry_prefix ): @@ -360,11 +348,11 @@ def test_nonexistent_model_returns_none( def test_config_hash_stability(): """Test that config_hash is stable for same inputs.""" columns = ["amount", "quantity", "discount"] - segment_by = ["region", "category"] + baseline_by = ["region", "category"] # Compute hash multiple times - hash1 = compute_config_hash(columns, segment_by) - hash2 = compute_config_hash(columns, segment_by) + hash1 = compute_config_hash(columns, baseline_by) + hash2 = compute_config_hash(columns, baseline_by) assert hash1 == hash2 assert len(hash1) == 16 # 16 hex characters @@ -386,8 +374,8 @@ def test_config_hash_differentiation(): hash3 = compute_config_hash(["amount", "quantity"], ["region"]) assert hash1 != hash2 # Different columns - assert hash1 != hash3 # Different segment_by - assert hash2 != hash3 # Different columns and segment_by + assert hash1 != hash3 # Different baseline_by + assert hash2 != hash3 # Different columns and baseline_by def test_config_hash_stored_during_training( @@ -407,11 +395,11 @@ def test_config_hash_stored_during_training( record = spark.table(registry_table).filter(f"identity.model_name = '{full_model_name}'").first() assert record is not None - assert record["segmentation"]["config_hash"] is not None + assert record["grouping"]["config_hash"] is not None # Verify hash matches expected expected_hash = compute_config_hash(columns, None) - assert record["segmentation"]["config_hash"] == expected_hash + assert record["grouping"]["config_hash"] == expected_hash def test_config_change_warning( @@ -474,7 +462,7 @@ def test_registry_active_model_and_archiving( baseline_stats={"amount": {"mean": 1.0, "std": 0.1}}, ), features=FeatureEngineering(mode="spark"), - segmentation=SegmentationConfig(is_global_model=True, config_hash="hash_v1"), + grouping=GroupingConfig(config_hash="hash_v1"), ) record_v2 = AnomalyModelRecord( @@ -492,7 +480,7 @@ def test_registry_active_model_and_archiving( metrics={"precision": 0.95}, ), features=FeatureEngineering(mode="spark"), - segmentation=SegmentationConfig(is_global_model=True, config_hash="hash_v2"), + grouping=GroupingConfig(config_hash="hash_v2"), ) registry.save_model(record_v1, registry_table) @@ -533,7 +521,7 @@ def test_save_model_when_table_does_not_exist_creates_table_no_archive( training_time=datetime.utcnow(), ), features=FeatureEngineering(mode="spark"), - segmentation=SegmentationConfig(is_global_model=True, config_hash="h"), + grouping=GroupingConfig(config_hash="h"), ) registry.save_model(record, registry_table) @@ -541,84 +529,3 @@ def test_save_model_when_table_does_not_exist_creates_table_no_archive( rows = spark.table(registry_table).collect() assert len(rows) == 1 assert rows[0]["identity"]["status"] == "active" - - -def test_registry_segment_lookup(spark: SparkSession, make_random: Callable[[int], str], anomaly_registry_prefix): - """Test segment model lookup and listing.""" - unique_id = make_random(8).lower() - registry_table = f"{anomaly_registry_prefix}.{unique_id}_registry" - - registry = AnomalyModelRegistry(spark) - base_name = f"{anomaly_registry_prefix}.seg_model_{make_random(4).lower()}" - segment_name = f"{base_name}__seg_region=US" - - record = AnomalyModelRecord( - identity=ModelIdentity( - model_name=segment_name, - model_uri="models:/seg_model/1", - algorithm="isolation_forest", - mlflow_run_id="run_seg", - ), - training=TrainingMetadata( - columns=["amount"], - hyperparameters={}, - training_rows=50, - training_time=datetime.utcnow(), - ), - features=FeatureEngineering(mode="spark"), - segmentation=SegmentationConfig( - segment_by=["region"], - segment_values={"region": "US"}, - is_global_model=False, - config_hash="hash_seg", - ), - ) - - registry.save_model(record, registry_table) - - fetched = registry.get_segment_model(registry_table, base_name, {"region": "US"}) - assert fetched is not None - assert fetched.identity.model_name == segment_name - - all_segments = registry.get_all_segment_models(registry_table, base_name) - assert len(all_segments) == 1 - - -def test_registry_segment_lookup_uses_canonical_order( - spark: SparkSession, make_random: Callable[[int], str], anomaly_registry_prefix -): - """Segment lookup should be deterministic regardless of input dictionary order.""" - unique_id = make_random(8).lower() - registry_table = f"{anomaly_registry_prefix}.{unique_id}_registry" - - registry = AnomalyModelRegistry(spark) - base_name = f"{anomaly_registry_prefix}.seg_model_{make_random(4).lower()}" - segment_name = f"{base_name}__seg_region=US_tier=gold" - - record = AnomalyModelRecord( - identity=ModelIdentity( - model_name=segment_name, - model_uri="models:/seg_model/1", - algorithm="isolation_forest", - mlflow_run_id="run_seg", - ), - training=TrainingMetadata( - columns=["amount"], - hyperparameters={}, - training_rows=50, - training_time=datetime.utcnow(), - ), - features=FeatureEngineering(mode="spark"), - segmentation=SegmentationConfig( - segment_by=["region", "tier"], - segment_values={"region": "US", "tier": "gold"}, - is_global_model=False, - config_hash="hash_seg", - ), - ) - - registry.save_model(record, registry_table) - - fetched = registry.get_segment_model(registry_table, base_name, {"tier": "gold", "region": "US"}) - assert fetched is not None - assert fetched.identity.model_name == segment_name diff --git a/tests/integration_anomaly/test_anomaly_segments.py b/tests/integration_anomaly/test_anomaly_segments.py deleted file mode 100644 index c9e69dcc7..000000000 --- a/tests/integration_anomaly/test_anomaly_segments.py +++ /dev/null @@ -1,525 +0,0 @@ -"""Integration tests for segment-based anomaly detection.""" - -import pytest -import pyspark.sql.functions as F -from pyspark.sql import SparkSession - -from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies -from databricks.labs.dqx.config import AnomalyParams -from databricks.labs.dqx.errors import InvalidParameterError -from databricks.labs.dqx.engine import DQEngine -from databricks.labs.dqx.rule import DQDatasetRule -from tests.constants import TEST_CATALOG -from tests.integration_anomaly.constants import ( - DQENGINE_SCORE_THRESHOLD, - SEGMENT_REGIONS, -) -from tests.integration_anomaly.conftest import create_anomaly_check_rule, train_model_with_params - - -def test_explicit_segment_training( - spark: SparkSession, - make_schema, - make_random, - anomaly_engine, -): - """Test explicit segment-based training.""" - # Create unique schema for test isolation - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - # Generate multi-region data - data = [] - for region in SEGMENT_REGIONS: - base = 100 if region == "US" else (200 if region == "EU" else 150) - for i in range(200): - data.append((region, base + i * 0.5, base * 0.8 + i * 0.3)) - - df = spark.createDataFrame(data, "region string, amount double, discount double") - table_name = f"{TEST_CATALOG}.{schema.name}.segment_test_{suffix}" - df.write.saveAsTable(table_name) - - # Train with explicit segments - model_name = f"{TEST_CATALOG}.{schema.name}.test_segments_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount", "discount"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - ) - - # Verify segmented models were created - registry = spark.table(registry_table) - models = registry.filter("identity.status = 'active'").collect() - assert len(models) == 3 # One per segment - - # Check segment model names - model_names = [row.identity.model_name for row in models] - assert any("region=US" in name for name in model_names) - assert any("region=EU" in name for name in model_names) - assert any("region=APAC" in name for name in model_names) - - -def test_segment_with_insufficient_data_skipped( - spark, - make_schema, - make_random, - anomaly_engine, - caplog, -): - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - # One segment with 5 rows (< 10), one with enough to train - data = [] - for i in range(5): - data.append(("small", 100.0 + i)) - for i in range(150): - data.append(("large", 200.0 + i * 0.5)) - - df = spark.createDataFrame(data, "region string, amount double") - table_name = f"{TEST_CATALOG}.{schema.name}.skip_segment_test_{suffix}" - df.write.saveAsTable(table_name) - - model_name = f"{TEST_CATALOG}.{schema.name}.test_skip_segment_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - with caplog.at_level("INFO"): - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - params=AnomalyParams(sample_fraction=1.0), - ) - - assert "Skipped" in caplog.text - assert "insufficient data" in caplog.text - assert "region=small" in caplog.text - - # Only the "large" segment should have a trained model - registry = spark.table(registry_table) - models = registry.filter("identity.status = 'active'").collect() - assert len(models) == 1 - assert any("region=large" in row.identity.model_name for row in models) - - -def test_all_segments_skipped_raises_invalid_parameter_error( - spark, - make_schema, - make_random, - anomaly_engine, -): - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - # Two segments, both with < 10 rows - data = [] - for i in range(5): - data.append(("a", 100.0 + i)) - for i in range(5): - data.append(("b", 200.0 + i)) - - df = spark.createDataFrame(data, "region string, amount double") - table_name = f"{TEST_CATALOG}.{schema.name}.all_skip_test_{suffix}" - df.write.saveAsTable(table_name) - - model_name = f"{TEST_CATALOG}.{schema.name}.test_all_skip_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - with pytest.raises(InvalidParameterError) as exc_info: - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - params=AnomalyParams(sample_fraction=1.0), - ) - - msg = str(exc_info.value) - assert "All 2 segments failed" in msg - assert "2 skipped" in msg - assert "0 errors" in msg - assert "Cannot train any models" in msg - - -def test_many_skipped_segments_logs_truncated_list( - spark, - make_schema, - make_random, - anomaly_engine, - caplog, -): - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - # Six segments, each with 5 rows - data = [] - for name in ("s1", "s2", "s3", "s4", "s5", "s6"): - for i in range(5): - data.append((name, 100.0 + i)) - - df = spark.createDataFrame(data, "region string, amount double") - table_name = f"{TEST_CATALOG}.{schema.name}.many_skip_test_{suffix}" - df.write.saveAsTable(table_name) - - model_name = f"{TEST_CATALOG}.{schema.name}.test_many_skip_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - with caplog.at_level("INFO"): - with pytest.raises(InvalidParameterError): - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - params=AnomalyParams(sample_fraction=1.0), - ) - - assert " and 1 more" in caplog.text - - -def test_segment_training_failure_logged( - spark, - make_schema, - make_random, - anomaly_engine, - caplog, -): - """When a segment fails during training, failure is logged""" - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - # One segment with enough data, one with constant values (can cause scaling/training to fail) - data = [] - for i in range(100): - data.append(("ok", 100.0 + i * 0.5, 10.0 + i * 0.1)) - for _ in range(15): - data.append(("bad", 1.0, 1.0)) # constant -> zero variance can break RobustScaler/IF - - df = spark.createDataFrame(data, "region string, amount double, quantity double") - table_name = f"{TEST_CATALOG}.{schema.name}.fail_segment_test_{suffix}" - df.write.saveAsTable(table_name) - - model_name = f"{TEST_CATALOG}.{schema.name}.test_fail_segment_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - with caplog.at_level("WARNING"): - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount", "quantity"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - params=AnomalyParams(sample_fraction=1.0), - ) - - # Either the "bad" segment failed (then we see the failure warning) or it succeeded; if failed: - if "failed during training" in caplog.text: - assert "region=bad" in caplog.text or "WARNING" in caplog.text - - -def test_segment_scoring( - ws, - spark: SparkSession, - make_schema, - make_random, - anomaly_engine, -): - """Test that segment scoring uses correct regional models.""" - # Create unique schema for test isolation - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - # Train - data = [] - for region in ("US", "EU"): - base = 100 if region == "US" else 200 - for i in range(200): - data.append((region, base + i * 0.5)) - - df = spark.createDataFrame(data, "region string, amount double") - table_name = f"{TEST_CATALOG}.{schema.name}.segment_score_test_{suffix}" - df.write.saveAsTable(table_name) - - model_name = f"{TEST_CATALOG}.{schema.name}.test_score_segments_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - ) - - # Score with anomalous data (use same case as training data for segment matching) - test_data = [ - (1, "US", 100.0), # Normal - (2, "US", 1000.0), # Strong anomaly (5x the max US training value) - (3, "EU", 200.0), # Normal - (4, "EU", 1500.0), # Strong anomaly (5x the max EU training value) - ] - test_df = spark.createDataFrame(test_data, "row_id int, region string, amount double") - - dq_engine = DQEngine(ws, spark) - check = DQDatasetRule( - criticality="error", - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": model_name, - "registry_table": registry_table, - "threshold": DQENGINE_SCORE_THRESHOLD, # Lowered from 0.7 to account for IsolationForest scoring characteristics - # Only scores are asserted; skip SHAP contributions + AI explanations (no LLM call). - "enable_contributions": False, - "enable_ai_explanation": False, - }, - ) - - result = dq_engine.apply_checks(test_df, [check]) - - # Access anomaly_score from _dq_info[0].anomaly.score (nested in DQEngine results) - result_with_score = result.select( - "row_id", - "region", - "amount", - F.col("_dq_info").getItem(0).getField("anomaly").getField("score").alias("anomaly_score"), - ) - rows = result_with_score.collect() - assert len(rows) == 4 - - # Verify we got scores for all rows - assert all(row.anomaly_score is not None for row in rows), "Some rows missing anomaly scores" - - # Verify at least the anomalous rows exceed the threshold - high_scorers = [row for row in rows if row.anomaly_score > 0.6] - assert ( - len(high_scorers) >= 2 - ), f"Expected at least 2 rows with score>0.6, got {len(high_scorers)}. All scores: {[(r.row_id, r.anomaly_score) for r in rows]}" - - -def test_multi_column_segments( - spark: SparkSession, - make_schema, - make_random, - anomaly_engine, -): - """Test segmentation with multiple segment columns.""" - # Create unique schema for test isolation - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - # Generate data with region + product_type - # Use 1500 rows per segment to ensure reliable probabilistic sampling - data = [] - for region in ("US", "EU"): - for product in ("A", "B"): - base = 100 + (50 if region == "EU" else 0) + (25 if product == "B" else 0) - for i in range(1500): - data.append((region, product, base + i * 0.5)) - - df = spark.createDataFrame(data, "region string, product_type string, amount double") - table_name = f"{TEST_CATALOG}.{schema.name}.multi_segment_test_{suffix}" - df.write.saveAsTable(table_name) - - # Train with multiple segment columns - model_name = f"{TEST_CATALOG}.{schema.name}.test_multi_segments_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - train_model_with_params( - engine=anomaly_engine, - df=spark.table(table_name), - model_name=model_name, - registry_table=registry_table, - columns=["amount"], - segment_by=["region", "product_type"], - params=AnomalyParams(sample_fraction=1.0), - ) - - # Verify segment models created (2 regions × 2 products = 4 segments). - # With sufficient data per segment, all segments should train successfully. - registry = spark.table(registry_table) - models = registry.filter("identity.status = 'active'").collect() - assert len(models) == 4 - - -def test_unknown_segment_handling( - ws, - spark: SparkSession, - make_schema, - make_random, - anomaly_engine, -): - """Test scoring with unknown segment values (not in training data).""" - # Create unique schema for test isolation - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - # Train on US and EU only - data = [] - for region in ("US", "EU"): - base = 100 if region == "US" else 200 - for i in range(200): - data.append((region, base + i * 0.5)) - - df = spark.createDataFrame(data, "region string, amount double") - table_name = f"{TEST_CATALOG}.{schema.name}.unknown_segment_test_{suffix}" - df.write.saveAsTable(table_name) - - model_name = f"{TEST_CATALOG}.{schema.name}.test_unknown_segments_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - ) - - # Score with unknown region "APAC" - test_data = [ - (1, "US", 100.0), - (2, "APAC", 300.0), # Unknown segment - ] - test_df = spark.createDataFrame(test_data, "row_id int, region string, amount double") - - dq_engine = DQEngine(ws, spark) - check = DQDatasetRule( - criticality="error", - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": model_name, - "registry_table": registry_table, - # Only scores are asserted; skip SHAP contributions + AI explanations (no LLM call). - "enable_contributions": False, - "enable_ai_explanation": False, - }, - ) - - result = dq_engine.apply_checks(test_df, [check]) - - # APAC row should have null score (access from _dq_info[0].anomaly.score) - - result_with_score = result.select( - "*", F.col("_dq_info").getItem(0).getField("anomaly").getField("score").alias("anomaly_score") - ) - apac_row = [row for row in result_with_score.collect() if row.region == "APAC"][0] - assert apac_row.anomaly_score is None - - -def test_all_unknown_segments_yield_null_scores( - ws, - spark: SparkSession, - make_schema, - make_random, - anomaly_engine, -): - """All rows should have null scores if none match trained segments.""" - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - train_data = [] - for region in ("US", "EU"): - for i in range(200): - base = 100 if region == "US" else 200 - train_data.append((region, base + i * 0.5)) - - df = spark.createDataFrame(train_data, "region string, amount double") - table_name = f"{TEST_CATALOG}.{schema.name}.unknown_only_test_{suffix}" - df.write.saveAsTable(table_name) - - model_name = f"{TEST_CATALOG}.{schema.name}.test_unknown_only_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - ) - - test_data = [ - (1, "APAC", 300.0), - (2, "LATAM", 400.0), - ] - test_df = spark.createDataFrame(test_data, "row_id int, region string, amount double") - - dq_engine = DQEngine(ws, spark) - check = DQDatasetRule( - criticality="error", - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": model_name, - "registry_table": registry_table, - "enable_contributions": True, - "enable_confidence_std": True, - # This test asserts contributions/null scores, not explanations; skip the LLM call. - "enable_ai_explanation": False, - }, - ) - - result = dq_engine.apply_checks(test_df, [check]) - result_with_score = result.select( - "*", F.col("_dq_info").getItem(0).getField("anomaly").getField("score").alias("anomaly_score") - ) - rows = result_with_score.collect() - assert all(row.anomaly_score is None for row in rows) - - -def test_try_segmented_fallback_when_global_missing( - spark: SparkSession, - make_schema, - make_random, - anomaly_engine, - ws, -): - """Fallback should score using segmented models when global record is missing.""" - schema = make_schema(catalog_name=TEST_CATALOG) - suffix = make_random(8).lower() - - data = [] - for region in ("US", "EU"): - base = 100 if region == "US" else 200 - for i in range(200): - data.append((region, base + i * 0.5)) - - df = spark.createDataFrame(data, "region string, amount double") - table_name = f"{TEST_CATALOG}.{schema.name}.fallback_test_{suffix}" - df.write.saveAsTable(table_name) - - model_name = f"{TEST_CATALOG}.{schema.name}.test_fallback_{suffix}" - registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount"], - segment_by=["region"], - model_name=model_name, - registry_table=registry_table, - ) - - test_df = spark.createDataFrame( - [(1, "US", 100.0), (2, "EU", 200.0)], - "transaction_id int, region string, amount double", - ) - - dq_engine = DQEngine(ws, spark) - check = create_anomaly_check_rule( - model_name=model_name, - registry_table=registry_table, - threshold=DQENGINE_SCORE_THRESHOLD, - ) - result = dq_engine.apply_checks(test_df, [check]) - rows = result.select( - "transaction_id", F.col("_dq_info").getItem(0).getField("anomaly").getField("score").alias("score") - ).collect() - - assert all(row.score is not None for row in rows) diff --git a/tests/integration_anomaly/test_anomaly_train_and_score.py b/tests/integration_anomaly/test_anomaly_train_and_score.py index f1d822433..025668274 100644 --- a/tests/integration_anomaly/test_anomaly_train_and_score.py +++ b/tests/integration_anomaly/test_anomaly_train_and_score.py @@ -233,7 +233,7 @@ def test_registry_table_auto_creation(spark: SparkSession, make_schema, make_ran # Verify table has expected schema (nested structs) registry_df = spark.table(registry_table) - expected_top_level_columns = ["identity", "training", "features", "segmentation"] + expected_top_level_columns = ["identity", "training", "features", "grouping"] for col in expected_top_level_columns: assert col in registry_df.columns diff --git a/tests/integration_anomaly/test_anomaly_training_validation.py b/tests/integration_anomaly/test_anomaly_training_validation.py index 18ceba129..9735d92c0 100644 --- a/tests/integration_anomaly/test_anomaly_training_validation.py +++ b/tests/integration_anomaly/test_anomaly_training_validation.py @@ -71,7 +71,6 @@ def test_build_context_raises_for_empty_model_name(spark, make_schema, make_rand model_name="", registry_table=registry_table, columns=["amount", "quantity"], - segment_by=None, params=None, exclude_columns=None, expected_anomaly_rate=0.02, @@ -91,7 +90,6 @@ def test_build_context_raises_for_empty_registry_table(spark, make_schema, make_ model_name=model_name, registry_table="", columns=["amount", "quantity"], - segment_by=None, params=None, exclude_columns=None, expected_anomaly_rate=0.02, @@ -114,7 +112,6 @@ def test_build_context_raises_for_empty_columns(spark, make_schema, make_random) model_name=model_name, registry_table=registry_table, columns=[], - segment_by=None, params=None, exclude_columns=None, expected_anomaly_rate=0.02, @@ -169,7 +166,6 @@ def test_build_context_logs_validation_warnings(spark, make_schema, make_random, model_name=model_name, registry_table=registry_table, columns=["amount", "quantity"], - segment_by=None, params=params, exclude_columns=None, expected_anomaly_rate=0.02, @@ -193,7 +189,6 @@ def test_build_context_excludes_columns_from_auto_discovery(spark, make_schema, model_name=model_name, registry_table=registry_table, columns=None, - segment_by=None, params=None, exclude_columns=["b"], expected_anomaly_rate=0.02, @@ -217,7 +212,6 @@ def test_build_context_raises_when_exclude_columns_not_in_dataframe(spark, make_ model_name=model_name, registry_table=registry_table, columns=["a", "b"], - segment_by=None, params=None, exclude_columns=["c"], expected_anomaly_rate=0.02, diff --git a/tests/unit/test_anomaly_check_funcs_validation.py b/tests/unit/test_anomaly_check_funcs_validation.py index e23704526..c4bd87656 100644 --- a/tests/unit/test_anomaly_check_funcs_validation.py +++ b/tests/unit/test_anomaly_check_funcs_validation.py @@ -15,7 +15,7 @@ AnomalyModelRecord, FeatureEngineering, ModelIdentity, - SegmentationConfig, + GroupingConfig, TrainingMetadata, compute_config_hash, ) @@ -231,7 +231,7 @@ def test_resolve_scoring_strategy_returns_strategy_for_isolation_forest(): def test_run_anomaly_scoring_raises_when_model_not_found_and_no_fallback(): - """When get_active_model returns None and segmented fallback returns None, InvalidParameterError is raised.""" + """When get_active_model returns None, InvalidParameterError is raised.""" mock_spark = create_autospec(SparkSession, instance=True) mock_df = create_autospec(DataFrame, instance=True) mock_df.sparkSession = mock_spark @@ -241,17 +241,14 @@ def test_run_anomaly_scoring_raises_when_model_not_found_and_no_fallback(): registry_table = "catalog.schema.registry" model_name = "catalog.schema.my_model" - # Minimal record so discovery (fetch_model_columns_and_segments) succeeds. + # Minimal record so discovery (fetch_model_columns) succeeds. record = create_autospec(AnomalyModelRecord, instance=True) record.training = create_autospec(TrainingMetadata, instance=True) record.training.columns = ["a", "b"] - record.segmentation = create_autospec(SegmentationConfig, instance=True) - record.segmentation.segment_by = None # global model for discovery mock_registry = create_autospec(AnomalyModelRegistry, instance=True) # Discovery calls get_active_model once -> return record; orchestrator calls it -> None. mock_registry.get_active_model.side_effect = [record, None] - mock_registry.get_all_segment_models.return_value = [] # fallback returns None with patch.object(model_discovery, "AnomalyModelRegistry") as mock_cls_disc: with patch.object(scoring_orchestrator, "AnomalyModelRegistry") as mock_cls_orch: mock_cls_disc.return_value = mock_registry @@ -269,70 +266,6 @@ def test_run_anomaly_scoring_raises_when_model_not_found_and_no_fallback(): assert "Train first" in str(exc_info.value) -def test_load_segment_models_raises_when_no_segments(): - """Loading segment models raises when get_all_segment_models returns empty on second call.""" - registry_table = "catalog.schema.registry" - model_name = "catalog.schema.my_model" - mock_spark = create_autospec(SparkSession, instance=True) - mock_df = create_autospec(DataFrame, instance=True) - mock_df.sparkSession = mock_spark - mock_df.withColumn.return_value = mock_df - mock_df.columns = ["a", "b"] - - segment = create_autospec(AnomalyModelRecord, instance=True) - segment.identity = create_autospec(ModelIdentity, instance=True) - segment.identity.model_name = model_name - segment.identity.algorithm = "IsolationForestV1" - segment.training = create_autospec(TrainingMetadata, instance=True) - segment.training.columns = ["a", "b"] - segment.training.training_time = datetime.min - segment.segmentation = create_autospec(SegmentationConfig, instance=True) - segment.segmentation.segment_by = ["region"] - - # Discovery and orchestrator both create a registry; patch both so the same mock is used. - mock_registry = create_autospec(AnomalyModelRegistry, instance=True) - mock_registry.get_active_model.return_value = None - mock_registry.get_all_segment_models.side_effect = [[segment], []] - with patch.object(model_discovery, "AnomalyModelRegistry") as mock_cls_disc: - with patch.object(scoring_orchestrator, "AnomalyModelRegistry") as mock_cls_orch: - mock_cls_disc.return_value = mock_registry - mock_cls_orch.return_value = mock_registry - - _, apply_fn, _ = has_no_row_anomalies( - model_name=model_name, - registry_table=registry_table, - ) - with pytest.raises(InvalidParameterError) as exc_info: - apply_fn(mock_df) - - assert "No segment models found for base model" in str(exc_info.value) - assert model_name in str(exc_info.value) - assert "Train segmented models first" in str(exc_info.value) - - -def test_score_segmented_raises_when_no_segments(): - """Scoring strategy raises when no segment models are passed (all_segments empty).""" - mock_df = create_autospec(DataFrame, instance=True) - registry_table = "catalog.schema.registry" - model_name = "catalog.schema.my_model" - config = ScoringConfig( - columns=["a", "b"], - model_name=model_name, - registry_table=registry_table, - threshold=95.0, - merge_columns=[], - ) - mock_registry = create_autospec(AnomalyModelRegistry, instance=True) - - strategy = resolve_scoring_strategy("IsolationForestV1") - with pytest.raises(InvalidParameterError) as exc_info: - strategy.score_segmented(mock_df, config, mock_registry, all_segments=[]) - - assert "No segment models found for base model" in str(exc_info.value) - assert model_name in str(exc_info.value) - assert "Train segmented models first" in str(exc_info.value) - - def test_extract_quantile_points_raises_when_score_quantiles_missing(): """get_quantile_points_for_severity raises when record.training.score_quantiles is missing or empty.""" record = create_autospec(AnomalyModelRecord, instance=True) @@ -387,13 +320,11 @@ def test_apply_fn_raises_when_global_model_has_no_feature_metadata(): record.training.columns = columns record.training.training_time = datetime(2024, 1, 1) record.features = FeatureEngineering(feature_metadata=None) - record.segmentation = create_autospec(SegmentationConfig, instance=True) - record.segmentation.segment_by = None - record.segmentation.config_hash = config_hash + record.grouping = create_autospec(GroupingConfig, instance=True) + record.grouping.config_hash = config_hash mock_registry = create_autospec(AnomalyModelRegistry, instance=True) mock_registry.get_active_model.return_value = record - mock_registry.get_all_segment_models.return_value = [] with patch.object(model_discovery, "AnomalyModelRegistry") as mock_cls_disc: with patch.object(scoring_orchestrator, "AnomalyModelRegistry") as mock_cls_orch: @@ -409,43 +340,3 @@ def test_apply_fn_raises_when_global_model_has_no_feature_metadata(): assert "missing feature_metadata" in str(exc_info.value) assert model_name in str(exc_info.value) - - -def test_apply_fn_raises_when_segment_model_has_no_segment_by(): - """When segment fallback returns a segment with segment_by None, apply_fn raises InvalidParameterError.""" - mock_spark = create_autospec(SparkSession, instance=True) - mock_df = create_autospec(DataFrame, instance=True) - mock_df.sparkSession = mock_spark - mock_df.withColumn.return_value = mock_df - mock_df.columns = ["a", "b"] - - registry_table = "catalog.schema.registry" - model_name = "catalog.schema.my_model" - - segment = create_autospec(AnomalyModelRecord, instance=True) - segment.identity = create_autospec(ModelIdentity, instance=True) - segment.identity.model_name = model_name - segment.identity.algorithm = "IsolationForestV1" - segment.training = create_autospec(TrainingMetadata, instance=True) - segment.training.columns = ["a", "b"] - segment.training.training_time = datetime.min - segment.segmentation = create_autospec(SegmentationConfig, instance=True) - segment.segmentation.segment_by = None - - mock_registry = create_autospec(AnomalyModelRegistry, instance=True) - mock_registry.get_active_model.return_value = None - mock_registry.get_all_segment_models.return_value = [segment] - - with patch.object(model_discovery, "AnomalyModelRegistry") as mock_cls_disc: - with patch.object(scoring_orchestrator, "AnomalyModelRegistry") as mock_cls_orch: - mock_cls_disc.return_value = mock_registry - mock_cls_orch.return_value = mock_registry - - _, apply_fn, _ = has_no_row_anomalies( - model_name=model_name, - registry_table=registry_table, - ) - with pytest.raises(InvalidParameterError) as exc_info: - apply_fn(mock_df) - - assert "Segment model must have segment_by" in str(exc_info.value) diff --git a/tests/unit/test_anomaly_configs.py b/tests/unit/test_anomaly_configs.py index 8e06714ed..4f1ba1013 100644 --- a/tests/unit/test_anomaly_configs.py +++ b/tests/unit/test_anomaly_configs.py @@ -1,6 +1,5 @@ """Unit tests for anomaly detection configuration classes.""" -from databricks.labs.dqx.anomaly.group_config import MAX_SEGMENT_MODELS from databricks.labs.dqx.config import ( AnomalyConfig, AnomalyParams, @@ -86,20 +85,9 @@ def test_anomaly_params_defaults(): assert params.sample_fraction == 0.3 assert params.max_rows == 1_000_000 assert params.train_ratio == 0.8 - assert params.max_segment_models == 50 assert isinstance(params.algorithm_config, IsolationForestConfig) -def test_max_group_models_default_matches_group_config(): - """The default must track the shared constant. - - ``config`` cannot import it, because the anomaly package requires the 'anomaly' extras - and ``config`` must stay importable without them, so the value is duplicated as a - literal. This test is the thing that stops the two drifting apart. - """ - assert AnomalyParams().max_segment_models == MAX_SEGMENT_MODELS - - def test_anomaly_params_custom_sample_fraction(): """Test AnomalyParams with custom sample fraction values.""" # Valid sample fractions @@ -204,21 +192,21 @@ def test_anomaly_config_defaults(): """Test AnomalyConfig defaults.""" cfg = AnomalyConfig() assert cfg.columns is None - assert cfg.segment_by is None + assert cfg.baseline_by is None assert cfg.model_name is None assert cfg.registry_table is None -def test_anomaly_config_with_columns_and_segments(): - """Test AnomalyConfig with custom columns and segmentation.""" +def test_anomaly_config_with_columns_and_baseline(): + """Test AnomalyConfig with custom columns and a baseline grouping.""" cfg = AnomalyConfig( columns=["a", "b"], - segment_by=["region"], + baseline_by=["region"], model_name="demo_model", registry_table="main.default.dqx_anomaly_models", ) assert cfg.columns == ["a", "b"] - assert cfg.segment_by == ["region"] + assert cfg.baseline_by == ["region"] assert cfg.model_name == "demo_model" assert cfg.registry_table == "main.default.dqx_anomaly_models" diff --git a/tests/unit/test_anomaly_model_record.py b/tests/unit/test_anomaly_model_record.py index 9eefbe4e9..f62facbcb 100644 --- a/tests/unit/test_anomaly_model_record.py +++ b/tests/unit/test_anomaly_model_record.py @@ -10,7 +10,7 @@ ModelIdentity, TrainingMetadata, FeatureEngineering, - SegmentationConfig, + GroupingConfig, ) @@ -30,17 +30,15 @@ def test_anomaly_model_record_creation_with_defaults(): training_time=datetime(2024, 1, 1, 12, 0, 0), ), features=FeatureEngineering(), - segmentation=SegmentationConfig(), + grouping=GroupingConfig(), ) assert record.identity.model_name == "test_model" assert record.identity.model_uri == "models:/test_model/1" assert record.identity.status == "active" # Default assert record.features.mode == "spark" # Default - assert record.segmentation.is_global_model is True # Default assert record.training.metrics is None # Default - assert record.segmentation.segment_by is None # Default - assert record.segmentation.segment_values is None # Default + assert record.grouping.baseline_by is None # Default def test_anomaly_model_record_with_all_fields(): @@ -71,24 +69,20 @@ def test_anomaly_model_record_with_all_fields(): column_types={"amount": "numeric", "quantity": "numeric", "discount": "numeric"}, feature_metadata='{"engineered_features": ["amount_scaled", "quantity_scaled"]}', ), - segmentation=SegmentationConfig( - segment_by=["region"], - segment_values={"region": "US"}, - is_global_model=False, + grouping=GroupingConfig( + baseline_by=["region"], ), ) assert record.identity.model_name == "segmented_model" assert record.identity.status == "archived" assert record.features.mode == "sklearn" - assert record.segmentation.is_global_model is False assert len(record.training.metrics) == 2 assert record.training.metrics["precision"] == 0.85 assert len(record.training.baseline_stats) == 2 assert record.training.baseline_stats["amount"]["mean"] == 150.0 assert len(record.features.feature_importance) == 3 - assert record.segmentation.segment_by == ["region"] - assert record.segmentation.segment_values == {"region": "US"} + assert record.grouping.baseline_by == ["region"] assert record.features.column_types["amount"] == "numeric" assert "engineered_features" in record.features.feature_metadata @@ -207,7 +201,7 @@ def test_anomaly_model_record_to_dict(): metrics={"precision": 0.85}, ), features=FeatureEngineering(), - segmentation=SegmentationConfig(), + grouping=GroupingConfig(), ) record_dict = record.__dict__ @@ -238,7 +232,7 @@ def test_anomaly_model_record_defaults_for_optional_fields(): training_time=datetime.now(), ), features=FeatureEngineering(), - segmentation=SegmentationConfig(), + grouping=GroupingConfig(), ) # These should all be None by default @@ -246,8 +240,7 @@ def test_anomaly_model_record_defaults_for_optional_fields(): assert record.training.baseline_stats is None assert record.features.feature_importance is None assert record.features.temporal_config is None - assert record.segmentation.segment_by is None - assert record.segmentation.segment_values is None + assert record.grouping.baseline_by is None assert record.features.column_types is None assert record.features.feature_metadata is None @@ -270,7 +263,7 @@ def test_anomaly_model_record_with_empty_collections(): baseline_stats={}, # Empty dict ), features=FeatureEngineering(), - segmentation=SegmentationConfig(), + grouping=GroupingConfig(), ) assert not record.training.columns diff --git a/tests/unit/test_anomaly_model_registry.py b/tests/unit/test_anomaly_model_registry.py index a493ede54..3fda305f9 100644 --- a/tests/unit/test_anomaly_model_registry.py +++ b/tests/unit/test_anomaly_model_registry.py @@ -8,7 +8,7 @@ AnomalyModelRecord, FeatureEngineering, ModelIdentity, - SegmentationConfig, + GroupingConfig, TrainingMetadata, ) @@ -19,12 +19,13 @@ def test_compute_config_hash_order_independent() -> None: - hash_a = compute_config_hash(["b", "a"], ["seg2", "seg1"]) - hash_b = compute_config_hash(["a", "b"], ["seg1", "seg2"]) + """Columns and baseline columns are sets: listing order must not change the hash.""" + hash_a = compute_config_hash(["b", "a"], ["g2", "g1"]) + hash_b = compute_config_hash(["a", "b"], ["g1", "g2"]) assert hash_a == hash_b -def test_compute_config_hash_handles_none_segment_by() -> None: +def test_compute_config_hash_handles_none_baseline_by() -> None: hash_a = compute_config_hash(["a", "b"], None) hash_b = compute_config_hash(["b", "a"], None) assert hash_a == hash_b @@ -37,21 +38,16 @@ def test_compute_config_hash_distinguishes_baseline_by() -> None: is part of the configuration -- it changes the feature list and the persisted baselines. """ ungrouped = compute_config_hash(["a", "b"], None) - grouped = compute_config_hash(["a", "b"], None, ["region"]) - grouped_wider = compute_config_hash(["a", "b"], None, ["region", "product"]) + grouped = compute_config_hash(["a", "b"], ["region"]) + grouped_wider = compute_config_hash(["a", "b"], ["region", "product"]) assert ungrouped != grouped assert grouped != grouped_wider -def test_compute_config_hash_is_baseline_order_independent() -> None: - """Baseline columns are a set: the key is built from them sorted, so listing order cannot matter.""" - assert compute_config_hash(["a"], None, ["p", "c"]) == compute_config_hash(["a"], None, ["c", "p"]) - - def test_compute_config_hash_treats_empty_baseline_as_ungrouped() -> None: """``baseline_by=[]`` is how a caller asks for whole-table comparison, which is the ungrouped case.""" - assert compute_config_hash(["a"], None, []) == compute_config_hash(["a"], None, None) + assert compute_config_hash(["a"], []) == compute_config_hash(["a"], None) def test_compute_config_hash_different_columns_produce_different_hash() -> None: @@ -61,20 +57,6 @@ def test_compute_config_hash_different_columns_produce_different_hash() -> None: assert hash_a != hash_b -def test_compute_config_hash_different_segments_produce_different_hash() -> None: - """Different segment_by should produce different hashes.""" - hash_a = compute_config_hash(["col1"], ["region"]) - hash_b = compute_config_hash(["col1"], ["country"]) - assert hash_a != hash_b - - -def test_compute_config_hash_with_vs_without_segments() -> None: - """Segmented vs non-segmented should produce different hashes.""" - hash_segmented = compute_config_hash(["col1"], ["region"]) - hash_global = compute_config_hash(["col1"], None) - assert hash_segmented != hash_global - - def test_compute_config_hash_empty_columns() -> None: """Empty columns should still produce a hash.""" hash_empty = compute_config_hash([], None) @@ -85,11 +67,11 @@ def test_compute_config_hash_empty_columns() -> None: def test_compute_config_hash_is_deterministic() -> None: """Same inputs should always produce same hash.""" columns = ["amount", "quantity"] - segment_by = ["region"] + baseline_by = ["region"] - hash_1 = compute_config_hash(columns, segment_by) - hash_2 = compute_config_hash(columns, segment_by) - hash_3 = compute_config_hash(columns, segment_by) + hash_1 = compute_config_hash(columns, baseline_by) + hash_2 = compute_config_hash(columns, baseline_by) + hash_3 = compute_config_hash(columns, baseline_by) assert hash_1 == hash_2 == hash_3 @@ -127,26 +109,16 @@ def test_training_metadata_required_fields() -> None: assert metadata.columns == ["amount", "quantity"] -def test_segmentation_config_for_global_model() -> None: - """Test SegmentationConfig for non-segmented model.""" - config = SegmentationConfig( - segment_by=None, - segment_values=None, - ) - - assert config.segment_by is None - assert config.segment_values is None - +def test_grouping_config_for_ungrouped_model() -> None: + """GroupingConfig for a model with no baseline conditioning.""" + config = GroupingConfig() + assert config.baseline_by is None -def test_segmentation_config_for_segment_model() -> None: - """Test SegmentationConfig for segmented model.""" - config = SegmentationConfig( - segment_by=["region"], - segment_values={"region": "US"}, - ) - assert config.segment_by == ["region"] - assert config.segment_values == {"region": "US"} +def test_grouping_config_for_conditioned_model() -> None: + """GroupingConfig for a model conditioned on a baseline grouping.""" + config = GroupingConfig(baseline_by=["region", "product"]) + assert config.baseline_by == ["region", "product"] # ============================================================================ @@ -175,22 +147,18 @@ def test_anomaly_model_record_full_construction() -> None: feature_importance=None, temporal_config=None, ), - segmentation=SegmentationConfig( - segment_by=None, - segment_values=None, - ), + grouping=GroupingConfig(baseline_by=None), ) assert record.identity.model_name == "catalog.schema.model" assert record.training.training_rows == 5000 assert record.training.columns == ["amount", "quantity"] - assert record.segmentation.segment_by is None + assert record.grouping.baseline_by is None -def test_is_segmented_property() -> None: - """Test is_segmented detection based on segment_by.""" - # Non-segmented model - global_record = AnomalyModelRecord( +def test_grouping_recorded_on_the_model() -> None: + """A conditioned model carries its baseline grouping; an unconditioned one carries None.""" + ungrouped = AnomalyModelRecord( identity=ModelIdentity( model_name="model", model_uri="uri", @@ -204,21 +172,13 @@ def test_is_segmented_property() -> None: training_rows=1000, training_time=datetime.now(), ), - features=FeatureEngineering( - feature_metadata=None, - feature_importance=None, - temporal_config=None, - ), - segmentation=SegmentationConfig( - segment_by=None, - segment_values=None, - ), + features=FeatureEngineering(feature_metadata=None, feature_importance=None, temporal_config=None), + grouping=GroupingConfig(baseline_by=None), ) - # Segmented model - segment_record = AnomalyModelRecord( + conditioned = AnomalyModelRecord( identity=ModelIdentity( - model_name="model__seg_region=US", + model_name="model", model_uri="uri", algorithm="IsolationForest", mlflow_run_id="run2", @@ -230,23 +190,12 @@ def test_is_segmented_property() -> None: training_rows=1000, training_time=datetime.now(), ), - features=FeatureEngineering( - feature_metadata=None, - feature_importance=None, - temporal_config=None, - ), - segmentation=SegmentationConfig( - segment_by=["region"], - segment_values={"region": "US"}, - ), + features=FeatureEngineering(feature_metadata=None, feature_importance=None, temporal_config=None), + grouping=GroupingConfig(baseline_by=["region"]), ) - # Global model has no segment_by - assert global_record.segmentation.segment_by is None - - # Segmented model has segment_by and segment_values - assert segment_record.segmentation.segment_by == ["region"] - assert segment_record.segmentation.segment_values == {"region": "US"} + assert ungrouped.grouping.baseline_by is None + assert conditioned.grouping.baseline_by == ["region"] # ============================================================================ @@ -275,5 +224,5 @@ def _minimal_record(model_name: str = "catalog.schema.model") -> AnomalyModelRec feature_importance=None, temporal_config=None, ), - segmentation=SegmentationConfig(segment_by=None, segment_values=None), + grouping=GroupingConfig(baseline_by=None), ) diff --git a/tests/unit/test_anomaly_scoring_run.py b/tests/unit/test_anomaly_scoring_run.py deleted file mode 100644 index b13848087..000000000 --- a/tests/unit/test_anomaly_scoring_run.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Unit tests for the budget-allocation helper used by *score_segmented*. - -Verifies that *_split_max_groups_budget* keeps the *total* LLM-call cap (per-segment -budget × num_segments) at or below the user-facing *max_groups* whenever the budget -is at least the segment count, and falls back to a documented finite bound (one call -per segment) when the budget is under-provisioned. -""" - -import pytest - -from databricks.labs.dqx.anomaly import scoring_run -from databricks.labs.dqx.errors import InvalidParameterError - - -@pytest.mark.parametrize( - "max_groups, num_segments, expected_per_segment", - [ - # Even split — total = 500, exactly at the cap. - pytest.param(500, 5, 100, id="even_split"), - # Uneven split — floor division yields 142, total = 994 < 1000. - pytest.param(1000, 7, 142, id="floor_divides_below_cap"), - # Single segment — gets the full budget. - pytest.param(500, 1, 500, id="single_segment_gets_full_budget"), - # Budget exactly equals segment count — each segment gets 1. - pytest.param(10, 10, 1, id="budget_equals_segments"), - ], -) -def test_split_keeps_total_at_or_below_cap(max_groups, num_segments, expected_per_segment): - per_segment = scoring_run._split_max_groups_budget(max_groups, num_segments) - assert per_segment == expected_per_segment - # The whole point of the helper: total LLM calls across segments stays bounded. - assert per_segment * num_segments <= max_groups - - -def test_split_floor_of_one_when_budget_under_provisioned(): - """When *max_groups* < num_segments the floor of 1 kicks in. - - Documented tradeoff: total calls become ``num_segments`` (> *max_groups*) but the - cap is still finite and proportional to the input — every segment gets a chance to - produce at least one explanation. Locks the documented behaviour. - """ - per_segment = scoring_run._split_max_groups_budget(max_groups=3, num_eligible_segments=10) - assert per_segment == 1 - # Total = 10 > max_groups=3, by design. - assert per_segment * 10 == 10 - - -def test_split_rejects_zero_segments(): - """Calling with no eligible segments is a programming error — *score_segmented* - skips the call entirely when *eligible* is empty. Surface it loudly here so a - refactor that drops the guard fails fast.""" - with pytest.raises(InvalidParameterError, match="num_eligible_segments must be positive"): - scoring_run._split_max_groups_budget(max_groups=500, num_eligible_segments=0) - - -def test_split_rejects_negative_segments(): - with pytest.raises(InvalidParameterError, match="num_eligible_segments must be positive"): - scoring_run._split_max_groups_budget(max_groups=500, num_eligible_segments=-1) diff --git a/tests/unit/test_anomaly_segment_naming.py b/tests/unit/test_anomaly_segment_naming.py deleted file mode 100644 index d90096a4c..000000000 --- a/tests/unit/test_anomaly_segment_naming.py +++ /dev/null @@ -1,234 +0,0 @@ -"""Unit tests for segment model naming convention consistency. - -These tests ensure that segment model names are consistent between training -(service.py) and querying (model_registry.py) to prevent lookup failures. -""" - -import pytest -from pyspark.sql import types as T - -from databricks.labs.dqx.anomaly import training_service as anomaly_training_service -from databricks.labs.dqx.anomaly.transformers import ColumnTypeInfo -from databricks.labs.dqx.anomaly.segment_utils import build_segment_name -from databricks.labs.dqx.anomaly.validation import validate_fully_qualified_name -from databricks.labs.dqx.errors import InvalidParameterError - - -class TestSegmentNamingConvention: - """Test segment model naming conventions match between training and registry.""" - - def test_segment_name_format_training_matches_registry_query(self) -> None: - """Verify training creates names that registry can find. - - This test documents the expected segment naming convention: - - Format: {base_model_name}__seg_{col1}={val1}_{col2}={val2} - - Example: my_model__seg_region=US - """ - # Simulate the naming logic from service.py (training) - base_model_name = "catalog.schema.my_model" - seg_values = {"region": "US"} - - # Training logic (service.py) - segment_name_training = build_segment_name(seg_values) - model_name_training = f"{base_model_name}__seg_{segment_name_training}" - - # Registry query logic (model_registry.py) - segment_name_registry = build_segment_name(seg_values) - model_name_registry = f"{base_model_name}__seg_{segment_name_registry}" - - # These MUST match for segment lookup to work - assert model_name_training == model_name_registry - assert model_name_training == "catalog.schema.my_model__seg_region=US" - - def test_multi_segment_naming_consistency(self) -> None: - """Test naming with multiple segment columns.""" - base_model_name = "catalog.schema.model" - seg_values = {"region": "US", "product": "A"} - - # Training creates this name - segment_name = build_segment_name(seg_values) - model_name = f"{base_model_name}__seg_{segment_name}" - - # Should contain all segment info - assert "__seg_" in model_name - assert "region=US" in model_name or "product=A" in model_name - - def test_segment_prefix_is_double_underscore(self) -> None: - """Ensure segment prefix uses double underscore for unambiguous parsing.""" - base_name = "my_model_name" - segment = "region=US" - - model_name = f"{base_name}__seg_{segment}" - - # Double underscore separates base name from segment info - assert "__seg_" in model_name - # Can reliably split on __seg_ to get base name - parts = model_name.split("__seg_") - assert len(parts) == 2 - assert parts[0] == base_name - assert parts[1] == segment - - def test_segment_value_format_uses_equals_sign(self) -> None: - """Verify segment values use = not _ between key and value.""" - seg_values = {"region": "US", "tier": "premium"} - - segment_name = build_segment_name(seg_values) - - # Must use = to separate key from value - assert "region=US" in segment_name or "tier=premium" in segment_name - # Not the old incorrect format - assert "region_US" not in segment_name.replace("region=US", "") - - def test_segment_name_startswith_query_pattern(self) -> None: - """Test that registry query pattern matches training output.""" - base_model_name = "catalog.schema.model" - seg_values = {"region": "APAC"} - - # Training output - segment_name = build_segment_name(seg_values) - trained_model_name = f"{base_model_name}__seg_{segment_name}" - - # Registry query pattern - query_prefix = f"{base_model_name}__seg_" - - # Training output must start with query prefix - assert trained_model_name.startswith(query_prefix) - - -class TestSegmentNameEdgeCases: - """Test edge cases in segment naming.""" - - def test_segment_value_with_special_characters(self) -> None: - """Test segment values containing underscores or other chars.""" - seg_values = {"region": "US_EAST", "product_line": "premium"} - - segment_name = build_segment_name(seg_values) - model_name = f"base__seg_{segment_name}" - - # Should handle underscores in values - assert "US_EAST" in model_name or "premium" in model_name - - def test_segment_value_case_sensitivity(self) -> None: - """Segment values should be case-sensitive.""" - seg_values_upper = {"region": "US"} - seg_values_lower = {"region": "us"} - - name_upper = build_segment_name(seg_values_upper) - name_lower = build_segment_name(seg_values_lower) - - # Different cases should produce different names - assert name_upper != name_lower - assert "region=US" in name_upper - assert "region=us" in name_lower - - def test_empty_segment_values_not_allowed(self) -> None: - """Empty segment dict would result in malformed name.""" - seg_values: dict[str, str] = {} - - segment_name = build_segment_name(seg_values) - - # Empty segment produces empty string - assert segment_name == "" - - def test_segment_order_consistency(self) -> None: - """Segment key order should be consistent for lookups.""" - # Same segments, different insertion order - seg_values_1 = {"region": "US", "tier": "gold"} - seg_values_2 = {"tier": "gold", "region": "US"} - - name_1 = build_segment_name(seg_values_1) - name_2 = build_segment_name(seg_values_2) - - assert name_1 == name_2 - assert name_1 == "region=US_tier=gold" - - -class TestValidateFullyQualifiedNameConsistency: - """Test that validation function is used consistently.""" - - def test_service_and_check_funcs_use_same_validation(self) -> None: - """Verify validation module is the single source of truth for fully qualified names.""" - assert anomaly_training_service.validate_fully_qualified_name is validate_fully_qualified_name - - def test_validation_accepts_three_part_names(self) -> None: - """Test validation accepts catalog.schema.name format.""" - # Should not raise - validate_fully_qualified_name("catalog.schema.model", label="model") - validate_fully_qualified_name("main.default.table", label="table") - - def test_validation_rejects_two_part_names(self) -> None: - """Test validation rejects schema.name format.""" - with pytest.raises(InvalidParameterError): - validate_fully_qualified_name("schema.model", label="model") - - def test_validation_rejects_single_part_names(self) -> None: - """Test validation rejects simple names.""" - with pytest.raises(InvalidParameterError): - validate_fully_qualified_name("model", label="model") - - -class TestFeatureEngineeringMetadataConsistency: - """Test feature engineering metadata is consistent between training and scoring.""" - - def test_column_type_info_required_fields(self) -> None: - """Test ColumnTypeInfo has all required fields for reconstruction.""" - info = ColumnTypeInfo( - name="test_col", - spark_type=T.DoubleType(), - category="numeric", - cardinality=None, - null_count=0, - encoding_strategy="none", - ) - - # All fields needed for feature reconstruction - assert info.name is not None - assert info.spark_type is not None - assert info.category is not None - - def test_column_type_info_category_values(self) -> None: - """Test ColumnTypeInfo category field has expected values.""" - valid_categories = ["numeric", "categorical", "datetime", "boolean", "unsupported"] - - for category in valid_categories: - assert category in valid_categories - - -class TestModelRegistryQueryPatterns: - """Test registry query patterns match what training produces.""" - - def test_global_model_query_by_exact_name(self) -> None: - """Global models are queried by exact name match.""" - model_name = "catalog.schema.my_model" - - # Query should match exact name - query_name = model_name - assert query_name == model_name - - def test_segment_model_query_by_prefix(self) -> None: - """Segment models are queried by prefix pattern.""" - base_name = "catalog.schema.my_model" - - # Query prefix for all segments of a base model - query_prefix = f"{base_name}__seg_" - - # All segment models should start with this prefix - segment_names = [ - f"{base_name}__seg_region=US", - f"{base_name}__seg_region=EU", - f"{base_name}__seg_region=APAC", - ] - - for name in segment_names: - assert name.startswith(query_prefix) - - def test_specific_segment_query_pattern(self) -> None: - """Test querying for a specific segment combination.""" - base_name = "catalog.schema.my_model" - segment_values = {"region": "US"} - - # Build expected segment model name - segment_name = build_segment_name(segment_values) - expected_model_name = f"{base_name}__seg_{segment_name}" - - assert expected_model_name == "catalog.schema.my_model__seg_region=US" diff --git a/tests/unit/test_anomaly_validation.py b/tests/unit/test_anomaly_validation.py index 2f3c963b6..80d5f1b74 100644 --- a/tests/unit/test_anomaly_validation.py +++ b/tests/unit/test_anomaly_validation.py @@ -11,7 +11,7 @@ AnomalyModelRecord, FeatureEngineering, ModelIdentity, - SegmentationConfig, + GroupingConfig, TrainingMetadata, ) from databricks.labs.dqx.anomaly.validation import validate_sklearn_compatibility, validate_training_params @@ -523,7 +523,7 @@ def _make_record(sklearn_version: str | None) -> AnomalyModelRecord: training_time=datetime.now(), ), features=FeatureEngineering(), - segmentation=SegmentationConfig(sklearn_version=sklearn_version), + grouping=GroupingConfig(sklearn_version=sklearn_version), ) From 7667ffed470e46cb32aa5c2613c0026e6ec8abd7 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 12:54:27 +0100 Subject: [PATCH 028/107] Add a reverse map from engineered feature name to source column Feature engineering expands each source column into engineered features by a fixed set of naming conventions (one-hot, frequency, null indicator, boolean, datetime cyclicals, numeric identity, baseline-relative). Nothing inverted those conventions, so two things were impossible: redacting every feature derived from a redacted column, and showing a reader a raw key like `event_count_rel_baseline` as a human phrase. New anomaly/feature_naming.py, pure functions over SparkFeatureMetadata: - source_column(name, metadata) -> str | None - human_label(name, metadata) -> str (falls back to the name; never raises, never hides a driver) - engineered_from(source, metadata) -> frozenset[str] (defined via source_column, so forward and reverse can never disagree) Resolution order is deliberate: one-hot first (matched against onehot_categories, since a value may contain an underscore or collide with a fixed suffix), then fixed suffixes accepted only when the remainder is a real source column, then numeric identity. Unit-tested across every convention, including a category value that collides with a suffix and a numeric column named like one. Unused until the next two commits wire it into redaction and prompt rendering. Co-authored-by: Isaac --- .../labs/dqx/anomaly/feature_naming.py | 139 ++++++++++++++++++ tests/unit/test_anomaly_feature_naming.py | 135 +++++++++++++++++ 2 files changed, 274 insertions(+) create mode 100644 src/databricks/labs/dqx/anomaly/feature_naming.py create mode 100644 tests/unit/test_anomaly_feature_naming.py diff --git a/src/databricks/labs/dqx/anomaly/feature_naming.py b/src/databricks/labs/dqx/anomaly/feature_naming.py new file mode 100644 index 000000000..dbdc94777 --- /dev/null +++ b/src/databricks/labs/dqx/anomaly/feature_naming.py @@ -0,0 +1,139 @@ +"""Reverse mapping from an engineered feature name back to the source column it came from. + +Feature engineering (see *transformers.py*) expands each source column into one or more engineered +features by a fixed set of naming conventions: one-hot ``{{col}}_{{value}}``, frequency ``{{col}}_freq``, +null indicator ``{{col}}_is_null``, boolean ``{{col}}_bool``, the datetime cyclicals +``{{col}}_hour_sin`` / ``_hour_cos`` / ``_dow_sin`` / ``_dow_cos`` / ``_month_sin`` / ``_month_cos`` / +``_is_weekend``, numeric identity (the feature *is* the column), and baseline-relative +``{{metric}}_rel_baseline``. This module inverts those conventions so that: + +- redaction can drop *every* feature derived from a redacted column, not just the column itself -- + a one-hot or frequency feature leaks the same information as the raw column, and +- a raw contribution key such as ``event_count_rel_baseline`` can be rendered for a person as + ``event_count vs its group baseline``. + +Pure functions over *SparkFeatureMetadata*: no Spark, no I/O, deterministic. + +Resolution order is deliberate and must not be reordered. One-hot names are matched first, against +the recorded ``onehot_categories``, because a category *value* may itself end in a fixed suffix +(a column *status* with a value *freq* produces ``status_freq``, which a suffix-first scan would +misread as frequency encoding of *status*). Fixed suffixes are tried next, and only accepted when +the remainder is a known source column, so a numeric column literally named ``revenue_freq`` is not +mistaken for the frequency encoding of a non-existent *revenue*. Numeric identity is matched last. +""" + +from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX, SparkFeatureMetadata + +# Fixed suffixes appended by the non-one-hot transforms, paired with a human-phrase template applied +# to the recovered source column. Order within this tuple does not affect correctness -- a suffix is +# only accepted when the remainder is a known source column, and no two suffixes strip the same name +# to the same valid column -- but the baseline-relative suffix is listed first because it is the one +# a reader is most likely to see in a contribution. +_SUFFIX_LABELS: tuple[tuple[str, str], ...] = ( + (BASELINE_RELATIVE_SUFFIX, "{col} vs its group baseline"), + ("_is_weekend", "{col} is a weekend"), + ("_hour_sin", "{col} hour"), + ("_hour_cos", "{col} hour"), + ("_dow_sin", "{col} day of week"), + ("_dow_cos", "{col} day of week"), + ("_month_sin", "{col} month"), + ("_month_cos", "{col} month"), + ("_is_null", "{col} is null"), + ("_freq", "{col} frequency"), + ("_bool", "{col}"), +) + + +def _source_column_names(metadata: SparkFeatureMetadata) -> frozenset[str]: + """Names of the analysed source columns, from the persisted column_infos.""" + return frozenset(info["name"] for info in metadata.column_infos if "name" in info) + + +def _match_onehot(engineered_name: str, metadata: SparkFeatureMetadata) -> tuple[str, str] | None: + """Return (source_column, value) if *engineered_name* is a recorded one-hot feature. + + Matched against ``onehot_categories`` rather than by splitting on ``_``, because a value may + contain an underscore and the column name may too, so no split point is reliable. + """ + for col, values in metadata.onehot_categories.items(): + for value in values: + if engineered_name == f"{col}_{value}": + return col, value + return None + + +def _match_suffix(engineered_name: str, source_names: frozenset[str]) -> tuple[str, str] | None: + """Return (source_column, label_template) if *engineered_name* ends in a known fixed suffix + and the remainder is a real source column.""" + for suffix, template in _SUFFIX_LABELS: + if engineered_name.endswith(suffix): + col = engineered_name[: -len(suffix)] + if col in source_names: + return col, template + return None + + +def source_column(engineered_name: str, metadata: SparkFeatureMetadata) -> str | None: + """The source column an engineered feature came from, or None if it cannot be resolved. + + Args: + engineered_name: A feature name as it appears in *engineered_feature_names* / a SHAP key. + metadata: The persisted feature metadata for the model that produced the feature. + + Returns: + The source column name, or None when the feature matches no known convention (a model + trained by a newer DQX with a convention this version does not know). + """ + onehot = _match_onehot(engineered_name, metadata) + if onehot is not None: + return onehot[0] + + source_names = _source_column_names(metadata) + suffix = _match_suffix(engineered_name, source_names) + if suffix is not None: + return suffix[0] + + if engineered_name in source_names: + return engineered_name + return None + + +def human_label(engineered_name: str, metadata: SparkFeatureMetadata) -> str: + """A short human phrase for an engineered feature, for display in prompts and info structs. + + Falls back to the engineered name unchanged when the feature matches no known convention, so + this never raises and never hides a driver from a reader. + + Args: + engineered_name: A feature name as it appears in *engineered_feature_names* / a SHAP key. + metadata: The persisted feature metadata for the model that produced the feature. + """ + onehot = _match_onehot(engineered_name, metadata) + if onehot is not None: + col, value = onehot + return f"{col} = {value}" + + source_names = _source_column_names(metadata) + suffix = _match_suffix(engineered_name, source_names) + if suffix is not None: + col, template = suffix + return template.format(col=col) + + if engineered_name in source_names: + return engineered_name + return engineered_name + + +def engineered_from(source: str, metadata: SparkFeatureMetadata) -> frozenset[str]: + """Every engineered feature derived from *source*. + + Defined in terms of *source_column* over the recorded feature list, so the forward and reverse + directions can never disagree: a feature is in this set exactly when it resolves back to + *source*. This is what lets redaction of a source column drop all of its derived features -- + one-hot, frequency, null indicator, baseline-relative and the rest. + + Args: + source: A source column name. + metadata: The persisted feature metadata for the model. + """ + return frozenset(name for name in metadata.engineered_feature_names if source_column(name, metadata) == source) diff --git a/tests/unit/test_anomaly_feature_naming.py b/tests/unit/test_anomaly_feature_naming.py new file mode 100644 index 000000000..c0f851e34 --- /dev/null +++ b/tests/unit/test_anomaly_feature_naming.py @@ -0,0 +1,135 @@ +"""Unit tests for the engineered-feature reverse map (no Spark).""" + +import pytest + +from databricks.labs.dqx.anomaly.feature_naming import engineered_from, human_label, source_column +from databricks.labs.dqx.anomaly.transformers import SparkFeatureMetadata + + +@pytest.fixture +def metadata() -> SparkFeatureMetadata: + """Covers every naming convention across distinct source columns. + + - amount: numeric identity + baseline-relative + - country: one-hot (US, DE) + a null indicator + - region: one-hot with an underscore in the value + - channel: frequency encoding + - is_active: boolean + - signup: the seven datetime cyclicals + """ + return SparkFeatureMetadata( + column_infos=[ + {"name": "amount", "category": "numeric"}, + {"name": "country", "category": "categorical"}, + {"name": "region", "category": "categorical"}, + {"name": "channel", "category": "categorical"}, + {"name": "is_active", "category": "boolean"}, + {"name": "signup", "category": "datetime"}, + ], + categorical_frequency_maps={"channel": {"web": 0.6, "app": 0.4}}, + onehot_categories={"country": ["US", "DE"], "region": ["north_america"]}, + engineered_feature_names=[ + "amount", + "country_US", + "country_DE", + "country_is_null", + "region_north_america", + "channel_freq", + "is_active_bool", + "signup_hour_sin", + "signup_hour_cos", + "signup_dow_sin", + "signup_dow_cos", + "signup_month_sin", + "signup_month_cos", + "signup_is_weekend", + "amount_rel_baseline", + ], + baseline_by=["country"], + ) + + +@pytest.mark.parametrize( + "engineered_name, expected_source, expected_label", + [ + ("amount", "amount", "amount"), + ("amount_rel_baseline", "amount", "amount vs its group baseline"), + ("country_US", "country", "country = US"), + ("country_DE", "country", "country = DE"), + ("country_is_null", "country", "country is null"), + ("region_north_america", "region", "region = north_america"), + ("channel_freq", "channel", "channel frequency"), + ("is_active_bool", "is_active", "is_active"), + ("signup_hour_sin", "signup", "signup hour"), + ("signup_hour_cos", "signup", "signup hour"), + ("signup_dow_sin", "signup", "signup day of week"), + ("signup_dow_cos", "signup", "signup day of week"), + ("signup_month_sin", "signup", "signup month"), + ("signup_month_cos", "signup", "signup month"), + ("signup_is_weekend", "signup", "signup is a weekend"), + ], +) +def test_every_convention_resolves( + metadata: SparkFeatureMetadata, engineered_name: str, expected_source: str, expected_label: str +): + assert source_column(engineered_name, metadata) == expected_source + assert human_label(engineered_name, metadata) == expected_label + + +def test_engineered_from_covers_all_derived_features(metadata: SparkFeatureMetadata): + assert engineered_from("country", metadata) == frozenset({"country_US", "country_DE", "country_is_null"}) + assert engineered_from("amount", metadata) == frozenset({"amount", "amount_rel_baseline"}) + assert engineered_from("channel", metadata) == frozenset({"channel_freq"}) + assert engineered_from("signup", metadata) == frozenset( + { + "signup_hour_sin", + "signup_hour_cos", + "signup_dow_sin", + "signup_dow_cos", + "signup_month_sin", + "signup_month_cos", + "signup_is_weekend", + } + ) + + +def test_engineered_from_is_empty_for_unknown_source(metadata: SparkFeatureMetadata): + assert engineered_from("not_a_column", metadata) == frozenset() + + +def test_onehot_value_that_collides_with_a_suffix(): + """A category value equal to a fixed suffix must resolve as one-hot, not as that suffix. + + Column *status* with a value *freq* produces ``status_freq``; a suffix-first scan would misread + it as frequency encoding of *status*. One-hot is matched first, so it resolves correctly. + """ + meta = SparkFeatureMetadata( + column_infos=[{"name": "status", "category": "categorical"}], + categorical_frequency_maps={}, + onehot_categories={"status": ["freq", "open"]}, + engineered_feature_names=["status_freq", "status_open"], + ) + assert source_column("status_freq", meta) == "status" + assert human_label("status_freq", meta) == "status = freq" + + +def test_numeric_column_named_like_a_suffix_is_identity_not_frequency(): + """A numeric column literally named ``revenue_freq`` must resolve to itself. + + Stripping ``_freq`` yields *revenue*, which is not a source column, so the suffix match is + rejected and numeric identity wins. + """ + meta = SparkFeatureMetadata( + column_infos=[{"name": "revenue_freq", "category": "numeric"}], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["revenue_freq"], + ) + assert source_column("revenue_freq", meta) == "revenue_freq" + assert human_label("revenue_freq", meta) == "revenue_freq" + + +def test_unknown_feature_resolves_to_none_and_labels_unchanged(metadata: SparkFeatureMetadata): + """A feature from a convention this version does not know never crashes and is never hidden.""" + assert source_column("mystery_feature", metadata) is None + assert human_label("mystery_feature", metadata) == "mystery_feature" From 9be52d4aae09b4de973f63a3a2e575aa159bfcc5 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 13:01:56 +0100 Subject: [PATCH 029/107] Close the redaction hole: drop every feature derived from a redacted column Redaction filters LLM-prompt contribution keys by exact match, and those keys are engineered feature names. A prior fix covered the baseline-relative feature (`amount` -> `amount_rel_baseline`) by reconstructing that one name from the column, but one-hot and frequency features could not be reconstructed from the column alone (`country` -> `country_US`, `country_DE`, `country_freq`, `country_is_null`), so they still reached an external serving endpoint after the user redacted `country`. Thread the model's feature metadata to the point of redaction and enumerate the derived features exactly: - ExplanationContext gains an optional `feature_metadata` field; `from_scoring_config` takes it and `scoring_run` passes the `parsed_metadata` already in scope at the call site. - `redaction_set(redact_columns, metadata)` now unions `engineered_from(column, metadata)` per redacted column, covering one-hot, frequency, null-indicator, baseline-relative and identity. This replaces the suffix-only expansion. With no metadata (a caller who built the context directly), it falls back to the baseline-relative feature only -- best effort, unchanged. Redaction still matches engineered keys, upstream of any human rendering, so the hole stays shut when 2c adds human labels. Tests: metadata-driven redaction over one-hot + frequency + null-indicator + baseline-relative; the existing metadata-less tests are unchanged and still pass. Co-authored-by: Isaac --- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 46 +++++++++++------- .../labs/dqx/anomaly/scoring_run.py | 2 +- .../test_anomaly_explanation_redaction.py | 47 ++++++++++++++++++- 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 23f579be5..17fbe5dbe 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -23,7 +23,8 @@ from pyspark.sql.types import DoubleType, LongType, StringType, StructField, StructType from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema -from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX +from databricks.labs.dqx.anomaly.feature_naming import engineered_from +from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX, SparkFeatureMetadata from databricks.labs.dqx.config import LLMModelConfig from databricks.labs.dqx.errors import InvalidParameterError @@ -194,13 +195,20 @@ class ExplanationContext: # the absolute cap on LLM calls for that one call. max_groups: int = 500 redact_columns: tuple[str, ...] = () - # Internal working-column name for the (segment, pattern) group key. Defaults to a fixed + # Internal working-column name for the (baseline group, pattern) group key. Defaults to a fixed # name for direct construction; production scoring passes a UUID-suffixed name so it can # never collide with a user-supplied column. pattern_col: str = _DEFAULT_PATTERN_COL + # The model's feature metadata, threaded through so redaction can drop every feature derived + # from a redacted column (not just the column itself) and so contribution keys can be rendered + # as human labels. Optional: a caller that builds the context directly without it falls back to + # best-effort redaction of the baseline-relative feature only, and to raw engineered keys. + feature_metadata: SparkFeatureMetadata | None = None @classmethod - def from_scoring_config(cls, config: "ScoringConfig") -> "ExplanationContext": + def from_scoring_config( + cls, config: "ScoringConfig", feature_metadata: SparkFeatureMetadata | None = None + ) -> "ExplanationContext": return cls( severity_col=config.severity_col, contributions_col=config.contributions_col, @@ -212,24 +220,30 @@ def from_scoring_config(cls, config: "ScoringConfig") -> "ExplanationContext": max_groups=config.max_groups, redact_columns=tuple(config.redact_columns or ()), pattern_col=config.pattern_col, + feature_metadata=feature_metadata, ) -def redaction_set(redact_columns: tuple[str, ...]) -> frozenset[str]: - """Columns to redact, plus the engineered features derived from them. +def redaction_set(redact_columns: tuple[str, ...], metadata: SparkFeatureMetadata | None = None) -> frozenset[str]: + """Columns to redact, plus every engineered feature derived from them. Redaction matches contribution keys exactly, and contribution keys are *engineered* feature - names. So redacting ``amount`` did not stop ``amount_rel_baseline`` -- a signed log-ratio of the - same column -- from reaching the LLM prompt. A caller naming a column sensitive means every - feature derived from it is sensitive too. - - Known remaining gap: one-hot and frequency-encoded features are not covered, because their names - cannot be reconstructed from the source column alone (``country`` becomes ``country_C3``, - ``country_DE`` and so on, one per observed value). Closing that needs the feature metadata at - prompt-construction time, which this function does not have. Tracked separately. + names. So redacting ``amount`` must also stop ``amount_rel_baseline`` -- a signed log-ratio of + the same column -- and redacting ``country`` must stop ``country_US``, ``country_DE``, + ``country_freq`` and ``country_is_null``. A caller naming a column sensitive means every feature + derived from it is sensitive too. + + With *metadata*, the derived features are enumerated exactly via *engineered_from*, which closes + the one-hot and frequency gap that the source column alone could not. Without it (a caller who + built the context directly and did not thread metadata through), only the baseline-relative + feature is reconstructable from the column name, so that alone is covered -- best effort. """ expanded = set(redact_columns) - expanded.update(f"{column}{BASELINE_RELATIVE_SUFFIX}" for column in redact_columns) + if metadata is not None: + for column in redact_columns: + expanded.update(engineered_from(column, metadata)) + else: + expanded.update(f"{column}{BASELINE_RELATIVE_SUFFIX}" for column in redact_columns) return frozenset(expanded) @@ -667,7 +681,7 @@ def _add_explanation_column_ai_query( the documented "one call per group per scoring run" cost model. The collected payload is small and bounded: at most ``max_groups`` rows, each holding three length-capped text fields. """ - redact_set = redaction_set(ctx.redact_columns) + redact_set = redaction_set(ctx.redact_columns, ctx.feature_metadata) anomalous = df_with_pattern.filter(F.col(ctx.severity_col) >= F.lit(ctx.threshold)) kept_sdf, dropped_groups_count, dropped_rows_count, total_groups = _aggregate_groups_spark( anomalous, @@ -733,7 +747,7 @@ def add_explanation_column( Raises: InvalidParameterError: When *model_name* does not resolve to a Databricks serving endpoint. """ - redact_set = redaction_set(ctx.redact_columns) + redact_set = redaction_set(ctx.redact_columns, ctx.feature_metadata) df_with_pattern = df.withColumn(ctx.pattern_col, _pattern_spark_expr(ctx.contributions_col, redact_set)) return _add_explanation_column_ai_query( df_with_pattern, ctx, "", is_ensemble, drift_summary, endpoint_reachable=endpoint_reachable diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index 55f10f0df..d07ceb09c 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -236,7 +236,7 @@ def score_global_model( if config.enable_ai_explanation: scored_df = add_explanation_column( scored_df, - ExplanationContext.from_scoring_config(config), + ExplanationContext.from_scoring_config(config, parsed_metadata), is_ensemble=record.identity.is_ensemble, drift_summary=format_drift_summary(drift_result, config.redact_columns), ) diff --git a/tests/unit/test_anomaly_explanation_redaction.py b/tests/unit/test_anomaly_explanation_redaction.py index 35e1775f5..d33f05d80 100644 --- a/tests/unit/test_anomaly_explanation_redaction.py +++ b/tests/unit/test_anomaly_explanation_redaction.py @@ -8,7 +8,7 @@ """ from databricks.labs.dqx.anomaly.anomaly_llm_explainer import redaction_set -from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX +from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX, SparkFeatureMetadata def test_redacting_a_column_also_redacts_its_baseline_relative_feature(): @@ -39,3 +39,48 @@ def test_similar_prefixes_are_not_swept_up(): assert "amount_paid" not in result assert f"amount_paid{BASELINE_RELATIVE_SUFFIX}" not in result + + +def _metadata() -> SparkFeatureMetadata: + """Feature metadata with a categorical column expanded to one-hot + frequency + null-indicator, + alongside a numeric column with a baseline-relative feature.""" + return SparkFeatureMetadata( + column_infos=[ + {"name": "amount", "category": "numeric"}, + {"name": "country", "category": "categorical"}, + ], + categorical_frequency_maps={"country": {"US": 0.7, "DE": 0.3}}, + onehot_categories={"country": ["US", "DE"]}, + engineered_feature_names=[ + "amount", + "country_US", + "country_DE", + "country_freq", + "country_is_null", + "amount_rel_baseline", + ], + ) + + +def test_metadata_driven_redaction_covers_onehot_and_frequency(): + """The gap this closed: one-hot and frequency features carry the same information as the raw + column, but their names cannot be reconstructed from the column alone. With metadata they are + enumerated exactly and dropped. + """ + result = redaction_set(("country",), _metadata()) + + assert result == {"country", "country_US", "country_DE", "country_freq", "country_is_null"} + + +def test_metadata_driven_redaction_covers_numeric_identity_and_baseline(): + result = redaction_set(("amount",), _metadata()) + + assert result == {"amount", "amount_rel_baseline"} + + +def test_metadata_driven_redaction_leaves_other_columns_untouched(): + """Redacting one column must not sweep up another column's features.""" + result = redaction_set(("country",), _metadata()) + + assert "amount" not in result + assert "amount_rel_baseline" not in result From 65c3d86a550218998d1cdec2f8d5d66beddb1b4f Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 13:24:00 +0100 Subject: [PATCH 030/107] Render contributions for a reader and reframe the prompt around baselines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contributions were shown to users and to the LLM as raw engineered names — a reader saw `event_count_rel_baseline (74%)` in `_dq_info` and in the prompt — and the prompt still spoke the removed language of segments. - feature_contributions in the prompt now renders human labels via the reverse map, after the redaction filter: `amount vs its group baseline (82%), quantity (11%)`. A driver whose label cannot be resolved falls back to its raw name, so nothing is ever hidden. The label lookup is a SQL map literal with escaped keys/values (column names are user-derived). - The ai_explanation struct gains a `top_drivers` string (the same human-labelled sentence) for display. `top_features` stays engineered names for grouping/tooling, and the top-level `contributions` map keeps engineered keys so redaction and downstream tooling still match. - Prompt vocabulary moves from segments to baselines: the dead `segment` field (always empty since the segment path was removed) becomes `baseline_grouping`, the columns forming each row's baseline group — the material fact for a contextual anomaly, and a per-run constant with no PII. Few-shot examples and instructions reframed accordingly. The dead `_format_segment` helper and its tests are removed; the committed prompt snapshot is regenerated. Billable ai_query surface: the fixed prompt header grows ~322 chars (~+80 est. tokens) per call from the richer baseline vocabulary and label guidance, plus a few chars per group for longer human labels. Tests: unit coverage for the baseline-grouping string and the human-label map (identity omitted, derived features labelled, empty without metadata); the AI-explanation integration test asserts top_drivers is populated and carries the top features with weights. Co-authored-by: Isaac --- .../labs/dqx/anomaly/anomaly_info_schema.py | 7 +- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 116 ++++++++++++------ .../test_anomaly_ai_explanation.py | 8 +- tests/resources/ai_query_prompt_header.txt | 14 +-- tests/unit/test_anomaly_llm_explainer.py | 57 +++++++-- 5 files changed, 140 insertions(+), 62 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py index dd6b17a64..dc46862bd 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py @@ -12,13 +12,16 @@ # Schema for the AI explanation sub-struct inside the anomaly info struct. # narrative / business_impact / action are LLM-generated; top_features is deterministic -# (the sorted top-2 contributing SHAP features that define the group). -# group_size / group_avg_severity describe the (segment, pattern) group this row belongs to. +# (the sorted top-2 contributing SHAP features that define the group, as engineered names, so it +# stays stable for grouping and tooling). top_drivers is the same drivers rendered as human labels +# with their weights, e.g. 'amount vs its group baseline (74%), quantity (12%)', for display. +# group_size / group_avg_severity describe the pattern group this row belongs to. ai_explanation_struct_schema = StructType( [ StructField("narrative", StringType(), True), StructField("business_impact", StringType(), True), StructField("top_features", StringType(), True), + StructField("top_drivers", StringType(), True), StructField("action", StringType(), True), StructField("group_size", LongType(), True), StructField("group_avg_severity", DoubleType(), True), diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 17fbe5dbe..c3c191d59 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -1,10 +1,10 @@ """LLM-based group explanation for row anomaly detection. The algorithm is group-based: anomalous rows are grouped by a deterministic -(segment, pattern) key — pattern being the sorted top-2 contributing features — -and the LLM is invoked once per group. Every row in a -group shares the same narrative/business_impact/action; group_size and -group_avg_severity signal that the explanation describes a pattern, not a row. +pattern key — the sorted top-2 contributing features — and the LLM is invoked +once per group. Every row in a group shares the same +narrative/business_impact/action; group_size and group_avg_severity signal that +the explanation describes a pattern, not a row. The LLM call runs entirely inside Spark via the SQL ``ai_query`` function against a Databricks Model Serving endpoint — no driver collect of LLM output, scales with the @@ -23,7 +23,7 @@ from pyspark.sql.types import DoubleType, LongType, StringType, StructField, StructType from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema -from databricks.labs.dqx.anomaly.feature_naming import engineered_from +from databricks.labs.dqx.anomaly.feature_naming import engineered_from, human_label from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX, SparkFeatureMetadata from databricks.labs.dqx.config import LLMModelConfig from databricks.labs.dqx.errors import InvalidParameterError @@ -39,13 +39,15 @@ "not a specific row.\n" "Be direct and concrete. Avoid hedging phrases like 'The data shows', 'It appears that', or " "'might indicate'. Do not restate the input field names back to the user, and do not invent " - "feature names, values, or segments that are not present in the input." + "feature names, values, or baseline groups that are not present in the input." ) _PROMPT_INPUT_FIELDS: tuple[tuple[str, str], ...] = ( ( "feature_contributions", - "Mean SHAP contributions across the group, e.g. 'amount (82%), quantity (11%), " - "discount (5%)'. These are aggregated relative importances — not raw data values.", + "Mean contributions across the group, already named for a reader, e.g. 'amount vs its " + "group baseline (82%), quantity (11%), discount (5%)'. A phrase like 'X vs its group " + "baseline' means X was unusual relative to its own baseline group, not in absolute terms. " + "These are aggregated relative importances — not raw data values.", ), ("group_size", "Number of rows in this group, e.g. '312 rows'."), ("severity_range", "Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'."), @@ -55,9 +57,10 @@ "for single-model scoring.", ), ( - "segment", - "Data segment this group belongs to, e.g. 'region=US, product=electronics'. Empty string " - "if no segmentation was used.", + "baseline_grouping", + "The columns whose values define each row's baseline group, e.g. 'region' or " + "'region, product'. Anomalies are judged relative to the row's own group baseline; " + "'none' when the model is not grouped.", ), ("threshold", "The severity percentile threshold configured by the user (0–100)."), ( @@ -88,23 +91,23 @@ # smaller serving models. Kept short so the prompt stays well within token budgets. _PROMPT_EXAMPLES = ( "Example (no drift):\n" - "feature_contributions: amount (61%), quantity (22%)\n" + "feature_contributions: amount vs its group baseline (61%), quantity (22%)\n" "group_size: 312 rows\n" "severity_range: mean 97.4, min 95.1, max 99.8\n" "confidence: high\n" - "segment: region=US\n" + "baseline_grouping: region\n" "threshold: 95.0\n" "drift_summary: none\n" - 'Response: {"narrative":"312 rows are driven mainly by amount (61%) with quantity secondary ' - '(22%); values sit far above the US-segment norm.","business_impact":"Inflated amount fields ' - 'overstate revenue if these rows are processed unchanged.","action":"Reconcile amount against ' - 'source orders for this US group."}\n\n' + 'Response: {"narrative":"312 rows are driven mainly by amount, which sits far above the norm ' + 'for its own region (61%), with quantity secondary (22%).","business_impact":"Inflated amount ' + 'fields overstate revenue if these rows are processed unchanged.","action":"Reconcile amount ' + 'against source orders within each affected region."}\n\n' "Example (with drift):\n" "feature_contributions: latency_ms (74%), retries (12%)\n" "group_size: 88 rows\n" "severity_range: mean 98.9, min 97.0, max 99.9\n" "confidence: mixed\n" - "segment: \n" + "baseline_grouping: none\n" "threshold: 95.0\n" "drift_summary: drift detected: latency_ms=4.12\n" 'Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from ' @@ -119,7 +122,7 @@ logger = logging.getLogger(__name__) _TOP_N = 5 -# Default working-column name for the (segment, pattern) group key. Production scoring overrides +# Default working-column name for the pattern group key. Production scoring overrides # this with a UUID-suffixed name via *ScoringConfig.pattern_col* (threaded through # *ExplanationContext.pattern_col*) so it can never collide with a user column; the constant is # only the fallback for direct *ExplanationContext* construction. @@ -273,17 +276,35 @@ def _pattern_spark_expr(contributions_col: str, redact_set: frozenset[str]) -> C return F.expr(sql) -def _format_segment(segment_values: dict[str, str] | None, redact_set: frozenset[str]) -> str: - """Format segment values as 'k1=v1, k2=v2' or empty string. +def _baseline_grouping_str(metadata: SparkFeatureMetadata | None) -> str: + """The baseline grouping columns as a prompt string, e.g. 'region, product' or 'none'. - Segment ``key=value`` pairs are sent verbatim to the LLM prompt, so any key listed in - *redact_set* is emitted as ``key=`` to keep sensitive segmentation values out of - the prompt (the value, not just the contribution, can be PII). + A per-run constant: the anomalies in this run are all judged against a group baseline defined by + these columns (or against a global baseline when the model is not grouped). The column *names* + are structural, not row values, so unlike the old segment values they carry no PII and need no + redaction. """ - if not segment_values: - return "" - parts = [f"{k}=" if k in redact_set else f"{k}={v}" for k, v in segment_values.items()] - return ", ".join(parts) + if metadata is None or not metadata.baseline_by: + return "none" + return ", ".join(metadata.baseline_by) + + +def _human_labels(metadata: SparkFeatureMetadata | None) -> dict[str, str]: + """Engineered-name -> human-label map for the model's features, omitting identity labels. + + Used to render contribution keys for a reader (``amount_rel_baseline`` -> + ``amount vs its group baseline``). Only entries whose label differs from the raw name are + included, so the SQL lookup stays small; anything not in the map falls back to its raw name. + Empty when no metadata was threaded through, in which case raw engineered names are shown. + """ + if metadata is None: + return {} + labels: dict[str, str] = {} + for name in metadata.engineered_feature_names: + label = human_label(name, metadata) + if label != name: + labels[name] = label + return labels def _build_empty_explanation_column() -> Column: @@ -351,12 +372,19 @@ def _resolve_ai_query_endpoint(model_name: str) -> str: return endpoint -def _format_contributions_sql(top_n: int) -> Column: +def _format_contributions_sql(top_n: int, labels: dict[str, str] | None = None) -> Column: """Spark expression producing 'feat_a (82%), feat_b (11%)' from a ``mean_contributions`` map. Mirrors *format_contributions_map* but stays inside Spark so per-group prompts can be assembled without a driver-side loop. Null/empty maps yield 'unknown'; entries are sorted by absolute value descending and percentages are normalised against the L1 sum of |value|. + + *labels* maps an engineered feature name to its human label; when supplied, each key is + rendered as its label ('amount_rel_baseline' -> 'amount vs its group baseline'), falling back + to the raw key for anything unmapped. The map keys and values are user-derived (column names), + so both are escaped before interpolation. Rendering happens here, after redaction has already + dropped sensitive keys upstream in *_aggregate_groups_spark*, so labelling never re-exposes a + redacted feature. """ entries = "filter(map_entries(`mean_contributions`), e -> e.value is not null)" sorted_entries = ( @@ -366,8 +394,13 @@ def _format_contributions_sql(top_n: int) -> Column: ) top = f"slice({sorted_entries}, 1, {int(top_n)})" abs_sum = f"aggregate({sorted_entries}, 0.0D, (acc, e) -> acc + abs(e.value))" + if labels: + pairs = ", ".join(f"'{_sql_string_literal(k)}', '{_sql_string_literal(v)}'" for k, v in labels.items()) + key_expr = f"coalesce(element_at(map({pairs}), e.key), e.key)" + else: + key_expr = "e.key" formatted = ( - f"transform({top}, e -> concat(e.key, ' (', " + f"transform({top}, e -> concat({key_expr}, ' (', " f"cast(round((abs(e.value) / case when {abs_sum} = 0 then 1 else {abs_sum} end) * 100) as int), '%)'))" ) sql = ( @@ -379,7 +412,6 @@ def _format_contributions_sql(top_n: int) -> Column: def _build_ai_query_prompt_column( ctx: ExplanationContext, - segment_str: str, is_ensemble: bool, drift_summary: str, ) -> Column: @@ -389,6 +421,7 @@ def _build_ai_query_prompt_column( constants for the whole call. The shared header (*_AI_QUERY_PROMPT_HEADER*) holds the instructions and field semantics. """ + baseline_grouping = _baseline_grouping_str(ctx.feature_metadata) confidence_expr = ( F.when((F.col("mean_std").isNull()) | F.lit(not is_ensemble), F.lit("n/a")) .when(F.col("mean_std") < F.lit(_CONFIDENCE_HIGH_BELOW), F.lit("high")) @@ -416,8 +449,8 @@ def _build_ai_query_prompt_column( F.lit("confidence: "), confidence_expr, F.lit("\n"), - F.lit("segment: "), - F.lit(segment_str), + F.lit("baseline_grouping: "), + F.lit(baseline_grouping), F.lit("\n"), F.lit("threshold: "), F.lit(str(ctx.threshold)), @@ -507,7 +540,6 @@ def _aggregate_groups_spark( def _call_llm_for_groups_ai_query( kept_groups_sdf: DataFrame, ctx: ExplanationContext, - segment_str: str, is_ensemble: bool, drift_summary: str, ) -> DataFrame: @@ -521,12 +553,14 @@ def _call_llm_for_groups_ai_query( llm_cfg = ctx.llm_model_config or LLMModelConfig() endpoint = _resolve_ai_query_endpoint(llm_cfg.model_name) pattern_col = ctx.pattern_col + # feature_contributions is rendered with human labels (empty map -> raw keys); it feeds both the + # prompt and the struct's top_drivers, so a reader sees the same human phrasing the LLM did. enriched = kept_groups_sdf.withColumn( "feature_contributions", - _format_contributions_sql(_TOP_N), + _format_contributions_sql(_TOP_N, _human_labels(ctx.feature_metadata)), ).withColumn( "__prompt", - _build_ai_query_prompt_column(ctx, segment_str, is_ensemble, drift_summary), + _build_ai_query_prompt_column(ctx, is_ensemble, drift_summary), ) # ai_query is parameterised through the SQL string. *endpoint* is matched against the strict @@ -581,6 +615,9 @@ def _sanitize(col_name: str) -> Column: _sanitize("narrative").alias("narrative"), _sanitize("business_impact").alias("business_impact"), _sanitize("action").alias("action"), + # Human-labelled drivers carried through for the struct's top_drivers. Built by us from the + # contributions map, not the LLM, so it needs no sanitisation. + F.col("feature_contributions").alias("top_drivers"), F.col("group_size").cast(LongType()).alias("group_size"), F.col("group_avg_severity").cast(DoubleType()).alias("group_avg_severity"), ) @@ -605,6 +642,7 @@ def _attach_explanation_struct( F.col("narrative").alias("narrative"), F.col("business_impact").alias("business_impact"), F.col(pattern_col).alias("top_features"), + F.col("top_drivers").alias("top_drivers"), F.col("action").alias("action"), F.col("group_size").alias("group_size"), F.col("group_avg_severity").alias("group_avg_severity"), @@ -614,6 +652,7 @@ def _attach_explanation_struct( pattern_col, "narrative", "business_impact", + "top_drivers", "action", "group_size", "group_avg_severity", @@ -667,7 +706,6 @@ def probe_endpoint_reachable(spark: object, llm_model_config: LLMModelConfig | N def _add_explanation_column_ai_query( df_with_pattern: DataFrame, ctx: ExplanationContext, - segment_str: str, is_ensemble: bool, drift_summary: str, endpoint_reachable: bool | None = None, @@ -710,7 +748,7 @@ def _add_explanation_column_ai_query( ctx.pattern_col ) _log_dropped_groups(dropped_groups_count, dropped_rows_count, ctx.max_groups) - result_sdf = _call_llm_for_groups_ai_query(kept_sdf, ctx, segment_str, is_ensemble, drift_summary) + result_sdf = _call_llm_for_groups_ai_query(kept_sdf, ctx, is_ensemble, drift_summary) # Pin the LLM responses: one ai_query execution per scoring run, regardless of how many # actions the caller takes on the returned DataFrame afterwards. result_rows = result_sdf.collect() @@ -750,5 +788,5 @@ def add_explanation_column( redact_set = redaction_set(ctx.redact_columns, ctx.feature_metadata) df_with_pattern = df.withColumn(ctx.pattern_col, _pattern_spark_expr(ctx.contributions_col, redact_set)) return _add_explanation_column_ai_query( - df_with_pattern, ctx, "", is_ensemble, drift_summary, endpoint_reachable=endpoint_reachable + df_with_pattern, ctx, is_ensemble, drift_summary, endpoint_reachable=endpoint_reachable ) diff --git a/tests/integration_anomaly/test_anomaly_ai_explanation.py b/tests/integration_anomaly/test_anomaly_ai_explanation.py index 59ec45929..3bad808f5 100644 --- a/tests/integration_anomaly/test_anomaly_ai_explanation.py +++ b/tests/integration_anomaly/test_anomaly_ai_explanation.py @@ -112,6 +112,12 @@ def test_ai_query_explanation_populated_for_anomalous_row( assert explanation["top_features"] for feat in explanation["top_features"].split("+"): assert feat in {"amount", "quantity", "discount"} + # top_drivers is the same drivers rendered for a reader, with weights. For these plain numeric + # features the human label is the column name itself, so each top feature appears with a percent. + assert explanation["top_drivers"] + assert "%" in explanation["top_drivers"] + for feat in explanation["top_features"].split("+"): + assert feat in explanation["top_drivers"] assert explanation["group_size"] == 1 # Single-row group: group_avg_severity is the row's severity. The struct's # severity_percentile is rounded to 1 decimal while group_avg_severity is full precision, @@ -171,7 +177,7 @@ def test_ai_query_explanation_redact_columns_filters_output( def test_ai_query_explanation_one_call_per_group( spark: SparkSession, shared_3d_model, test_df_factory, anomaly_scorer, ai_query_endpoint ): - """Multiple identical anomalous rows collapse into a single (segment, pattern) group. + """Multiple identical anomalous rows collapse into a single pattern group. The ai_query call runs on executors so we can't intercept it directly; instead we assert the *observable* contract: every flagged row in the group shares the same narrative and diff --git a/tests/resources/ai_query_prompt_header.txt b/tests/resources/ai_query_prompt_header.txt index 9634b8c23..700b15bed 100644 --- a/tests/resources/ai_query_prompt_header.txt +++ b/tests/resources/ai_query_prompt_header.txt @@ -1,12 +1,12 @@ You are a data quality analyst. Given aggregate metadata for a GROUP of anomalous rows sharing the same root-cause pattern, explain in plain business language why this group was flagged. Your explanation will be shown for every row in the group — describe the pattern, not a specific row. -Be direct and concrete. Avoid hedging phrases like 'The data shows', 'It appears that', or 'might indicate'. Do not restate the input field names back to the user, and do not invent feature names, values, or segments that are not present in the input. +Be direct and concrete. Avoid hedging phrases like 'The data shows', 'It appears that', or 'might indicate'. Do not restate the input field names back to the user, and do not invent feature names, values, or baseline groups that are not present in the input. Inputs: -- feature_contributions: Mean SHAP contributions across the group, e.g. 'amount (82%), quantity (11%), discount (5%)'. These are aggregated relative importances — not raw data values. +- feature_contributions: Mean contributions across the group, already named for a reader, e.g. 'amount vs its group baseline (82%), quantity (11%), discount (5%)'. A phrase like 'X vs its group baseline' means X was unusual relative to its own baseline group, not in absolute terms. These are aggregated relative importances — not raw data values. - group_size: Number of rows in this group, e.g. '312 rows'. - severity_range: Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'. - confidence: Model confidence label across the group. 'high' / 'mixed' / 'low' for ensemble, 'n/a' for single-model scoring. -- segment: Data segment this group belongs to, e.g. 'region=US, product=electronics'. Empty string if no segmentation was used. +- baseline_grouping: The columns whose values define each row's baseline group, e.g. 'region' or 'region, product'. Anomalies are judged relative to the row's own group baseline; 'none' when the model is not grouped. - threshold: The severity percentile threshold configured by the user (0–100). - drift_summary: Baseline drift signal from the scoring run, e.g. 'drift detected: amount=4.12; quantity=3.55' or 'none'. If drift is present, explicitly frame the narrative vs baseline. @@ -16,21 +16,21 @@ Respond with ONLY a JSON object. Field rules: - action: One sentence, max 20 words. What a data analyst should investigate for this group. Example (no drift): -feature_contributions: amount (61%), quantity (22%) +feature_contributions: amount vs its group baseline (61%), quantity (22%) group_size: 312 rows severity_range: mean 97.4, min 95.1, max 99.8 confidence: high -segment: region=US +baseline_grouping: region threshold: 95.0 drift_summary: none -Response: {"narrative":"312 rows are driven mainly by amount (61%) with quantity secondary (22%); values sit far above the US-segment norm.","business_impact":"Inflated amount fields overstate revenue if these rows are processed unchanged.","action":"Reconcile amount against source orders for this US group."} +Response: {"narrative":"312 rows are driven mainly by amount, which sits far above the norm for its own region (61%), with quantity secondary (22%).","business_impact":"Inflated amount fields overstate revenue if these rows are processed unchanged.","action":"Reconcile amount against source orders within each affected region."} Example (with drift): feature_contributions: latency_ms (74%), retries (12%) group_size: 88 rows severity_range: mean 98.9, min 97.0, max 99.9 confidence: mixed -segment: +baseline_grouping: none threshold: 95.0 drift_summary: drift detected: latency_ms=4.12 Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from baseline; retries contribute modestly (12%).","business_impact":"Elevated latency risks SLA breaches for downstream consumers.","action":"Investigate latency_ms regressions against the training baseline."} diff --git a/tests/unit/test_anomaly_llm_explainer.py b/tests/unit/test_anomaly_llm_explainer.py index fe33898d8..b5265a96c 100644 --- a/tests/unit/test_anomaly_llm_explainer.py +++ b/tests/unit/test_anomaly_llm_explainer.py @@ -1,8 +1,8 @@ """Unit tests for the ai_query-based group explainer in anomaly_llm_explainer. Spark is never started — these exercise the pure helpers: prompt rendering, endpoint -resolution, segment redaction, SQL-literal escaping, the structured-output schema, and the -pattern-column threading through ExplanationContext. +resolution, baseline-grouping and human-label rendering, SQL-literal escaping, the +structured-output schema, and the pattern-column threading through ExplanationContext. """ from pathlib import Path @@ -11,6 +11,7 @@ from databricks.labs.dqx.anomaly import anomaly_llm_explainer as llm_explainer from databricks.labs.dqx.anomaly.anomaly_llm_explainer import ExplanationContext +from databricks.labs.dqx.anomaly.transformers import SparkFeatureMetadata from databricks.labs.dqx.anomaly.scoring_config import ScoringConfig, ScoringOutputColumns from databricks.labs.dqx.config import LLMModelConfig from databricks.labs.dqx.errors import InvalidParameterError @@ -153,21 +154,51 @@ def test_ai_query_response_format_is_strict_json_schema_built_from_output_fields assert llm_explainer._build_ai_query_response_format() == schema -def test_format_segment_empty_returns_empty_string(): - assert llm_explainer._format_segment(None, frozenset()) == "" - assert llm_explainer._format_segment({}, frozenset()) == "" +def test_baseline_grouping_str_reports_the_grouping_columns(): + """The prompt's baseline_grouping field is the baseline_by columns, a per-run constant.""" + metadata = SparkFeatureMetadata( + column_infos=[{"name": "amount", "category": "numeric"}], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["amount"], + baseline_by=["region", "product"], + ) + assert llm_explainer._baseline_grouping_str(metadata) == "region, product" + +def test_baseline_grouping_str_is_none_when_ungrouped_or_metadataless(): + metadata = SparkFeatureMetadata( + column_infos=[{"name": "amount", "category": "numeric"}], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["amount"], + ) + assert llm_explainer._baseline_grouping_str(metadata) == "none" + assert llm_explainer._baseline_grouping_str(None) == "none" + + +def test_human_labels_map_omits_identity_and_labels_derived_features(): + """Only features whose label differs from the raw name are in the map; identities are dropped + so the SQL lookup stays small and unmapped keys fall back to the raw name.""" + metadata = SparkFeatureMetadata( + column_infos=[ + {"name": "amount", "category": "numeric"}, + {"name": "country", "category": "categorical"}, + ], + categorical_frequency_maps={"country": {"US": 0.7}}, + onehot_categories={"country": ["US"]}, + engineered_feature_names=["amount", "amount_rel_baseline", "country_US", "country_freq"], + ) + labels = llm_explainer._human_labels(metadata) -def test_format_segment_formats_key_value_pairs(): - out = llm_explainer._format_segment({"region": "US", "product": "electronics"}, frozenset()) - assert out == "region=US, product=electronics" + assert "amount" not in labels # identity, omitted + assert labels["amount_rel_baseline"] == "amount vs its group baseline" + assert labels["country_US"] == "country = US" + assert labels["country_freq"] == "country frequency" -def test_format_segment_redacts_listed_keys(): - """A segment key in redact_columns must never leak its value into the prompt.""" - out = llm_explainer._format_segment({"region": "US", "customer_id": "C-42"}, frozenset({"customer_id"})) - assert out == "region=US, customer_id=" - assert "C-42" not in out +def test_human_labels_map_is_empty_without_metadata(): + assert not llm_explainer._human_labels(None) def test_sql_string_literal_escapes_quote_and_backslash(): From e6d52a7ce30562fce32c4c169e10c2fc330fbb0f Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 15:28:26 +0100 Subject: [PATCH 031/107] Pin the group-conditioning invariants with tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exploration found no defects in the conditioning design; this adds the tests that would catch a regression in the invariants it rests on, and documents one coupling. New unit pins (no Spark), tests/unit/test_anomaly_baseline_invariants.py: - validate_baseline_columns accepts exactly the types whose string rendering agrees between Spark and Python (string, integral, boolean, date) and rejects float/decimal, where the two diverge and a group key built at training would miss at scoring. Uses create_autospec for the schema, no session. - A structural guard that every scoring module routes features through the shared feature_prep entry points before scoring; a scorer that fed raw columns would score on a different feature list than it trained on — silent, since shapes can still line up. New integration pins, test_anomaly_group_relative_features.py: - With multiple metrics, the _rel_baseline features are a contiguous trailing block (strengthens the single-metric append-last case); the sklearn pipeline relies on that positional order. - A null-scored row keeps its info struct through join_filtered_results_back: max_by(info, score) is null for a null score, and the coalesce with first() is what preserves is_new_baseline on unseen-group rows. Documented the drift coupling in prepare_drift_df: drift is measured over engineered features including _rel_baseline, so a wholesale shift in a group's baseline is absorbed into the relative feature and by design not flagged — that is conditioning working, not a gap. Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/drift.py | 11 ++- .../test_anomaly_group_relative_features.py | 50 ++++++++++++ .../unit/test_anomaly_baseline_invariants.py | 77 +++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_anomaly_baseline_invariants.py diff --git a/src/databricks/labs/dqx/anomaly/drift.py b/src/databricks/labs/dqx/anomaly/drift.py index 8f5ede477..37e0e178b 100644 --- a/src/databricks/labs/dqx/anomaly/drift.py +++ b/src/databricks/labs/dqx/anomaly/drift.py @@ -199,7 +199,16 @@ def prepare_drift_df( columns: list[str], record: AnomalyModelRecord, ) -> tuple[DataFrame, list[str]]: - """Prepare drift DataFrame and columns aligned to training baseline stats.""" + """Prepare drift DataFrame and columns aligned to training baseline stats. + + Drift is measured over the *engineered* feature list, which for a grouped model includes the + ``_rel_baseline`` features. Those are each already a deviation from the group baseline, so a + shift in the baseline itself moves the raw metric and its baseline together and leaves the + relative feature unchanged. Consequence, by design: drift on a grouped metric reports a change + in how far rows sit *from their group norm*, not a change in the norm. A wholesale shift of a + group's level is absorbed into the baseline and is not flagged here — that is the point of + conditioning, not a gap in drift detection. + """ feature_metadata_json = record.features.feature_metadata if not feature_metadata_json: return df.select(*columns), columns diff --git a/tests/integration_anomaly/test_anomaly_group_relative_features.py b/tests/integration_anomaly/test_anomaly_group_relative_features.py index dc1387724..ff0026b5d 100644 --- a/tests/integration_anomaly/test_anomaly_group_relative_features.py +++ b/tests/integration_anomaly/test_anomaly_group_relative_features.py @@ -14,6 +14,7 @@ from pyspark.sql import functions as F from pyspark.sql import types as T +from databricks.labs.dqx.anomaly.scoring_utils import join_filtered_results_back from databricks.labs.dqx.anomaly.segment_utils import ( BASELINE_KEY_NULL, build_baseline_key, @@ -138,6 +139,28 @@ def test_relative_feature_is_appended_last(spark: SparkSession): assert metadata.engineered_feature_names[-1] == "amount_rel_baseline" +def test_all_relative_features_form_the_trailing_block(spark: SparkSession): + """With several metrics, every _rel_baseline feature is a trailing entry, one per metric. + + Strengthens the single-metric case above: an already-trained model is handed features in this + order, so the relative block must stay contiguous at the tail — not interleaved with base + features, which would silently reorder the model's inputs. + """ + df = spark.createDataFrame( + [("DE", 100.0, 5.0), ("DE", 110.0, 6.0), ("IT", 20.0, 1.0), ("IT", 22.0, 2.0)], + "country string, amount double, quantity double", + ) + + _, metadata = apply_feature_engineering( + df, [_numeric_info("amount"), _numeric_info("quantity")], baseline_by=["country"] + ) + + names = metadata.engineered_feature_names + relative = [name for name in names if name.endswith("_rel_baseline")] + assert set(relative) == {"amount_rel_baseline", "quantity_rel_baseline"} + assert names[-len(relative) :] == relative # a contiguous trailing block + + def test_feature_prefix_matches_an_ungrouped_model(spark: SparkSession): """Everything before the appended tail must be identical to the ungrouped feature list. @@ -230,3 +253,30 @@ def test_unseen_group_falls_back_to_the_global_baseline(spark: SparkSession): value = _first(engineered)["amount_rel_baseline"] assert value is not None + + +# ============================================================================ +# is_new_baseline survives the row_filter merge-back +# ============================================================================ + + +def test_unscored_row_keeps_its_info_when_merged_back(spark: SparkSession): + """An unseen-group row carries a null score but a real is_new_baseline flag to report. + + join_filtered_results_back groups by the row id and aggregates with + ``max_by(info, score)`` — which is null when the row's only score is null — so the merge would + drop the info struct for exactly those rows. The ``coalesce(..., first(info))`` is what rescues + it. This pins that a null-scored row keeps its info; regressing the coalesce would silently lose + is_new_baseline on every unseen-group row. + """ + df = spark.createDataFrame([(1,), (2,)], "row_id long") + result = spark.createDataFrame( + [(1, None, "unseen-group"), (2, 0.9, "scored")], + "row_id long, score double, info string", + ) + + merged = join_filtered_results_back(df, result, ["row_id"], "score", "info") + + info_by_id = {row["row_id"]: row["info"] for row in merged.collect()} + assert info_by_id[1] == "unseen-group" # kept despite a null score + assert info_by_id[2] == "scored" diff --git a/tests/unit/test_anomaly_baseline_invariants.py b/tests/unit/test_anomaly_baseline_invariants.py new file mode 100644 index 000000000..0ce8a6ef7 --- /dev/null +++ b/tests/unit/test_anomaly_baseline_invariants.py @@ -0,0 +1,77 @@ +"""Unit pins for two invariants group conditioning rests on (no Spark). + +- The group-column *type* contract: only types whose string rendering agrees between Spark and + Python may be a baseline column, because the group key is built in both places and a divergence + silently misses every lookup (see test_anomaly_group_key for the key itself, and + test_anomaly_group_relative_features for the live Python/Spark parity). +- The feature-engineering contract: every scorer must run features through the shared feature_prep + entry points before scoring, or a model would score on a different feature list than it trained on. +""" + +import inspect +from unittest.mock import create_autospec + +import pytest +from pyspark.sql import DataFrame +from pyspark.sql import types as T + +from databricks.labs.dqx.anomaly import ensemble_scorer, single_model_scorer +from databricks.labs.dqx.anomaly.validation import validate_baseline_columns +from databricks.labs.dqx.errors import InvalidParameterError + + +def _fake_df(schema: dict[str, T.DataType]) -> DataFrame: + """A DataFrame stand-in exposing only the schema/columns validate_baseline_columns reads. + + Uses create_autospec rather than a real session: the function under test only inspects + ``df.schema.fields`` and ``df.columns``, so no Spark is needed to exercise the type contract. + """ + df = create_autospec(DataFrame, instance=True) + df.schema = T.StructType([T.StructField(name, dtype, True) for name, dtype in schema.items()]) + df.columns = list(schema) + return df + + +@pytest.mark.parametrize( + "dtype", + [ + T.StringType(), + T.ByteType(), + T.ShortType(), + T.IntegerType(), + T.LongType(), + T.BooleanType(), + T.DateType(), + ], +) +def test_validate_baseline_columns_accepts_types_that_render_identically(dtype: T.DataType): + """String, integral, boolean and date render the same in Spark and Python, so they are allowed.""" + validate_baseline_columns(_fake_df({"g": dtype}), ["g"], []) # must not raise + + +@pytest.mark.parametrize("dtype", [T.FloatType(), T.DoubleType(), T.DecimalType(10, 2)]) +def test_validate_baseline_columns_rejects_types_spark_and_python_format_differently(dtype: T.DataType): + """Floating-point and decimal render differently between Spark and Python, so a group key built + at training would not match the one built at scoring. They must be rejected up front.""" + with pytest.raises(InvalidParameterError, match="unsupported types"): + validate_baseline_columns(_fake_df({"g": dtype}), ["g"], []) + + +def test_every_scorer_applies_feature_engineering_before_scoring(): + """Structural guard: every scoring module must route features through the shared feature_prep + entry points (which call apply_feature_engineering_from_metadata) before scoring. + + A scorer that fed raw columns to the model would score on a different feature list than the + model trained on — the shapes can still line up, so it is a silent correctness bug, not a crash. + If you add a scoring module, add it to this list and make it apply feature engineering. + """ + fe_entry_points = ( + "apply_feature_engineering_for_scoring", + "apply_feature_engineering_with_row_passthrough", + "apply_feature_engineering_from_metadata", + ) + for module in (single_model_scorer, ensemble_scorer): + source = inspect.getsource(module) + assert any( + entry in source for entry in fe_entry_points + ), f"{module.__name__} must apply feature engineering before scoring" From 0568704bc4d396b9e264ef43b5e9551c53c8b78d Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 15:42:23 +0100 Subject: [PATCH 032/107] Performance: one baseline join for all metrics; drop count() emptiness scans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two score-neutral optimisations on the training path. Both verified to leave the group-relative feature values bit-identical (the exact signed-log-ratio, multi-metric independence, and trailing feature-order tests all pass) — no change to what the model sees. P1 — Baseline-relative feature engineering did one broadcast join per numeric metric. Build a single lookup carrying every metric's baseline as its own column and join once. The per-metric append order is preserved exactly (the feature list is positional), and each value is unchanged: the same per-group median, the same coalesce to the global baseline on a group miss, the same signed-log ratio. A group absent for one metric is null in that column and coalesces to global, exactly as the per-metric left join did. New test pins two metrics getting independent group baselines through the single join. P4 — Three validation/quantile guards used `df.count() == 0` purely to test emptiness, each a full frame scan. Replaced with `not df.take(1)`. Same branch taken, no count materialised. Not done, deliberately: - The plan's discovery optimisations (per-candidate groupBy().count(); four separate discovery aggregations) were already eliminated by the segment_by-removal discovery rework — batched distinct-count helpers and a single total count. Nothing left to do there. - Single-pass score quantiles via grouping sets: held. It would replace approxQuantile with a grouping-set percentile_approx for the global quantiles, changing the derived quantile values and so the severity calibration — that breaks the scores-identical requirement. - Reusing the engineered frame for post-training baseline stats: skipped. On serverless the reused frame cannot be cached, so it re-executes on the next action regardless; the saving is negligible and the change would thread a Spark frame back through the training-strategy contract. Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/core.py | 6 +-- .../labs/dqx/anomaly/transformers.py | 46 +++++++++++++------ .../test_anomaly_group_relative_features.py | 30 ++++++++++++ 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/core.py b/src/databricks/labs/dqx/anomaly/core.py index 36d57e3d2..895abe7df 100644 --- a/src/databricks/labs/dqx/anomaly/core.py +++ b/src/databricks/labs/dqx/anomaly/core.py @@ -266,7 +266,7 @@ def compute_validation_metrics( model: Pipeline, val_df: DataFrame, feature_cols: list[str], feature_metadata: SparkFeatureMetadata ) -> dict[str, float]: """Compute validation metrics and distribution statistics.""" - if val_df.count() == 0: + if not val_df.take(1): # emptiness only — take(1) avoids a full-frame count scan return {"validation_rows": 0} scored = score_with_model(model, val_df, feature_cols, feature_metadata) @@ -305,7 +305,7 @@ def compute_score_quantiles( Also populates ``feature_metadata.baseline_score_quantiles`` when the model is grouped, so scoring can calibrate severity against each group's own distribution. """ - if df.count() == 0: + if not df.take(1): # emptiness only — take(1) avoids a full-frame count scan return {} scored = score_with_model(model, df, feature_cols, feature_metadata) @@ -319,7 +319,7 @@ def compute_score_quantiles_ensemble( Also populates ``feature_metadata.baseline_score_quantiles`` when the model is grouped. """ - if df.count() == 0: + if not df.take(1): # emptiness only — take(1) avoids a full-frame count scan return {} scored = score_with_ensemble_models(models, df, feature_cols, feature_metadata) diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index f929469e3..5eba1fcde 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -807,10 +807,19 @@ def _process_baseline_relative_features( baseline_medians.update(computed_group) global_medians.update(computed_global) + # One broadcast join carrying every metric's baseline as its own column, rather than one join + # per metric. The per-metric append order below is preserved exactly (the feature list is + # positional), and each feature's value is unchanged: the same per-group baseline, the same + # coalesce to the global baseline on a group miss, the same signed-log-ratio. + metrics_with_baselines = [metric for metric in metrics if baseline_medians.get(metric)] + baseline_cols = {metric: f"__dqx_{metric}_baseline" for metric in metrics_with_baselines} + if metrics_with_baselines: + lookup_df = _baseline_lookup_df(transformed_df, baseline_medians, metrics_with_baselines, group_key_col) + transformed_df = transformed_df.join(broadcast(lookup_df), on=group_key_col, how="left") + for metric in metrics: feature_name = f"{metric}{BASELINE_RELATIVE_SUFFIX}" baselines = baseline_medians.get(metric, {}) - global_baseline = global_medians.get(metric, 0.0) if not baselines: # No baseline for this metric at all: the deviation is undefined, so emit a constant @@ -819,16 +828,16 @@ def _process_baseline_relative_features( engineered_features.append(feature_name) continue - baseline_col = f"__dqx_{metric}_baseline" - lookup_df = _baseline_lookup_df(transformed_df, baselines, group_key_col, baseline_col) - transformed_df = transformed_df.join(broadcast(lookup_df), on=group_key_col, how="left") - resolved_baseline = coalesce(col(baseline_col), lit(global_baseline)) + global_baseline = global_medians.get(metric, 0.0) + resolved_baseline = coalesce(col(baseline_cols[metric]), lit(global_baseline)) transformed_df = transformed_df.withColumn( feature_name, _signed_log1p(col(metric)) - _signed_log1p(resolved_baseline) ) - transformed_df = transformed_df.drop(baseline_col) engineered_features.append(feature_name) + if baseline_cols: + transformed_df = transformed_df.drop(*baseline_cols.values()) + # The key column deliberately survives: it is the frame's only remaining record of the # grouping once the raw group columns are dropped, and training-time severity calibration # runs on the scored frame, downstream of here. It is not a feature — see the explicit @@ -866,15 +875,26 @@ def _compute_baseline_medians( return baseline_medians, global_medians -def _baseline_lookup_df(df: DataFrame, baselines: dict[str, float], group_key_col: str, baseline_col: str) -> DataFrame: - """Build a broadcastable ``(group key, baseline)`` lookup for one metric.""" +def _baseline_lookup_df( + df: DataFrame, + baseline_medians: dict[str, dict[str, float]], + metrics: list[str], + group_key_col: str, +) -> DataFrame: + """Build one broadcastable lookup: ``(group key, _baseline per metric)``. + + One row per group key seen for any metric, with each metric's baseline in its own column and + null where that metric never saw the group — which the caller coalesces to the global baseline, + exactly as a per-metric left join would. Replaces N single-metric lookups with one. + """ + all_keys = sorted({key for metric in metrics for key in baseline_medians[metric]}) + baseline_cols = {metric: f"__dqx_{metric}_baseline" for metric in metrics} + rows = [(key, *(baseline_medians[metric].get(key) for metric in metrics)) for key in all_keys] schema = T.StructType( - [ - T.StructField(group_key_col, T.StringType(), False), - T.StructField(baseline_col, T.DoubleType(), False), - ] + [T.StructField(group_key_col, T.StringType(), False)] + + [T.StructField(baseline_cols[metric], T.DoubleType(), True) for metric in metrics] ) - return df.sparkSession.createDataFrame(list(baselines.items()), schema=schema) + return df.sparkSession.createDataFrame(rows, schema=schema) def apply_feature_engineering( diff --git a/tests/integration_anomaly/test_anomaly_group_relative_features.py b/tests/integration_anomaly/test_anomaly_group_relative_features.py index ff0026b5d..e4f0d46b6 100644 --- a/tests/integration_anomaly/test_anomaly_group_relative_features.py +++ b/tests/integration_anomaly/test_anomaly_group_relative_features.py @@ -224,6 +224,36 @@ def test_relative_value_is_the_signed_log_ratio_to_the_group_median(spark: Spark assert row["amount_rel_baseline"] == pytest.approx(math.log1p(200.0) - math.log1p(100.0), abs=1e-6) +def test_multiple_metrics_get_independent_group_baselines(spark: SparkSession): + """Each metric deviates from its own per-group median, computed in one joined lookup. + + Exercises the single-join path directly: two metrics, two groups, and one row per metric bumped + to twice its group median. Each relative value must be the signed-log ratio for that metric's + own baseline — no cross-contamination between the metrics' baseline columns. + """ + rows = [ + ("DE", 100.0, 4.0), + ("DE", 100.0, 4.0), + ("DE", 200.0, 4.0), # amount at 2x its DE median; quantity on its median + ("IT", 10.0, 40.0), + ("IT", 10.0, 40.0), + ("IT", 10.0, 80.0), # quantity at 2x its IT median; amount on its median + ] + df = spark.createDataFrame(rows, "country string, amount double, quantity double") + + engineered, _ = apply_feature_engineering( + df, [_numeric_info("amount"), _numeric_info("quantity")], baseline_by=["country"] + ) + + bumped_amount = _first(engineered.filter((F.col("country") == "DE") & (F.col("amount") == 200.0))) + assert bumped_amount["amount_rel_baseline"] == pytest.approx(math.log1p(200.0) - math.log1p(100.0), abs=1e-6) + assert bumped_amount["quantity_rel_baseline"] == pytest.approx(0.0, abs=1e-6) + + bumped_quantity = _first(engineered.filter((F.col("country") == "IT") & (F.col("quantity") == 80.0))) + assert bumped_quantity["quantity_rel_baseline"] == pytest.approx(math.log1p(80.0) - math.log1p(40.0), abs=1e-6) + assert bumped_quantity["amount_rel_baseline"] == pytest.approx(0.0, abs=1e-6) + + def test_negative_metrics_stay_finite(spark: SparkSession): """Plain log1p is NaN for x <= -1; the signed form must survive signed metrics.""" df = spark.createDataFrame( From ca5a6c28b01c5a25fb433700e30e7bad67f77446 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 21:57:22 +0100 Subject: [PATCH 033/107] Add the SMD estimator bake-off, and move the train/test loader to datasets Harness-only; no src behaviour changes. This is the evidence behind giving DQX a second anomaly algorithm, so it belongs in the same PR rather than in a scratch branch. The bake-off varies four axes at once on SMD -- featuriser, estimator, scope, metric -- under a chronological protocol: fit on SMD's own unlabelled train split, score the labelled test split, no point adjustment. What it found, and what motivates the rest of this work: raw/iforest (ships today) PR-AUC 0.0649 event recall @1% budget 0.359 raw/pca_recon PR-AUC 0.1189 event recall @1% budget 0.769 raw/mahalanobis PR-AUC 0.1079 event recall @1% budget 0.744 raw/mlp_ae (reference) PR-AUC 0.1403 event recall @1% budget 0.885 IsolationForest is the weakest estimator measured here, and windowed features do not rescue it: raw features beat windowed ones on incident coverage, so the gap is the estimator's inductive bias rather than missing feature engineering. `event_recall_at_budget` is introduced because point-wise PR-AUC is the wrong lens on this data. SMD's 3,732 anomalous rows sit in only 39 incidents of median length 6 but maximum length 1041, so PR-AUC is dominated by whether the one long incident was found -- the best-PR-AUC configuration covers 2 of 39 incidents. Event recall counts incidents while leaving precision strictly point-wise, so it is not point adjustment by another name: the alert budget is capped at 1% of rows, and a detector cannot game it by firing everywhere. `load_smd_split` moves out of a scratch script and into `datasets/real.py`, next to `_load_smd`, where a dataset loader belongs. `_load_smd` fits and scores the same labelled rows, which measures separability; `load_smd_split` also loads the unlabelled train split so a model can be fitted on train and scored on test, which is what any claim about generalisation requires. Co-authored-by: Isaac --- .../anomaly_conditioning/datasets/real.py | 25 ++ .../results/smd-bakeoff.json | 30 ++ .../anomaly_conditioning/smd_bakeoff.py | 338 ++++++++++++++++++ 3 files changed, 393 insertions(+) create mode 100644 benchmarks/anomaly_conditioning/results/smd-bakeoff.json create mode 100644 benchmarks/anomaly_conditioning/smd_bakeoff.py diff --git a/benchmarks/anomaly_conditioning/datasets/real.py b/benchmarks/anomaly_conditioning/datasets/real.py index ee2e320ad..565061997 100644 --- a/benchmarks/anomaly_conditioning/datasets/real.py +++ b/benchmarks/anomaly_conditioning/datasets/real.py @@ -78,6 +78,31 @@ def _load_smd() -> tuple[np.ndarray, np.ndarray, np.ndarray]: return np.vstack(values_list), np.concatenate(labels_list), np.concatenate(entity_list) +def load_smd_split() -> tuple[dict[str, np.ndarray], dict[str, np.ndarray], dict[str, np.ndarray]]: + """Return ``(train values, test values, test labels)`` keyed by entity, in file (time) order. + + The counterpart to :func:`_load_smd`, and the honest one for any claim about generalisation. + ``_load_smd`` fits and scores the labelled *test* split, which measures separability; this loads + the **unlabelled train split as well**, so a model can be fitted on train and scored on test. + Train precedes test in time, so that is a chronological protocol for free. + + Rows stay in file order, which for SMD is time order. Both splits are capped at the same + per-entity row limit the rest of the harness uses, so a run stays in minutes. The train split + carries no labels, which is exactly the semi-supervised setup DQX targets: learn what normal looks + like, then score unseen rows. + """ + train, test, labels = {}, {}, {} + for entity in SMD_ENTITIES: + tr = np.loadtxt(_fetch(f"{SMD_BASE}/train/{entity}.txt", f"smd/train-{entity}.txt"), delimiter=",") + te = np.loadtxt(_fetch(f"{SMD_BASE}/test/{entity}.txt", f"smd/test-{entity}.txt"), delimiter=",") + lb = np.loadtxt(_fetch(f"{SMD_BASE}/test_label/{entity}.txt", f"smd/label-{entity}.txt"), delimiter=",") + take_test = min(len(te), len(lb), SMD_MAX_ROWS_PER_ENTITY) + train[entity] = tr[:SMD_MAX_ROWS_PER_ENTITY] + test[entity] = te[:take_test] + labels[entity] = lb[:take_test] + return train, test, labels + + def _load_nslkdd() -> tuple[np.ndarray, np.ndarray, dict[str, np.ndarray]]: """Return ``(numeric values, labels, {grouping name: group labels})``. diff --git a/benchmarks/anomaly_conditioning/results/smd-bakeoff.json b/benchmarks/anomaly_conditioning/results/smd-bakeoff.json new file mode 100644 index 000000000..fe8a17546 --- /dev/null +++ b/benchmarks/anomaly_conditioning/results/smd-bakeoff.json @@ -0,0 +1,30 @@ +{ + "window": 60, + "seeds": 1, + "results": [ + { + "featuriser": "raw", + "estimator": "iforest", + "scope": "pooled", + "pr_auc": 0.06701655758365027, + "roc_auc": 0.7150735941232091, + "precision_at_n": 0.09163987138263666, + "event_recall_at_1pct": 0.3333333333333333, + "n_features": 38, + "n_models": 1, + "seconds": 0.7 + }, + { + "featuriser": "win_stats", + "estimator": "iforest", + "scope": "pooled", + "pr_auc": 0.082534382387672, + "roc_auc": 0.7446868279028607, + "precision_at_n": 0.11709539121114684, + "event_recall_at_1pct": 0.10256410256410256, + "n_features": 190, + "n_models": 1, + "seconds": 2.6 + } + ] +} \ No newline at end of file diff --git a/benchmarks/anomaly_conditioning/smd_bakeoff.py b/benchmarks/anomaly_conditioning/smd_bakeoff.py new file mode 100644 index 000000000..6857725df --- /dev/null +++ b/benchmarks/anomaly_conditioning/smd_bakeoff.py @@ -0,0 +1,338 @@ +"""SMD bake-off: what would it take to be in the ballpark on time-series anomaly detection? + +An earlier experiment asked "do lag and rolling-window features help?" and answered no: +9% PR-AUC for +190 extra features, with ROC-AUC regressing, because IsolationForest degrades as the feature count +grows. This asks the harder question — **what does it actually take** — across four axes at once, and +its answer is why DQX gained a second algorithm: the estimator was the problem, not the features. + +Why this script exists rather than a comparison against published numbers: SMD's headline results +(~0.80 F1) are computed with *point adjustment*, under which a random score reaches state of the art +(Kim et al., AAAI 2022). They are not a target and cannot be scaled to "80% of". So this script builds +its own honest reference — the best windowed reconstruction detector it can — and everything else is +measured against that. + +Axes: + +1. **Unit of analysis.** DQX scores one row at a time. SMD anomalies are subsequences, so a single row + is the wrong unit. ``win_stats`` makes the trailing window the sample: current value plus trailing + mean, standard deviation, min and max. Strictly trailing, so no leakage. +2. **Estimator.** Isolation Forest (what ships) against PCA reconstruction error, an MLP + autoencoder, and Mahalanobis distance. PCA and the autoencoder are the interesting candidates + because reconstruction degrades gracefully as the feature count grows, which is where Isolation + Forest was measured to fail. +3. **Scope.** Pooled across all 28 machines (what DQX does) versus one model per machine (what the SMD + literature does). Per-entity here uses a **global** score threshold rather than per-entity + contamination — the latter is what made an all-normal entity flag its own most-unusual rows and + produced the 56% false-alarm result in earlier work. +4. **Metric.** Point-wise PR-AUC is reported throughout, and it is harsh: an anomaly is a *range*, so a + detector that fires two timesteps late scores as a miss plus a false positive. Alongside it, + **event recall at a fixed alert budget**: of the true anomaly ranges, how many contain at least one + alerted row, when the alert budget is capped. Precision stays strictly point-wise, so this is not + point adjustment — it is the operational question a data-quality user actually has ("of the + incidents, how many did I surface inside my review queue?"). + +Run: uv run python benchmarks/anomaly_conditioning/smd_bakeoff.py --seeds 2 +""" + +import argparse +import json +import time + +import numpy as np + +from datasets.real import SMD_ENTITIES, load_smd_split +from metrics import pr_auc, precision_at_n, roc_auc +from sklearn.decomposition import PCA +from sklearn.ensemble import IsolationForest +from sklearn.neural_network import MLPRegressor +from sklearn.preprocessing import StandardScaler + +N_TREES = 200 +CONTAMINATION = 0.02 +# Trailing window length in rows. SMD is one-minute cadence, so 60 rows is the last hour: long enough +# to characterise "normal recently" without so much lag that a short incident is averaged away. +WINDOW = 60 + + +# -------------------------------------------------------------------------------------- +# Axis 1: the unit of analysis +# -------------------------------------------------------------------------------------- + + +def window_stats(values: np.ndarray, window: int = WINDOW) -> np.ndarray: + """Current value plus trailing mean, stddev, min and max over the preceding *window* rows. + + Feature count is ``5 x n_metrics`` regardless of window length, unlike flattening the window, + which multiplies by the window itself. Strictly trailing: the statistics cover rows + ``[i-window, i-1]``, so the current row never contributes to its own baseline and nothing is + computed from the future. + + Warm-up rows (fewer than two rows of history) fall back to the current value with zero spread, + which is the honest neutral: with no history there is no deviation to report. A production + implementation should mark them unscoreable instead, the way an unseen baseline group already is. + """ + n_rows, n_cols = values.shape + padded = np.vstack([np.zeros((1, n_cols)), values]) + csum = np.cumsum(padded, axis=0) + csum_sq = np.cumsum(padded**2, axis=0) + + mean = np.zeros_like(values) + std = np.zeros_like(values) + lo = np.zeros_like(values) + hi = np.zeros_like(values) + + for i in range(n_rows): + start = max(0, i - window) + count = i - start + if count < 2: + mean[i], std[i], lo[i], hi[i] = values[i], 0.0, values[i], values[i] + continue + total = csum[i] - csum[start] + total_sq = csum_sq[i] - csum_sq[start] + m = total / count + mean[i] = m + std[i] = np.sqrt(np.maximum(total_sq / count - m**2, 0.0)) + chunk = values[start:i] + lo[i] = chunk.min(axis=0) + hi[i] = chunk.max(axis=0) + + return np.hstack([values, mean, std, lo, hi]) + + +FEATURISERS = { + "raw": lambda v: v, + "win_stats": window_stats, +} + + +# -------------------------------------------------------------------------------------- +# Axis 2: estimators. Each returns test scores, higher = more anomalous. +# -------------------------------------------------------------------------------------- + + +def score_iforest(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: + model = IsolationForest(n_estimators=N_TREES, contamination=CONTAMINATION, random_state=seed, n_jobs=-1) + model.fit(train) + return -model.score_samples(test) + + +def score_pca_recon(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: + """Reconstruction error after projecting onto the dominant linear subspace of *train*. + + A linear autoencoder converges to exactly this, so it is the honest cheap stand-in for one, and it + is the candidate most likely to survive a wide feature matrix: extra correlated columns add to the + subspace rather than diluting a random split. + """ + scaler = StandardScaler().fit(train) + tr, te = scaler.transform(train), scaler.transform(test) + n_components = max(1, min(int(0.5 * tr.shape[1]), tr.shape[1] - 1)) + pca = PCA(n_components=n_components, random_state=seed).fit(tr) + return np.sum((te - pca.inverse_transform(pca.transform(te))) ** 2, axis=1) + + +def score_mlp_autoencoder(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: + """Non-linear reconstruction error from a small MLP trained to reproduce its input. + + Stands in for the deep autoencoders the SMD literature uses; no torch in this environment, so this + is the closest available reference for "what a learned reconstruction achieves". Deliberately + small and capped in iterations — this is a reference point, not a tuned model. + """ + scaler = StandardScaler().fit(train) + tr, te = scaler.transform(train), scaler.transform(test) + width = tr.shape[1] + bottleneck = max(2, width // 4) + model = MLPRegressor( + hidden_layer_sizes=(max(4, width // 2), bottleneck, max(4, width // 2)), + random_state=seed, + max_iter=60, + early_stopping=False, + learning_rate_init=0.005, + ) + model.fit(tr, tr) + return np.sum((te - model.predict(te)) ** 2, axis=1) + + +def score_mahalanobis(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: + """Covariance-aware distance from the training centre. Ledoit-Wolf style ridge for stability.""" + del seed + mean = train.mean(axis=0) + centred = train - mean + cov = np.cov(centred, rowvar=False) + cov = np.atleast_2d(cov) + np.eye(cov.shape[0] if cov.ndim else 1) * 1e-3 + inv = np.linalg.pinv(cov) + delta = test - mean + return np.einsum("ij,jk,ik->i", delta, inv, delta) + + +ESTIMATORS = { + "iforest": score_iforest, + "pca_recon": score_pca_recon, + "mlp_ae": score_mlp_autoencoder, + "mahalanobis": score_mahalanobis, +} + + +# -------------------------------------------------------------------------------------- +# Axis 4: an honest event-level metric +# -------------------------------------------------------------------------------------- + + +def event_recall_at_budget(labels: np.ndarray, scores: np.ndarray, budget_frac: float = 0.01) -> tuple[float, int]: + """Fraction of true anomaly *ranges* containing at least one alerted row, within an alert budget. + + Why this is not point adjustment: point adjustment rewrites every point of a detected range as a + true positive, which inflates precision and is why a random scorer reaches state of the art under + it. Here precision is never touched — the alert budget is fixed at *budget_frac* of all rows, and + only recall is counted per event. A detector cannot game it by firing everywhere, because the + budget caps how much it may fire. + + Returns ``(event_recall, n_events)``. + """ + flags = np.zeros(len(labels), dtype=bool) + budget = max(1, int(len(labels) * budget_frac)) + flags[np.argsort(scores)[::-1][:budget]] = True + + # Contiguous runs of label == 1 are events. + padded = np.concatenate([[0], (labels > 0).astype(int), [0]]) + edges = np.diff(padded) + starts = np.flatnonzero(edges == 1) + ends = np.flatnonzero(edges == -1) + if len(starts) == 0: + return float("nan"), 0 + detected = sum(1 for s, e in zip(starts, ends, strict=False) if flags[s:e].any()) + return detected / len(starts), len(starts) + + +# -------------------------------------------------------------------------------------- +# Axis 3: scope, and the driver +# -------------------------------------------------------------------------------------- + + +def run_cell( + train: dict[str, np.ndarray], + test: dict[str, np.ndarray], + labels: dict[str, np.ndarray], + featuriser: str, + estimator: str, + per_entity: bool, + seed: int, +) -> dict: + """One configuration: featurise per entity, then fit either one model or one per entity.""" + started = time.perf_counter() + fx = FEATURISERS[featuriser] + scorer = ESTIMATORS[estimator] + + tr_by_entity = {e: np.nan_to_num(fx(train[e]), nan=0.0, posinf=0.0, neginf=0.0) for e in SMD_ENTITIES} + te_by_entity = {e: np.nan_to_num(fx(test[e]), nan=0.0, posinf=0.0, neginf=0.0) for e in SMD_ENTITIES} + + if per_entity: + # One model per machine. Scores are z-normalised per entity against that entity's own *training* + # score distribution, so the numbers are comparable across models and a single global threshold + # applies -- this is what stops an all-normal entity from flagging its own quietest rows. + score_parts = [] + for entity in SMD_ENTITIES: + tr, te = tr_by_entity[entity], te_by_entity[entity] + raw_test = scorer(tr, te, seed) + raw_train = scorer(tr, tr, seed) + centre, spread = float(np.mean(raw_train)), float(np.std(raw_train)) or 1.0 + score_parts.append((raw_test - centre) / spread) + scores = np.concatenate(score_parts) + n_models = len(SMD_ENTITIES) + else: + tr_all = np.vstack([tr_by_entity[e] for e in SMD_ENTITIES]) + te_all = np.vstack([te_by_entity[e] for e in SMD_ENTITIES]) + scores = scorer(tr_all, te_all, seed) + n_models = 1 + + y = np.concatenate([labels[e] for e in SMD_ENTITIES]) + recall, n_events = event_recall_at_budget(y, scores) + return { + "pr_auc": pr_auc(y, scores), + "roc_auc": roc_auc(y, scores), + "precision_at_n": precision_at_n(y, scores), + "event_recall_at_1pct": recall, + "n_events": n_events, + "n_features": int(next(iter(tr_by_entity.values())).shape[1]), + "n_models": n_models, + "seconds": round(time.perf_counter() - started, 1), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seeds", type=int, default=2) + parser.add_argument("--estimators", nargs="+", default=list(ESTIMATORS), choices=list(ESTIMATORS)) + parser.add_argument("--skip-per-entity", action="store_true", help="pooled scope only (faster)") + args = parser.parse_args() + + print("Loading SMD (cached)...", flush=True) + train, test, labels = load_smd_split() + y = np.concatenate([labels[e] for e in SMD_ENTITIES]) + _, n_events = event_recall_at_budget(y, np.random.default_rng(0).random(len(y))) + print(f" 28 entities | {len(y):,} test rows | {y.mean():.2%} anomalous points | {n_events} events") + print(" protocol: fit on SMD train split, score test split (chronological), no point adjustment") + print(f" event_recall_at_1pct = of {n_events} events, share with >=1 row in a 1%-of-rows alert budget\n") + + scopes = [False] if args.skip_per_entity else [False, True] + header = f"{'featuriser':11s} {'estimator':12s} {'scope':11s} {'PR-AUC':>8s} {'ROC':>7s} {'P@n':>7s} {'EvRec@1%':>9s} {'feat':>5s} {'s':>5s}" + print(header) + print("-" * len(header)) + + results: list[dict] = [] + for featuriser in FEATURISERS: + for estimator in args.estimators: + for per_entity in scopes: + per_seed = [ + run_cell(train, test, labels, featuriser, estimator, per_entity, seed) for seed in range(args.seeds) + ] + agg = { + "featuriser": featuriser, + "estimator": estimator, + "scope": "per_entity" if per_entity else "pooled", + **{ + k: float(np.mean([m[k] for m in per_seed])) + for k in ("pr_auc", "roc_auc", "precision_at_n", "event_recall_at_1pct") + }, + "n_features": per_seed[0]["n_features"], + "n_models": per_seed[0]["n_models"], + "seconds": float(np.mean([m["seconds"] for m in per_seed])), + } + results.append(agg) + print( + f"{featuriser:11s} {estimator:12s} {agg['scope']:11s} " + f"{agg['pr_auc']:8.4f} {agg['roc_auc']:7.4f} {agg['precision_at_n']:7.4f} " + f"{agg['event_recall_at_1pct']:9.3f} {agg['n_features']:5d} {agg['seconds']:5.0f}", + flush=True, + ) + + shipped = next( + (r for r in results if r["featuriser"] == "raw" and r["estimator"] == "iforest" and r["scope"] == "pooled"), + None, + ) + best_pr = max(results, key=lambda r: r["pr_auc"]) + best_ev = max(results, key=lambda r: r["event_recall_at_1pct"]) + + print("\n--- reading ---") + if shipped: + print(f"shipped today : PR-AUC {shipped['pr_auc']:.4f} EvRec@1% {shipped['event_recall_at_1pct']:.3f}") + print( + f"best PR-AUC : {best_pr['pr_auc']:.4f} " + f"({best_pr['featuriser']}/{best_pr['estimator']}/{best_pr['scope']}, {best_pr['n_features']} feat)" + ) + print( + f"best event recall : {best_ev['event_recall_at_1pct']:.3f} " + f"({best_ev['featuriser']}/{best_ev['estimator']}/{best_ev['scope']})" + ) + if shipped and shipped["pr_auc"]: + print(f"headroom on PR-AUC : {best_pr['pr_auc'] / shipped['pr_auc']:.2f}x over what ships today") + print("\nThe best row here IS the ballpark: published SMD figures use point adjustment and are not") + print("a valid target. '80% of the ballpark' means 80% of the best honest configuration above.") + + out = "benchmarks/anomaly_conditioning/results/smd-bakeoff.json" + with open(out, "w", encoding="utf-8") as handle: + json.dump({"window": WINDOW, "seeds": args.seeds, "results": results}, handle, indent=2) + print(f"\nwrote {out}") + + +if __name__ == "__main__": + main() From 65cc55bede97f79482645865124571ada52ab52a Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 22:03:25 +0100 Subject: [PATCH 034/107] Step 0: settle the time-series estimator by measurement, and drop Ledoit-Wolf Three pre-registered gates decided the estimator before any src/ code. Results on raw features, pooled scope, chronological protocol, two seeds. Committed in results/smd-bakeoff.json and results/smd-bakeoff-contaminated-0.033.json. clean contaminated 3.3% estimator PR-AUC EvRec@1% PR-AUC EvRec@1% iforest (ships) 0.0649 0.359 0.0620 0.333 pca_recon 0.1189 0.769 0.1013 0.795 mahalanobis + LW 0.1017 0.769 0.1033 0.795 maha_ridge 0.1264 0.821 0.1176 0.795 G1 (does StandardScaler + Ledoit-Wolf reach >=0.75 incident coverage?) PASSES at 0.769 -- but a plain scale-free ridge floor is better on both metrics, 0.821 / 0.1264. **Ledoit-Wolf over-regularises here, so it is not the default.** Note what SMD can and cannot say about that: at 38 features with ~4000 rows per entity the covariance is well conditioned, which is exactly the regime where shrinkage has nothing to add. Ledoit-Wolf earns its place in the small-sample regime (n close to p) and against the collinear one-hot columns DQX's feature engineering produces -- neither of which SMD exercises. So the implementation should use the ridge floor by default and fall back to Ledoit-Wolf when the sample is too small for a stable covariance, rather than choosing one globally. G2 (does Mahalanobis beat PCA reconstruction on identical footing?) PASSES for the ridge variant: 0.821 vs 0.769 incident coverage and 0.1264 vs 0.1189 PR-AUC. It does NOT pass for the Ledoit-Wolf variant, which ties on coverage and loses on PR-AUC -- another reason the shrinkage choice mattered. G3 (does it survive training on data that contains anomalies?) PASSES, and this is the gate that could have invalidated the whole approach. DQX fits on a random sample of the user's table, anomalies included, and a moment-based estimator is non-robust: extreme rows inflate the covariance along the anomaly direction, which is the direction that must stay tight. Measured, that masking effect is small -- incident coverage 0.821 -> 0.795 and PR-AUC 0.1264 -> 0.1176 at a realistic 3.3% contamination rate, comparable to the noise between seeds. A trim-and-refit mitigation was implemented, measured, and **removed**: it was worse than doing nothing in both conditions (contaminated PR-AUC 0.0944 trimmed vs 0.1176 untrimmed), because on data that is mostly clean it discards good rows to guard against a problem that is not biting. The cure cost more than the disease. Also adds `contaminate()`, without which the clean SMD train split would have made G3 unanswerable -- the earlier run measured only the cost of trimming, never the failure it was meant to prevent. Co-authored-by: Isaac --- .../smd-bakeoff-contaminated-0.033.json | 126 ++++++++++++++++++ .../results/smd-bakeoff.json | 91 +++++++++++-- .../anomaly_conditioning/smd_bakeoff.py | 107 +++++++++++++-- 3 files changed, 305 insertions(+), 19 deletions(-) create mode 100644 benchmarks/anomaly_conditioning/results/smd-bakeoff-contaminated-0.033.json diff --git a/benchmarks/anomaly_conditioning/results/smd-bakeoff-contaminated-0.033.json b/benchmarks/anomaly_conditioning/results/smd-bakeoff-contaminated-0.033.json new file mode 100644 index 000000000..c19a93d1f --- /dev/null +++ b/benchmarks/anomaly_conditioning/results/smd-bakeoff-contaminated-0.033.json @@ -0,0 +1,126 @@ +{ + "window": 60, + "seeds": 2, + "results": [ + { + "featuriser": "raw", + "estimator": "iforest", + "scope": "pooled", + "pr_auc": 0.06199381240530498, + "roc_auc": 0.7006709662321806, + "precision_at_n": 0.08989817792068595, + "event_recall_at_1pct": 0.33333333333333337, + "n_features": 38, + "n_models": 1, + "seconds": 0.7 + }, + { + "featuriser": "raw", + "estimator": "pca_recon", + "scope": "pooled", + "pr_auc": 0.10134410255674345, + "roc_auc": 0.7196872372518814, + "precision_at_n": 0.11655948553054662, + "event_recall_at_1pct": 0.7948717948717948, + "n_features": 38, + "n_models": 1, + "seconds": 0.2 + }, + { + "featuriser": "raw", + "estimator": "mahalanobis", + "scope": "pooled", + "pr_auc": 0.10334293809801756, + "roc_auc": 0.7121257515440129, + "precision_at_n": 0.11629153269024652, + "event_recall_at_1pct": 0.7948717948717948, + "n_features": 38, + "n_models": 1, + "seconds": 0.2 + }, + { + "featuriser": "raw", + "estimator": "maha_ridge", + "scope": "pooled", + "pr_auc": 0.117612954966071, + "roc_auc": 0.7206672197976749, + "precision_at_n": 0.11629153269024652, + "event_recall_at_1pct": 0.7948717948717948, + "n_features": 38, + "n_models": 1, + "seconds": 0.2 + }, + { + "featuriser": "raw", + "estimator": "maha_trimmed", + "scope": "pooled", + "pr_auc": 0.09442408579801306, + "roc_auc": 0.7184355028890834, + "precision_at_n": 0.12031082529474812, + "event_recall_at_1pct": 0.7692307692307693, + "n_features": 38, + "n_models": 1, + "seconds": 0.35 + }, + { + "featuriser": "win_stats", + "estimator": "iforest", + "scope": "pooled", + "pr_auc": 0.08202921622489219, + "roc_auc": 0.7465907092087116, + "precision_at_n": 0.1137459807073955, + "event_recall_at_1pct": 0.11538461538461538, + "n_features": 190, + "n_models": 1, + "seconds": 2.6500000000000004 + }, + { + "featuriser": "win_stats", + "estimator": "pca_recon", + "scope": "pooled", + "pr_auc": 0.14662545881271646, + "roc_auc": 0.7625117429216082, + "precision_at_n": 0.19640943193997856, + "event_recall_at_1pct": 0.20512820512820512, + "n_features": 190, + "n_models": 1, + "seconds": 2.25 + }, + { + "featuriser": "win_stats", + "estimator": "mahalanobis", + "scope": "pooled", + "pr_auc": 0.16345061641209957, + "roc_auc": 0.7954142272533906, + "precision_at_n": 0.21864951768488747, + "event_recall_at_1pct": 0.48717948717948717, + "n_features": 190, + "n_models": 1, + "seconds": 2.25 + }, + { + "featuriser": "win_stats", + "estimator": "maha_ridge", + "scope": "pooled", + "pr_auc": 0.16312010506359406, + "roc_auc": 0.7908960262990758, + "precision_at_n": 0.21757770632368703, + "event_recall_at_1pct": 0.3333333333333333, + "n_features": 190, + "n_models": 1, + "seconds": 2.2 + }, + { + "featuriser": "win_stats", + "estimator": "maha_trimmed", + "scope": "pooled", + "pr_auc": 0.1907541178601119, + "roc_auc": 0.8134753965498104, + "precision_at_n": 0.219989281886388, + "event_recall_at_1pct": 0.46153846153846156, + "n_features": 190, + "n_models": 1, + "seconds": 2.75 + } + ] +} \ No newline at end of file diff --git a/benchmarks/anomaly_conditioning/results/smd-bakeoff.json b/benchmarks/anomaly_conditioning/results/smd-bakeoff.json index fe8a17546..d0b26e0f7 100644 --- a/benchmarks/anomaly_conditioning/results/smd-bakeoff.json +++ b/benchmarks/anomaly_conditioning/results/smd-bakeoff.json @@ -1,30 +1,103 @@ { "window": 60, - "seeds": 1, + "seeds": 2, + "contaminate": 0.0, "results": [ { "featuriser": "raw", "estimator": "iforest", "scope": "pooled", - "pr_auc": 0.06701655758365027, - "roc_auc": 0.7150735941232091, - "precision_at_n": 0.09163987138263666, - "event_recall_at_1pct": 0.3333333333333333, + "pr_auc": 0.06485182445285484, + "roc_auc": 0.7049237121424422, + "precision_at_n": 0.09016613076098606, + "event_recall_at_1pct": 0.358974358974359, "n_features": 38, "n_models": 1, "seconds": 0.7 }, + { + "featuriser": "raw", + "estimator": "pca_recon", + "scope": "pooled", + "pr_auc": 0.11893518451393884, + "roc_auc": 0.7445973923190322, + "precision_at_n": 0.1270096463022508, + "event_recall_at_1pct": 0.7692307692307693, + "n_features": 38, + "n_models": 1, + "seconds": 0.2 + }, + { + "featuriser": "raw", + "estimator": "mahalanobis", + "scope": "pooled", + "pr_auc": 0.10169012776870497, + "roc_auc": 0.7075271236542118, + "precision_at_n": 0.1152197213290461, + "event_recall_at_1pct": 0.7692307692307693, + "n_features": 38, + "n_models": 1, + "seconds": 0.2 + }, + { + "featuriser": "raw", + "estimator": "maha_ridge", + "scope": "pooled", + "pr_auc": 0.12636238430070718, + "roc_auc": 0.7276285562827283, + "precision_at_n": 0.11843515541264737, + "event_recall_at_1pct": 0.8205128205128205, + "n_features": 38, + "n_models": 1, + "seconds": 0.2 + }, { "featuriser": "win_stats", "estimator": "iforest", "scope": "pooled", - "pr_auc": 0.082534382387672, - "roc_auc": 0.7446868279028607, - "precision_at_n": 0.11709539121114684, - "event_recall_at_1pct": 0.10256410256410256, + "pr_auc": 0.083889006916833, + "roc_auc": 0.7513011959257863, + "precision_at_n": 0.11454983922829581, + "event_recall_at_1pct": 0.14102564102564102, "n_features": 190, "n_models": 1, "seconds": 2.6 + }, + { + "featuriser": "win_stats", + "estimator": "pca_recon", + "scope": "pooled", + "pr_auc": 0.15240356108314648, + "roc_auc": 0.7672142449816186, + "precision_at_n": 0.2015005359056806, + "event_recall_at_1pct": 0.23076923076923078, + "n_features": 190, + "n_models": 1, + "seconds": 2.2 + }, + { + "featuriser": "win_stats", + "estimator": "mahalanobis", + "scope": "pooled", + "pr_auc": 0.17142117053734277, + "roc_auc": 0.8027929809443131, + "precision_at_n": 0.2237406216505895, + "event_recall_at_1pct": 0.5128205128205128, + "n_features": 190, + "n_models": 1, + "seconds": 2.2 + }, + { + "featuriser": "win_stats", + "estimator": "maha_ridge", + "scope": "pooled", + "pr_auc": 0.16850907884294952, + "roc_auc": 0.7968172772094938, + "precision_at_n": 0.227491961414791, + "event_recall_at_1pct": 0.358974358974359, + "n_features": 190, + "n_models": 1, + "seconds": 2.1500000000000004 } ] } \ No newline at end of file diff --git a/benchmarks/anomaly_conditioning/smd_bakeoff.py b/benchmarks/anomaly_conditioning/smd_bakeoff.py index 6857725df..1bbf25ae6 100644 --- a/benchmarks/anomaly_conditioning/smd_bakeoff.py +++ b/benchmarks/anomaly_conditioning/smd_bakeoff.py @@ -42,6 +42,7 @@ from datasets.real import SMD_ENTITIES, load_smd_split from metrics import pr_auc, precision_at_n, roc_auc +from sklearn.covariance import LedoitWolf from sklearn.decomposition import PCA from sklearn.ensemble import IsolationForest from sklearn.neural_network import MLPRegressor @@ -152,16 +153,55 @@ def score_mlp_autoencoder(train: np.ndarray, test: np.ndarray, seed: int) -> np. return np.sum((te - model.predict(te)) ** 2, axis=1) +def _mahalanobis_sq(train: np.ndarray, test: np.ndarray, *, shrinkage: str | float, ridge: float) -> np.ndarray: + """Squared Mahalanobis distance from the training centre, scaled and regularised. + + Two facts govern every variant below, and they are easy to get wrong. + + **The scaler is a no-op in exact arithmetic.** Mahalanobis distance is invariant under any + invertible linear map: standardising *x* and using the standardised covariance gives bit-identical + distances to using the raw covariance. So ``StandardScaler`` cannot change the answer directly -- + it changes it only *through the regulariser*, because a shrinkage target is not affine-equivariant. + A fixed ``1e-3`` ridge is negligible against a column of variance 1e6 and dominant against one of + variance 1e-3; standardising first makes one ridge value mean the same thing for every column. This + is the mirror image of the argument in ``core.py`` for why RobustScaler was removed for + IsolationForest: there the scaler was measured to be a genuine no-op, here it earns its place only + by fixing the regulariser's basis. + + **Ledoit-Wolf minimises the error of the covariance, not the conditioning of its inverse.** So a + ridge floor is still applied on top, expressed relative to the average variance so it is + scale-free. + """ + scaler = StandardScaler().fit(train) + tr, te = scaler.transform(train), scaler.transform(test) + + if shrinkage == "ledoit_wolf": + cov = LedoitWolf(assume_centered=False).fit(tr).covariance_ + else: + cov = np.atleast_2d(np.cov(tr - tr.mean(axis=0), rowvar=False)) + if shrinkage: # explicit convex shrink toward the average-variance diagonal + target = np.eye(cov.shape[0]) * (np.trace(cov) / cov.shape[0]) + cov = (1.0 - float(shrinkage)) * cov + float(shrinkage) * target + + cov = np.atleast_2d(cov) + cov = cov + np.eye(cov.shape[0]) * ridge * (np.trace(cov) / cov.shape[0]) + delta = te - tr.mean(axis=0) + # Cholesky solve rather than an explicit inverse: same answer, better conditioned. + factor = np.linalg.cholesky(cov) + solved = np.linalg.solve(factor, delta.T) + return np.sum(solved**2, axis=0) + + def score_mahalanobis(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: - """Covariance-aware distance from the training centre. Ledoit-Wolf style ridge for stability.""" + """The candidate under test: StandardScaler + Ledoit-Wolf + a scale-free ridge floor (G1).""" del seed - mean = train.mean(axis=0) - centred = train - mean - cov = np.cov(centred, rowvar=False) - cov = np.atleast_2d(cov) + np.eye(cov.shape[0] if cov.ndim else 1) * 1e-3 - inv = np.linalg.pinv(cov) - delta = test - mean - return np.einsum("ij,jk,ik->i", delta, inv, delta) + return _mahalanobis_sq(train, test, shrinkage="ledoit_wolf", ridge=1e-6) + + +def score_mahalanobis_ridge(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: + """Same, with no Ledoit-Wolf: the fallback if data-estimated shrinkage over-regularises (G1).""" + del seed + return _mahalanobis_sq(train, test, shrinkage=0.0, ridge=1e-6) ESTIMATORS = { @@ -169,6 +209,7 @@ def score_mahalanobis(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndar "pca_recon": score_pca_recon, "mlp_ae": score_mlp_autoencoder, "mahalanobis": score_mahalanobis, + "maha_ridge": score_mahalanobis_ridge, } @@ -208,6 +249,38 @@ def event_recall_at_budget(labels: np.ndarray, scores: np.ndarray, budget_frac: # -------------------------------------------------------------------------------------- +def contaminate( + train: dict[str, np.ndarray], + test: dict[str, np.ndarray], + labels: dict[str, np.ndarray], + rate: float, + seed: int = 0, +) -> dict[str, np.ndarray]: + """Return a copy of *train* with anomalous rows mixed in, at approximately *rate*. + + Without this, the whole exercise measures the wrong thing. SMD ships a **clean** train split, but + DQX fits on ``sample_df`` -- a random sample of the user's table, anomalies included. Sample mean and + covariance are non-robust, so a few extreme rows inflate the covariance *along the anomaly + direction*, which is precisely the direction that must stay tight. That is masking, and it is the + difference between a benchmark number and a number a user will see. + + IsolationForest resists this (it has ``contamination`` and subsampling); a moment-based estimator + does not. Anomalous rows are drawn from each entity's own test split so they are realistic + anomalies for that machine rather than synthetic noise. + """ + rng = np.random.default_rng(seed) + out = {} + for entity, rows in train.items(): + anomalous = test[entity][labels[entity] > 0] + n_inject = int(len(rows) * rate) + if len(anomalous) == 0 or n_inject == 0: + out[entity] = rows + continue + picks = rng.choice(len(anomalous), size=min(n_inject, len(anomalous)), replace=n_inject > len(anomalous)) + out[entity] = np.vstack([rows, anomalous[picks]]) + return out + + def run_cell( train: dict[str, np.ndarray], test: dict[str, np.ndarray], @@ -263,10 +336,19 @@ def main() -> None: parser.add_argument("--seeds", type=int, default=2) parser.add_argument("--estimators", nargs="+", default=list(ESTIMATORS), choices=list(ESTIMATORS)) parser.add_argument("--skip-per-entity", action="store_true", help="pooled scope only (faster)") + parser.add_argument( + "--contaminate", + type=float, + default=0.0, + help="Mix this fraction of anomalous rows into the TRAIN split (G3: masking under contamination)", + ) args = parser.parse_args() print("Loading SMD (cached)...", flush=True) train, test, labels = load_smd_split() + if args.contaminate: + train = contaminate(train, test, labels, args.contaminate) + print(f" TRAIN CONTAMINATED at {args.contaminate:.1%} (G3: does the estimator survive masking?)") y = np.concatenate([labels[e] for e in SMD_ENTITIES]) _, n_events = event_recall_at_budget(y, np.random.default_rng(0).random(len(y))) print(f" 28 entities | {len(y):,} test rows | {y.mean():.2%} anomalous points | {n_events} events") @@ -328,9 +410,14 @@ def main() -> None: print("\nThe best row here IS the ballpark: published SMD figures use point adjustment and are not") print("a valid target. '80% of the ballpark' means 80% of the best honest configuration above.") - out = "benchmarks/anomaly_conditioning/results/smd-bakeoff.json" + suffix = f"-contaminated-{args.contaminate:g}" if args.contaminate else "" + out = f"benchmarks/anomaly_conditioning/results/smd-bakeoff{suffix}.json" with open(out, "w", encoding="utf-8") as handle: - json.dump({"window": WINDOW, "seeds": args.seeds, "results": results}, handle, indent=2) + json.dump( + {"window": WINDOW, "seeds": args.seeds, "contaminate": args.contaminate, "results": results}, + handle, + indent=2, + ) print(f"\nwrote {out}") From 56a423d95635875f2ca0ffb0f0db43091ea719b6 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 26 Aug 2026 23:47:56 +0100 Subject: [PATCH 035/107] Add the Mahalanobis detector, with non-negative leave-one-out attribution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The estimator itself. Nothing is wired to it yet, so the IsolationForest path is untouched: this commit adds one module, its unit tests, and two names to pylint's good-names list. Why a second algorithm at all: IsolationForest splits on one randomly chosen feature at a time, which is why it is strong on tabular data and weak on multivariate metrics whose anomalies are broken *correlations* rather than extreme single values. Measured on SMD, incident coverage inside a 1%-of-rows alert budget is 0.359 for IsolationForest and 0.821 here. Three decisions worth recording: **Standardisation lives inside the estimator, not as a pipeline step.** Mahalanobis distance is invariant under any invertible linear map, so standardising and using the standardised covariance is bit-identical to using the raw one. A scaler therefore cannot change the answer directly -- only through the regulariser, whose shrinkage target is not affine-equivariant. Since its whole job is to give the ridge a sensible basis, it belongs inside the distance computation. That also keeps the sklearn pipeline single-step and identical in shape to the IsolationForest one, so `named_steps["model"]` unwrapping and everything downstream needs no knowledge that a scaler exists. **Regularisation adapts rather than being chosen globally.** A plain scale-free ridge floor measured better than Ledoit-Wolf on SMD (0.821 vs 0.769), but SMD has 38 features and thousands of rows per entity -- the well-conditioned regime where shrinkage has nothing to add. Ledoit-Wolf earns its place in the small-sample regime and against the collinear one-hot columns DQX's feature engineering produces, neither of which SMD exercises. So: ridge floor by default, Ledoit-Wolf when there are fewer than ten rows per feature, and a hard refusal when the covariance is outright singular (n <= p) rather than silently returning a model that scores everything as anomalous. **Attribution is leave-one-out, and non-negative by construction:** `aᵢ = zᵢ²/(Σ⁻¹)ᵢᵢ`, which by the Schur-complement identity is exactly the drop in squared distance from marginalising feature i out -- "how much of the anomaly disappears if we stop looking at this feature". The tempting alternative was rejected, and the rejection is a test rather than a comment. The signed decomposition `cᵢ = (x−μ)ᵢ·zᵢ` sums exactly to d², but its terms go negative on correlated features: with Σ = [[1,0.9],[0.9,1]] and x−μ = (1.0, 0.5) it yields (2.895, −1.053). Every consumer downstream takes abs() and renormalises (format_shap_contributions, _pattern_spark_expr, _format_contributions_sql), so a feature that *reduced* the distance would be presented to an LLM as a 27% driver and written into a narrative. Additivity buys nothing here because nothing downstream consumes it; non-negativity is what correctness requires. Constant-in-training features (null indicators for columns that had no nulls, single-category one-hots) are excluded from the distance rather than divided by a ~zero spread, but are still reported as 0.0 so the output width keeps matching engineered_feature_names -- the persisted scoring contract. Uses numpy rather than scipy for the Cholesky solves: scipy is not a declared DQX dependency, only a transitive one via scikit-learn, and at p <= 50 the difference is unmeasurable. "X" and "y" are added to pylint's good-names with a rationale, matching how that list already documents "df" and "ws". They are mandated by the scikit-learn estimator contract, which is the third-party-API case the project's linting policy allows an exception for. Co-authored-by: Isaac --- pyproject.toml | 3 +- .../labs/dqx/anomaly/timeseries_detector.py | 203 ++++++++++++++++++ .../unit/test_anomaly_mahalanobis_detector.py | 175 +++++++++++++++ 3 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 src/databricks/labs/dqx/anomaly/timeseries_detector.py create mode 100644 tests/unit/test_anomaly_mahalanobis_detector.py diff --git a/pyproject.toml b/pyproject.toml index 5b9695f51..3013b6479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -445,7 +445,8 @@ good-names = [ "_", # use for ignores "a", # use for databricks.sdk.AccountClient "w", "ws", # use for databricks.sdk.WorkspaceClient - "me" # use for current user + "me", # use for current user + "X", "y" # required by the scikit-learn estimator contract (fit/predict/score_samples) ] # Good variable names regexes, separated by a comma. If names match any regex, diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py new file mode 100644 index 000000000..12aa6ad55 --- /dev/null +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -0,0 +1,203 @@ +"""A correlation-aware detector for multivariate metrics, and its exact feature attribution. + +IsolationForest splits on one randomly chosen feature at a time, which is why it is strong on tabular +data and weak on multivariate metrics whose anomalies are *broken correlations* rather than extreme +single values. Measured on SMD (28 machines, 38 metrics, fit on the train split and scored on the test +split, no point adjustment) it catches 36% of incidents inside a 1%-of-rows alert budget; the +Mahalanobis detector here catches 82%. See ``benchmarks/anomaly_conditioning/smd_bakeoff.py`` and the +committed results next to it. + +The distance is the ordinary squared Mahalanobis distance from the training centre, +``d² = (x−μ)ᵀ Σ⁻¹ (x−μ)``, with three deliberate choices. + +**Standardisation is internal, not a pipeline step.** Mahalanobis distance is invariant under any +invertible linear map: standardising *x* and using the standardised covariance gives bit-identical +distances to using the raw covariance. So a scaler cannot change the answer directly — it changes it +only *through the regulariser*, whose shrinkage target is not affine-equivariant. A fixed ridge is +negligible against a column of variance 1e6 and dominant against one of variance 1e-3, so the ridge is +expressed relative to the average variance and the standardisation is kept inside this estimator, where +it belongs. Keeping it internal also leaves the sklearn pipeline single-step and therefore identical in +shape to the IsolationForest one, so nothing downstream has to know a scaler exists. + +**Regularisation adapts to the sample.** A plain ridge floor measured better than Ledoit-Wolf on SMD +(incident coverage 0.821 vs 0.769) — but SMD has 38 features and thousands of rows per entity, which is +exactly the well-conditioned regime where shrinkage has nothing to add. Ledoit-Wolf earns its place in +the small-sample regime and against the collinear one-hot columns DQX's feature engineering produces, +neither of which SMD exercises. So the ridge floor is the default and Ledoit-Wolf is used when the +sample is too small for a stable empirical covariance. + +**Attribution is leave-one-out, and non-negative by construction.** See +:meth:`MahalanobisDetector.feature_contributions`. +""" + +import logging + +import numpy as np +from sklearn.base import BaseEstimator, OutlierMixin +from sklearn.covariance import LedoitWolf + +logger = logging.getLogger(__name__) + +# Below this many rows per feature the empirical covariance is too noisy to invert meaningfully, so +# Ledoit-Wolf shrinkage is used instead. Ten is the usual rule of thumb for a stable covariance. +SMALL_SAMPLE_ROWS_PER_FEATURE = 10 +# A feature whose training standard deviation is at or below this is treated as constant and excluded +# from the distance. Null indicators for columns that had no nulls, and single-category one-hot columns, +# are exactly this: constant, and therefore contributing a zero row and column to the covariance. +CONSTANT_FEATURE_TOLERANCE = 1e-12 +# Ridge added to the covariance diagonal, as a fraction of the average variance so it is scale-free. +DEFAULT_RIDGE = 1e-6 + + +class MahalanobisDetector(BaseEstimator, OutlierMixin): + """Squared-Mahalanobis outlier detector with exact per-feature attribution. + + Implements the scikit-learn outlier-detector contract in full — ``fit``, ``score_samples``, + ``decision_function``, ``predict`` — because DQX depends on all of it: scoring calls + ``-model.score_samples(X)`` (so **higher must mean more normal**, matching IsolationForest), and + ``core.score_with_model`` calls ``predict`` on the training path, not only MLflow's signature + inference. + + Args: + contamination: Expected fraction of anomalies, used only to place ``offset_`` so that + ``predict`` labels roughly that fraction as outliers. Does not affect ``score_samples`` + and therefore does not affect ranking or DQX's severity calibration. + ridge: Ridge added to the covariance diagonal as a fraction of the average variance. + """ + + def __init__(self, contamination: float = 0.02, ridge: float = DEFAULT_RIDGE) -> None: + # sklearn convention: __init__ only stores arguments, never validates or transforms them, so + # that get_params/set_params/clone round-trip exactly. + self.contamination = contamination + self.ridge = ridge + + def fit(self, X: np.ndarray, y: object = None) -> "MahalanobisDetector": + """Estimate the centre, the regularised covariance and its Cholesky factor. + + Args: + X: Training features, shape ``(n_samples, n_features)``. + y: Ignored; present for scikit-learn compatibility. + """ + del y + data = np.asarray(X, dtype=float) + if data.ndim != 2: + raise ValueError(f"expected a 2-D feature matrix, got shape {data.shape}") + n_samples, n_features = data.shape + + self.location_ = data.mean(axis=0) + spread = data.std(axis=0) + # Constant features are excluded rather than divided by: dividing by ~0 would turn any change + # at scoring time into an astronomical distance. Their contribution is reported as 0.0 so that + # the emitted feature list still matches engineered_feature_names exactly. + self.active_ = spread > CONSTANT_FEATURE_TOLERANCE + self.scale_ = np.where(self.active_, spread, 1.0) + n_active = int(self.active_.sum()) + if n_active == 0: + raise ValueError("every feature is constant in the training data; nothing to model") + + standardised = self._standardise(data) + self.n_features_in_ = n_features + + covariance = self._covariance(standardised, n_samples, n_active) + # Ledoit-Wolf minimises the error of the covariance, not the conditioning of its inverse, so a + # floor is applied either way. Scale-free: a fraction of the average variance. + average_variance = float(np.trace(covariance)) / n_active + covariance = covariance + np.eye(n_active) * self.ridge * average_variance + + self.cholesky_ = np.linalg.cholesky(covariance) + # diag(Σ⁻¹) without forming Σ⁻¹: with Σ = L Lᵀ, (Σ⁻¹)ᵢᵢ is the squared norm of column i of L⁻¹. + inverse_factor = np.linalg.solve(self.cholesky_, np.eye(n_active)) + self.precision_diagonal_ = np.sum(inverse_factor**2, axis=0) + + # offset_ places the predict/decision_function boundary, mirroring IsolationForest: the + # contamination-th percentile of the training scores. + training_scores = self.score_samples(data) + self.offset_ = float(np.percentile(training_scores, 100.0 * self.contamination)) + return self + + def _covariance(self, standardised: np.ndarray, n_samples: int, n_active: int) -> np.ndarray: + """Empirical covariance, or Ledoit-Wolf shrinkage when the sample is too small to trust one.""" + if n_samples <= n_active: + raise ValueError( + f"cannot fit a correlation-aware detector on {n_samples} rows with {n_active} " + "informative features: the covariance is singular. Provide more training rows, or " + "reduce the feature count." + ) + if n_samples < SMALL_SAMPLE_ROWS_PER_FEATURE * n_active: + logger.warning( + f"Only {n_samples} training rows for {n_active} features " + f"(<{SMALL_SAMPLE_ROWS_PER_FEATURE} per feature): using Ledoit-Wolf shrinkage, " + "which is more stable but less sharp. More training data would detect better." + ) + return np.atleast_2d(LedoitWolf(assume_centered=False).fit(standardised).covariance_) + centred = standardised - standardised.mean(axis=0) + return np.atleast_2d(np.cov(centred, rowvar=False)) + + def _standardise(self, data: np.ndarray) -> np.ndarray: + """Centre and scale, keeping only the features that varied during training.""" + return ((data - self.location_) / self.scale_)[:, self.active_] + + def _whitened(self, X: np.ndarray) -> np.ndarray: + """``L⁻¹ z`` for each row, whose squared norm is the squared Mahalanobis distance.""" + standardised = self._standardise(np.asarray(X, dtype=float)) + # numpy's general solve rather than a triangular one: scipy is not a declared dependency of + # DQX, and at the feature counts involved (p <= 50) the difference is unmeasurable. + return np.linalg.solve(self.cholesky_, standardised.T).T + + def mahalanobis_squared(self, X: np.ndarray) -> np.ndarray: + """Squared Mahalanobis distance per row. Higher means more unusual.""" + return np.sum(self._whitened(X) ** 2, axis=1) + + def score_samples(self, X: np.ndarray) -> np.ndarray: + """Anomaly score per row, **higher meaning more normal**. + + The sign matters: DQX negates this (``-model.score_samples(X)``) exactly as it does for + IsolationForest, so returning the negated distance is what makes the two interchangeable + everywhere downstream, including severity calibration. + """ + return -self.mahalanobis_squared(X) + + def decision_function(self, X: np.ndarray) -> np.ndarray: + """``score_samples`` shifted so that negative means outlier, as scikit-learn expects.""" + return self.score_samples(X) - self.offset_ + + def predict(self, X: np.ndarray) -> np.ndarray: + """``-1`` for outliers and ``1`` for inliers. + + Required beyond MLflow's signature inference: ``core.score_with_model`` calls this and maps + ``-1`` to a flag, so anything other than the sklearn convention would silently mislabel rows. + """ + return np.where(self.decision_function(X) < 0, -1, 1) + + def feature_contributions(self, X: np.ndarray) -> np.ndarray: + """Per-feature contribution to each row's distance: **non-negative**, leave-one-out. + + ``aᵢ = zᵢ² / (Σ⁻¹)ᵢᵢ`` where ``z = Σ⁻¹(x−μ)``. By the Schur-complement identity this is exactly + the drop in squared distance from marginalising feature *i* out — "how much of the anomaly + disappears if we stop looking at this feature" — so it is always ``>= 0`` and independent of the + order features are considered in. + + The tempting alternative, the exactly-additive ``cᵢ = (x−μ)ᵢ·zᵢ`` with ``Σᵢ cᵢ = d²``, is + **wrong for this pipeline**: its terms can be negative when features are correlated. With + ``Σ = [[1, 0.9], [0.9, 1]]`` and ``x−μ = (1.0, 0.5)`` it gives ``(2.895, −1.053)`` — the second + feature *reduced* the distance. Every consumer downstream takes ``abs()`` and renormalises + (``explainability.format_shap_contributions``, ``_pattern_spark_expr``, + ``_format_contributions_sql``), so that term would be presented to an LLM as a 27% *driver* of + the anomaly and written into a narrative. Additivity buys nothing here, because nothing + downstream consumes it; non-negativity is what correctness requires. + + Constant-in-training features are excluded from the distance and reported as ``0.0``, so the + returned width always matches the trained feature count and therefore + ``engineered_feature_names``. + + Returns: + Array of shape ``(n_samples, n_features_in_)``, non-negative. + """ + whitened = self._whitened(X) + # z = Σ⁻¹(x−μ) = L⁻ᵀ(L⁻¹ z̃), reusing the whitened rows rather than forming Σ⁻¹. + precision_delta = np.linalg.solve(self.cholesky_.T, whitened.T).T + active_contributions = precision_delta**2 / self.precision_diagonal_ + + contributions = np.zeros((active_contributions.shape[0], self.n_features_in_), dtype=float) + contributions[:, self.active_] = active_contributions + return contributions diff --git a/tests/unit/test_anomaly_mahalanobis_detector.py b/tests/unit/test_anomaly_mahalanobis_detector.py new file mode 100644 index 000000000..a20033fe0 --- /dev/null +++ b/tests/unit/test_anomaly_mahalanobis_detector.py @@ -0,0 +1,175 @@ +"""Unit tests for the correlation-aware detector and its attribution (no Spark, no workspace). + +The attribution tests are the most valuable ones here. A distance decomposition that is merely +*additive* would pass a naive "sums to the total" check and still feed a false claim into an LLM +narrative, so the rejected alternative is asserted explicitly rather than described in a comment. +""" + +import numpy as np +import pytest +from sklearn.base import clone +from sklearn.pipeline import Pipeline + +from databricks.labs.dqx.anomaly.timeseries_detector import MahalanobisDetector + +# The correlated 2x2 case used throughout: rho = 0.9, so the off-diagonal precision terms are large +# enough that the signed decomposition goes negative. +_CORRELATED = np.array([[1.0, 0.9], [0.9, 1.0]]) + + +def _sample_from(covariance: np.ndarray, n_samples: int = 4000, seed: int = 0) -> np.ndarray: + rng = np.random.default_rng(seed) + return rng.multivariate_normal(np.zeros(len(covariance)), covariance, size=n_samples) + + +@pytest.fixture +def fitted() -> MahalanobisDetector: + """Fitted on a well-conditioned correlated sample, so the empirical covariance path is used.""" + return MahalanobisDetector(ridge=0.0).fit(_sample_from(_CORRELATED)) + + +# --------------------------------------------------------------------------- +# The sklearn outlier contract, which DQX depends on in full +# --------------------------------------------------------------------------- + + +def test_score_samples_is_higher_for_more_normal_rows(fitted: MahalanobisDetector): + """The sign convention is load-bearing: DQX computes ``-model.score_samples(X)``, so this must + match IsolationForest's orientation or every score downstream is inverted.""" + normal, extreme = np.array([[0.0, 0.0]]), np.array([[6.0, -6.0]]) + + assert fitted.score_samples(normal)[0] > fitted.score_samples(extreme)[0] + assert fitted.score_samples(normal)[0] <= 0.0 # negated distance, so never positive + + +def test_predict_returns_the_sklearn_outlier_labels(fitted: MahalanobisDetector): + """``core.score_with_model`` maps -1 to a flag, so any other encoding silently mislabels rows.""" + labels = fitted.predict(np.array([[0.0, 0.0], [6.0, -6.0]])) + + assert set(np.unique(labels)).issubset({-1, 1}) + assert labels[0] == 1 and labels[1] == -1 + + +def test_decision_function_is_negative_exactly_for_outliers(fitted: MahalanobisDetector): + points = np.array([[0.0, 0.0], [6.0, -6.0]]) + + decisions = fitted.decision_function(points) + assert np.array_equal(np.where(decisions < 0, -1, 1), fitted.predict(points)) + + +def test_get_params_round_trips_so_clone_and_pipeline_work(fitted: MahalanobisDetector): + """Needed by sklearn ``Pipeline`` and by ``mlflow.sklearn``.""" + detector = MahalanobisDetector(contamination=0.07, ridge=1e-5) + + assert clone(detector).get_params() == {"contamination": 0.07, "ridge": 1e-5} + + +def test_works_as_the_model_step_of_a_single_step_pipeline(): + """DQX wraps the estimator as ``Pipeline([('model', ...)])`` and reaches it via + ``named_steps['model']``; keeping the shape single-step is what makes this detector a drop-in.""" + pipeline = Pipeline([("model", MahalanobisDetector())]).fit(_sample_from(_CORRELATED)) + + assert list(pipeline.named_steps) == ["model"] + assert pipeline.score_samples(np.array([[0.0, 0.0]])).shape == (1,) + + +# --------------------------------------------------------------------------- +# Attribution +# --------------------------------------------------------------------------- + + +def test_contributions_are_never_negative(fitted: MahalanobisDetector): + """The property the whole choice of formula rests on: every consumer downstream takes abs() and + renders the result as a percentage driver, so a negative term would become a false claim.""" + rng = np.random.default_rng(7) + points = rng.normal(size=(500, 2)) * 3.0 + + assert (fitted.feature_contributions(points) >= 0.0).all() + + +def test_contribution_equals_the_drop_from_marginalising_that_feature_out(): + """Pins the leave-one-out identity ``aᵢ = d²(all) − d²(all but i)`` against a hand-built covariance. + + Computed directly from the 2x2 case rather than through the estimator's internals, so the test + would still fail if the implementation and the intended identity drifted apart. + """ + detector = MahalanobisDetector(ridge=0.0).fit(_sample_from(_CORRELATED, n_samples=20000, seed=3)) + delta = np.array([[1.0, 0.5]]) + standardised = (delta - detector.location_) / detector.scale_ + + contributions = detector.feature_contributions(delta)[0] + + covariance = detector.cholesky_ @ detector.cholesky_.T + full = float(standardised @ np.linalg.inv(covariance) @ standardised.T) + for index in (0, 1): + kept = [j for j in range(2) if j != index] + sub = covariance[np.ix_(kept, kept)] + without = float(standardised[:, kept] @ np.linalg.inv(sub) @ standardised[:, kept].T) + assert contributions[index] == pytest.approx(full - without, rel=1e-6) + + +def test_the_signed_additive_decomposition_would_go_negative(): + """The rejected alternative, asserted so the rejection is executable rather than a comment. + + ``cᵢ = (x−μ)ᵢ·zᵢ`` sums exactly to ``d²`` and is therefore tempting. On correlated features one + term is negative — the feature *reduced* the distance — and since every downstream consumer takes + ``abs()`` and renormalises, it would be reported as a substantial positive driver of the anomaly. + """ + precision = np.linalg.inv(_CORRELATED) + delta = np.array([1.0, 0.5]) + signed = delta * (precision @ delta) + + assert signed.sum() == pytest.approx(delta @ precision @ delta) # exactly additive... + assert signed.min() < 0.0 # ...and still unusable + # The leave-one-out form on the same input is strictly positive. + leave_one_out = (precision @ delta) ** 2 / np.diag(precision) + assert (leave_one_out > 0.0).all() + + +def test_the_deviating_feature_is_ranked_first(fitted: MahalanobisDetector): + """A single feature pushed far from its correlated partner must dominate the attribution.""" + contributions = fitted.feature_contributions(np.array([[5.0, 0.0]]))[0] + + assert contributions.argmax() == 0 + + +def test_constant_features_are_excluded_but_still_reported(): + """Null indicators for columns that had no nulls, and single-category one-hots, are constant. + + They must not be divided by a ~zero spread, and they must still appear in the output so the width + keeps matching ``engineered_feature_names`` — the persisted scoring contract. + """ + varying = _sample_from(_CORRELATED) + train = np.hstack([varying, np.full((len(varying), 1), 3.0)]) + + detector = MahalanobisDetector().fit(train) + contributions = detector.feature_contributions(np.array([[1.0, 0.5, 3.0]])) + + assert contributions.shape == (1, 3) + assert contributions[0, 2] == 0.0 + assert np.isfinite(detector.score_samples(np.array([[1.0, 0.5, 999.0]]))).all() + + +def test_all_constant_training_data_is_refused(): + with pytest.raises(ValueError, match="every feature is constant"): + MahalanobisDetector().fit(np.ones((50, 3))) + + +def test_more_features_than_rows_is_refused_with_an_actionable_message(): + """Silent failure here would produce a model that scores everything as wildly anomalous.""" + rng = np.random.default_rng(0) + + with pytest.raises(ValueError, match="covariance is singular"): + MahalanobisDetector().fit(rng.normal(size=(8, 20))) + + +def test_small_samples_fall_back_to_shrinkage(caplog): + """A plain ridge floor measured better on SMD, but SMD is well conditioned. Shrinkage is what + keeps a near-square sample usable, so the fallback must actually engage and say so.""" + rng = np.random.default_rng(1) + + with caplog.at_level("WARNING"): + detector = MahalanobisDetector().fit(rng.normal(size=(40, 8))) + + assert "Ledoit-Wolf" in caplog.text + assert np.isfinite(detector.score_samples(rng.normal(size=(5, 8)))).all() From 45e0adcb6ded756a0bf21c46c82bf08b220bbef1 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 09:45:48 +0100 Subject: [PATCH 036/107] Pin the IsolationForest path against change, before wiring anything to it Written now rather than after the new algorithm is wired up, so "the tabular path is untouched" is an executable claim instead of a review argument. Six assertions, each describing behaviour that predates the Mahalanobis detector and must survive it. The strongest is a committed array of scores from a fixed seeded frame. Compared with pytest.approx rather than a hash: scikit-learn is pinned only as >=1.0,<2.0, so a patch release may legitimately move the last bits of a float without changing the model -- a hash would fail on noise, this fails on a behaviour change. Verified sensitive: changing the seed to 43 or the tree count to 100 both break it. Alongside it, the pipeline must stay a single "model" step (that is how the SHAP explainer reaches the forest, and the absence of a scaler is a measured decision recorded in fit_sklearn_model's docstring), and the persisted hyperparameters dict is asserted whole rather than key by key. That second one guards a specific mistake: adding a shared key such as an "algorithm" discriminator would rewrite training.hyperparameters for every existing IsolationForest registry row, which is why the discriminator lives in identity.algorithm instead. Also pinned: AnomalyParams' field order, because it is a plain dataclass and anything constructing it positionally rebinds silently if a field is inserted rather than appended; and that both existing algorithm strings, "IsolationForest" and "IsolationForest_Ensemble_3", still resolve -- every model already in a registry table carries one of them, so widening the resolver must not stop it matching. The battery was validated by deliberately breaking the path: inserting a StandardScaler step and adding a hyperparameters key made exactly the two relevant tests fail. Worth recording what did *not* fail -- the scores were unchanged by the scaler, because IsolationForest is scale-invariant. That is the same property that justified removing RobustScaler in the first place, reconfirmed by accident, and the reason the pipeline-shape assertion has to exist separately from the score assertion. One discovery while writing this: `fit_sklearn_model(frame, AnomalyParams())` cannot be called at all. IsolationForestConfig.contamination defaults to None and scikit-learn rejects it, so production always fills it from expected_anomaly_rate first. The test therefore uses 0.02 explicitly, which is the real effective default rather than the declared one. Co-authored-by: Isaac --- ...test_anomaly_isolation_forest_inertness.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 tests/unit/test_anomaly_isolation_forest_inertness.py diff --git a/tests/unit/test_anomaly_isolation_forest_inertness.py b/tests/unit/test_anomaly_isolation_forest_inertness.py new file mode 100644 index 000000000..bc2fea42e --- /dev/null +++ b/tests/unit/test_anomaly_isolation_forest_inertness.py @@ -0,0 +1,130 @@ +"""The IsolationForest path must not change when a second algorithm is added. + +Written before any wiring, so "the tabular path is untouched" is an executable claim rather than a +review argument. Everything here describes behaviour that existed before the Mahalanobis detector and +must survive it: the pipeline's shape, the exact hyperparameters persisted into every registry row, the +positional field order of ``AnomalyParams``, the algorithm strings that resolve to the existing scoring +strategy, and — the literal claim — the scores themselves. + +If a change makes one of these fail, the change is not additive, whatever else it looks like. +""" + +import dataclasses + +import numpy as np +import pandas as pd +import pytest + +from databricks.labs.dqx.anomaly.core import fit_sklearn_model +from databricks.labs.dqx.anomaly.scoring_strategies import resolve_scoring_strategy +from databricks.labs.dqx.config import AnomalyParams, IsolationForestConfig + +# Fixed seed, fixed shape: the reference scores below were generated from exactly this frame. +_TRAIN_SEED = 1234 +_TRAIN_ROWS = 500 +_TRAIN_COLUMNS = ["a", "b", "c"] + +# A spread of ordinary and extreme points, scored against the reference model. +_SCORE_GRID = [ + [0.0, 0.0, 0.0], + [1.0, -1.0, 0.5], + [4.0, 4.0, 4.0], + [-3.0, 2.0, -2.0], + [0.5, 0.5, -0.5], +] + +# Committed reference scores. Compared with pytest.approx rather than an exact hash: scikit-learn is +# pinned only as >=1.0,<2.0, and a patch release can legitimately shift the last bits of a float +# without changing the model. A hash would fail on noise; this fails on a behaviour change. +_REFERENCE_SCORES = [ + -0.3914786863, + -0.45006446042, + -0.764524071473, + -0.672658509853, + -0.411859435027, +] + + +def _reference_params() -> AnomalyParams: + """Params as production actually presents them to ``fit_sklearn_model``. + + ``contamination`` is explicit because a bare ``AnomalyParams()`` cannot be fitted at all: + ``IsolationForestConfig.contamination`` defaults to None and scikit-learn rejects that. Production + fills it from *expected_anomaly_rate* (default 0.02) before training, so 0.02 is the real default. + """ + return AnomalyParams(algorithm_config=IsolationForestConfig(contamination=0.02)) + + +def _reference_training_frame() -> pd.DataFrame: + rng = np.random.default_rng(_TRAIN_SEED) + return pd.DataFrame(rng.normal(size=(_TRAIN_ROWS, len(_TRAIN_COLUMNS))), columns=_TRAIN_COLUMNS) + + +def test_scores_match_the_committed_reference(): + """The literal inertness claim: same data in, same scores out. + + This is the assertion that catches an accidentally inserted preprocessing step, a changed default, + or a different seed — none of which would necessarily break any other test, and all of which would + silently move every severity percentile for every deployed tabular model. + """ + pipeline, _ = fit_sklearn_model(_reference_training_frame(), _reference_params()) + + scores = pipeline.score_samples(pd.DataFrame(_SCORE_GRID, columns=_TRAIN_COLUMNS)) + + assert list(scores) == pytest.approx(_REFERENCE_SCORES, rel=1e-9) + + +def test_pipeline_stays_a_single_model_step(): + """``explainability`` reaches the forest through ``named_steps["model"]``, and the absence of a + scaler is a measured decision recorded in ``fit_sklearn_model``'s docstring, not an oversight.""" + pipeline, _ = fit_sklearn_model(_reference_training_frame(), _reference_params()) + + assert list(pipeline.named_steps) == ["model"] + + +def test_persisted_hyperparameters_are_exactly_these_keys(): + """These land in ``training.hyperparameters`` on every registry row and in ``mlflow.log_params``. + + Adding a shared key — an "algorithm" discriminator, say — would change the persisted row for every + existing IsolationForest model, which is why the discriminator lives in ``identity.algorithm`` + instead. Asserted as a whole dict, so an addition fails rather than passing unnoticed. + """ + _, hyperparams = fit_sklearn_model(_reference_training_frame(), _reference_params()) + + assert hyperparams == { + "contamination": 0.02, + "num_trees": 200, + "max_samples": None, + "random_seed": 42, + "feature_scaling": "none", + } + + +def test_anomaly_params_field_order_and_defaults_are_stable(): + """``AnomalyParams`` is a plain dataclass, so field order is part of its public API: anything + constructing it positionally rebinds silently if a new field is inserted rather than appended.""" + assert [f.name for f in dataclasses.fields(AnomalyParams)] == [ + "sample_fraction", + "max_rows", + "train_ratio", + "ensemble_size", + "algorithm_config", + "feature_engineering", + "baseline_by", + ] + + defaults = AnomalyParams() + assert defaults.sample_fraction == 0.3 + assert defaults.max_rows == 1_000_000 + assert defaults.train_ratio == 0.8 + assert defaults.ensemble_size == 3 + assert defaults.baseline_by is None + + +@pytest.mark.parametrize("algorithm", ["IsolationForest", "IsolationForest_Ensemble_3"]) +def test_existing_algorithm_strings_still_resolve(algorithm: str): + """Every model already in a registry table carries one of these strings. Widening the resolver for + a new algorithm must not stop it matching the old ones.""" + strategy = resolve_scoring_strategy(algorithm) + + assert strategy.supports(algorithm) From f01c43ea43a7cd32f1294ab9c7a5c73127a1dbb5 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 09:51:34 +0100 Subject: [PATCH 037/107] Dispatch attribution by estimator, so contributions work without SHAP The contributions path was hard-bound to trees: `SHAP.TreeExplainer(tree_model)` was the only source of attribution. That mattered more than it looks, because `check_funcs._resolve_ai_explanation_flag` silently disables AI explanations when contributions are unavailable -- so a non-tree algorithm would have quietly lost both features rather than failing loudly. `compute_shap_values` becomes `compute_row_attributions` and gains one branch: an estimator exposing `feature_contributions` supplies its own attribution, otherwise TreeExplainer runs exactly as before. The name changes because the function no longer always computes SHAP; it had no callers outside this module, so the rename is free. Dispatch is by duck typing rather than `isinstance`, deliberately. This module is imported at rule-registration time and by both scorers, so importing a concrete estimator here would pull it into all of them. `hasattr(estimator, "feature_contributions")` keeps the dependency pointing one way, and IsolationForest lacks the attribute so the tree branch is unreachable for it either way. The values feed the existing `format_shap_contributions` unchanged, which is the point: `abs()` is the identity on non-negative input, so its normalisation, its 1/num_features zero-total fallback, its rounding and its None-filling all apply as-is. The emitted map is therefore indistinguishable in shape from the SHAP path's, and redaction, human labels, the LLM prompt and the _dq_info schema need no changes. Writing a parallel formatter would have been the obvious mistake here. One trap avoided: this module already contains a dead `compute_feature_contributions(model_uri, df, columns) -> DataFrame` with no callers anywhere. Naming the new helper that would have silently shadowed it, since the later definition wins. Hence `compute_row_attributions`. The dead function is left alone rather than deleted in a commit about dispatch. Six new tests cover the branch through the real gating entry point rather than in isolation: the gate still returns None below threshold, the map has the same keys and sums to ~100, values are never negative, the deviating feature dominates, and NaN rows still produce an all-None map. The never-negative one is the load-bearing assertion -- it is why the leave-one-out decomposition was chosen over the exactly-additive signed one, which would have had `abs()` present a distance-reducing feature as a driver. Unit suite 2540 passed, up from 2515, no regressions. Co-authored-by: Isaac --- .../labs/dqx/anomaly/explainability.py | 44 +++++++--- tests/unit/test_anomaly_shap_gating.py | 88 +++++++++++++++++++ 2 files changed, 118 insertions(+), 14 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/explainability.py b/src/databricks/labs/dqx/anomaly/explainability.py index b0e2eef73..07982373c 100644 --- a/src/databricks/labs/dqx/anomaly/explainability.py +++ b/src/databricks/labs/dqx/anomaly/explainability.py @@ -63,27 +63,43 @@ def format_shap_contributions( return contributions -def compute_shap_values( +def compute_row_attributions( model_local: Any, feature_matrix: pd.DataFrame, engineered_feature_cols: list[str], ) -> tuple[np.ndarray, np.ndarray]: - """Compute SHAP values for a model and feature matrix.""" + """Per-feature attribution for each row, from whichever estimator the model wraps. + + Two sources, one output shape. A tree model goes through ``SHAP.TreeExplainer``, which is + approximate and the only SHAP explainer fast enough to be worth running here. An estimator that + exposes ``feature_contributions`` supplies its own *exact* attribution instead -- the Mahalanobis + detector's leave-one-out decomposition, which needs no SHAP at all. + + Dispatch is by duck typing rather than an ``isinstance`` check on purpose: this module is imported + at rule-registration time and by both scorers, so importing a concrete estimator here would drag it + into all of them and make the dependency direction harder to reason about. + + Whatever the source, the values feed the same *format_shap_contributions*, so the emitted map has + identical keys, scaling and null handling either way, and everything downstream -- redaction, + human labels, the LLM prompt, the ``_dq_info`` schema -- is unaffected by which branch ran. + """ scaler = getattr(model_local, "named_steps", {}).get("scaler") - tree_model = getattr(model_local, "named_steps", {}).get("model", model_local) + estimator = getattr(model_local, "named_steps", {}).get("model", model_local) - shap_data = scaler.transform(feature_matrix) if scaler else feature_matrix.values - valid_indices = ~pd.isna(shap_data).any(axis=1) + feature_values = scaler.transform(feature_matrix) if scaler else feature_matrix.values + valid_indices = ~pd.isna(feature_values).any(axis=1) - shap_values = np.array([]) + attribution = np.array([]) if valid_indices.any(): if len(engineered_feature_cols) == 1: - shap_values = np.ones((len(shap_data[valid_indices]), 1)) + attribution = np.ones((len(feature_values[valid_indices]), 1)) + elif hasattr(estimator, "feature_contributions"): + attribution = estimator.feature_contributions(feature_values[valid_indices]) else: - explainer = SHAP.TreeExplainer(tree_model) - shap_values = explainer.shap_values(shap_data[valid_indices]) + explainer = SHAP.TreeExplainer(estimator) + attribution = explainer.shap_values(feature_values[valid_indices]) - return shap_values, valid_indices + return attribution, valid_indices # Severity-gating margin for in-UDF SHAP computation. The UDF recomputes severity from raw @@ -123,17 +139,17 @@ def compute_gated_shap_contributions( """ num_rows = len(feature_matrix) if not quantile_points or threshold is None: - shap_values, valid_indices = compute_shap_values(model_local, feature_matrix, engineered_feature_cols) - return list(format_shap_contributions(shap_values, valid_indices, num_rows, engineered_feature_cols)) + attribution, valid_indices = compute_row_attributions(model_local, feature_matrix, engineered_feature_cols) + return list(format_shap_contributions(attribution, valid_indices, num_rows, engineered_feature_cols)) severity = severity_from_scores(np.asarray(scores, dtype=float), quantile_points) anomalous_positions = np.flatnonzero(severity >= (float(threshold) - _SEVERITY_GATE_EPSILON)) contributions: list[dict[str, float | None] | None] = [None] * num_rows if anomalous_positions.size: subset = feature_matrix.iloc[anomalous_positions] - shap_values, valid_indices = compute_shap_values(model_local, subset, engineered_feature_cols) + attribution, valid_indices = compute_row_attributions(model_local, subset, engineered_feature_cols) subset_contributions = format_shap_contributions( - shap_values, valid_indices, len(subset), engineered_feature_cols + attribution, valid_indices, len(subset), engineered_feature_cols ) for position, contribution in zip(anomalous_positions.tolist(), subset_contributions): contributions[position] = contribution diff --git a/tests/unit/test_anomaly_shap_gating.py b/tests/unit/test_anomaly_shap_gating.py index 10ffd9cfb..39bbd5ed4 100644 --- a/tests/unit/test_anomaly_shap_gating.py +++ b/tests/unit/test_anomaly_shap_gating.py @@ -4,7 +4,9 @@ import pandas as pd import pytest from sklearn.ensemble import IsolationForest +from sklearn.pipeline import Pipeline +from databricks.labs.dqx.anomaly.timeseries_detector import MahalanobisDetector from databricks.labs.dqx.anomaly.explainability import ( compute_gated_shap_contributions, severity_from_scores, @@ -66,3 +68,89 @@ def test_gated_contributions_epsilon_includes_threshold_boundary(fitted_model_an ) assert contributions[:3] == [None, None, None] assert isinstance(contributions[3], dict) + + +# --------------------------------------------------------------------------- +# Dispatch: an estimator that supplies its own attribution instead of SHAP +# --------------------------------------------------------------------------- + + +@pytest.fixture(name="fitted_mahalanobis_and_features") +def fitted_mahalanobis_and_features_fixture(): + """The Mahalanobis detector wrapped exactly as DQX wraps it: a single-step pipeline.""" + rng = np.random.default_rng(42) + train = pd.DataFrame({"amount": rng.normal(100, 5, 400), "quantity": rng.normal(2, 0.5, 400)}) + model = Pipeline([("model", MahalanobisDetector())]).fit(train) + features = pd.DataFrame({"amount": [100.0, 101.0, 99.0, 9999.0], "quantity": [2.0, 2.1, 1.9, 1.0]}) + return model, features + + +def test_gating_behaves_identically_for_a_non_shap_estimator(fitted_mahalanobis_and_features): + """The gate is estimator-agnostic: rows below the threshold get None either way. + + This is the assertion the AI-explanation feature depends on. ``_resolve_ai_explanation_flag`` + silently disables explanations when contributions are unavailable, so a null map here would turn + off explanations for every model using this algorithm without raising anything. + """ + model, features = fitted_mahalanobis_and_features + scores = np.array([1.0, 1.2, 1.1, 10.0]) + + contributions = compute_gated_shap_contributions( + model, features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=85.0 + ) + + assert contributions[:3] == [None, None, None] + assert isinstance(contributions[3], dict) + + +def test_non_shap_contributions_are_map_compatible_with_the_shap_path(fitted_mahalanobis_and_features): + """Same keys, same 0-100 scaling, same total. Everything downstream -- redaction, human labels, + the LLM prompt, the ``_dq_info`` schema -- reads this map, so it has to be indistinguishable in + shape from the SHAP path's.""" + model, features = fitted_mahalanobis_and_features + scores = np.array([1.0, 1.2, 1.1, 10.0]) + + contributions = compute_gated_shap_contributions( + model, features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=85.0 + ) + anomalous = contributions[3] + + assert set(anomalous) == {"amount", "quantity"} + assert sum(v for v in anomalous.values() if v is not None) == pytest.approx(100.0, abs=0.5) + + +def test_non_shap_contributions_are_never_negative(fitted_mahalanobis_and_features): + """The reason the leave-one-out decomposition was chosen over the exactly-additive signed one: + a negative term would be rendered by ``abs()`` downstream as a positive driver of the anomaly.""" + model, features = fitted_mahalanobis_and_features + + contributions = compute_gated_shap_contributions( + model, features, ["amount", "quantity"], np.array([9.0, 9.0, 9.0, 10.0]), QUANTILE_POINTS, threshold=10.0 + ) + + for row in contributions: + assert row is not None + assert all(value >= 0.0 for value in row.values() if value is not None) + + +def test_the_deviating_feature_dominates_the_attribution(fitted_mahalanobis_and_features): + """A useful explanation has to name the right feature: `amount` is the column pushed to 9999.""" + model, features = fitted_mahalanobis_and_features + + contributions = compute_gated_shap_contributions( + model, features, ["amount", "quantity"], np.array([1.0, 1.2, 1.1, 10.0]), QUANTILE_POINTS, threshold=85.0 + ) + + assert contributions[3]["amount"] > contributions[3]["quantity"] + + +def test_rows_with_nulls_get_an_all_none_map(fitted_mahalanobis_and_features): + """NaN handling is shared with the SHAP path, and must survive the new branch.""" + model, _ = fitted_mahalanobis_and_features + features = pd.DataFrame({"amount": [9999.0, np.nan], "quantity": [1.0, 2.0]}) + + contributions = compute_gated_shap_contributions( + model, features, ["amount", "quantity"], np.array([10.0, 10.0]), QUANTILE_POINTS, threshold=10.0 + ) + + assert all(value is None for value in contributions[1].values()) From fe946a151c6bfafc54c0c785222f8df384f5c1a4 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 10:29:27 +0100 Subject: [PATCH 038/107] Select the detector with a `profile`, defaulted so nothing changes Wires the Mahalanobis detector to the public API. `AnomalyEngine.train(..., profile=...)` takes one of three words, and the default reproduces today's behaviour exactly: auto (default) the tabular detector, plus permission for the profiler to *advise* switching when the data looks temporal. That advisory is what makes "auto" honest -- it chooses the default and says when the default looks wrong, rather than choosing the algorithm. tabular the same detector, chosen deliberately; suppresses that advice. timeseries the correlation-aware detector. Needs no timestamp column: it models cross-metric correlation, not time. Why the profile is defaulted rather than required, even though a required argument would guarantee a deliberate choice: making it required is a breaking change to every existing caller, and this PR has already spent the one breaking change that Experimental status permitted. Additive-only is the commitment from here. Why the algorithm is never chosen automatically: on the ten tabular benchmarks neither dimensionality nor base rate separates the regimes (thyroid at 6 features favours IsolationForest, covertype at 10 favours a z-score; mnist at 100 favours the z-score, satellite at 36 favours the forest), so picking correctly needs labels DQX does not have. And a silent estimator change would move every score users have calibrated thresholds against. Deliberately no new config dataclass. `contamination` is read from `algorithm_config`, which is where `apply_expected_anomaly_rate_if_default_contamination` already puts *expected_anomaly_rate* -- one source of truth for "how many anomalies do we expect", no Union type for `ConfigSerializer` to disambiguate, and no parallel validation branch. The IsolationForest-specific fields beside it are simply not read. `resolve_training_profile` is a pure function, and two of its properties are load-bearing: - for the tabular profiles it returns the caller's params object *by identity*, so choosing a profile explicitly cannot perturb an existing configuration; - `strategy_override` makes an injected strategy win over the profile. That precedence lives in the resolver rather than inside the service specifically so it can be asserted without a Spark session and without a test reaching past a private boundary. Scoring gains no second strategy class. A class whose body was also `return score_global_model(...)` would read as design and test as duplication, so the existing one is renamed `GlobalModelScoringStrategy` and its `supports` widened over a prefix tuple. Both existing algorithm strings still resolve, which the inertness battery asserts. Two things lint caught that improved the design rather than being suppressed: - an import-outside-toplevel, added to dodge a circular import. The real fix was to move the Spark-side orchestration into training_strategies.py, which already imports `prepare_training_features` at top level. Consequence worth having: `timeseries_detector` now imports no pyspark at all, and it is the module cloudpickled to executors, so every import it carries is one they must satisfy. - two mypy errors in the bake-off ported earlier: `FEATURISERS` and `ESTIMATORS` mixed a lambda with named functions, widening to `object` so every call through them was untyped. Missed first time because only black and ruff were run on that file, not the full gate. Also fixed while wiring: `profile` was first added to `AnomalyTrainingContext` *before* its non-defaulted fields, which stopped five test modules from even importing. Appended instead -- the same rule the inertness battery pins for `AnomalyParams`. Unit suite 2552 passed, mypy clean over 348 files, pylint 10.00/10. Co-authored-by: Isaac --- .../anomaly_conditioning/smd_bakeoff.py | 7 +- .../labs/dqx/anomaly/anomaly_engine.py | 11 ++ .../labs/dqx/anomaly/scoring_strategies.py | 20 ++- .../labs/dqx/anomaly/timeseries_detector.py | 53 ++++++++ .../labs/dqx/anomaly/training_service.py | 31 ++++- .../labs/dqx/anomaly/training_strategies.py | 120 ++++++++++++++++++ src/databricks/labs/dqx/anomaly/types.py | 5 + ...test_anomaly_isolation_forest_inertness.py | 63 +++++++++ 8 files changed, 299 insertions(+), 11 deletions(-) diff --git a/benchmarks/anomaly_conditioning/smd_bakeoff.py b/benchmarks/anomaly_conditioning/smd_bakeoff.py index 1bbf25ae6..7cd7e028f 100644 --- a/benchmarks/anomaly_conditioning/smd_bakeoff.py +++ b/benchmarks/anomaly_conditioning/smd_bakeoff.py @@ -37,6 +37,7 @@ import argparse import json import time +from collections.abc import Callable import numpy as np @@ -100,7 +101,9 @@ def window_stats(values: np.ndarray, window: int = WINDOW) -> np.ndarray: return np.hstack([values, mean, std, lo, hi]) -FEATURISERS = { +# Annotated so the values keep a callable type: an unannotated dict mixing a lambda with a +# named function widens to object, and every call through it becomes untyped. +FEATURISERS: dict[str, Callable[[np.ndarray], np.ndarray]] = { "raw": lambda v: v, "win_stats": window_stats, } @@ -204,7 +207,7 @@ def score_mahalanobis_ridge(train: np.ndarray, test: np.ndarray, seed: int) -> n return _mahalanobis_sq(train, test, shrinkage=0.0, ridge=1e-6) -ESTIMATORS = { +ESTIMATORS: dict[str, Callable[[np.ndarray, np.ndarray, int], np.ndarray]] = { "iforest": score_iforest, "pca_recon": score_pca_recon, "mlp_ae": score_mlp_autoencoder, diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index 4f99635be..12f790bcd 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -64,6 +64,7 @@ def train( exclude_columns: list[str] | None = None, expected_anomaly_rate: float = 0.02, baseline_by: list[str] | None = None, + profile: str | None = None, ) -> str: """ Train a row anomaly detection model with intelligent auto-discovery. @@ -83,6 +84,15 @@ def train( registry_table: Registry table (REQUIRED). Must be fully qualified Unity Catalog table as 'catalog.schema.table'. columns: Columns to use for row anomaly detection (auto-discovered if omitted). + profile: What kind of data this is, which selects the detector. Defaults to ``"auto"``, + which is the tabular detector -- exactly the behaviour before this option existed -- + and additionally lets the profiler *advise* switching when the data looks temporal. + ``"tabular"`` is the same detector chosen deliberately, which suppresses that advice. + ``"timeseries"`` selects a correlation-aware detector suited to multivariate metrics, + where anomalies are broken correlations rather than extreme single values; measured on + the SMD benchmark it surfaces 82% of incidents inside a 1%-of-rows alert budget against + 36% for the tabular detector. It needs no timestamp column, and trains a single model + rather than an ensemble because it is deterministic. baseline_by: Columns identifying the group a row belongs to, so a metric is judged against its own group's baseline rather than against the whole table. Each numeric metric gains its deviation from that baseline as an extra feature on @@ -167,6 +177,7 @@ def train( exclude_columns=exclude_columns, expected_anomaly_rate=expected_anomaly_rate, baseline_by=baseline_by, + profile=profile, ) log_telemetry(self.ws, "anomaly_num_features", str(len(context.columns))) diff --git a/src/databricks/labs/dqx/anomaly/scoring_strategies.py b/src/databricks/labs/dqx/anomaly/scoring_strategies.py index f90714628..2d719d3f5 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_strategies.py +++ b/src/databricks/labs/dqx/anomaly/scoring_strategies.py @@ -27,17 +27,29 @@ def score_global(self, df: DataFrame, record: AnomalyModelRecord, config: Scorin """Score the model.""" -class IsolationForestScoringStrategy(AnomalyScoringStrategy): - """IsolationForest scoring strategy (default).""" +# Algorithms scored by one pooled sklearn-compatible model. All of them reach the model through +# ``score_samples`` and are therefore indistinguishable to the scoring path — the estimator differs, +# the scoring mechanics do not. Existing registry rows carry "IsolationForest" or +# "IsolationForest_Ensemble_N", so prefix matching keeps them resolving unchanged. +_GLOBAL_ALGORITHM_PREFIXES = ("IsolationForest", "Mahalanobis") + + +class GlobalModelScoringStrategy(AnomalyScoringStrategy): + """Scoring for any algorithm represented by a single pooled model. + + One class rather than one per algorithm on purpose: a second class whose body was also + ``return score_global_model(...)`` would read as design while testing as duplication. The strategy + interface still earns its place for a future path that genuinely scores differently. + """ def supports(self, algorithm: str) -> bool: - return algorithm.startswith("IsolationForest") + return algorithm.startswith(_GLOBAL_ALGORITHM_PREFIXES) def score_global(self, df: DataFrame, record: AnomalyModelRecord, config: ScoringConfig) -> DataFrame: return score_global_model(df, record, config) -_SCORING_STRATEGIES: list[AnomalyScoringStrategy] = [IsolationForestScoringStrategy()] +_SCORING_STRATEGIES: list[AnomalyScoringStrategy] = [GlobalModelScoringStrategy()] def resolve_scoring_strategy(algorithm: str) -> AnomalyScoringStrategy: diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py index 12aa6ad55..2b9858b6d 100644 --- a/src/databricks/labs/dqx/anomaly/timeseries_detector.py +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -31,10 +31,17 @@ """ import logging +import sys +from typing import Any +import cloudpickle import numpy as np +import pandas as pd from sklearn.base import BaseEstimator, OutlierMixin from sklearn.covariance import LedoitWolf +from sklearn.pipeline import Pipeline + +from databricks.labs.dqx.config import AnomalyParams, IsolationForestConfig logger = logging.getLogger(__name__) @@ -47,6 +54,9 @@ CONSTANT_FEATURE_TOLERANCE = 1e-12 # Ridge added to the covariance diagonal, as a fraction of the average variance so it is scale-free. DEFAULT_RIDGE = 1e-6 +# Fallback expected anomaly rate, matching AnomalyEngine.train's expected_anomaly_rate default. Only +# used if contamination somehow reached this point unset; production fills it in before training. +DEFAULT_CONTAMINATION = 0.02 class MahalanobisDetector(BaseEstimator, OutlierMixin): @@ -129,7 +139,9 @@ def _covariance(self, standardised: np.ndarray, n_samples: int, n_active: int) - f"(<{SMALL_SAMPLE_ROWS_PER_FEATURE} per feature): using Ledoit-Wolf shrinkage, " "which is more stable but less sharp. More training data would detect better." ) + self.used_shrinkage_ = True return np.atleast_2d(LedoitWolf(assume_centered=False).fit(standardised).covariance_) + self.used_shrinkage_ = False centred = standardised - standardised.mean(axis=0) return np.atleast_2d(np.cov(centred, rowvar=False)) @@ -201,3 +213,44 @@ def feature_contributions(self, X: np.ndarray) -> np.ndarray: contributions = np.zeros((active_contributions.shape[0], self.n_features_in_), dtype=float) contributions[:, self.active_] = active_contributions return contributions + + +# cloudpickle serialises classes **by reference** by default, which would make any pickled payload +# containing this estimator require ``databricks.labs.dqx`` to be importable on every executor. +# Contributions-disabled scoring deliberately needs nothing but sklearn and numpy on the workers today +# — the ai_query explainer's docstring records that as a design goal — so this module is registered by +# value and the class travels inside the pickle instead. +cloudpickle.register_pickle_by_value(sys.modules[__name__]) + + +def fit_mahalanobis_model(train_pandas: pd.DataFrame, params: AnomalyParams) -> tuple[Pipeline, dict[str, Any]]: + """Fit the detector on pre-engineered pandas features, wrapped as DQX wraps every model. + + Deliberately a sibling of ``core.fit_sklearn_model`` rather than a branch inside it: that keeps the + IsolationForest fit function literally untouched, so "the tabular path is unchanged" is a fact a + reviewer reads off the diff rather than a claim to verify. + + The pipeline is single-step, matching the IsolationForest one, because standardisation lives inside + the estimator — so ``named_steps["model"]`` resolves identically for both algorithms. + + *contamination* is read from ``algorithm_config``, which is where + ``training_service.apply_expected_anomaly_rate_if_default_contamination`` puts + *expected_anomaly_rate*. Reusing that field rather than adding a parallel one keeps one source of + truth for "how many anomalies do we expect"; the genuinely IsolationForest-specific fields beside it + (tree count, subsampling) are simply not read here. + """ + algo_cfg = params.algorithm_config or IsolationForestConfig() + contamination = algo_cfg.contamination if algo_cfg.contamination else DEFAULT_CONTAMINATION + + detector = MahalanobisDetector(contamination=contamination, ridge=DEFAULT_RIDGE) + pipeline = Pipeline([("model", detector)]) + pipeline.fit(train_pandas) + + hyperparams: dict[str, Any] = { + "contamination": contamination, + "ridge": DEFAULT_RIDGE, + "covariance": "ledoit_wolf" if detector.used_shrinkage_ else "empirical", + "informative_features": int(detector.active_.sum()), + "feature_scaling": "standardised_internally", + } + return pipeline, hyperparams diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index c1e811da1..5de4a5004 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -29,7 +29,11 @@ TrainingMetadata, ) from databricks.labs.dqx.anomaly.profiler import auto_discover_columns -from databricks.labs.dqx.anomaly.training_strategies import AnomalyTrainingStrategy, IsolationForestTrainingStrategy +from databricks.labs.dqx.anomaly.training_strategies import ( + PROFILE_AUTO, + AnomalyTrainingStrategy, + resolve_training_profile, +) from databricks.labs.dqx.anomaly.transformers import ( SparkFeatureMetadata, apply_feature_engineering_from_metadata, @@ -59,9 +63,17 @@ class AnomalyTrainingService: """ def __init__(self, spark: SparkSession, strategy: AnomalyTrainingStrategy | None = None) -> None: - """Initialize the training service.""" + """Initialize the training service. + + Args: + spark: Active Spark session. + strategy: Explicit training strategy. When given it overrides the profile, which is what + keeps an injected test double authoritative; when omitted the strategy is resolved from + the context's *profile* at training time. Kept as None rather than defaulted here so + those two cases stay distinguishable. + """ self._spark = spark - self._strategy = strategy or IsolationForestTrainingStrategy() + self._strategy = strategy @staticmethod def _perform_auto_discovery(df_filtered: DataFrame) -> tuple[list[str], list[str] | None]: @@ -196,6 +208,7 @@ def build_context( exclude_columns: list[str] | None, expected_anomaly_rate: float, baseline_by: list[str] | None = None, + profile: str | None = None, ) -> AnomalyTrainingContext: """Build training context with all validated inputs.""" validate_spark_version(self._spark) @@ -258,6 +271,7 @@ def build_context( exclude_columns=exclude_columns, auto_discovery_used=auto_discovery_used, baseline_by=baseline_by, + profile=profile, ) def train(self, context: AnomalyTrainingContext) -> str: @@ -317,11 +331,18 @@ def _train_global(self, context: AnomalyTrainingContext) -> str: train_df, val_df = train_validation_split(sampled_df, context.params) - result = self._strategy.train( + # An explicitly injected strategy wins over the profile, so a test double is never silently + # bypassed. Otherwise the profile decides, and may tighten parameters (the correlation-aware + # detector collapses the ensemble to one model); for the tabular profiles the returned params + # are the very same object, so nothing is perturbed. + strategy, params = resolve_training_profile(context.profile, context.params, self._strategy) + logger.info(f"profile={context.profile or PROFILE_AUTO} -> algorithm strategy '{strategy.name}'") + + result = strategy.train( train_df, val_df, context.columns, - context.params, + params, context.model_name, allow_ensemble=True, ) diff --git a/src/databricks/labs/dqx/anomaly/training_strategies.py b/src/databricks/labs/dqx/anomaly/training_strategies.py index 447fdc30a..9d9571811 100644 --- a/src/databricks/labs/dqx/anomaly/training_strategies.py +++ b/src/databricks/labs/dqx/anomaly/training_strategies.py @@ -9,6 +9,8 @@ - Potential for alternative backends """ +import dataclasses +import logging from abc import ABC, abstractmethod from pyspark.sql import DataFrame @@ -18,11 +20,27 @@ compute_validation_metrics, fit_isolation_forest, prepare_engineered_pandas, + prepare_training_features, ) from databricks.labs.dqx.anomaly.ensemble_training import train_ensemble from databricks.labs.dqx.anomaly.mlflow_registry import ModelRegistryBase, get_default_registry +from databricks.labs.dqx.anomaly.timeseries_detector import fit_mahalanobis_model from databricks.labs.dqx.anomaly.types import TrainingResult from databricks.labs.dqx.config import AnomalyParams +from databricks.labs.dqx.errors import InvalidParameterError + +logger = logging.getLogger(__name__) + +# Persisted in ModelIdentity.algorithm and matched by the scoring resolver, so it is a stored contract: +# changing it would orphan every model already trained with this algorithm. +MAHALANOBIS_ALGORITHM = "Mahalanobis" + +# The public profile vocabulary. "auto" is the default so that adding this option changes nothing for +# any existing caller. +PROFILE_AUTO = "auto" +PROFILE_TABULAR = "tabular" +PROFILE_TIMESERIES = "timeseries" +SUPPORTED_PROFILES = (PROFILE_AUTO, PROFILE_TABULAR, PROFILE_TIMESERIES) class AnomalyTrainingStrategy(ABC): @@ -122,3 +140,105 @@ def train( ensemble_size=ensemble_size, algorithm=algorithm, ) + + +class MahalanobisTrainingStrategy(AnomalyTrainingStrategy): + """Correlation-aware training strategy, for multivariate metrics such as time series. + + Same feature engineering, same registry, same metrics as the IsolationForest strategy — only the + estimator differs. See ``timeseries_detector`` for why: IsolationForest splits one feature at a + time, so anomalies that are broken *correlations* rather than extreme single values are close to + invisible to it. Measured on SMD, incident coverage inside a 1%-of-rows alert budget is 0.359 for + IsolationForest and 0.821 here. + """ + + name = "mahalanobis" + + def train( + self, + train_df: DataFrame, + val_df: DataFrame, + columns: list[str], + params: AnomalyParams, + model_name: str, + *, + allow_ensemble: bool, + ) -> TrainingResult: + """Train a single correlation-aware model. + + *allow_ensemble* is accepted and ignored. The estimator is deterministic, so the ensemble -- + which exists to average away the randomness of differently-seeded forests and to report a + confidence standard deviation from their disagreement -- would train N identical models, pay N + times the cost, and report a spread of exactly zero. Declining it is the honest behaviour, and + it is logged rather than silently dropped. + """ + if allow_ensemble and params.ensemble_size and params.ensemble_size > 1: + logger.info( + f"Ignoring ensemble_size={params.ensemble_size} for the {self.name} algorithm: it is " + "deterministic, so every ensemble member would be an identical model." + ) + + # Feature engineering is deliberately the shared implementation: this algorithm changes how + # rows are scored, not how columns become features, so it inherits one-hot encoding, frequency + # encoding, datetime cyclicals, null indicators and the group-relative baseline features as-is. + train_pandas, feature_metadata = prepare_training_features(train_df, columns, params) + model, hyperparams = fit_mahalanobis_model(train_pandas, params) + validation_metrics = compute_validation_metrics(model, val_df, columns, feature_metadata) + score_quantiles = compute_score_quantiles(model, train_df, columns, feature_metadata) + + self._registry.ensure_registry_configured() + train_pandas = prepare_engineered_pandas(train_df, feature_metadata) + model_uri, run_id = self._registry.register_model_with_signature_inference( + model, model_name, train_pandas, hyperparams, validation_metrics + ) + + return TrainingResult( + model_uri=model_uri, + run_id=run_id, + hyperparams=hyperparams, + validation_metrics=validation_metrics, + score_quantiles=score_quantiles, + feature_metadata=feature_metadata, + ensemble_size=1, + algorithm=MAHALANOBIS_ALGORITHM, + ) + + +def resolve_training_profile( + profile: str | None, + params: AnomalyParams, + strategy_override: AnomalyTrainingStrategy | None = None, +) -> tuple[AnomalyTrainingStrategy, AnomalyParams]: + """Map a *profile* to the strategy that implements it, and any parameter defaults it implies. + + *strategy_override*, when given, replaces the resolved strategy — this is how an explicitly + injected strategy (including a test double) stays authoritative over the profile. Parameters still + follow the profile, because the parameter defaults are declared by the profile rather than by + whichever object ends up doing the training. The precedence rule lives here, in a pure function, + rather than inside the training service, so it can be asserted without a Spark session. + + Pure: it returns parameters rather than mutating the caller's. For the tabular profiles it returns + the **same object**, so choosing a profile explicitly cannot perturb an existing configuration. + + The profiles describe the data a user has, not the algorithm DQX picks for it: + + * ``auto`` (the default) — the tabular detector, exactly as before this option existed. Distinct + from ``tabular`` only in that the profiler may *advise* switching when the data looks temporal; + the choice itself is never made automatically, because it cannot be verified without labels and a + silent estimator change would move every score a user has calibrated thresholds against. + * ``tabular`` — the same detector, chosen deliberately. Suppresses that advice. + * ``timeseries`` — the correlation-aware detector, with the ensemble collapsed to a single model. + Needs no time column: it models cross-metric correlation, not time. + """ + requested = (profile or PROFILE_AUTO).strip().lower() + + if requested in (PROFILE_AUTO, PROFILE_TABULAR): + strategy: AnomalyTrainingStrategy = IsolationForestTrainingStrategy() + resolved_params = params + elif requested == PROFILE_TIMESERIES: + strategy = MahalanobisTrainingStrategy() + resolved_params = dataclasses.replace(params, ensemble_size=1) + else: + raise InvalidParameterError(f"Unknown profile {profile!r}. Choose one of: {', '.join(SUPPORTED_PROFILES)}.") + + return (strategy_override or strategy), resolved_params diff --git a/src/databricks/labs/dqx/anomaly/types.py b/src/databricks/labs/dqx/anomaly/types.py index a87f6c794..2db8c6d1d 100644 --- a/src/databricks/labs/dqx/anomaly/types.py +++ b/src/databricks/labs/dqx/anomaly/types.py @@ -87,6 +87,11 @@ class AnomalyTrainingContext: exclude_columns: list[str] | None auto_discovery_used: bool baseline_by: list[str] | None = None + # Which kind of data the user says this is, which selects the detector. None means "auto", i.e. + # today's tabular behaviour plus permission for the profiler to advise otherwise. Appended last: + # a defaulted field cannot precede a non-defaulted one, and appending also keeps positional + # construction stable for anything building this directly. + profile: str | None = None @dataclass(frozen=True) diff --git a/tests/unit/test_anomaly_isolation_forest_inertness.py b/tests/unit/test_anomaly_isolation_forest_inertness.py index bc2fea42e..cd184d350 100644 --- a/tests/unit/test_anomaly_isolation_forest_inertness.py +++ b/tests/unit/test_anomaly_isolation_forest_inertness.py @@ -17,7 +17,13 @@ from databricks.labs.dqx.anomaly.core import fit_sklearn_model from databricks.labs.dqx.anomaly.scoring_strategies import resolve_scoring_strategy +from databricks.labs.dqx.anomaly.training_strategies import ( + IsolationForestTrainingStrategy, + MahalanobisTrainingStrategy, + resolve_training_profile, +) from databricks.labs.dqx.config import AnomalyParams, IsolationForestConfig +from databricks.labs.dqx.errors import InvalidParameterError # Fixed seed, fixed shape: the reference scores below were generated from exactly this frame. _TRAIN_SEED = 1234 @@ -128,3 +134,60 @@ def test_existing_algorithm_strings_still_resolve(algorithm: str): strategy = resolve_scoring_strategy(algorithm) assert strategy.supports(algorithm) + + +@pytest.mark.parametrize("profile", [None, "auto", "tabular", "AUTO", " Tabular "]) +def test_tabular_profiles_select_isolation_forest_and_leave_params_untouched(profile: str | None): + """The default must resolve to today's behaviour, and must not perturb the caller's parameters. + + Identity, not equality: returning a copy would be harmless here but would mean the resolver is + rewriting configuration on a path that is supposed to be a no-op, which is the kind of thing that + later grows a surprise. Case and whitespace are tolerated because this is user-typed. + """ + params = _reference_params() + + strategy, resolved = resolve_training_profile(profile, params) + + assert isinstance(strategy, IsolationForestTrainingStrategy) + assert resolved is params + + +def test_timeseries_profile_collapses_the_ensemble_without_mutating_the_caller(): + """The detector is deterministic, so an ensemble would be N identical models. The caller's params + must survive unchanged even so -- the resolver returns a new object rather than editing theirs.""" + params = _reference_params() + assert params.ensemble_size == 3 + + strategy, resolved = resolve_training_profile("timeseries", params) + + assert strategy.name == "mahalanobis" + assert resolved.ensemble_size == 1 + assert params.ensemble_size == 3, "the caller's params were mutated" + + +def test_an_unknown_profile_is_rejected_by_name(): + with pytest.raises(InvalidParameterError, match="Unknown profile"): + resolve_training_profile("timeseries-ish", _reference_params()) + + +@pytest.mark.parametrize("profile", [None, "auto", "tabular", "timeseries"]) +def test_an_injected_strategy_wins_over_every_profile(profile: str | None): + """``AnomalyTrainingService(spark, strategy=...)`` is how a caller substitutes a strategy, and how + existing tests substitute a double. If profile resolution overrode it, those tests would keep + passing while exercising entirely different code. + + The precedence rule lives in this pure function rather than inside the service, so it is assertable + without a Spark session and without reaching past any boundary. + """ + injected = MahalanobisTrainingStrategy() + + strategy, _ = resolve_training_profile(profile, _reference_params(), injected) + + assert strategy is injected + + +def test_parameter_defaults_still_follow_the_profile_when_a_strategy_is_injected(): + """An override replaces the strategy, not the profile's declared parameter defaults.""" + _, resolved = resolve_training_profile("timeseries", _reference_params(), IsolationForestTrainingStrategy()) + + assert resolved.ensemble_size == 1 From 88218e0b968db37209511689a8b4a4927dcec369 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 13:57:32 +0100 Subject: [PATCH 039/107] Decline automatic profile detection, and drop the "auto" name that implied it Two halves of one decision: DQX will not choose the algorithm for a user, and the public vocabulary should not contain a word suggesting it might. ## The advisory, measured and declined The plan was for DQX to notice when a table looks like a time series and suggest `profile="timeseries"`. Pre-registered gate: the signal must separate temporal from tabular data before any warning ships. `benchmarks/anomaly_conditioning/profile_advisory_gate.py` records the attempt so it is not re-argued from scratch. Not wired into DQX. Timestamp presence was rejected before measuring: nearly every Delta table has a created_at, so "a timestamp exists" fires on almost everything. The candidate worth testing was mean absolute lag-1 autocorrelation of the numeric columns -- genuine time series have serially correlated metrics, i.i.d. tabular rows should not. TEMPORAL SMD, 28 entities in time order mean 0.660 min 0.426 max 0.791 TABULAR satellite 0.823 | cardio 0.728 | covertype 0.573 | spambase 0.104 | mnist 0.086 thyroid 0.068 | fraud 0.065 | mammography 0.056 | shuttle 0.005 | campaign 0.004 Three of ten tabular datasets score above the weakest SMD entity and two score above SMD's mean, so no threshold admits every time series while rejecting every tabular table. The reason is the part worth remembering: the statistic is confounded by **any ordering correlated with the values**, not only a temporal one. These datasets are stored sorted -- satellite at 0.823 is almost certainly ordered by class -- and sorted storage produces serial correlation with no time series underneath. That transfers directly to real data: batch loads, sorted ETL output and backfills all leave created_at correlated with the values beside it. And it fails in the expensive direction, advising an algorithm change on data that does not need one. ## The consequence for the API `profile` previously defaulted to "auto", justified as "the tabular detector *plus* the advisory". With the advisory declined, "auto" and "tabular" became behaviourally identical, and "auto" was left promising a selection that never happens. So the vocabulary is now two honest values, `tabular` and `timeseries`, `tabular` is the default and the meaning of an unset profile, and "auto" is rejected with a message naming the valid choices. Caught by a reader asking the obvious question -- if there is no auto-detection, must the user be explicit? -- which the parameter name was quietly answering wrongly. Nothing is released, so removing the value costs nothing. Behaviour is unchanged either way: an unset profile trains exactly what it trained before this option existed. What ships instead is the honest minimum: the resolved profile is logged at INFO on every training run so the default is visible rather than implicit, and the documentation carries the choice and the measured difference (36% against 82% incident coverage) so a user who knows their data can make it. Choosing for them would have to be verified, and verifying it needs labels DQX does not have. Better signals may exist -- per-entity cadence regularity, or autocorrelation against a permutation baseline that controls for the ordering itself. Neither is attempted: the point is that the cheap version was tried, measured and declined. Unit suite 2550 passed, mypy clean over 349 files, pylint 10.00/10. Co-authored-by: Isaac --- .../profile_advisory_gate.py | 119 ++++++++++++++++++ .../labs/dqx/anomaly/anomaly_engine.py | 10 +- .../labs/dqx/anomaly/training_service.py | 4 +- .../labs/dqx/anomaly/training_strategies.py | 31 +++-- src/databricks/labs/dqx/anomaly/types.py | 4 +- ...test_anomaly_isolation_forest_inertness.py | 6 +- 6 files changed, 151 insertions(+), 23 deletions(-) create mode 100644 benchmarks/anomaly_conditioning/profile_advisory_gate.py diff --git a/benchmarks/anomaly_conditioning/profile_advisory_gate.py b/benchmarks/anomaly_conditioning/profile_advisory_gate.py new file mode 100644 index 000000000..e07a49101 --- /dev/null +++ b/benchmarks/anomaly_conditioning/profile_advisory_gate.py @@ -0,0 +1,119 @@ +"""Should DQX warn a user that their data looks temporal? Measured answer: not from this signal. + +`profile="timeseries"` selects a correlation-aware detector that measured far better on multivariate +metrics (incident coverage 0.82 against 0.36 on SMD). The obvious next step is for DQX to notice when a +table looks temporal and say so. This module exists so that proposal stays measured rather than +re-argued each time. **It is not wired into DQX**, and the numbers below are why. + +## What was rejected first, and why it was never viable + +Detecting a timestamp column. Nearly every Delta table has `created_at`, `updated_at` or `ingested_at`, +so "a timestamp exists, therefore this is a time series" fires on almost everything. Presence of a +column is not evidence of intent. + +## What was actually measured + +Mean absolute lag-1 autocorrelation of the numeric columns, under the row ordering given. The +hypothesis: genuine time series have serially correlated metrics, i.i.d. tabular rows do not, so the +statistic should separate the two even though timestamp presence cannot. + + TEMPORAL SMD, 28 entities in time order mean 0.660 min 0.426 max 0.791 + + TABULAR satellite 0.823 + cardio 0.728 + covertype 0.573 + spambase 0.104 + mnist 0.086 + thyroid 0.068 + fraud 0.065 + mammography 0.056 + shuttle 0.005 + campaign 0.004 + +## Result: it does not separate + +Three of ten tabular datasets score above the weakest SMD entity, and two score above SMD's *mean*. +There is no threshold that admits every time series and rejects every tabular table, so a warning built +on this would fire on ordinary tabular data. + +## Why it fails, which is the part worth remembering + +The statistic is confounded by **any ordering correlated with the values**, not just a temporal one. +These datasets are stored sorted -- `satellite` at 0.823 is almost certainly ordered by class -- and +sorted storage produces serial correlation without any time series underneath. + +That is not an artefact of the benchmark. It is the common case in a warehouse: batch loads, sorted ETL +output and backfills all leave `created_at` correlated with the values beside it. So the failure mode +transfers directly to the data a user would actually run this on, and it fails in the expensive +direction -- advising a change of algorithm on data that does not need one. + +## What DQX does instead + +Nothing automatic, and nothing silent. The resolved profile is logged at INFO on every training run, so +`auto` is visible rather than invisible, and the documentation states the choice and the measured +difference so a user who knows their data can make it. Choosing the algorithm for them would need to be +verified, and verifying it needs labels DQX does not have. + +A better signal may exist -- regular cadence per entity, or autocorrelation compared against a +permutation baseline that controls for the ordering itself. Neither is attempted here: the point of this +module is that the cheap version was tried, measured, and declined. + +Run: uv run python benchmarks/anomaly_conditioning/profile_advisory_gate.py +""" + +import numpy as np + +from datasets.real import SMD_ENTITIES, load_smd_split +from datasets.tabular import DATASETS, load + + +def mean_abs_lag1_autocorr(values: np.ndarray) -> float: + """Mean absolute lag-1 autocorrelation across columns, in the row order given. + + Constant columns are skipped: they have no correlation to measure, and including them as zeros + would dilute the statistic by however many one-hot or indicator columns a frame happens to carry. + """ + scores = [] + for column in range(values.shape[1]): + series = values[:, column] + if series.std() <= 1e-12 or len(series) < 3: + continue + current, previous = series[1:], series[:-1] + if current.std() <= 1e-12 or previous.std() <= 1e-12: + continue + scores.append(abs(float(np.corrcoef(current, previous)[0, 1]))) + return float(np.mean(scores)) if scores else 0.0 + + +def main() -> None: + print("Temporal reference: SMD, per entity, in file (time) order") + train, _, _ = load_smd_split() + temporal = [mean_abs_lag1_autocorr(train[entity]) for entity in SMD_ENTITIES] + print( + f" {len(SMD_ENTITIES)} entities | mean {np.mean(temporal):.3f} " + f"| min {min(temporal):.3f} | max {max(temporal):.3f}\n" + ) + + print("Tabular comparison: classical benchmarks, in stored order") + tabular: dict[str, float] = {} + for name in DATASETS: + values, _ = load(name) + tabular[name] = mean_abs_lag1_autocorr(values) + for name, score in sorted(tabular.items(), key=lambda item: -item[1]): + print(f" {name:14s} {score:.3f}") + + weakest_temporal = min(temporal) + strongest_tabular = max(tabular.values()) + print("\nSeparation") + print(f" weakest temporal {weakest_temporal:.3f}") + print(f" strongest tabular {strongest_tabular:.3f} ({max(tabular, key=lambda k: tabular[k])})") + if weakest_temporal > strongest_tabular: + print(f" SEPARATES: any threshold in ({strongest_tabular:.3f}, {weakest_temporal:.3f}) works") + else: + above = sorted(name for name, score in tabular.items() if score > weakest_temporal) + print(f" DOES NOT SEPARATE: {len(above)} tabular datasets score above the weakest time series") + print(f" overlapping: {', '.join(above)}") + + +if __name__ == "__main__": + main() diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index 12f790bcd..147aa823c 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -84,15 +84,15 @@ def train( registry_table: Registry table (REQUIRED). Must be fully qualified Unity Catalog table as 'catalog.schema.table'. columns: Columns to use for row anomaly detection (auto-discovered if omitted). - profile: What kind of data this is, which selects the detector. Defaults to ``"auto"``, - which is the tabular detector -- exactly the behaviour before this option existed -- - and additionally lets the profiler *advise* switching when the data looks temporal. - ``"tabular"`` is the same detector chosen deliberately, which suppresses that advice. + profile: What kind of data this is, which selects the detector. Defaults to + ``"tabular"`` -- IsolationForest, exactly the behaviour before this option existed. ``"timeseries"`` selects a correlation-aware detector suited to multivariate metrics, where anomalies are broken correlations rather than extreme single values; measured on the SMD benchmark it surfaces 82% of incidents inside a 1%-of-rows alert budget against 36% for the tabular detector. It needs no timestamp column, and trains a single model - rather than an ensemble because it is deterministic. + rather than an ensemble because it is deterministic. There is no automatic option: DQX + never changes the algorithm on your behalf, because the choice cannot be verified + without labels. The resolved profile is logged on every run. baseline_by: Columns identifying the group a row belongs to, so a metric is judged against its own group's baseline rather than against the whole table. Each numeric metric gains its deviation from that baseline as an extra feature on diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index 5de4a5004..e10cde75d 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -30,7 +30,7 @@ ) from databricks.labs.dqx.anomaly.profiler import auto_discover_columns from databricks.labs.dqx.anomaly.training_strategies import ( - PROFILE_AUTO, + DEFAULT_PROFILE, AnomalyTrainingStrategy, resolve_training_profile, ) @@ -336,7 +336,7 @@ def _train_global(self, context: AnomalyTrainingContext) -> str: # detector collapses the ensemble to one model); for the tabular profiles the returned params # are the very same object, so nothing is perturbed. strategy, params = resolve_training_profile(context.profile, context.params, self._strategy) - logger.info(f"profile={context.profile or PROFILE_AUTO} -> algorithm strategy '{strategy.name}'") + logger.info(f"profile={context.profile or DEFAULT_PROFILE} -> algorithm strategy '{strategy.name}'") result = strategy.train( train_df, diff --git a/src/databricks/labs/dqx/anomaly/training_strategies.py b/src/databricks/labs/dqx/anomaly/training_strategies.py index 9d9571811..42b670113 100644 --- a/src/databricks/labs/dqx/anomaly/training_strategies.py +++ b/src/databricks/labs/dqx/anomaly/training_strategies.py @@ -35,12 +35,19 @@ # changing it would orphan every model already trained with this algorithm. MAHALANOBIS_ALGORITHM = "Mahalanobis" -# The public profile vocabulary. "auto" is the default so that adding this option changes nothing for -# any existing caller. -PROFILE_AUTO = "auto" +# The public profile vocabulary. It describes the *data a user has*, not the algorithm DQX picks for it. +# +# There is deliberately no "auto": DQX never selects the algorithm on a user's behalf. Choosing +# correctly cannot be verified without labels, which an unsupervised tool does not have, and the one +# cheap signal for "this looks temporal" was measured and rejected -- lag-1 autocorrelation is +# confounded by any ordering correlated with the values, which sorted warehouse storage produces +# routinely (see benchmarks/anomaly_conditioning/profile_advisory_gate.py). A value named "auto" would +# therefore have promised a selection that never happens. PROFILE_TABULAR = "tabular" PROFILE_TIMESERIES = "timeseries" -SUPPORTED_PROFILES = (PROFILE_AUTO, PROFILE_TABULAR, PROFILE_TIMESERIES) +SUPPORTED_PROFILES = (PROFILE_TABULAR, PROFILE_TIMESERIES) +# Unset means the tabular detector: exactly the behaviour that predates this option. +DEFAULT_PROFILE = PROFILE_TABULAR class AnomalyTrainingStrategy(ABC): @@ -222,17 +229,19 @@ def resolve_training_profile( The profiles describe the data a user has, not the algorithm DQX picks for it: - * ``auto`` (the default) — the tabular detector, exactly as before this option existed. Distinct - from ``tabular`` only in that the profiler may *advise* switching when the data looks temporal; - the choice itself is never made automatically, because it cannot be verified without labels and a - silent estimator change would move every score a user has calibrated thresholds against. - * ``tabular`` — the same detector, chosen deliberately. Suppresses that advice. + * ``tabular`` (the default, and what an unset profile means) — IsolationForest, exactly the + behaviour that predates this option. * ``timeseries`` — the correlation-aware detector, with the ensemble collapsed to a single model. Needs no time column: it models cross-metric correlation, not time. + + There is no automatic option. DQX will not switch algorithms on a user's behalf: the choice cannot + be verified without labels, and a silent estimator change would move every score a user has + calibrated thresholds against. The resolved profile is logged on every run so the default is visible + rather than implicit. """ - requested = (profile or PROFILE_AUTO).strip().lower() + requested = (profile or DEFAULT_PROFILE).strip().lower() - if requested in (PROFILE_AUTO, PROFILE_TABULAR): + if requested == PROFILE_TABULAR: strategy: AnomalyTrainingStrategy = IsolationForestTrainingStrategy() resolved_params = params elif requested == PROFILE_TIMESERIES: diff --git a/src/databricks/labs/dqx/anomaly/types.py b/src/databricks/labs/dqx/anomaly/types.py index 2db8c6d1d..251d8eede 100644 --- a/src/databricks/labs/dqx/anomaly/types.py +++ b/src/databricks/labs/dqx/anomaly/types.py @@ -87,8 +87,8 @@ class AnomalyTrainingContext: exclude_columns: list[str] | None auto_discovery_used: bool baseline_by: list[str] | None = None - # Which kind of data the user says this is, which selects the detector. None means "auto", i.e. - # today's tabular behaviour plus permission for the profiler to advise otherwise. Appended last: + # Which kind of data the user says this is, which selects the detector. None means "tabular", + # i.e. exactly the behaviour that predates this option. Appended last: # a defaulted field cannot precede a non-defaulted one, and appending also keeps positional # construction stable for anything building this directly. profile: str | None = None diff --git a/tests/unit/test_anomaly_isolation_forest_inertness.py b/tests/unit/test_anomaly_isolation_forest_inertness.py index cd184d350..2841177d9 100644 --- a/tests/unit/test_anomaly_isolation_forest_inertness.py +++ b/tests/unit/test_anomaly_isolation_forest_inertness.py @@ -136,9 +136,9 @@ def test_existing_algorithm_strings_still_resolve(algorithm: str): assert strategy.supports(algorithm) -@pytest.mark.parametrize("profile", [None, "auto", "tabular", "AUTO", " Tabular "]) +@pytest.mark.parametrize("profile", [None, "tabular", "TABULAR", " Tabular "]) def test_tabular_profiles_select_isolation_forest_and_leave_params_untouched(profile: str | None): - """The default must resolve to today's behaviour, and must not perturb the caller's parameters. + """An unset profile must resolve to today's behaviour, and must not perturb the caller's params. Identity, not equality: returning a copy would be harmless here but would mean the resolver is rewriting configuration on a path that is supposed to be a no-op, which is the kind of thing that @@ -170,7 +170,7 @@ def test_an_unknown_profile_is_rejected_by_name(): resolve_training_profile("timeseries-ish", _reference_params()) -@pytest.mark.parametrize("profile", [None, "auto", "tabular", "timeseries"]) +@pytest.mark.parametrize("profile", [None, "tabular", "timeseries"]) def test_an_injected_strategy_wins_over_every_profile(profile: str | None): """``AnomalyTrainingService(spark, strategy=...)`` is how a caller substitutes a strategy, and how existing tests substitute a double. If profile resolution overrode it, those tests would keep From 3535f18e3cf15d2b834c225c078264a0423082c2 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 15:19:43 +0100 Subject: [PATCH 040/107] Verify the timeseries profile against a real workspace, settling two unknowns Everything so far was unit-level numpy or the offline SMD bake-off. Neither can answer the two questions that only a workspace can, and both were live risks rather than formalities: 1. Does MLflow round-trip a DQX-defined estimator class at all? `log_sklearn_model_compatible` passes no `code_paths` and no `pip_requirements` (mlflow_registry.py:167). The sklearn flavour defaults to cloudpickle and `timeseries_detector` registers itself for pickle-by-value, so the class *should* travel inside the artifact -- but scoring goes through `mlflow.sklearn.load_model` (model_loader.py:34), where a failure would surface as a load error. 2. Does signature inference accept it? `register_model_with_signature_inference` calls `predict`. Unit tests pin that it returns {-1, +1}; only MLflow can say whether that satisfies inference. Both hold. The test passes end to end on a live workspace in 254s: two models trained and registered in Unity Catalog, scored through the pandas UDF, attributed, and explained by ai_query. ## The fixture is the load-bearing part `generate_correlated_multivariate_data` drives metrics from shared latent factors, then produces anomalies by permuting the broken columns *among the anomalous rows* (a cyclic shift by one, so no row keeps its own values). That preserves each column's marginal distribution exactly -- the same multiset before and after -- which is what makes the comparison mean something: marginals preserved exactly True max-abs-z PR-AUC 0.0724 (random floor 0.0500) IsolationForest PR-AUC 0.1457 Mahalanobis PR-AUC 0.8731 top contributor in broken set 26/30 rows No per-column statistic can separate these positives -- not a threshold, not a z-score, not a quantile -- by construction rather than by tuning. So a detector has to model correlation to see anything at all, and the 6x gap measures the estimator rather than which one was tuned harder. Measured offline before any workspace time was spent, which is also where the thresholds below come from. ## Thresholds, and why they are where they are MIN_PR_AUC_GAIN = 0.25 against a measured gap of 0.73: loose on purpose, because the claim is the direction and rough size, not a tripwire on the forest's RNG. MIN_TOP_CONTRIBUTOR_HIT_RATE = 0.6 against a measured 26/30. Deliberately not "every row": the permutation moves values *between* anomalous rows, so a row can receive a value that happens to fit, leaving a correlated column as its largest single term. Requiring all 30 would have encoded an accident of the fixture as a contract. The gating assertion is "not every row got contributions", not "exactly the flagged rows did. The UDF-side gate is intentionally over-inclusive by `_SEVERITY_GATE_EPSILON` so drift between the numpy and Spark severity computations can never leave a flagged row without a map -- an exact set-equality assertion would have been a latent flake at the threshold boundary. ## What each assertion protects - Registry `algorithm == "Mahalanobis"`: scoring resolves its strategy from this string, so a wrong value would not fail training, it would fail every future scoring run against the model. - Contributions non-negative: the leave-one-out attribution is non-negative because the precision matrix is PSD, and that is precisely what lets it reuse `format_shap_contributions` unchanged. A negative value here would mean the formula regressed, not the formatting. - Keys a subset of the persisted `engineered_feature_names`, read back through `SparkFeatureMetadata.from_json` so the test reads the contract the way scoring does. - `ai_explanation.narrative` non-null: proves a non-SHAP attribution reaches the LLM prompt intact. Load-bearing on the assertion above it, because `_resolve_ai_explanation_flag` disables explanations silently when contributions are absent. - Finite scores: a singular covariance surfaces as NaN or inf rather than as an exception, so every metric above would silently degrade instead of failing. ## Shared fixture rather than a cross-module import `ai_query_endpoint` and its probe moved from `test_anomaly_ai_explanation.py` into `conftest.py`, and became public (`ai_query_llm_config`, `ai_query_endpoint_available`). Importing a fixture across test modules needs a `# noqa: F401` on the import and `# noqa: F811` on the parameter, and both are forbidden by AGENTS.md -- suppressing the lint would have been the hack, not the fix. Duplicating the probe was the other option and is forbidden too (fixtures belong in conftest). Nothing about that file's behaviour changes; it now takes the fixture from where pytest already looks. Pylint's too-many-locals (36/24, then 30/24) was likewise fixed rather than silenced, by extracting `_train_both_profiles`, `_assert_registry_records_profile`, `_assert_beats_tabular_and_baselines`, `_assert_contribution_contract`, `_top_contributor_hits` and `_assert_ai_explanation_present`. The test body reads as six numbered claims now, which is what it should have been first time. Gates: unit 2550 passed, mypy clean over 350 files, pylint 10.00/10, integration test green on a live workspace. Co-authored-by: Isaac --- tests/integration_anomaly/conftest.py | 61 +++- .../synthetic_generators.py | 72 +++++ .../test_anomaly_ai_explanation.py | 58 +--- .../test_anomaly_timeseries_profile.py | 282 ++++++++++++++++++ 4 files changed, 425 insertions(+), 48 deletions(-) create mode 100644 tests/integration_anomaly/test_anomaly_timeseries_profile.py diff --git a/tests/integration_anomaly/conftest.py b/tests/integration_anomaly/conftest.py index bf1ddd1ba..382d25df1 100644 --- a/tests/integration_anomaly/conftest.py +++ b/tests/integration_anomaly/conftest.py @@ -16,11 +16,18 @@ from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies -from databricks.labs.dqx.config import AnomalyConfig, AnomalyParams, InputConfig, IsolationForestConfig +from databricks.labs.dqx.config import ( + AnomalyConfig, + AnomalyParams, + InputConfig, + IsolationForestConfig, + LLMModelConfig, +) from databricks.labs.dqx.rule import DQDatasetRule from databricks.labs.pytester.fixtures.baseline import factory from tests.constants import TEST_CATALOG from tests.integration_anomaly.constants import ( + DEFAULT_AI_QUERY_ENDPOINT, DEFAULT_SCORE_THRESHOLD, OUTLIER_AMOUNT, OUTLIER_QUANTITY, @@ -44,6 +51,49 @@ # ----------------------------------------------------------------------------- +# ----------------------------------------------------------------------------- +# ai_query (LLM) endpoint probing — shared by every test that needs a live endpoint +# ----------------------------------------------------------------------------- + +AI_QUERY_TEST_ENDPOINT = os.environ.get("DQX_AI_QUERY_TEST_ENDPOINT", DEFAULT_AI_QUERY_ENDPOINT) + + +def ai_query_llm_config(endpoint: str) -> LLMModelConfig: + """LLMModelConfig pointing at a Model Serving endpoint reached through SQL ``ai_query``.""" + return LLMModelConfig(model_name=endpoint) + + +def ai_query_endpoint_available(session: SparkSession) -> tuple[bool, str | None]: + """Cheap probe — does ai_query against the configured endpoint succeed? + + Returns ``(available, error_message)``. The error message is surfaced in the skip reason so + a failing probe doesn't masquerade as 'endpoint not provisioned' — knowing why the probe + failed (auth, missing entitlement, wrong name) is what lets the user decide whether to set + DQX_AI_QUERY_TEST_ENDPOINT. + """ + try: + session.sql( + f"SELECT ai_query('{AI_QUERY_TEST_ENDPOINT}', 'reply with the single word: ok', " + f"modelParameters => named_struct('max_tokens', 8, 'temperature', 0.0)) AS r" + ).collect() + return True, None + except Exception as exc: + return False, repr(exc) + + +@pytest.fixture +def ai_query_endpoint(ws, spark): + """Skip the test when the workspace cannot reach the configured ai_query endpoint.""" + assert ws.current_user.me() is not None # fail-fast if workspace auth is broken + available, error = ai_query_endpoint_available(spark) + if not available: + pytest.skip( + f"ai_query endpoint {AI_QUERY_TEST_ENDPOINT!r} not reachable; " + f"set DQX_AI_QUERY_TEST_ENDPOINT to override. Probe error: {error}" + ) + return AI_QUERY_TEST_ENDPOINT + + def qualify_model_name(model_name: str, registry_table: str) -> str: """Return a fully qualified model name using the registry table prefix.""" if model_name.count(".") >= 2: @@ -140,6 +190,7 @@ def train_model_with_params( params: AnomalyParams, expected_anomaly_rate: float = 0.02, baseline_by: list[str] | None = None, + profile: str | None = None, ) -> str: """Train a model with internal params (test-only).""" return engine.train( @@ -150,6 +201,7 @@ def train_model_with_params( baseline_by=baseline_by, params=params, expected_anomaly_rate=expected_anomaly_rate, + profile=profile, ) @@ -785,7 +837,7 @@ def quick_model_factory(ws, make_random, make_schema): """ Factory for training lightweight models with custom parameters. - Use when tests need specific training params (internal, e.g., AnomalyParams, segment_by). + Use when tests need specific training params (internal, e.g., AnomalyParams, profile). For simple 2D scoring tests, prefer function-scoped shared_2d_model instead. Returns a callable that accepts spark and training parameters. @@ -801,6 +853,7 @@ def _train( schema: str | None = None, baseline_by: list[str] | None = None, train_schema: str | None = None, + profile: str | None = None, ): """ Train a quick test model. @@ -812,6 +865,8 @@ def _train( train_data (list[tuple] | None): Custom training data tuples (overrides train_size) params (AnomalyParams | None): Internal training params (test-only) baseline_by (list[str] | None): Group columns for baseline-conditioned models + profile (str | None): Which detector to train ("tabular" / "timeseries"); None means + the default, so existing callers keep the IsolationForest path untouched. train_schema (str | None): Explicit DDL for train_data (needed when group columns are not doubles) catalog (str): Catalog name @@ -853,6 +908,7 @@ def _train( model_name=model_name, registry_table=registry_table, baseline_by=baseline_by, + profile=profile, ) else: full_model_name = train_model_with_params( @@ -863,6 +919,7 @@ def _train( columns=columns, params=params, baseline_by=baseline_by, + profile=profile, ) return full_model_name, registry_table, columns diff --git a/tests/integration_anomaly/synthetic_generators.py b/tests/integration_anomaly/synthetic_generators.py index 0099e79b2..0e5c3384c 100644 --- a/tests/integration_anomaly/synthetic_generators.py +++ b/tests/integration_anomaly/synthetic_generators.py @@ -252,6 +252,78 @@ def generate_group_conditional_data( return ["event_count"], train_df, test_df, incident_key +def generate_correlated_multivariate_data( + spark, + *, + seed: int = 42, + n_train: int = 1200, + n_test: int = 600, + n_features: int = 8, + n_factors: int = 2, + noise_scale: float = 0.15, + anomaly_frac: float = 0.05, + n_broken_features: int = 3, +) -> tuple[list[str], DataFrame, DataFrame, list[str]]: + """Generate the *correlation-break* anomaly that motivates ``profile="timeseries"``. + + Metrics are driven by a small number of shared latent factors, so they move together the way + machine telemetry does -- CPU, memory and queue depth all rising when load rises. An anomaly here + is not a metric leaving its range; it is metrics that always moved together **stopping**. + + The break is produced by permuting the broken columns' values **across the anomalous rows**, which + is the standard way to null out dependence while leaving every marginal distribution untouched. + That is the whole point of the fixture, and it is a stronger construction than shifting a value: + + * Each broken column's values are the *same multiset* before and after, so no per-column statistic + can separate the anomalies -- not a threshold, not a z-score, not a quantile. A univariate + detector cannot do better than chance here, by construction rather than by tuning. + * Only the *joint* distribution changes, so a detector has to model correlation to see anything. + + That makes this the fixture that distinguishes the two profiles rather than merely exercising one. + Isolation Forest splits one feature at a time, so it is close to blind to this; a correlation-aware + detector sees it as a large distance in the whitened space. + + Returns ``(feature_columns, train_df, test_df, broken_columns)``. Both frames carry an + ``is_anomaly`` label (all zero in training). *broken_columns* names the columns whose correlation + was severed, so a test can assert the attribution points at them rather than merely being non-null. + """ + rng = np.random.default_rng(seed) + feature_cols = [f"metric_{i}" for i in range(n_features)] + + # Loadings are strictly positive so every metric rises and falls with the shared factors. Mixed + # signs would also be correlated, but positively-coupled telemetry is the case users recognise. + loadings = rng.uniform(0.6, 1.4, size=(n_factors, n_features)) + + def draw(n_rows: int) -> np.ndarray: + factors = rng.normal(0.0, 1.0, size=(n_rows, n_factors)) + return factors @ loadings + rng.normal(0.0, noise_scale, size=(n_rows, n_features)) + + train_values = draw(n_train) + test_values = draw(n_test) + + n_anomalies = max(2, int(n_test * anomaly_frac)) + broken_columns = feature_cols[:n_broken_features] + labels = np.zeros(n_test) + labels[-n_anomalies:] = 1.0 + + # Permute within the anomalous block only. A derangement (no row keeps its own values) guarantees + # every anomalous row actually had its correlation severed -- a plain shuffle can leave rows fixed, + # which would plant unlabelled normal rows among the positives and understate any detector. + block = test_values[-n_anomalies:, :n_broken_features] + test_values[-n_anomalies:, :n_broken_features] = np.roll(block, shift=1, axis=0) + + train_rows = np.hstack([train_values, np.zeros((n_train, 1))]).tolist() + test_rows = np.hstack([test_values, labels.reshape(-1, 1)]).tolist() + + schema = ", ".join(f"{col} double" for col in feature_cols) + ", is_anomaly double" + return ( + feature_cols, + spark.createDataFrame(train_rows, schema), + spark.createDataFrame(test_rows, schema), + broken_columns, + ) + + def inject_missingness_spike( df: DataFrame, *, diff --git a/tests/integration_anomaly/test_anomaly_ai_explanation.py b/tests/integration_anomaly/test_anomaly_ai_explanation.py index 3bad808f5..7378f7fd5 100644 --- a/tests/integration_anomaly/test_anomaly_ai_explanation.py +++ b/tests/integration_anomaly/test_anomaly_ai_explanation.py @@ -6,7 +6,6 @@ configured endpoint (override with the DQX_AI_QUERY_TEST_ENDPOINT env var). """ -import os import re import pytest @@ -16,20 +15,18 @@ from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies from databricks.labs.dqx.config import LLMModelConfig from databricks.labs.dqx.errors import InvalidParameterError -from tests.integration_anomaly.conftest import qualify_model_name +from tests.integration_anomaly.conftest import ( + AI_QUERY_TEST_ENDPOINT, + ai_query_llm_config, + qualify_model_name, +) from tests.integration_anomaly.constants import ( - DEFAULT_AI_QUERY_ENDPOINT, DEFAULT_SCORE_THRESHOLD, OUTLIER_AMOUNT, OUTLIER_QUANTITY, ) _LLM_EXPLAINER_LOGGER = "databricks.labs.dqx.anomaly.anomaly_llm_explainer" -_AI_QUERY_TEST_ENDPOINT = os.environ.get("DQX_AI_QUERY_TEST_ENDPOINT", DEFAULT_AI_QUERY_ENDPOINT) - - -def _ai_query_llm_cfg(endpoint: str) -> LLMModelConfig: - return LLMModelConfig(model_name=endpoint) def _score_with_explanation(scorer, df, model_meta, *, llm_model_config, **overrides): @@ -55,37 +52,6 @@ def _make_outlier_df(spark, factory, *, repeat: int = 1): ) -def _ai_query_endpoint_available(spark: SparkSession) -> tuple[bool, str | None]: - """Cheap probe — does ai_query against the configured endpoint succeed? - - Returns ``(available, error_message)``. The error message is surfaced in the skip reason so - a failing probe doesn't masquerade as 'endpoint not provisioned' — knowing why the probe - failed (auth, missing entitlement, wrong name) is what lets the user decide whether to set - DQX_AI_QUERY_TEST_ENDPOINT. - """ - try: - spark.sql( - f"SELECT ai_query('{_AI_QUERY_TEST_ENDPOINT}', 'reply with the single word: ok', " - f"modelParameters => named_struct('max_tokens', 8, 'temperature', 0.0)) AS r" - ).collect() - return True, None - except Exception as exc: - return False, repr(exc) - - -@pytest.fixture -def ai_query_endpoint(ws, spark): - """Skip the test when the workspace cannot reach the configured ai_query endpoint.""" - assert ws.current_user.me() is not None # fail-fast if workspace auth is broken - available, error = _ai_query_endpoint_available(spark) - if not available: - pytest.skip( - f"ai_query endpoint {_AI_QUERY_TEST_ENDPOINT!r} not reachable; " - f"set DQX_AI_QUERY_TEST_ENDPOINT to override. Probe error: {error}" - ) - return _AI_QUERY_TEST_ENDPOINT - - def test_ai_query_explanation_populated_for_anomalous_row( spark: SparkSession, shared_3d_model, test_df_factory, anomaly_scorer, ai_query_endpoint ): @@ -96,7 +62,7 @@ def test_ai_query_explanation_populated_for_anomalous_row( """ test_df = _make_outlier_df(spark, test_df_factory) result_df = _score_with_explanation( - anomaly_scorer, test_df, shared_3d_model, llm_model_config=_ai_query_llm_cfg(ai_query_endpoint) + anomaly_scorer, test_df, shared_3d_model, llm_model_config=ai_query_llm_config(ai_query_endpoint) ) row = result_df.collect()[0] anomaly_info = row["_dq_info"][0]["anomaly"] @@ -141,7 +107,7 @@ def test_ai_query_explanation_null_for_non_anomalous_row( columns_schema="amount double, quantity double, discount double", ) result_df = _score_with_explanation( - anomaly_scorer, test_df, shared_3d_model, llm_model_config=_ai_query_llm_cfg(_AI_QUERY_TEST_ENDPOINT) + anomaly_scorer, test_df, shared_3d_model, llm_model_config=ai_query_llm_config(AI_QUERY_TEST_ENDPOINT) ) row = result_df.collect()[0] anomaly_info = row["_dq_info"][0]["anomaly"] @@ -160,7 +126,7 @@ def test_ai_query_explanation_redact_columns_filters_output( anomaly_scorer, test_df, shared_3d_model, - llm_model_config=_ai_query_llm_cfg(ai_query_endpoint), + llm_model_config=ai_query_llm_config(ai_query_endpoint), redact_columns=["amount"], ) row = result_df.collect()[0] @@ -186,7 +152,7 @@ def test_ai_query_explanation_one_call_per_group( """ test_df = _make_outlier_df(spark, test_df_factory, repeat=5) result_df = _score_with_explanation( - anomaly_scorer, test_df, shared_3d_model, llm_model_config=_ai_query_llm_cfg(ai_query_endpoint) + anomaly_scorer, test_df, shared_3d_model, llm_model_config=ai_query_llm_config(ai_query_endpoint) ) rows = result_df.collect() explanations = [ @@ -211,7 +177,7 @@ def test_ai_query_explanation_references_dominant_feature_within_word_caps( """ test_df = _make_outlier_df(spark, test_df_factory) result_df = _score_with_explanation( - anomaly_scorer, test_df, shared_3d_model, llm_model_config=_ai_query_llm_cfg(ai_query_endpoint) + anomaly_scorer, test_df, shared_3d_model, llm_model_config=ai_query_llm_config(ai_query_endpoint) ) explanation = result_df.collect()[0]["_dq_info"][0]["anomaly"]["ai_explanation"] assert explanation is not None @@ -272,7 +238,7 @@ def test_ai_query_response_shape_portability( test_df = _make_outlier_df(spark, test_df_factory) result_df = _score_with_explanation( - anomaly_scorer, test_df, shared_3d_model, llm_model_config=_ai_query_llm_cfg(endpoint) + anomaly_scorer, test_df, shared_3d_model, llm_model_config=ai_query_llm_config(endpoint) ) row = result_df.collect()[0] anomaly_info = row["_dq_info"][0]["anomaly"] @@ -344,7 +310,7 @@ def test_ai_query_explanation_disabled_without_contributions( anomaly_scorer, test_df, shared_3d_model, - llm_model_config=_ai_query_llm_cfg(_AI_QUERY_TEST_ENDPOINT), + llm_model_config=ai_query_llm_config(AI_QUERY_TEST_ENDPOINT), enable_contributions=False, # overrides the default-True; explanation is downgraded off ) anomaly = result_df.collect()[0]["_dq_info"][0]["anomaly"] diff --git a/tests/integration_anomaly/test_anomaly_timeseries_profile.py b/tests/integration_anomaly/test_anomaly_timeseries_profile.py new file mode 100644 index 000000000..4cf4f64d8 --- /dev/null +++ b/tests/integration_anomaly/test_anomaly_timeseries_profile.py @@ -0,0 +1,282 @@ +"""``profile="timeseries"`` end to end: registry, contributions, AI explanation, detection quality. + +This is the only place the correlation-aware detector is exercised through the real pipeline. Everything +before it is unit-level (numpy) or offline (the SMD bake-off), and neither can answer the two questions +that only a workspace can: + +1. **Does MLflow round-trip a DQX-defined estimator class at all?** ``log_sklearn_model_compatible`` + passes no *code_paths* and no *pip_requirements* (``mlflow_registry.py:167``). The sklearn flavour + defaults to cloudpickle, and ``timeseries_detector`` registers itself for pickle-by-value, so the + class should travel inside the artifact -- but "should" is the word this test exists to remove. + Scoring calls ``mlflow.sklearn.load_model`` (``model_loader.py:34``), so a failure here surfaces as a + load error rather than as a wrong number. +2. **Does signature inference accept it?** ``register_model_with_signature_inference`` calls + ``predict``. Unit tests pin that it returns ``{-1, +1}``; only MLflow can say whether that satisfies + its inference. + +Deliberately **one test with several assertions**, in the style of ``test_anomaly_quality.py``: every +property is read off the same pair of trained models, and each model is registered in Unity Catalog. +Splitting the assertions would retrain that pair once per test for no extra coverage. pytester's +``spark`` fixture is function-scoped, so a module-scoped fixture cannot hold the pair either. Each +assertion carries its own message, so a failure still says which property broke. + +The fixture is ``generate_correlated_multivariate_data``, whose anomalies are produced by permuting +columns among the anomalous rows. That preserves every marginal distribution exactly, so the positives +are unreachable by any per-column statistic -- which is what makes a comparison between the two profiles +mean something rather than measuring which detector is better tuned. +""" + +import numpy as np +import pytest +from pyspark.sql import SparkSession +from pyspark.sql import functions as F + +from databricks.labs.dqx.anomaly.training_strategies import MAHALANOBIS_ALGORITHM +from databricks.labs.dqx.anomaly.transformers import SparkFeatureMetadata +from databricks.labs.dqx.config import AnomalyParams + +from tests.integration_anomaly.constants import DEFAULT_SCORE_THRESHOLD +from tests.integration_anomaly.quality_metrics import pr_auc, trivial_baselines +from tests.integration_anomaly.synthetic_generators import generate_correlated_multivariate_data +from tests.integration_anomaly.conftest import ai_query_llm_config + +# The gap the correlation-aware detector must clear against IsolationForest on data whose anomalies are +# *only* joint. Loose on purpose: the claim is the direction and rough size, not a tripwire on the +# forest's RNG. Measured offline on this exact fixture, PR-AUC was 0.146 for IsolationForest and 0.873 +# for the correlation-aware detector, so 0.25 leaves a wide margin. The offline run also confirmed the +# fixture is univariately invisible: max-abs-z reached 0.072 against a 0.050 random floor. +MIN_PR_AUC_GAIN = 0.25 + +# Contributions must point at the columns whose correlation was actually severed, not merely be +# non-null. Not every row: the permutation moves values between anomalous rows, so a row can +# occasionally receive a value that happens to fit, leaving a *correlated* column as the largest single +# term. Measured 26 of 30 rows offline, so a simple majority is a wide margin and still fails loudly if +# the attribution were pointing somewhere arbitrary. +MIN_TOP_CONTRIBUTOR_HIT_RATE = 0.6 + +TRAIN_ROWS = 1200 +TEST_ROWS = 600 + +pytestmark = pytest.mark.slow + + +def _scored_frame(anomaly_scorer, test_df, model, registry, columns, **check_kwargs): + """Score, then collect the columns the metrics need into pandas.""" + result = anomaly_scorer(test_df, model, registry, extract_score=False, **check_kwargs) + anomaly = F.element_at(F.col("_dq_info"), 1).getField("anomaly") + return ( + result.select( + *[F.col(c) for c in columns], + F.col("is_anomaly").alias("label"), + anomaly.getField("score").alias("score"), + anomaly.getField("is_anomaly").cast("double").alias("flagged"), + anomaly.getField("contributions").alias("contributions"), + anomaly.getField("ai_explanation").alias("ai_explanation"), + ) + .toPandas() + .dropna(subset=["score"]) + ) + + +def _engineered_feature_names(spark: SparkSession, registry_table: str, model_name: str) -> list[str]: + """Read the persisted feature contract back out of the registry. + + Goes through ``SparkFeatureMetadata.from_json`` rather than parsing the JSON here, so this reads the + contract the same way scoring does. + """ + row = ( + spark.table(registry_table) + .filter(F.col("identity.model_name") == model_name) + .select("features.feature_metadata") + .collect()[0] + ) + return SparkFeatureMetadata.from_json(row["feature_metadata"]).engineered_feature_names + + +def _train_both_profiles(spark, quick_model_factory, columns, train_rows): + """Train one model per profile on identical data, returning ``{profile: (model, registry)}``.""" + train_schema = ", ".join(f"{col} double" for col in columns) + ", is_anomaly double" + models = {} + for profile in ("timeseries", "tabular"): + model, registry, _ = quick_model_factory( + spark, + columns=columns, + train_data=train_rows, + train_schema=train_schema, + # sample_fraction=1.0 so the comparison is between estimators rather than between two + # different random subsets. baseline_by=[] suppresses grouping discovery: there is no + # grouping column in this fixture, and an empty list keeps that explicit rather than + # relying on discovery happening to find nothing. + params=AnomalyParams(sample_fraction=1.0), + baseline_by=[], + profile=profile, + ) + models[profile] = (model, registry) + return models + + +def _assert_contribution_contract(contributions_series, engineered_names: set[str]) -> None: + """Every map is keyed by the persisted contract, non-negative, and normalised to 100. + + Non-negativity is the load-bearing one: the leave-one-out attribution is non-negative by + construction (the precision matrix is PSD), which is what lets it reuse the SHAP formatter + unchanged. A negative value here would mean the formula regressed, not the formatting. + """ + for contributions in contributions_series: + assert contributions is not None, "a flagged row carried no contributions map" + unknown = sorted(set(contributions) - engineered_names) + assert not unknown, f"contribution keys {unknown} are not in the persisted engineered feature names" + values = [v for v in contributions.values() if v is not None] + assert values, "a flagged row's contributions map held only nulls" + assert min(values) >= 0.0, f"leave-one-out contributions must be non-negative, got {min(values)}" + assert abs(sum(values) - 100.0) < 0.5, f"contributions should be normalised to 100, summed to {sum(values)}" + + +def _top_contributor_hits(contributions_series, expected_columns: list[str]) -> int: + """How many rows name one of *expected_columns* as their single largest contributor.""" + expected = set(expected_columns) + hits = 0 + for contributions in contributions_series: + ranked = [(k, v) for k, v in contributions.items() if v is not None] + if ranked and max(ranked, key=lambda item: item[1])[0] in expected: + hits += 1 + return hits + + +def _assert_registry_records_profile(spark: SparkSession, registry_table: str, model_name: str) -> None: + """The registry names the algorithm the profile selected, and how the covariance was estimated. + + Scoring reads *algorithm* back off the registry to resolve a scoring strategy, so a wrong value here + would not fail training -- it would fail every future scoring run against this model. + """ + row = ( + spark.table(registry_table) + .filter(F.col("identity.model_name") == model_name) + .select("identity.algorithm", "training.hyperparameters") + .collect()[0] + ) + assert row["algorithm"] == MAHALANOBIS_ALGORITHM, ( + f"registry recorded algorithm {row['algorithm']!r}; scoring resolves its strategy from this " + f"string, so anything else orphans the model" + ) + covariance = row["hyperparameters"].get("covariance") + assert covariance in {"empirical", "ledoit_wolf"}, ( + f"expected the covariance estimator to be recorded, got {covariance!r}; this is how a reader " + f"tells whether shrinkage was applied" + ) + + +def _assert_beats_tabular_and_baselines(timeseries, tabular, columns: list[str]) -> None: + """The detector beats the default profile, and beats doing almost nothing.""" + timeseries_pr_auc = pr_auc(timeseries["label"], timeseries["score"]) + tabular_pr_auc = pr_auc(tabular["label"], tabular["score"]) + + # The design claim: a broken correlation is close to invisible to a per-feature splitter. + assert timeseries_pr_auc > tabular_pr_auc + MIN_PR_AUC_GAIN, ( + f"the timeseries profile should beat the tabular one on anomalies that are purely joint " + f"(tabular PR-AUC {tabular_pr_auc:.4f}, timeseries {timeseries_pr_auc:.4f})" + ) + + # And it must beat doing almost nothing. max_abs_z is the honest floor here: the fixture preserves + # every marginal exactly, so a univariate statistic cannot separate these positives -- which also + # means failing this assertion would say the fixture broke rather than that the model did. + baselines = trivial_baselines(timeseries, columns) + assert ( + timeseries_pr_auc > baselines["random"] + ), f"timeseries {timeseries_pr_auc:.4f} did not beat random {baselines['random']:.4f}" + assert timeseries_pr_auc > baselines["max_abs_z"], ( + f"timeseries {timeseries_pr_auc:.4f} did not beat a max-abs-z baseline {baselines['max_abs_z']:.4f}; " + f"the fixture's anomalies may have stopped being purely joint" + ) + + +def _assert_ai_explanation_present(flagged) -> None: + """The AI explanation survives the new attribution path. + + It reads the contributions map, so this is what proves a non-SHAP attribution reaches the LLM prompt + intact. Asserted structurally -- a non-empty narrative -- so the test does not depend on wording. + """ + explained = flagged[flagged["ai_explanation"].notna()] + assert not explained.empty, "no flagged row carried an ai_explanation struct" + narrative = explained["ai_explanation"].iloc[0]["narrative"] + assert isinstance(narrative, str) and narrative.strip(), "ai_explanation.narrative was empty" + + +def test_timeseries_profile_end_to_end( + spark: SparkSession, + quick_model_factory, + anomaly_scorer, + ai_query_endpoint, +): + """The timeseries profile trains, registers, scores, attributes, explains, and beats the default. + + Trains two models on identical data -- one per profile -- and compares them on the same rows. + """ + columns, train_df, test_df, broken_columns = generate_correlated_multivariate_data( + spark, + n_train=TRAIN_ROWS, + n_test=TEST_ROWS, + ) + train_rows = [tuple(r) for r in train_df.collect()] + + models = _train_both_profiles(spark, quick_model_factory, columns, train_rows) + timeseries_model, timeseries_registry = models["timeseries"] + tabular_model, tabular_registry = models["tabular"] + + # 1. The profile's choice is persisted and readable back. + _assert_registry_records_profile(spark, timeseries_registry, timeseries_model) + + # Contributions and the AI explanation are only produced when asked for, and the explanation + # depends on the contributions map — _resolve_ai_explanation_flag disables it silently when + # contributions are off, so assertion 4 is load-bearing for assertion 5. + timeseries = _scored_frame( + anomaly_scorer, + test_df, + timeseries_model, + timeseries_registry, + columns, + threshold=DEFAULT_SCORE_THRESHOLD, + enable_contributions=True, + enable_ai_explanation=True, + ai_explanation_llm_model_config=ai_query_llm_config(ai_query_endpoint), + ) + tabular = _scored_frame(anomaly_scorer, test_df, tabular_model, tabular_registry, columns) + + # 2. Detection quality: better than the default profile, and better than a one-liner. + _assert_beats_tabular_and_baselines(timeseries, tabular, columns) + + # 3. Contributions honour the persisted feature contract on every flagged row. + engineered_names = set(_engineered_feature_names(spark, timeseries_registry, timeseries_model)) + flagged = timeseries[timeseries["flagged"] == 1.0] + assert not flagged.empty, "no row was flagged, so the contributions assertions would pass vacuously" + + _assert_contribution_contract(flagged["contributions"], engineered_names) + + # Gating must actually have happened. Attribution costs an order of magnitude more than scoring, + # so computing it for every row is a performance regression rather than a cosmetic one. Asserted as + # "not all rows" rather than "exactly the flagged rows" because the UDF-side gate is deliberately + # over-inclusive by a small epsilon, so drift between the numpy and Spark severity computations can + # never leave a flagged row without a map. + with_contributions = int(timeseries["contributions"].notna().sum()) + assert with_contributions < len(timeseries), ( + f"all {len(timeseries)} rows received contributions, so severity gating did not run; " + f"attribution is far more expensive than scoring, so this is a cost regression" + ) + + # 4. The attribution points at the columns whose correlation was actually severed. Without this the + # map could be uniform noise and every assertion above would still pass. + positives = timeseries[(timeseries["label"] == 1.0) & timeseries["contributions"].notna()] + assert not positives.empty, "no labelled anomaly received contributions" + hits = _top_contributor_hits(positives["contributions"], broken_columns) + hit_rate = hits / len(positives) + assert hit_rate >= MIN_TOP_CONTRIBUTOR_HIT_RATE, ( + f"the top contributor named a column from {broken_columns} on only {hit_rate:.0%} of labelled " + f"anomalies ({hits}/{len(positives)}); the attribution is not tracking the severed correlation" + ) + + # 5. The AI explanation still reaches the user with a non-SHAP attribution behind it. + _assert_ai_explanation_present(flagged) + + # 6. Scores must be finite everywhere. A singular covariance would surface as NaN or inf rather + # than as an exception, and every metric above would silently degrade instead of failing. + assert np.isfinite(timeseries["score"]).all(), "the timeseries model produced non-finite scores" From f2cf126e84fe020bc9485d97216b87eb413c8f8c Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 15:31:45 +0100 Subject: [PATCH 041/107] Fix the docs build, which commit 07cae7e1 in this PR broke `make docs-build` fails on this branch: Error: MDX compilation failed for file ".../docs/reference/api/anomaly/check_funcs.md" Could not parse expression with acorn (line 106, column 28) Found by building the docs, which nothing in this repo does automatically -- there is no docs-build in CI and no target runs it -- so it had been broken since 07cae7e1 ("Judge each metric against its own group's baseline, via baseline_by") without anything noticing. ## The mechanism, because the two-line diff does not explain itself 07cae7e1 added *single*-backtick object names to the `has_no_row_anomalies` docstring: `` `is_new_baseline` ``, `` `foreign_key` ``, `` `is_in_list` ``. pydoc-markdown's crossref processor treats those as cross-references, and in doing so it corrupts the **double**-backtick spans elsewhere in the same docstring, rewriting every one of their backticks into `` `foreign_key`N `` (N counting up). So this docstring line, which predates this PR: with keys ``{"model_name", "api_key", "api_base"}`` was emitted as with keys `foreign_key`2{"model_name", "api_key", "api_base"}`foreign_key`2 The code span is destroyed, which is what matters: inside a code span MDX leaves `{` alone, and outside one it parses `{...}` as a JSX expression. `{"model_name", "api_key", "api_base"}` is not valid JavaScript, so acorn fails and the whole site build dies. The braces were never the bug -- they had been safely inside a code span since #1129 in June. Losing the span is the bug. This is precisely the case AGENTS.md warns about, in the "Writing Docstrings" rules: **No backticks** around object names -- use italics instead (e.g., *arg1*, *column*). Backticks cause rendering issues in API docs. The rule was stated, and violating it broke the build. Fixed as the rule prescribes: italics. Kept as its own commit rather than folded into the docs work that found it, because it repairs an earlier commit in this PR and someone may want to read or cherry-pick it on its own. ## Verified by building, not by reading uv run --group docs pydoc-markdown -> the `foreign_key`N mangling is gone make docs-build -> SUCCESS, 156 documents, no broken links Two notes for whoever touches this next. Regenerating the API docs must go through the Makefile or carry UV_FROZEN=1: a bare `uv run --group docs pydoc-markdown` rewrote uv.lock with internal registry URLs (4,442 lines), exactly what AGENTS.md warns about. And `docs/dqx/docs/reference/api` is gitignored, so the corrupted output never appeared in a diff -- only a build reveals it. Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/check_funcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/check_funcs.py b/src/databricks/labs/dqx/anomaly/check_funcs.py index c22fb867f..11fb8b162 100644 --- a/src/databricks/labs/dqx/anomaly/check_funcs.py +++ b/src/databricks/labs/dqx/anomaly/check_funcs.py @@ -156,12 +156,12 @@ def has_no_row_anomalies( DQX aligns scored rows back to the input using an internal row id and removes it before returning. Baseline conditioning is inferred from the trained model's metadata. - Rows whose group was never seen in training are reported (`is_new_baseline`) but are **not** + Rows whose group was never seen in training are reported (*is_new_baseline*) but are **not** flagged as violations: neither categorical encoder can represent an unseen value honestly — one-hot makes it look maximally normal, frequency encoding maximally extreme — so DQX cannot judge the row, and "could not judge" is not the same claim as "is anomalous". If an unrecognised group value is itself a problem worth failing on, that is a membership - question rather than an anomaly one: use `foreign_key` or `is_in_list` on the group column + question rather than an anomaly one: use *foreign_key* or *is_in_list* on the group column against your set of known values, which is the check built for it. Args: From 3758eed3187977c23461a9f6483003bafd86e1d7 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 15:50:07 +0100 Subject: [PATCH 042/107] Document the profile, and rename the contract from "SHAP" to "feature contributions" Two changes that have to land together, because the second is only true once the first exists. ## 1. `profile` is documented, with generated numbers rather than adjectives New "Choosing a profile" section in the user guide, a "Which detector to use" section in the quality reference, and a `profile` row in the `train()` reference table. Both sections lead with the question a user can actually answer -- *what kind of data do I have?* -- rather than with the algorithm, because that is the only form in which the choice is answerable without ML knowledge. Every number comes from `emit_docs.py` reading the committed bake-off JSONs. That needed a new generator and marker region, fed by a `--bakeoff` argument because the estimator bake-off and the conditioning sweep are separate runs over different datasets: | `profile` | detector | clean split | contaminated training | | `"tabular"` | Isolation Forest | 36% | 33% | | `"timeseries"` | correlation-aware | **82%** | **79%** | `emit_docs.py` now writes **two** pages. The guide's table had been hand-typed on the first pass, and a hand-typed copy of a measured number drifts the first time the measurement is refreshed -- exactly the failure that script exists to prevent. The guide gets the short form (contaminated only, the column that describes a real run); the reference page gets both columns and the methodology. Contaminated training is the honest column: DQX fits a random sample of the user's table with anomalies still in it, so that is what a user gets, while the clean split is the upper bound a curated benchmark would report. Covariance estimates are sensitive to precisely the extreme rows they are meant to find, so a collapse here would have ruled the approach out. It holds: 82% -> 79%. Publishing only the clean number would have been the flattering choice and the wrong one. Every docstring quoting these figures was updated to match. The first draft had the source saying 36/82 while the docs it generated said 33/79 -- a disagreement a reader has no way to resolve. ## The quality page stays simple, and no number on it is typed by hand The first draft of that section explained the methodology with hand-typed statistics: incident counts, the incident-length distribution, how many incidents the best-PR-AUC configuration covered. Those are one-off numbers -- nothing regenerates them, so they rot silently the moment the measurement is refreshed -- and they made a page someone reads to make a decision read like a paper. They are gone. The methodology detail still lives where it belongs, in the docstrings of `event_recall_at_budget` and `smd_bakeoff.py`, and the page now says in plain words what "incidents surfaced" counts and why it is not point-adjusted F1, with no figures in the prose at all. What remains is one generated table and the few sentences needed to read it. The framing was wrong on the first pass too. It put the table first, so a reader met "33%" cold and read it as a score for DQX. It is not: SMD is telemetry, which is what `"timeseries"` is for, and the tabular benchmarks further down the same page show the default detector strong on the data it *is* for. The interpretation now comes **before** the table -- which tool for which job -- so the number is never read without it. This is not softening the result: the weak number is still published, beside the strong one, because that comparison is the entire reason two profiles exist. Both sections also state where DQX is **weak**, in the user's own terms and without figures: no seasonality, no trend, no forecasting, and nothing to correlate against on a single series. Someone deciding whether to adopt this is better served knowing that up front than discovering it on their data. ## 2. "SHAP contributions" was the contract; now it is one implementation of it SHAP has not been removed and nothing about the default path changed: `profile="tabular"` is still Isolation Forest explained by `SHAP.TreeExplainer`, and `shap` is still a dependency of the `[anomaly]` extra. What changed is that a second detector now produces contributions *without* SHAP, so naming SHAP in the contract is simply wrong. The `contributions` map's keys and types are unchanged, making this a wording correction rather than a schema change: 8 places in the user guide (including deck slide 9, the most visible "SHAP" in the whole documentation set), the `has_no_row_anomalies` docstring, and the group-key comment in `anomaly_info_schema.py`. Where SHAP genuinely *is* the mechanism -- the tree branch, the dependency itself -- it stays named. One accuracy fix rather than a rename: `_validate_explanation_flags` raises "enable_contributions=True requires the 'shap' dependency", now false for the timeseries detector, whose leave-one-out attribution needs no SHAP. The message names the tabular detector instead. The *check* stays in place deliberately, even though it can now reject a legitimate combination (a timeseries model with SHAP absent): it runs at rule-construction time where only the model *name* is known, so making it algorithm-aware would mean a registry read on every rule built, and deferring it to scoring would trade fail-fast for a failure deep inside a UDF. The combination it wrongly rejects is unreachable through the supported install anyway, since `shap` ships in the `[anomaly]` extra. ## Two removed parameters were still documented as live `quality_checks.mdx` listed `segment_by` on `train()` and `AnomalyParams.max_segment_models`, both deleted earlier in this PR. Documenting a parameter that no longer exists is worse than omitting it: a reader who follows the table gets a TypeError. Replaced with the parameters that do exist (`baseline_by`, `profile`), and `ensemble_size`'s note about segmented training -- which no longer exists either -- now describes the live caveat, the deterministic timeseries detector. `anomaly_detection_quality.mdx` described `segment_by` in the present tense; past tense now. The guide's architecture overview said training always builds an Isolation Forest ensemble, which is only true of the default profile. Verified by building rather than reading: `make docs-build` SUCCESS, 156 documents, no broken links -- which is what confirms the cross-page anchors actually resolve, including the one that broke when the reference section was renamed mid-edit. The regenerated conditioning, per-group and cost tables came out byte-identical, so the only content changes are the intended ones. Gates: mypy clean, pylint 10.00/10, unit 2550 passed. Co-authored-by: Isaac --- CHANGELOG.md | 1 + benchmarks/anomaly_conditioning/emit_docs.py | 119 +++++++++++++++--- .../guide/row_anomaly_detection/index.mdx | 107 ++++++++++++++-- .../reference/anomaly_detection_quality.mdx | 64 +++++++++- docs/dqx/docs/reference/quality_checks.mdx | 6 +- .../labs/dqx/anomaly/anomaly_engine.py | 5 +- .../labs/dqx/anomaly/anomaly_info_schema.py | 2 +- .../labs/dqx/anomaly/check_funcs.py | 28 +++-- .../labs/dqx/anomaly/timeseries_detector.py | 5 +- .../labs/dqx/anomaly/training_strategies.py | 4 +- 10 files changed, 291 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a101515c1..7ec755b00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 0.16.0 * Added baseline conditioning to row anomaly detection ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). Anomaly detection could not detect a **contextual** anomaly — a value that is unremarkable across the table but wrong for its own group. On the measurements in the issue, one group's volume dropping 80% behind a flat daily total scored 45.1, the 45th percentile, so no threshold recovered it. `AnomalyEngine.train()` now takes `baseline_by`: each numeric metric gains its deviation from that metric's own baseline within the row's group, as a signed log-ratio, on a **single** pooled model — so the cost does not grow with the group count, and the same collapse scores above 95. Measured offline in the unit suite, a contextual collapse goes from PR-AUC 0.0028 (chance) to 0.6962, while an anomaly that was already globally extreme is unchanged at 1.0000, so conditioning costs nothing measurable when there is nothing to gain. Baseline columns must be string, integral, boolean or date; floating-point and decimal types are rejected because Spark and Python format them differently, which would silently break the key lookup that matches persisted baselines to rows. Grouping auto-discovery is no longer coupled to column discovery, so passing explicit `columns` no longer silently gives up conditioning, and a discovered grouping is used as `baseline_by` rather than turning into one model per group. Measured against v0.16.0 on identical tables through the real pipeline, a contextual collapse goes from PR-AUC 0.0376 to 0.5703, and the untouched ungrouped path scores identically on both builds. Across a wider sweep — 1,545 configurations over synthetic data, the Server Machine Dataset, NSL-KDD and ten classical tabular benchmarks — conditioning is worth a median +0.0742 PR-AUC where anomalies are contextual and nothing measurable where they are not, and beats one-model-per-group in every case measured; see [Anomaly detection quality](https://databrickslabs.github.io/dqx/docs/reference/anomaly_detection_quality) for the full results, the licences, and what these numbers do not mean. +* Added a `profile` argument to row anomaly training, and a second detector behind it ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). Row anomaly detection had one algorithm, scikit-learn's Isolation Forest, which splits on one feature at a time. That is why it is strong on tabular data and weak when the anomaly *is* a broken relationship between metrics that each stay inside their own range: on the Server Machine Dataset (28 machines, 38 metrics, chronological split, unadjusted metrics) it surfaced only 33% of labelled incidents inside an alert budget of 1% of rows. `AnomalyEngine.train()` now takes `profile`, which describes the data you have rather than the algorithm: `"tabular"` — the default, and exactly the behaviour that predates this option — keeps Isolation Forest, while `"timeseries"` selects a correlation-aware detector (Mahalanobis distance with Ledoit–Wolf shrinkage where the sample warrants it, standardised internally) that surfaces **79%** of the same incidents. (Those figures are measured with anomalies present in the training data, which is what DQX does since it fits a sample of your table; on a curated clean training split the same comparison is 36% against 82%.) It needs no timestamp column, because it models correlation between metrics rather than behaviour over time, and it trains a single model rather than an ensemble because it is deterministic, so `ensemble_size` is ignored and `confidence_std` is unavailable for it. Feature engineering, the registry, `has_no_row_anomalies`, contributions and AI explanations are all unchanged; the algorithm is persisted in the registry so scoring inherits the choice with no scoring-side API change. Contributions for the new detector are an exact leave-one-out decomposition (`aᵢ = zᵢ²/(Σ⁻¹)ᵢᵢ`, the Schur-complement drop from marginalising feature *i*) rather than SHAP, which needs no SHAP dependency and is non-negative by construction — the naive signed decomposition was rejected because its terms can go negative and every downstream consumer takes `abs()`, so a distance-*reducing* feature would have been rendered to the LLM as a driver. There is deliberately **no automatic option**: choosing correctly cannot be verified without labels, and the one cheap signal for "this looks temporal" (lag-1 autocorrelation) was measured and rejected because it is confounded by any ordering correlated with the values, which sorted warehouse storage produces routinely — three of ten classical tabular benchmarks scored above the weakest SMD entity. The resolved profile is logged on every training run. Published SMD figures near 0.80 F1 are **not** a comparable target: they use point adjustment, which Kim et al. (AAAI 2022) showed random scores also reach. * Added a pluggable actions and alerting subsystem ([#1289](https://github.com/databrickslabs/dqx/issues/1289)). DQX now supports extensible *actions* that run when checked data violates an optional condition evaluated against the summary metrics produced by `DQMetricsObserver`. The built-in `DQAlert` action can send notifications to Slack, Microsoft Teams, a generic HTTPS webhook, or the log, so pipelines can react to data quality regressions without custom plumbing. You can create your own custom actions as well, and custom alerting is possible via the callback destination, which invokes an in-process Python callable for each alert. * Added an MCP (Model Context Protocol) server for DQX ([#1252](https://github.com/databrickslabs/dqx/issues/1252)). The server exposes DQX's data quality capabilities as tools that any MCP-compatible AI agent (Claude, Genie Code, Cursor, Mosaic AI) can discover and orchestrate. It runs as a Databricks App with on-behalf-of (OBO) authentication, so all data access is governed by the calling user's Unity Catalog permissions. * Added support for summary metrics in Lakeflow Declarative Pipelines (LDP/DLT) ([#1301](https://github.com/databrickslabs/dqx/issues/1301)). A new `DQEngine.compute_summary_metrics(...)` produces the same row counts, per-check breakdown, and custom observer metrics as a lazy aggregation over the results DataFrame, so metrics can be computed inside Spark Declarative Pipelines where the observer- and streaming-listener-based paths cannot be used. diff --git a/benchmarks/anomaly_conditioning/emit_docs.py b/benchmarks/anomaly_conditioning/emit_docs.py index 5b61164d4..67989cc76 100644 --- a/benchmarks/anomaly_conditioning/emit_docs.py +++ b/benchmarks/anomaly_conditioning/emit_docs.py @@ -1,6 +1,13 @@ """Refresh the generated tables in the user-facing quality page from a results JSON. - python benchmarks/anomaly_conditioning/emit_docs.py results/2026-08-25-abc1234.json + python benchmarks/anomaly_conditioning/emit_docs.py results/2026-08-25-abc1234.json \ + [--bakeoff results/smd-bakeoff.json results/smd-bakeoff-contaminated-0.033.json] + +The conditioning sweep and the estimator bake-off are separate runs over different datasets, so the +profile table is fed from its own file(s) rather than from the conditioning results. Pass the clean +run first and the contaminated run second: contaminated training is what DQX actually does (it fits a +random sample of the user's table, anomalies included), so publishing only the clean number would +overstate what a user gets. Only the regions between marker comments are replaced, so the hand-written prose around them -- which is most of the page, and the part that tells a reader what to do about a number -- survives @@ -21,16 +28,20 @@ "per_group": ("", ""), "tabular": ("", ""), "cost": ("", ""), + "profile": ("", ""), } -PAGE = ( - pathlib.Path(__file__).resolve().parents[2] - / "docs" - / "dqx" - / "docs" - / "reference" - / "anomaly_detection_quality.mdx" -) +# The estimator each profile ships. `maha_ridge` is the bake-off name for the configuration in +# ``anomaly/timeseries_detector.py``: standardised internally, with a small ridge floor on the +# covariance. Kept as a mapping rather than inlined so the table cannot silently start reporting a +# configuration DQX does not ship. +PROFILE_ESTIMATORS = {"tabular": "iforest", "timeseries": "maha_ridge"} + +_DOCS = pathlib.Path(__file__).resolve().parents[2] / "docs" / "dqx" / "docs" +PAGE = _DOCS / "reference" / "anomaly_detection_quality.mdx" +# The user guide carries the same profile table, in a shorter form. Generated rather than copied: a +# hand-typed duplicate of a measured number drifts the first time the measurement is refreshed. +GUIDE = _DOCS / "guide" / "row_anomaly_detection" / "index.mdx" MECHANISM_LABELS = { "contextual": "**contextual** — ordinary for the table, wrong for their group", @@ -135,19 +146,83 @@ def tabular_table(baselines: list[dict]) -> str: return "\n".join(rows) -def replace_region(text: str, name: str, body: str) -> str: +def profile_table(runs: list[tuple[str, list[dict]]]) -> str: + """Incident coverage per profile, on raw features and pooled scope, across the runs given. + + Reports **event recall at a 1%-of-rows alert budget**, not PR-AUC, and not point-adjusted F1. + SMD's 3,732 anomalous rows sit in 39 incidents of median length 6 and maximum length 1041, so + point-wise PR-AUC is dominated by "did you find the one huge incident" -- the best-PR-AUC + configuration in this sweep covers 2 of 39 incidents. Counting incidents inside a fixed budget + keeps precision point-wise and answers the operational question instead. Point adjustment is + excluded on purpose: Kim et al. (AAAI 2022) showed random scores reach state-of-the-art under it, + which is why published SMD figures near 0.80 F1 are not a comparable target. + """ + header = ["| `profile` | detector |"] + divider = ["|---|---|"] + for label, _ in runs: + header.append(f" incidents surfaced ({label}) |") + divider.append("---|") + rows = ["".join(header), "".join(divider)] + + for profile, estimator in PROFILE_ESTIMATORS.items(): + detector = "Isolation Forest" if profile == "tabular" else "correlation-aware" + cells = [f"| `\"{profile}\"` | {detector} |"] + for _, results in runs: + match = [ + r + for r in results + if r["estimator"] == estimator and r["featuriser"] == "raw" and r["scope"] == "pooled" + ] + if not match: + cells.append(" n/a |") + continue + recall = match[0]["event_recall_at_1pct"] + emphasis = f"**{recall:.0%}**" if profile == "timeseries" else f"{recall:.0%}" + cells.append(f" {emphasis} |") + rows.append("".join(cells)) + return "\n".join(rows) + + +def guide_profile_table(runs: list[tuple[str, list[dict]]]) -> str: + """The same measurement, trimmed for the page where a user chooses. + + Shows only the contaminated-training column -- the number a user actually gets, since DQX fits a + random sample of their table with anomalies still in it. The clean-split upper bound and the + methodology stay on the reference page, where a reader has come to interrogate the numbers rather + than to make a choice. + """ + preferred = [r for label, r in runs if "contaminated" in label] or [runs[-1][1]] + results = preferred[0] + rows = ["| `profile` | Incidents surfaced |", "|---|---|"] + for profile, estimator in PROFILE_ESTIMATORS.items(): + match = [ + r for r in results if r["estimator"] == estimator and r["featuriser"] == "raw" and r["scope"] == "pooled" + ] + recall = f"{match[0]['event_recall_at_1pct']:.0%}" if match else "n/a" + emphasis = f"**{recall}**" if profile == "timeseries" else recall + rows.append(f'| `"{profile}"` | {emphasis} |') + return "\n".join(rows) + + +def replace_region(text: str, name: str, body: str, page: pathlib.Path = PAGE) -> str: start, end = MARKERS[name] if start not in text or end not in text: - raise SystemExit(f"marker pair for {name!r} missing from {PAGE.name}; add {start} / {end}") + raise SystemExit(f"marker pair for {name!r} missing from {page.name}; add {start} / {end}") head = text[: text.index(start) + len(start)] tail = text[text.index(end) :] return f"{head}\n{body}\n{tail}" def main() -> int: - if len(sys.argv) < 2: - raise SystemExit(f"usage: {pathlib.Path(sys.argv[0]).name} ") - data = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")) + argv = sys.argv[1:] + bakeoff_paths: list[str] = [] + if "--bakeoff" in argv: + cut = argv.index("--bakeoff") + bakeoff_paths = argv[cut + 1 :] + argv = argv[:cut] + if not argv: + raise SystemExit(f"usage: {pathlib.Path(sys.argv[0]).name} [--bakeoff ...]") + data = json.loads(pathlib.Path(argv[0]).read_text(encoding="utf-8")) cells = data["cells"] baselines = data.get("tabular_baselines") or [] @@ -155,6 +230,22 @@ def main() -> int: text = replace_region(text, "conditioning", conditioning_table(cells)) text = replace_region(text, "per_group", per_group_table(cells)) text = replace_region(text, "cost", cost_table(cells)) + if bakeoff_paths: + runs = [] + for raw_path in bakeoff_paths: + bakeoff = json.loads(pathlib.Path(raw_path).read_text(encoding="utf-8")) + # `contaminate` is null in a run that trained on data containing anomalies, and 0.0 in one + # that trained on the clean split. Label from the file rather than from argument order, so a + # mislabelled column cannot outlive a typo on the command line. + contaminate = bakeoff.get("contaminate") + label = "clean training split" if contaminate == 0.0 else "contaminated training" + runs.append((label, bakeoff["results"])) + text = replace_region(text, "profile", profile_table(runs)) + + guide_text = GUIDE.read_text(encoding="utf-8") + guide_text = replace_region(guide_text, "profile", guide_profile_table(runs), GUIDE) + GUIDE.write_text(guide_text, encoding="utf-8") + print(f"refreshed the profile table in {GUIDE}") if baselines: text = replace_region(text, "tabular", tabular_table(baselines)) else: diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 99b1ab81b..6ed427d89 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -43,8 +43,8 @@ Use row anomaly detection to automatically find unusual rows in your data using An unusual banana gets separated in just a few steps, so it sticks out. A normal banana is buried in the crowd and takes many steps to single out. - - A score alone isn't enough. You want to know *why*. SHAP breaks it down: "too brown", "wrong size". So you can act on the insight straight away. + + A score alone isn't enough. You want to know *why*. DQX breaks the score down per column: "too brown", "wrong size". So you can act on the insight straight away. Everything you need to catch unusual rows, no ML expertise required. @@ -143,7 +143,7 @@ DQM and DQX each provide distinct capabilities. Together, they complement one an Each row is scored and enriched with: - **Severity percentile (0–100)**: how unusual the row is compared to training data. - **Anomaly flag**: whether it crosses your chosen score threshold (default 95). You can tune this to control how many alerts you get. -- **Top contributors (explainability)**: which fields most influenced the anomaly score, so you can see *why* a row was flagged. Powered by SHAP, this turns a black-box score into an actionable insight. +- **Top contributors (explainability)**: which fields most influenced the anomaly score, so you can see *why* a row was flagged. This turns a black-box score into an actionable insight. You can tune the threshold and other options later if you need to reduce alert noise or catch more edge cases, but the defaults should work well for most use cases. @@ -267,7 +267,7 @@ However, when you run a row anomaly detection check, DQX also adds a `_dq_info` The info column is an array of structs, with one element per anomaly detection check that was applied. -Feature contributions are on by default and are computed only for anomalous rows (severity at or above the threshold). Non-anomalous rows carry a `null` contributions map, so the SHAP cost scales with the number of anomalies, not the table size. Set `enable_contributions=False` to skip them entirely for the fastest scoring. Severity percentiles are always computed for every row, so you can still count would-be anomalies at other thresholds after scoring; but if you want contributions (and AI explanations) for rows below your enforcement threshold, score with the lower exploration threshold and filter afterwards: +Feature contributions are on by default and are computed only for anomalous rows (severity at or above the threshold). Non-anomalous rows carry a `null` contributions map, so the cost of computing them scales with the number of anomalies, not the table size. Set `enable_contributions=False` to skip them entirely for the fastest scoring. Severity percentiles are always computed for every row, so you can still count would-be anomalies at other thresholds after scoring; but if you want contributions (and AI explanations) for rows below your enforcement threshold, score with the lower exploration threshold and filter afterwards: ```python DQDatasetRule( @@ -276,7 +276,7 @@ DQDatasetRule( check_func_kwargs={ "model_name": "catalog.schema.orders_monitor", # fully qualified name "registry_table": "catalog.schema.dqx_anomaly_models", # fully qualified name - # "enable_contributions": False, # optional: skip SHAP (also disables AI explanations) + # "enable_contributions": False, # optional: skip contributions (also disables AI explanations) } ) ``` @@ -339,6 +339,85 @@ It is not flagged as a violation. Neither categorical encoder can represent an u If an unrecognised group value is itself something you want to fail on, that is a membership question rather than an anomaly one: use `foreign_key` or `is_in_list` on the baseline column against your set of known values. +## Choosing a profile + + + + + +`profile` tells DQX what kind of data you have. It is the one modelling decision DQX asks you to make, +and it exists because a single algorithm cannot cover both cases well. + +| Your data | `profile` | What an anomaly looks like | +|---|---|---| +| Independent records — transactions, orders, customers, events | `"tabular"` (default) | A row whose *values*, or whose combination of values, is unusual | +| Multivariate metrics — machine telemetry, service metrics, sensor readings | `"timeseries"` | Metrics that normally move **together** stop doing so, while each one stays inside its usual range | + +```python +anomaly_engine.train( + df, + model_name="catalog.schema.fleet_model", + registry_table="catalog.schema.dqx_anomaly_models", + profile="timeseries", +) +``` + +Nothing else changes: the same automatic feature engineering, the same registry, the same +`has_no_row_anomalies` check, the same contributions and AI explanations. Scoring reads the choice back +off the model, so you never repeat it. + +### Why the choice matters + +The default detector, Isolation Forest, splits on one feature at a time. That is why it is strong on +tabular data and weak when the anomaly *is* a broken relationship: if CPU is normal and memory is +normal, no single-feature split separates the row, even though "high CPU with idle memory" never happens +on a healthy machine. + +Measured on the Server Machine Dataset — real machine telemetry with labelled incidents — this is the +share of incidents each detector surfaces while the alert budget is capped at 1% of rows: + + +| `profile` | Incidents surfaced | +|---|---| +| `"tabular"` | 33% | +| `"timeseries"` | **79%** | + + +That is telemetry, which is what `"timeseries"` is for — on ordinary tabular data the ranking reverses. +See [Anomaly detection quality](/docs/reference/anomaly_detection_quality#which-detector-to-use) for both +detectors measured on both kinds of data, and what this metric counts. + + +DQX does not detect which profile you need. Getting it right cannot be verified without labelled +anomalies, which an unsupervised tool does not have. The one cheap signal — serial correlation between +consecutive rows — was measured and rejected: it is produced just as readily by *sorted storage*, so on +ordinary tabular tables loaded in batches it fires constantly. Rather than guess, DQX defaults to +`"tabular"` and logs the resolved profile on every training run. + + +### What `"timeseries"` does and does not need + +- **No timestamp column.** It models correlation *between* metrics, not behaviour over time. Rows may + arrive in any order. +- **No ensemble.** The detector is deterministic, so `ensemble_size` is ignored and `confidence_std` is + unavailable — averaging identical models would cost N times as much and report a spread of zero. +- **Enough rows.** It estimates how the metrics co-vary, which needs many more rows than features. + Training warns when the sample is thin relative to the feature count. + +### What neither profile covers + +Being straight about the edges is more useful than a longer feature list: + +- **Seasonality and trend.** Neither profile models "Sunday is always quiet" or "traffic has grown 40% + this quarter". A predictable weekly cycle will read as unusual until it is in the training data, and + gradual growth eventually reads as drift. Use `baseline_by` for the periodic case where the cycle is a + column you have, and Databricks Data Quality Monitoring for volume and freshness over time. +- **Forecasting.** DQX judges rows against learned normal; it does not predict the next value and compare. +- **Single-series anomaly detection.** With one metric and no others to correlate against, `"timeseries"` + has nothing to model. A rule or a threshold is the right tool. +- **Labelled anomaly classification.** If you have labels, train a classifier — it will beat any + unsupervised detector on the pattern it was taught. + ## Upgrading and breaking changes Row anomaly detection was **Experimental** in earlier releases and is now **Beta**. This release replaces the old per-group model path (`segment_by`) with `baseline_by`, which measured better on detection, reliability, and cost. If you never used `segment_by`, nothing here affects you. @@ -356,14 +435,16 @@ For full parameter and schema details, see [Row Anomaly Detection in Quality Che ### Architecture overview 1. **Feature engineering** (automatic): DQX detects column types and creates features (numerical standardized, categorical one-hot or frequency-encoded, temporal expanded to hour/day/month/weekend). -2. **Smart sampling and training**: DQX samples your data (default 30%, capped at 1M rows), trains an ensemble of Isolation Forest models, and captures baseline statistics for drift detection. +2. **Smart sampling and training**: DQX samples your data (default 30%, capped at 1M rows), trains the detector your `profile` selects — an ensemble of Isolation Forest models by default, or a single correlation-aware model for `profile="timeseries"` — and captures baseline statistics for drift detection. 3. **Model registry**: Models and metadata live in MLflow and a Delta table; each `train()` call registers one model under its name and archives the previously active version. -4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. SHAP contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. +4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. Feature contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. How they are computed depends on the detector — SHAP for Isolation Forest, an exact leave-one-out decomposition for the correlation-aware detector — but the output is the same map either way. 5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping (categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group, for example region or product category), and this happens whether or not you passed `columns`, since what to measure and what to compare it against are independent questions. A discovered grouping is used as `baseline_by` (see [Group-aware anomaly detection](#group-aware-anomaly-detection)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. -### Why Isolation Forest? +### Which algorithm, and why + +**Isolation Forest** (`profile="tabular"`, the default) measures how "easy" it is to isolate a data point; anomalies are isolated in few splits, normal points need many. It is fast, handles mixed types, is robust to noise, and explains itself through SHAP. DQX uses it by default because it fits data quality use cases without tuning. -Isolation Forest measures how "easy" it is to isolate a data point; anomalies are isolated in few splits, normal points need many. It is fast, handles mixed types, is robust to noise, and is explainable via SHAP. DQX uses it by default because it fits data quality use cases without tuning. +**A correlation-aware detector** (`profile="timeseries"`) measures how far a row sits from normal *once the relationships between metrics are accounted for* — a distance in a space where the metrics have been decorrelated. That is exactly the case Isolation Forest is weakest on, because a broken relationship between two in-range values cannot be separated by splitting either one. It explains itself by leaving each feature out in turn and reporting how much of the anomaly disappears, so it needs no SHAP. See [Choosing a profile](#choosing-a-profile). ### Output structure and options @@ -408,7 +489,7 @@ The nested `ai_explanation` struct (populated when AI explanations are on, which ### AI explanations -AI explanations are **on by default**: each anomalous row gets a plain-language, LLM-generated explanation in `_dq_info[0].anomaly.ai_explanation` answering *why was this flagged, what's the impact, and what should I do*, without anyone reading raw SHAP percentages. Set `enable_ai_explanation=False` to turn it off (or `enable_contributions=False`, which disables explanations too, since they use the SHAP contributions as input). The explanation is AI-generated from the anomaly signal (feature names + SHAP + severity); it is **not** grounded in your catalog's table/column descriptions, so treat the business-impact and action as a starting point, not authoritative. +AI explanations are **on by default**: each anomalous row gets a plain-language, LLM-generated explanation in `_dq_info[0].anomaly.ai_explanation` answering *why was this flagged, what's the impact, and what should I do*, without anyone reading raw contribution percentages. Set `enable_ai_explanation=False` to turn it off (or `enable_contributions=False`, which disables explanations too, since they use the contributions as input). The explanation is AI-generated from the anomaly signal (feature names + contributions + severity); it is **not** grounded in your catalog's table/column descriptions, so treat the business-impact and action as a starting point, not authoritative. The call runs inside Spark via the SQL `ai_query` function against a Databricks Model Serving endpoint, so it needs no extra setup and scales with the cluster. This requires **Databricks serverless compute or Databricks Runtime 15.4 LTS or above** (where `ai_query` is available); on older runtimes explanations are skipped with a warning and scoring still completes. Similar anomalous rows are grouped together and the model is called **once per group** rather than once per row, so cost stays predictable on large datasets. @@ -500,7 +581,7 @@ When you train without specifying a grouping, DQX looks for one: columns that lo 1. Train model on historical batch data 2. Apply checks to streaming DataFrame -3. **Recommended**: Disable contributions (default): `enable_contributions=False` (SHAP is too slow for real-time processing) +3. **Recommended**: Disable contributions (default): `enable_contributions=False` (computing them is too slow for real-time processing)
@@ -514,7 +595,7 @@ When you train without specifying a grouping, DQX looks for one: columns that lo - There are no external API calls. **Consider**: -- SHAP contributions may expose sensitive patterns in explanations. +- Feature contributions may expose sensitive patterns in explanations. - Model metadata includes column names and statistics. - Use column-level security for model registry if needed. @@ -533,7 +614,7 @@ When you train without specifying a grouping, DQX looks for one: columns that lo **DQX Row Anomaly Detection**: - Row-level anomaly scores - Cross-column pattern detection -- Per-row explanations (SHAP contributions) +- Per-row explanations (feature contributions) - Can be applied together with rule-based checks **Use both**: Data Quality Monitoring for table health + DQX for row-level issues inside the data. diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index 66668a873..86f020cce 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -6,6 +6,8 @@ sidebar_position: 509 --- +import Admonition from '@theme/Admonition'; + # Anomaly Detection Quality How well does row anomaly detection actually detect? This page reports measured detection quality; @@ -53,10 +55,68 @@ chance. Conditioned on the group, 0.6962. To compare against the whole table anyway, pass `baseline_by=[]`. +## Which detector to use + +DQX ships two detectors, chosen with `profile`. They answer different questions, so neither replaces +the other: + +| `profile` | detector | finds | +|---|---|---| +| `"tabular"` (default) | Isolation Forest | rows whose values, or combination of values, are unusual | +| `"timeseries"` | correlation-aware | metrics that normally move together and stopped | + +The second exists because the first cannot see the second kind of problem. Isolation Forest splits on +one feature at a time, so a row where every metric sits inside its usual range — but in a combination +that never happens on healthy data — never gets separated by any single split. + +The **Server Machine Dataset** is exactly that kind of data — real machine telemetry with labelled +incidents — so it shows the gap at its widest. Read the table as *which tool for which job*: it says +`"timeseries"` is the right choice for telemetry, not that either detector is good or bad in general. On +ordinary tabular data the ranking reverses, which is what the +[tabular benchmarks](#plain-tabular-benchmarks) below measure. + + +| `profile` | detector | incidents surfaced (clean training split) | incidents surfaced (contaminated training) | +|---|---|---|---| +| `"tabular"` | Isolation Forest | 36% | 33% | +| `"timeseries"` | correlation-aware | **82%** | **79%** | + + +Two columns, because they answer different questions. **Contaminated training is the one that describes +your run**: DQX fits a random sample of your table with the anomalies still in it. The clean-split column +is what a benchmark with a hand-curated training set would report — an upper bound you would not see in +practice. The correlation-aware detector holds up across both, which is the part worth noting: covariance +estimates are sensitive to exactly the extreme rows they are meant to find, so this was the result that +could have ruled the approach out. + +### What "incidents surfaced" means + +Of the labelled incidents in the data, the share that produce **at least one alert while the alert budget +is capped at 1% of all rows**. + +It counts *problems that reach a human* at an alert volume they will tolerate. Counting anomalous rows +instead would let a single long incident dominate the result, and would reward a detector that alerts on +everything. Fixing the budget prevents both. + +It is deliberately **not** point-adjusted F1, the metric most published time-series results use. Point +adjustment credits an entire incident as detected from a single lucky row, and Kim et al. +([AAAI 2022](https://arxiv.org/abs/2109.05257)) showed that random scores reach state-of-the-art under +it. Published figures produced that way are not comparable with the ones above, in either direction. + + +Neither profile models seasonality, trend, or forecasting: a predictable weekly cycle reads as unusual +until it is in the training data, and steady growth eventually reads as drift. With a *single* metric, +`"timeseries"` has nothing to correlate against, and a threshold is the better tool. DQX also does not +detect which profile you need — [Choosing a +profile](/docs/guide/row_anomaly_detection#choosing-a-profile) explains why the obvious heuristic was +measured and rejected. + + ## One model, not one per group -The legacy `segment_by` trains a separate model per group. Conditioning expresses the grouping as -features on a single model instead, and wins on both axes. +The `segment_by` path, removed in this release, trained a separate model per group. Conditioning +expresses the grouping as features on a single model instead, and won on both axes — which is why the +comparison is recorded here rather than left to memory. | your anomalies are... | median ΔPR-AUC vs one-model-per-group | diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index ad31f8873..e923b7a13 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -3496,7 +3496,8 @@ Required arguments: `df`, `model_name`, `registry_table`. Optional parameters: | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `columns` | list[str] | None | Columns to use for row anomaly detection (auto-discovered if omitted) | -| `segment_by` | list[str] | None | Train separate models per segment. When both `columns` and `segment_by` are omitted, DQX may auto-discover segment columns (e.g. categorical, 2–50 distinct values) and train a segmented model. Use `segment_by=[]` to force a single global model. | +| `baseline_by` | list[str] | None | Columns identifying the group a row belongs to, so each metric is judged against its own group's baseline rather than against the whole table. One model whatever the group count. Auto-discovered when both `columns` and `baseline_by` are omitted; pass `baseline_by=[]` to suppress discovery and compare against the whole table. | +| `profile` | str | `"tabular"` | Which detector to train. `"tabular"` is Isolation Forest — the behaviour that predates this option. `"timeseries"` selects a correlation-aware detector for multivariate metrics whose anomalies are broken *correlations* rather than extreme single values; it needs no timestamp column and trains a single model rather than an ensemble. There is no automatic option: DQX never changes the algorithm on your behalf, because the choice cannot be verified without labels. See [Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile). | | `exclude_columns` | list[str] | None | Columns to exclude from training (e.g., IDs, labels, ground truth) | | `expected_anomaly_rate` | float | 0.02 | Expected fraction of anomalies in your data (0.02 = 2%). Sets model contamination parameter. | | `params` | AnomalyParams | None | Optional. Advanced tuning parameters. See sections below for details. | @@ -3512,9 +3513,8 @@ Pass an `AnomalyParams` object to the `params` argument to customize training be | `sample_fraction` | float | 0.3 | Fraction of data to sample for training (30%). Reduce for faster training on large datasets. | | `max_rows` | int | 1,000,000 | Maximum rows to use for training. Caps memory usage for very large datasets. | | `train_ratio` | float | 0.8 | Train/validation split ratio (80% train, 20% validation). | -| `ensemble_size` | int or None | 3 | Number of models in ensemble. Set to None for single model. More models = more robust but slower training. Applies to a single global model only: **segmented training always trains one model per segment and ignores this setting**, so `confidence_std` is unavailable for segmented models. | +| `ensemble_size` | int or None | 3 | Number of models in ensemble. Set to None for single model. More models = more robust but slower training. Ignored by `profile="timeseries"`: that detector is deterministic, so every ensemble member would be an identical model and the reported spread would be exactly zero. `confidence_std` is therefore unavailable for it. | | `baseline_by` | list[str] or None | None | Columns identifying the group a row belongs to, so each metric is judged against its own group's baseline rather than against the whole table. Adds one feature per metric — its signed log-ratio to that group's median — on a **single** model, so cost does not grow with the group count. Normally set by passing `baseline_by` to `train()`. See [Group-aware anomaly detection](/docs/guide/row_anomaly_detection#group-aware-anomaly-detection). | -| `max_segment_models` | int | 50 | Ceiling on per-segment models one run will attempt, guarding the legacy `segment_by` path. Cost there is linear in the segment count and segmented training does not ensemble, so 90 segments measures roughly 88 minutes. Exceeding this raises rather than warns. Irrelevant to `baseline_by`. | #### IsolationForestConfig (Algorithm Parameters) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index 147aa823c..7d778f6bb 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -88,8 +88,9 @@ def train( ``"tabular"`` -- IsolationForest, exactly the behaviour before this option existed. ``"timeseries"`` selects a correlation-aware detector suited to multivariate metrics, where anomalies are broken correlations rather than extreme single values; measured on - the SMD benchmark it surfaces 82% of incidents inside a 1%-of-rows alert budget against - 36% for the tabular detector. It needs no timestamp column, and trains a single model + the SMD benchmark it surfaces 79% of incidents inside a 1%-of-rows alert budget against + 33% for the tabular detector, both trained on data that still contains anomalies as DQX + does (82% against 36% on a clean training split). It needs no timestamp column, and trains a single model rather than an ensemble because it is deterministic. There is no automatic option: DQX never changes the algorithm on your behalf, because the choice cannot be verified without labels. The resolved profile is logged on every run. diff --git a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py index dc46862bd..8366791ce 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py @@ -12,7 +12,7 @@ # Schema for the AI explanation sub-struct inside the anomaly info struct. # narrative / business_impact / action are LLM-generated; top_features is deterministic -# (the sorted top-2 contributing SHAP features that define the group, as engineered names, so it +# (the sorted top-2 contributing features that define the group, as engineered names, so it # stays stable for grouping and tooling). top_drivers is the same drivers rendered as human labels # with their weights, e.g. 'amount vs its group baseline (74%), quantity (12%)', for display. # group_size / group_avg_severity describe the pattern group this row belongs to. diff --git a/src/databricks/labs/dqx/anomaly/check_funcs.py b/src/databricks/labs/dqx/anomaly/check_funcs.py index 11fb8b162..7680c54e1 100644 --- a/src/databricks/labs/dqx/anomaly/check_funcs.py +++ b/src/databricks/labs/dqx/anomaly/check_funcs.py @@ -73,20 +73,20 @@ def _validate_thresholds(threshold: float, drift_threshold: float | None) -> Non def _validate_explanation_flags(enable_contributions: bool) -> None: if enable_contributions and not SHAP_AVAILABLE: raise InvalidParameterError( - "enable_contributions=True requires the 'shap' dependency. " - "Install anomaly extras: pip install databricks-labs-dqx[anomaly]" + "enable_contributions=True requires the 'shap' dependency for the default tabular " + "detector. Install anomaly extras: pip install databricks-labs-dqx[anomaly]" ) def _resolve_ai_explanation_flag(enable_contributions: bool, enable_ai_explanation: bool) -> bool: - """AI explanations use SHAP contributions as their input, so they require + """AI explanations use the feature contributions as their input, so they require *enable_contributions*. Both default to True; if a caller turns contributions off (e.g. to - skip the SHAP cost) we disable explanations with a warning rather than raising, so the cheap + skip the attribution cost) we disable explanations with a warning rather than raising, so the cheap opt-out stays frictionless. """ if enable_ai_explanation and not enable_contributions: logger.warning( - "AI explanations require SHAP contributions; disabling enable_ai_explanation because " + "AI explanations require feature contributions; disabling enable_ai_explanation because " "enable_contributions=False." ) return False @@ -144,7 +144,7 @@ def has_no_row_anomalies( - _dq_info[0].anomaly.is_anomaly: Boolean flag - _dq_info[0].anomaly.threshold: Severity percentile threshold used (0–100) - _dq_info[0].anomaly.model: Model name - - _dq_info[0].anomaly.contributions: SHAP contributions as percentages (0–100); populated + - _dq_info[0].anomaly.contributions: feature contributions as percentages (0–100); populated only for anomalous rows, null otherwise - _dq_info[0].anomaly.confidence_std: Ensemble std (if requested) - _dq_info[0].anomaly.is_new_baseline: True when the row's group was absent from training, @@ -178,11 +178,13 @@ def has_no_row_anomalies( drift_threshold: Drift detection threshold, in standard deviations of the training baseline (default None, which disables drift detection). Set a positive value such as 3.0 to enable it. - enable_contributions: Include SHAP feature contributions for explainability (default True). + enable_contributions: Include per-feature contributions for explainability (default True). Per-feature contributions are added to _dq_info for anomalous rows only (severity at or - above the threshold; other rows get a null map), so the SHAP cost scales with the number - of anomalies rather than the table size. Requires the SHAP library (installed with the - anomaly extra). Set False to skip the SHAP cost entirely (this also disables AI + above the threshold; other rows get a null map), so the attribution cost scales with the + number of anomalies rather than the table size. How they are computed depends on the + detector: SHAP for the default tabular one (installed with the anomaly extra), an exact + leave-one-out decomposition for the timeseries one, which needs no SHAP at all. The emitted + map is identical either way. Set False to skip the cost entirely (this also disables AI explanations, since they use contributions as input). enable_confidence_std: Include ensemble confidence scores in _dq_info and top-level (default False). Automatically available when training with ensemble_size > 1 (default is 3). @@ -193,7 +195,7 @@ def has_no_row_anomalies( endpoint is unreachable (e.g. no Foundation Model APIs in the workspace), explanations are skipped with a warning and scoring still completes. Output is in _dq_info[0].anomaly.ai_explanation, and is AI-generated from the anomaly signal (feature - names + SHAP + severity), not grounded in catalog metadata. + names + contributions + severity), not grounded in catalog metadata. ai_explanation_llm_model_config: LLM model configuration for AI explanations (named distinctly from the check's *model_name* to avoid confusion). Defaults to LLMModelConfig() (model_name='databricks/databricks-claude-sonnet-4-5'). Its *model_name* @@ -211,7 +213,7 @@ def has_no_row_anomalies( LLMModelConfig instance is accepted. The simplest dict form sets only *model_name* to a Databricks Model Serving endpoint. See the AI Explanations section of the Row Anomaly Detection reference docs for a full example. - redact_columns: Column names to exclude from the LLM prompt. Filters SHAP contribution + redact_columns: Column names to exclude from the LLM prompt. Filters the contribution map keys, the top-2 pattern key, and — when the scored model is segmented — any matching segment key (emitted as ``key=`` so sensitive segmentation values never reach the prompt). @@ -234,7 +236,7 @@ def has_no_row_anomalies( >>> df_scored.filter(col("_dq_info").getItem(0).getField("anomaly").getField("is_anomaly")) """ llm_model_config = _coerce_llm_model_config(ai_explanation_llm_model_config) - # AI explanations need SHAP contributions; if contributions are off, disable explanations + # AI explanations need the contributions map; if contributions are off, disable explanations # (with a warning) rather than failing — both default on, so this only triggers when a caller # explicitly opts out of contributions. enable_ai_explanation = _resolve_ai_explanation_flag(enable_contributions, enable_ai_explanation) diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py index 2b9858b6d..51d942627 100644 --- a/src/databricks/labs/dqx/anomaly/timeseries_detector.py +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -4,7 +4,10 @@ data and weak on multivariate metrics whose anomalies are *broken correlations* rather than extreme single values. Measured on SMD (28 machines, 38 metrics, fit on the train split and scored on the test split, no point adjustment) it catches 36% of incidents inside a 1%-of-rows alert budget; the -Mahalanobis detector here catches 82%. See ``benchmarks/anomaly_conditioning/smd_bakeoff.py`` and the +Mahalanobis detector here catches 82%. Refitting on training data that still contains anomalies -- what +DQX actually does -- costs both of them a few points and does not change the conclusion: 33% against +79%. That was the result that could have sunk the approach, because sample covariance is not robust and +a few extreme rows inflate it along the very direction that needs to stay tight. See ``benchmarks/anomaly_conditioning/smd_bakeoff.py`` and the committed results next to it. The distance is the ordinary squared Mahalanobis distance from the training centre, diff --git a/src/databricks/labs/dqx/anomaly/training_strategies.py b/src/databricks/labs/dqx/anomaly/training_strategies.py index 42b670113..5f6ca7ecf 100644 --- a/src/databricks/labs/dqx/anomaly/training_strategies.py +++ b/src/databricks/labs/dqx/anomaly/training_strategies.py @@ -156,7 +156,9 @@ class MahalanobisTrainingStrategy(AnomalyTrainingStrategy): estimator differs. See ``timeseries_detector`` for why: IsolationForest splits one feature at a time, so anomalies that are broken *correlations* rather than extreme single values are close to invisible to it. Measured on SMD, incident coverage inside a 1%-of-rows alert budget is 0.359 for - IsolationForest and 0.821 here. + IsolationForest and 0.821 here on a clean training split, and 0.333 against 0.795 when the training + data itself contains anomalies -- which is the case that matters, because DQX fits a random sample of + the user's table rather than a curated one. """ name = "mahalanobis" From 9778ee10c704b63e1a307501d4ea7f2287f5cf62 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 16:03:16 +0100 Subject: [PATCH 043/107] Add two deck slides: comparing like with like, and correlation breaking The banana deck told one story -- features, then Isolation Forest, then contributions -- and after this PR that story is missing the two things a reader most needs to decide anything. One slide each. ## Slide 5: "Normal for whom? Compare like with like." `baseline_by` shipped earlier in this PR and the deck never mentioned it, which left the deck teaching whole-table comparison as if it were the only option. The visual makes the argument with a single number rather than a diagram. One 18cm banana arrives and is judged three times: normal for a Cavendish (16-22cm), odd for a lady finger (10-14cm), and small for a plantain (22-30cm). Across the whole crate the range is 10-30cm, so 18cm looks unremarkable and a model comparing against the whole crate never flags it. That is the entire case for conditioning, and it needs no notion of statistics to follow. Placed after `row-level`, because it refines the claim that slide just made about what "normal" means. ## Slide 10: "When nothing is odd, but the combination is." The visual argument for why a second detector exists. Crate weight tracks banana count, because bananas have a weight -- until one crate holds 33 bananas and weighs 1.6kg. Both numbers are ordinary in isolation; only the relationship is broken. The two bars alternate between "they move together" and the flagged crate, so the reader sees the pattern before the exception. This is the honest way to justify `profile="timeseries"`: it shows the shape of anomaly Isolation Forest cannot reach by splitting one feature at a time, rather than asserting that one algorithm is better. Placed after `isolation-forest`, so it reads as the limitation of the mechanism just explained. ## Two things worth knowing before editing this deck again **A `visual` key with no `case` renders an empty slide, not an error.** `SlideVisuals` dispatches through a `switch` with no default, so a typo between the `VisualKey` union, the `` prop and the `case` costs you a blank panel that no build catches. Verified all three by hand, then confirmed both components and their keys reached the client bundle -- the deck renders only the *current* slide, so grepping the static HTML finds slide 1 and nothing else, and cannot verify slides 5 or 10. **An incremental docs build can serve stale CSS.** The first build after adding these styles produced a stylesheet with `bv-iso` present and `bv-baseline` absent, from webpack's PackFileCacheStrategy. The source file was correct the whole time. A CSS change therefore needs `rm -rf docs/dqx/.docusaurus docs/dqx/node_modules/.cache docs/dqx/build` before the build means anything -- and a stale-CSS failure is invisible: unstyled elements still render, just wrong. Slide 9's title was reworded from "SHAP explains" to "Contributions explain" in the previous commit, where the rest of that rename lives. Verified: `make docs-build` SUCCESS after a full cache clear, 156 documents, no broken links; both components, both `case` keys and both CSS blocks present in the built bundles; the slide bodies' inline code renders as `` rather than literal backticks. Deck is now 13 slides. `npx tsc --noEmit` reports only the four pre-existing `Cannot find namespace 'JSX'` errors in `FeatureTags.tsx`, none in `SlideVisuals.tsx`. pylint 10.00/10. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 6 + docs/dqx/src/components/SlideVisuals.tsx | 114 +++++++++++++++++- docs/dqx/src/css/custom.css | 89 ++++++++++++++ 3 files changed, 208 insertions(+), 1 deletion(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 6ed427d89..e217a4f54 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -37,12 +37,18 @@ Use row anomaly detection to automatically find unusual rows in your data using DQX learns what "normal" looks like from your good data, then checks every single row. No labels needed; it figures out what's unusual on its own. *"Is this banana weird?"* + + A 30cm plantain is perfectly normal; a 30cm lady finger does not exist. `baseline_by` judges each row against **its own group's** normal instead of the whole table's, so a value that is unremarkable overall can still be flagged where it does not belong. + Each banana becomes a set of numbers: size, colour, spots, bend. An Isolation Forest then tries to separate each point from the rest. If a banana is easy to isolate, it's probably odd. An unusual banana gets separated in just a few steps, so it sticks out. A normal banana is buried in the crowd and takes many steps to single out. + + Some problems are not extreme values at all — they are a *relationship* breaking. Count and weight always move together, until a crate holds 33 bananas and weighs 1.6kg. Splitting one number at a time cannot see that, so `profile="timeseries"` switches to a detector that models how metrics move together. + A score alone isn't enough. You want to know *why*. DQX breaks the score down per column: "too brown", "wrong size". So you can act on the insight straight away. diff --git a/docs/dqx/src/components/SlideVisuals.tsx b/docs/dqx/src/components/SlideVisuals.tsx index a56a99c5d..e3f285b08 100644 --- a/docs/dqx/src/components/SlideVisuals.tsx +++ b/docs/dqx/src/components/SlideVisuals.tsx @@ -5,7 +5,7 @@ import type { Token, RenderProps } from 'prism-react-renderer'; export type VisualKey = | 'title' | 'known-unknowns' | 'dqx-rules' | 'trucks-dqm' | 'gap' | 'row-level' | 'features' | 'isolation-forest' - | 'shap' | 'summary' | 'coming-soon'; + | 'baseline-groups' | 'correlation' | 'shap' | 'summary' | 'coming-soon'; // ── Data ──────────────────────────────────────────────────────────── @@ -49,6 +49,29 @@ const TRUCK_CONFIGS: Array<{ bananaCount: number; label: string; isAnomaly: bool const BANANA_FEATURES = ['size', 'colour', 'shape', 'flavour', 'smell'] as const; +// Baseline conditioning: one arriving size, judged three times. 18cm is unremarkable across the whole +// crate -- it sits inside the combined range -- and clearly wrong for a lady finger. That is the whole +// argument for `baseline_by` in one number. +const ARRIVING_SIZE_CM = 18; +const BANANA_VARIETIES: Array<{ name: string; emoji: string; scale: number; low: number; high: number }> = [ + { name: 'Plantain', emoji: '🍌', scale: 1.3, low: 22, high: 30 }, + { name: 'Cavendish', emoji: '🍌', scale: 1.0, low: 16, high: 22 }, + { name: 'Lady finger', emoji: '🍌', scale: 0.72, low: 10, high: 14 }, +]; + +// Correlation break: crate weight normally tracks the banana count, because bananas have a weight. +// The last crate keeps both numbers inside their usual ranges and breaks the relationship between them. +const CRATE_READINGS: Array<{ count: number; weight: number; broken?: boolean }> = [ + { count: 30, weight: 3.6 }, + { count: 34, weight: 4.1 }, + { count: 28, weight: 3.4 }, + { count: 36, weight: 4.3 }, + { count: 31, weight: 3.7 }, + { count: 35, weight: 4.2 }, + { count: 29, weight: 3.5 }, + { count: 33, weight: 1.6, broken: true }, +]; + const DQX_RULES_CODE = `from databricks.labs.dqx.rule import DQRowRule from databricks.labs.dqx.check_funcs import is_not_null, is_in_range, is_in_list @@ -560,6 +583,93 @@ function ComingSoonSlide() { // ── Export ─────────────────────────────────────────────────────────── +function BaselineGroups() { + const [activeIdx, setActiveIdx] = useState(BANANA_VARIETIES.length - 1); + useEffect(() => { + const id = setInterval(() => setActiveIdx(i => (i + 1) % BANANA_VARIETIES.length), 2200); + return () => clearInterval(id); + }, []); + + const combinedLow = Math.min(...BANANA_VARIETIES.map(v => v.low)); + const combinedHigh = Math.max(...BANANA_VARIETIES.map(v => v.high)); + + return ( +
+

+ A {ARRIVING_SIZE_CM}cm banana arrives. Is it odd? +

+
+ {BANANA_VARIETIES.map((variety, i) => { + const isOdd = ARRIVING_SIZE_CM < variety.low || ARRIVING_SIZE_CM > variety.high; + const isActive = i === activeIdx; + return ( +
+ + {variety.emoji} + + {variety.name} + + usually {variety.low}–{variety.high}cm + + {isOdd ? 'odd here' : 'normal here'} +
+ ); + })} +
+

+ Across the whole crate: {combinedLow}–{combinedHigh}cm — so {ARRIVING_SIZE_CM}cm looks fine, and a + model comparing against the whole crate never flags it. +

+
+ ); +} + +function CorrelationBreak() { + const [revealed, setRevealed] = useState(false); + useEffect(() => { + const id = setInterval(() => setRevealed(r => !r), 2600); + return () => clearInterval(id); + }, []); + + const maxCount = Math.max(...CRATE_READINGS.map(r => r.count)); + const maxWeight = Math.max(...CRATE_READINGS.map(r => r.weight)); + + return ( +
+
+ {CRATE_READINGS.map((reading, i) => ( +
+ + +
+ ))} +
+
+ bananas counted + crate weight +
+

+ {revealed ? ( + <> + The last crate: 33 bananas, 1.6kg. Both numbers are ordinary on their own — and + 33 bananas have never weighed 1.6kg. + + ) : ( + <>Count and weight always rise and fall together. Until one crate stops. + )} +

+
+ ); +} + export default function SlideVisuals({ visual }: { visual: VisualKey }) { switch (visual) { case 'title': return ; @@ -570,6 +680,8 @@ export default function SlideVisuals({ visual }: { visual: VisualKey }) { case 'row-level': return ; case 'features': return ; case 'isolation-forest': return ; + case 'baseline-groups': return ; + case 'correlation': return ; case 'shap': return ; case 'summary': return ; case 'coming-soon': return ; diff --git a/docs/dqx/src/css/custom.css b/docs/dqx/src/css/custom.css index abbc392ca..528bcc7ee 100644 --- a/docs/dqx/src/css/custom.css +++ b/docs/dqx/src/css/custom.css @@ -702,6 +702,95 @@ button { .bv-iso__crowd { display: flex; flex-wrap: wrap; gap: 0.2rem; justify-content: center; font-size: 1.5rem; } .bv-iso__vs { font-size: 1.25rem; color: #a8a29e; } +/* ── Baseline conditioning (`baseline_by`) ───────────────────────── */ +.bv-baseline { margin-top: 0.75rem; } +.bv-baseline__intro { font-size: 0.9rem; color: #44403c; margin-bottom: 0.6rem; text-align: center; } +.bv-baseline__groups { + display: flex; + gap: 0.75rem; + justify-content: center; + flex-wrap: wrap; +} +.bv-baseline__card { + display: flex; + flex-direction: column; + align-items: center; + gap: 0.15rem; + padding: 0.6rem 0.75rem; + border-radius: 0.5rem; + min-width: 8rem; + /* Transition only the properties the active cycle changes, so the border colour set by the + --odd / --normal modifiers is not animated on first paint. */ + transition: transform 0.4s ease, box-shadow 0.4s ease; +} +.bv-baseline__card--normal { background: rgba(220, 252, 231, 0.8); border: 2px solid rgba(74, 222, 128, 0.5); } +.bv-baseline__card--odd { background: rgba(254, 243, 199, 0.8); border: 2px solid rgba(251, 191, 36, 0.6); } +.bv-baseline__card--active { transform: translateY(-3px); box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1); } +.bv-baseline__emoji { line-height: 1.2; } +.bv-baseline__name { font-size: 0.85rem; color: #292524; } +.bv-baseline__range { font-size: 0.72rem; color: #57534e; } +.bv-baseline__verdict { font-size: 0.75rem; font-weight: 600; margin-top: 0.15rem; } +.bv-baseline__card--normal .bv-baseline__verdict { color: #166534; } +.bv-baseline__card--odd .bv-baseline__verdict { color: #92400e; } +.bv-baseline__whole { + margin-top: 0.7rem; + font-size: 0.8rem; + color: #57534e; + text-align: center; +} + +/* ── Correlation break (`profile="timeseries"`) ──────────────────── */ +.bv-corr { margin-top: 0.75rem; } +.bv-corr__chart { + display: flex; + align-items: flex-end; + gap: 0.5rem; + height: 7rem; + padding: 0 0.25rem; + /* A narrow viewport must scroll the chart, not the page. */ + overflow-x: auto; +} +.bv-corr__col { + display: flex; + align-items: flex-end; + gap: 2px; + flex: 1 1 0; + min-width: 1.75rem; + height: 100%; + padding: 0.15rem; + border-radius: 0.25rem; + transition: background 0.4s ease; +} +.bv-corr__col--flagged { background: rgba(254, 243, 199, 0.9); outline: 2px solid rgba(251, 191, 36, 0.6); } +.bv-corr__bar { flex: 1; border-radius: 2px 2px 0 0; transition: height 0.4s ease; } +.bv-corr__bar--count { background: #a8a29e; } +.bv-corr__bar--weight { background: #fbbf24; } +.bv-corr__legend { + display: flex; + gap: 1rem; + justify-content: center; + margin-top: 0.5rem; + font-size: 0.75rem; + color: #57534e; +} +.bv-corr__key { display: inline-flex; align-items: center; gap: 0.3rem; } +.bv-corr__key::before { + content: ''; + width: 0.6rem; + height: 0.6rem; + border-radius: 2px; +} +.bv-corr__key--count::before { background: #a8a29e; } +.bv-corr__key--weight::before { background: #fbbf24; } +.bv-corr__caption { + margin-top: 0.6rem; + font-size: 0.82rem; + color: #44403c; + text-align: center; + /* Both captions are close in length; a fixed floor stops the slide jumping as they swap. */ + min-height: 2.6em; +} + /* ── SHAP carousel ────────────────────────────────────────────────── */ .bv-shap { margin-top: 0.75rem; width: 100%; } .bv-shap__subtitle { font-size: 0.8rem; color: #78716c; text-align: center; margin: 0 0 0.5rem; } From 966e833efb5dbee8172ad5d40ffdb67279aa6105 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 27 Aug 2026 16:08:41 +0100 Subject: [PATCH 044/107] Repair the row anomaly demo, which this PR broke, and verify it by running it `demos/dqx_row_anomaly_detection_demo.py` raised `AnalysisException` about a third of the way through on this branch, leaving every cell after it dead. The flagship demo for the feature was left behind by the `segment_by` removal earlier in this PR: demos/dqx_row_anomaly_detection_demo.py:257 "segmentation.segment_by" column gone demos/dqx_row_anomaly_detection_demo.py:258 "segmentation.segment_values" column gone demos/dqx_row_anomaly_detection_demo.py:705 documents `segment_by` as a live parameter `ANOMALY_MODEL_TABLE_SCHEMA` (model_registry.py:32) declares `grouping struct` now; the `segmentation` struct is gone. So the `display(spark.table(registry_table).select(...))` could not resolve, and a documented BREAKING CHANGE shipped without its demo being updated. Nothing catches this: `grep -rn demos Makefile .github/workflows/*.yml` returns nothing. No CI job and no make target has ever executed a demo notebook, so the only thing keeping them correct is someone noticing. ## Fixed - The registry `display` selects `grouping.baseline_by`. - The `segment_by` parameter entry is deleted; the `baseline_by` entry above it already describes the replacement correctly. `profile` is documented in its place. - The `shap` line in the extras list says it covers tree-based models, since the timeseries detector computes its own contributions. - The threshold summary said "threshold 95 flags the top 5% most unusual records", and the demo then reported 9.3%. Both are right and the sentence was wrong: severity is a percentile of the *training* distribution, and this data has anomalies injected while the training data did not, so more than 5% of new rows exceeding it is the intended lesson rather than an error. Said so explicitly. ## Verified by running it, not by reading it Built a wheel from this branch, uploaded it to a UC volume, and ran the notebook as a serverless job through the demo's own `test_library_ref` widget -- so it exercised *this* code rather than the released package, which a `%pip install databricks-labs-dqx[anomaly]` would have pulled instead: job 174820891805646, run 153244550281680, task run 482661657570726 TERMINATED / SUCCESS, 34 cells, ~7 min on serverless catalog dqx, schema dqx_demo_verify Then read every cell's rendered output out of the run export, because an exit code says only that nothing raised -- it says nothing about a cell that prints a header and an empty table, which is how these break in practice. What the cells actually produced: - **The formerly-broken registry cell** returns one row: `['dqx.dqx_demo_verify.sales_auto', ['amount','quantity','date'], ['region','category'], 1191, '2026-08-27T14:56:25Z', 'active']` -- `grouping.baseline_by` populated, which is the fix confirmed against a real registry table rather than against the schema constant. - **Auto-discovery** found `['amount','quantity','date']` plus a baseline grouping of `['region','category']` (20 groups, ~250 rows/group) and logged `profile=tabular -> algorithm strategy 'isolation_forest'` -- the new profile log line reading correctly on the default path. - **Detection**: 1,000 new rows, 24 injected anomalies, **24 caught (100% recall)**, 93 flagged at threshold 95; the sweep reports 136 / 94 / 58 rows at thresholds 90 / 95 / 98. - **Contributions** are populated and show conditioning working: `{amount: 1.2, quantity: 34.5, amount_rel_baseline: 22.3, quantity_rel_baseline: 41.9}` -- the group-relative features carry most of the weight, which is what `baseline_by` is for. - **AI explanations** are populated with real narratives, and they read the baseline features: *"40 rows are driven by quantity (31%) and quantity sitting far above its region-category baseline (25%)"*. That is the whole chain -- engineered baseline feature to attribution to LLM prompt to narrative -- working on a live workspace. - Both further models (manual columns, contributions) trained and scored; 3-model ensembles registered in Unity Catalog as expected. Zero cells errored. Nothing printed an empty table. Co-authored-by: Isaac --- demos/dqx_row_anomaly_detection_demo.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/demos/dqx_row_anomaly_detection_demo.py b/demos/dqx_row_anomaly_detection_demo.py index 1087e2e3a..acdb7d495 100644 --- a/demos/dqx_row_anomaly_detection_demo.py +++ b/demos/dqx_row_anomaly_detection_demo.py @@ -53,7 +53,8 @@ # MAGIC **What's included in `[anomaly]` extras:** # MAGIC - `scikit-learn` - Machine learning algorithms used for row anomaly detection # MAGIC - `mlflow` - Model tracking and registry -# MAGIC - `shap` - Feature contributions for explainability +# MAGIC - `shap` - Feature contributions for tree-based models (the `timeseries` detector computes its +# MAGIC own contributions and needs no SHAP) # MAGIC - `cloudpickle` - Model serialization # MAGIC # MAGIC **Note**: If you are using ML Runtime or Serverless compute, most dependencies are already pre-installed. @@ -253,9 +254,8 @@ def generate_historical_sales_data( .filter(F.col("identity.model_name").contains(model_name_auto)) .select( "identity.model_name", - "training.columns", - "segmentation.segment_by", - "segmentation.segment_values", + "training.columns", + "grouping.baseline_by", "training.training_rows", "training.training_time", "identity.status" @@ -417,7 +417,9 @@ def inject_anomalies_and_dq_issues( print("\n💡 Summary:") print(" • We trained on historical data and applied checks on new data.") -print(" • Default threshold 95 flags the top 5% most unusual records.") +print(" • Default threshold 95 flags rows above the 95th percentile of the *training* data.") +print(" • So more than 5% of new rows can be flagged — that is the point: this data has") +print(" anomalies injected into it, and the training data did not.") print(" • Threshold is a percentile cutoff — tune it based on your data and alert tolerance.") # COMMAND ---------- @@ -702,7 +704,9 @@ def inject_anomalies_and_dq_issues( # MAGIC - `baseline_by` (list[str]): columns identifying a row's group, so each metric is judged # MAGIC against its own group's baseline rather than the whole table — catches values that are # MAGIC ordinary globally but wrong in context. One model, whatever the group count. -# MAGIC - `segment_by` (list[str]): legacy, trains one model per group; prefer `baseline_by` +# MAGIC - `profile` (str): which detector to train — `"tabular"` (the default: Isolation Forest) or +# MAGIC `"timeseries"` for multivariate metrics whose anomalies are broken correlations rather than +# MAGIC extreme single values. No timestamp column needed. DQX never picks this for you. # MAGIC - `sample_fraction`, `max_rows`: training sample controls # MAGIC - `ensemble_size`: number of models in the ensemble # MAGIC - `expected_anomaly_rate`: expected anomaly rate for calibration From e4fed0adf5402c7c9343d1107e6af234edb7d40f Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 10:23:15 +0100 Subject: [PATCH 045/107] Correct a false claim: DQX does model calendar seasonality The previous commit's "what neither detector does" wording said neither profile models seasonality, and illustrated it with "Sunday is always quiet". That is wrong, and it understated the feature in the one place a reader goes to decide whether to adopt it. `_process_datetime_columns` (transformers.py:671-713) derives **seven** features per datetime column: hour_sin, hour_cos, dow_sin, dow_cos, month_sin, month_cos, is_weekend. So "quiet on Sundays" is exactly what dow_sin/dow_cos/is_weekend encode -- the model learns the joint distribution of phase and value, so a low volume on a Sunday is normal *for a Sunday* while the same volume on a Wednesday still stands out. The sine/cosine pairing is what makes it work: it puts 23:00 next to 00:00 instead of maximally far apart, which a raw hour integer would not. Verified against the model the demo trained on a live workspace rather than from the source alone -- `features.feature_metadata.engineered_feature_names` reads: date_hour_sin, date_hour_cos, date_dow_sin, date_dow_cos, date_month_sin, date_month_cos, date_is_weekend, amount, quantity, amount_rel_baseline, quantity_rel_baseline Seven of eleven features are calendar features. And the demo run already demonstrated them working: its training data is business-hours weekdays only (weekend dates are moved to Friday, hours drawn from 9am-6pm), one injected anomaly type is "off-hours + large spike" at hours 2/3/4/22/23, and the run caught 24 of 24 injected anomalies. That anomaly's off-hours signal is only visible through the hour cyclicals. ## Where the boundary actually is **Trend** is genuinely not modelled, and for a specific reason worth recording: the raw datetime column is **dropped** after the cyclical features are extracted (transformers.py:712), and nothing monotonic replaces it. A datetime therefore contributes only *where in the cycle* a row sits, never *how far along* it is. So sustained growth drifts away from the training baseline with no feature able to represent it -- which is what `drift_threshold` is for, and the honest remedy is a retrain. **Cycles outside hour/day-of-week/month** are also not covered: a six-week promotional cycle, or a fiscal quarter that does not align to calendar months, has no matching encoding. The remedy is a column plus `baseline_by`, which judges each row against its own phase -- so the guidance is now actionable instead of being a flat "not supported". Forecasting and single-series detection were correct as stated and are unchanged. ## Both pages The guide gains a **"What is handled automatically"** subsection, placed *before* the limitations, because the previous ordering implied absence by omission -- a reader met the gaps without ever being told what works. It names calendar seasonality, group context and categorical/null handling, all of which need no configuration. The reference page's admonition now leads with what is handled and then narrows to trend and non-calendar periods. Verified: `make docs-build` SUCCESS after a full cache clear, 156 documents, no broken links, and the new `#what-is-handled-automatically` anchor resolves from both references to it. pylint 10.00/10. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 26 ++++++++++++++++--- .../reference/anomaly_detection_quality.mdx | 10 ++++--- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index e217a4f54..506b6d49b 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -410,14 +410,32 @@ ordinary tabular tables loaded in batches it fires constantly. Rather than guess - **Enough rows.** It estimates how the metrics co-vary, which needs many more rows than features. Training warns when the sample is thin relative to the feature count. +### What is handled automatically + +Both profiles get the same feature engineering, so these need no configuration: + +- **Calendar seasonality.** Include a date or timestamp column and DQX derives cyclical hour-of-day, + day-of-week and month features plus a weekend flag — seven features per datetime column, encoded as + sine/cosine pairs so that 23:00 and 00:00 are adjacent rather than maximally far apart. A quiet Sunday + is therefore learned as normal *for a Sunday*, and the same volume on a Wednesday still stands out. +- **Group context.** `baseline_by` judges each metric against its own group's baseline; see + [Group-aware anomaly detection](#group-aware-anomaly-detection). +- **Categoricals, booleans and nulls.** One-hot or frequency encoding by cardinality, 0/1 mapping, and an + explicit indicator for columns that contain nulls. + ### What neither profile covers Being straight about the edges is more useful than a longer feature list: -- **Seasonality and trend.** Neither profile models "Sunday is always quiet" or "traffic has grown 40% - this quarter". A predictable weekly cycle will read as unusual until it is in the training data, and - gradual growth eventually reads as drift. Use `baseline_by` for the periodic case where the cycle is a - column you have, and Databricks Data Quality Monitoring for volume and freshness over time. +- **Trend.** Neither profile models "traffic has grown 40% this quarter". Nothing in the feature set + represents elapsed time — a datetime column contributes only *where in the cycle* a row sits, never + *how far along* it is — so sustained growth gradually drifts away from the training baseline. Enable + `drift_threshold` to be told when that has happened, and retrain. For volume and freshness over time, + Databricks Data Quality Monitoring is the right tool. +- **Cycles that are not hourly, daily, weekly, or monthly.** The calendar features cover those (see + [What is handled automatically](#what-is-handled-automatically) below). A six-week promotional cycle, or + a fiscal quarter that does not align to calendar months, is not one of them — pass the cycle as a + column and use `baseline_by`, which judges each row against its own phase of the cycle. - **Forecasting.** DQX judges rows against learned normal; it does not predict the next value and compare. - **Single-series anomaly detection.** With one metric and no others to correlate against, `"timeseries"` has nothing to model. A rule or a threshold is the right tool. diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index 86f020cce..7562c07af 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -104,10 +104,12 @@ adjustment credits an entire incident as detected from a single lucky row, and K it. Published figures produced that way are not comparable with the ones above, in either direction. -Neither profile models seasonality, trend, or forecasting: a predictable weekly cycle reads as unusual -until it is in the training data, and steady growth eventually reads as drift. With a *single* metric, -`"timeseries"` has nothing to correlate against, and a threshold is the better tool. DQX also does not -detect which profile you need — [Choosing a +Calendar seasonality *is* handled, by both profiles: a datetime column becomes cyclical hour-of-day, +day-of-week and month features plus a weekend flag, so "quiet on Sundays" is learned rather than flagged. +What is **not** modelled is **trend** — no feature represents elapsed time, so steady growth eventually +reads as drift and calls for a retrain — nor cycles whose period is not one of those three, nor +forecasting the next value. With a *single* metric, `"timeseries"` has nothing to correlate against and a +threshold is the better tool. DQX also does not detect which profile you need — [Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile) explains why the obvious heuristic was measured and rejected. From 7d438deb9a53198f3844f9b912bdf6bd4ff934d9 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 10:35:53 +0100 Subject: [PATCH 046/107] Withdraw a false safety claim about trend, and measure why neither fix works The previous commit told users that steady growth "reads as drift and calls for a retrain" -- i.e. that `drift_threshold` would warn them. It does not. That is a false safety guarantee, which is worse than saying nothing, because a user who relies on it is not warned and does not know they were not warned. Prompted by the question of whether the trend limitation should be fixed or dropped. It turns out to be neither: it is real, it is not cheaply fixable, and the guidance attached to it was wrong. ## Both obvious fixes were measured, and both fail **Give the model an elapsed-time feature so it can learn the slope.** Makes false flagging *worse*: 95.8% -> 100.0% of a batch that merely continues the trend. New rows carry time values outside the training range, isolating an out-of-range value takes very few splits, and that is exactly what Isolation Forest scores as anomalous. The feature meant to explain the growth away becomes the strongest evidence of anomaly. **Condition on a time bucket via `baseline_by`.** Structurally impossible, and I had this wrong for a while: an intermediate test that computed group medians *from the scored batch* showed conditioning fixing the trend completely (95.8% -> 0.0% false flags). DQX does not work that way. It persists a median per group at training time and looks it up at scoring time; a row whose group was absent from training has its score, severity and contributions nulled (`scoring_utils._null_unseen_group_scores`). Every future time bucket is by definition absent from training, so such a model would score nothing at all. Checking the scoring path rather than trusting the local result is what caught this. ## And the safety net cannot fire `drift_threshold=3.0` is the documented setting. The drift score saturates below 1.8 however steep the trend: trend over window false flags drift score warns at 3.0? 0% 3.0% 0.04 no (correct) 10% 19.6% 1.17 NO 40% 80.2% 1.68 NO 200% 99.2% 1.74 NO At 200% growth, 99.2% of a batch in which nothing is wrong is flagged and drift detection stays silent. The reason is mechanical rather than a matter of tuning: `_compute_column_drift_score` divides the batch's mean shift by the *training window's* standard deviation, and a linear ramp inflates that standard deviation in proportion to the growth. Numerator and denominator scale together, so the ratio is bounded no matter how steep the slope. No threshold rescues it. That is a pre-existing property of drift detection, not something this PR introduced, and fixing it needs a trend-aware statistic rather than a z-score against an inflated baseline. Out of scope here, and deliberately recorded rather than quietly worked around. ## What the docs say now The limitation stays -- it is real -- but it becomes actionable instead of a bare confession, which is the only version worth keeping: - **Model a quantity that does not trend.** `orders_per_customer` rather than `daily_orders`, `revenue_per_order` rather than `cumulative_revenue`. A rate or a ratio stays comparable as the business grows, and is usually what the user actually wanted to watch. - **Where the level itself matters, retrain on a schedule** rather than waiting to be told, *because* the drift signal is suppressed for exactly this failure. The reason is stated, so nobody re-derives it. `benchmarks/anomaly_conditioning/trend_limits.py` carries the measurement and both rejected fixes, following the `profile_advisory_gate.py` precedent: a measurement that decided a documentation claim belongs in the tree, not only in a commit message. Verified: `make docs-build` SUCCESS after a full cache clear, 156 documents, no broken links; the withdrawn claim appears nowhere in the built output. pylint 10.00/10. Co-authored-by: Isaac --- .../anomaly_conditioning/trend_limits.py | 138 ++++++++++++++++++ .../guide/row_anomaly_detection/index.mdx | 13 +- .../reference/anomaly_detection_quality.mdx | 14 +- 3 files changed, 156 insertions(+), 9 deletions(-) create mode 100644 benchmarks/anomaly_conditioning/trend_limits.py diff --git a/benchmarks/anomaly_conditioning/trend_limits.py b/benchmarks/anomaly_conditioning/trend_limits.py new file mode 100644 index 000000000..ab6d868bf --- /dev/null +++ b/benchmarks/anomaly_conditioning/trend_limits.py @@ -0,0 +1,138 @@ +"""Why trend is a documented limitation, and why `drift_threshold` is not the safety net for it. + +Row anomaly detection learns "normal" from a window and compares later rows against it. A steadily +growing metric therefore leaves that window, and ordinary rows start being flagged. This module measures +how quickly that happens, and tests the two fixes that suggest themselves -- both of which fail. **It is +not wired into DQX**; it exists so the documented limitation stays measured rather than re-argued. + +## Fix 1: give the model an elapsed-time feature so it can learn the slope + +Measured: it makes false flagging *worse*, not better -- 95.8% -> 100.0% on a batch that simply continues +the trend. New rows carry time values outside the training range, and isolating an out-of-range value +takes very few splits, which is precisely what Isolation Forest scores as anomalous. The feature intended +to explain the growth away becomes the strongest evidence of anomaly. + +## Fix 2: condition on a time bucket with `baseline_by` + +Cannot work, for a structural reason rather than a numerical one. `baseline_by` persists a median per +group at training time and looks it up while scoring; a row whose group was absent from training is +reported via ``is_new_baseline`` and has its score, severity and contributions **nulled** +(``scoring_utils._null_unseen_group_scores``). Every future time bucket is by definition absent from +training, so a model conditioned on one would score nothing at all. The null-on-unseen behaviour is +correct for categorical groups -- an unseen category cannot be judged honestly -- which is exactly why a +time bucket is the wrong thing to condition on. + +## So the remaining question: does drift detection warn before scoring degrades? + +No, and it cannot. `drift_threshold=3.0` is the documented setting; the drift score saturates below 1.8 +however steep the trend, because ``_compute_column_drift_score`` divides the batch's mean shift by the +*training window's* standard deviation, and a linear ramp inflates that standard deviation in proportion +to the growth. Both numerator and denominator scale together, so the ratio is bounded: + + trend over window false flags drift score warns at 3.0? + 0% 3.0% 0.04 no (correct) + 4% 5.8% 0.60 NO + 10% 19.6% 1.17 NO + 20% 45.4% 1.53 NO + 40% 80.2% 1.68 NO + 100% 95.8% 1.74 NO + 200% 99.2% 1.74 NO + +At 200% growth, 99.2% of a batch in which nothing is wrong is flagged, and drift detection stays silent. + +## What the documentation says because of this + +Not "enable drift_threshold and you will be warned" -- that would be a false safety guarantee, and a user +relying on it gets burned without notice. Instead: model a quantity that does not trend (a rate or a +ratio, not a running level), and where the level itself matters, retrain on a schedule. + +Fixing drift detection for this case is a separate change -- it needs a trend-aware statistic rather than +a z-score against an inflated baseline -- and is deliberately not attempted here. + +Run: uv run python benchmarks/anomaly_conditioning/trend_limits.py +""" + +import numpy as np +from sklearn.ensemble import IsolationForest + +N_TRAIN = 2000 +N_SCORE = 500 +CONTAMINATION = 0.02 +NOISE = 3.0 +# The setting the user guide documents, and what DQX compares its max-across-columns score against. +DRIFT_THRESHOLD = 3.0 +# Trend slopes per row, spanning "flat" to "the metric tripled across the training window". +SLOPES = (0.0, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1) + + +def _series(slope: float, start: int, count: int, rng: np.random.Generator) -> np.ndarray: + """A metric growing at *slope* per row, with constant noise.""" + t = np.arange(start, start + count, dtype=float) + return 100.0 + slope * t + rng.normal(0, NOISE, count) + + +def false_flag_rate(train: np.ndarray, score: np.ndarray) -> float: + """Share of a nothing-is-wrong batch that gets flagged. The correct answer is ~CONTAMINATION.""" + model = IsolationForest(contamination=CONTAMINATION, random_state=42).fit(train.reshape(-1, 1)) + return float((model.predict(score.reshape(-1, 1)) == -1).mean()) + + +def drift_score(train: np.ndarray, score: np.ndarray) -> float: + """DQX's per-column drift score, mirroring ``drift._compute_column_drift_score``.""" + baseline_mean, baseline_std = float(train.mean()), float(train.std()) + current_mean, current_std = float(score.mean()), float(score.std()) + if baseline_std == 0: + return abs(current_mean - baseline_mean) + z_score = abs(current_mean - baseline_mean) / baseline_std + std_change = abs(current_std - baseline_std) / baseline_std + return (z_score * 0.7) + (std_change * 0.3) + + +def elapsed_time_feature_makes_it_worse() -> tuple[float, float]: + """Fix 1, measured: adding a monotonic time feature raises the false-flag rate.""" + rng = np.random.default_rng(42) + t_train = np.arange(N_TRAIN, dtype=float) + metric_train = _series(0.05, 0, N_TRAIN, rng) + t_score = np.arange(N_TRAIN, N_TRAIN + N_SCORE, dtype=float) + metric_score = _series(0.05, N_TRAIN, N_SCORE, rng) + + without = false_flag_rate(metric_train, metric_score) + + model = IsolationForest(contamination=CONTAMINATION, random_state=42).fit(np.column_stack([metric_train, t_train])) + with_time = float((model.predict(np.column_stack([metric_score, t_score])) == -1).mean()) + return without, with_time + + +def main() -> None: + print("Does a trend break scoring, and does drift detection warn?\n") + print(f"{'trend over window':>18} {'false flags':>12} {'drift score':>12} warns?") + print("-" * 60) + for slope in SLOPES: + rng = np.random.default_rng(42) + train = _series(slope, 0, N_TRAIN, rng) + score = _series(slope, N_TRAIN, N_SCORE, rng) + flags = false_flag_rate(train, score) + drift = drift_score(train, score) + # A flat series *should* stay silent, so only a missed warning on a real trend is a failure. + if drift >= DRIFT_THRESHOLD: + warns = "yes" + else: + warns = "NO" if slope > 0 else "no (correct)" + print(f"{slope * N_TRAIN:>17.0f}% {flags:>11.1%} {drift:>12.2f} {warns}") + + print("\nfalse flags = share of a batch that merely continues the trend and is flagged anomalous") + print(f" (the correct answer is ~{CONTAMINATION:.0%}, the contamination rate)") + + without, with_time = elapsed_time_feature_makes_it_worse() + print("\nFix 1 -- add an elapsed-time feature so the model can learn the slope:") + print(f" metric only {without:6.1%} falsely flagged") + print(f" metric + elapsed time {with_time:6.1%} falsely flagged <- worse, not better") + + print("\nFix 2 -- condition on a time bucket via baseline_by:") + print(" Not measurable, and not viable: every future bucket is an unseen group, whose score,") + print(" severity and contributions are nulled. Such a model would score nothing. See the") + print(" module docstring.") + + +if __name__ == "__main__": + main() diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 506b6d49b..564ea99b0 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -429,9 +429,16 @@ Being straight about the edges is more useful than a longer feature list: - **Trend.** Neither profile models "traffic has grown 40% this quarter". Nothing in the feature set represents elapsed time — a datetime column contributes only *where in the cycle* a row sits, never - *how far along* it is — so sustained growth gradually drifts away from the training baseline. Enable - `drift_threshold` to be told when that has happened, and retrain. For volume and freshness over time, - Databricks Data Quality Monitoring is the right tool. + *how far along* it is — so a steadily growing metric eventually sits outside the range it was trained + on, and ordinary rows start being flagged. + + **What to do:** give the model a quantity that does not trend. `orders_per_customer` instead of + `daily_orders`, `revenue_per_order` instead of `cumulative_revenue` — a rate or a ratio stays + comparable as the business grows, and is usually the thing you actually wanted to watch. Where the + level itself matters, retrain on a schedule rather than waiting to be told: a gradual trend inflates + the spread of the training window it is measured against, which suppresses the drift signal, so + `drift_threshold` is not a reliable tripwire for this particular failure. For volume and freshness over + time, Databricks Data Quality Monitoring is the tool built for it. - **Cycles that are not hourly, daily, weekly, or monthly.** The calendar features cover those (see [What is handled automatically](#what-is-handled-automatically) below). A six-week promotional cycle, or a fiscal quarter that does not align to calendar months, is not one of them — pass the cycle as a diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index 7562c07af..e4c3053ef 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -106,12 +106,14 @@ it. Published figures produced that way are not comparable with the ones above, Calendar seasonality *is* handled, by both profiles: a datetime column becomes cyclical hour-of-day, day-of-week and month features plus a weekend flag, so "quiet on Sundays" is learned rather than flagged. -What is **not** modelled is **trend** — no feature represents elapsed time, so steady growth eventually -reads as drift and calls for a retrain — nor cycles whose period is not one of those three, nor -forecasting the next value. With a *single* metric, `"timeseries"` has nothing to correlate against and a -threshold is the better tool. DQX also does not detect which profile you need — [Choosing a -profile](/docs/guide/row_anomaly_detection#choosing-a-profile) explains why the obvious heuristic was -measured and rejected. +Cycles whose period is not one of those three are not, and neither is **trend** — a steadily growing +metric leaves the range it was trained on, and DQX has no feature representing elapsed time to explain the +growth away. Feed it a quantity that does not trend (a rate or a ratio rather than a running level), and +retrain on a schedule; see [Trend](/docs/guide/row_anomaly_detection#what-neither-profile-covers) for what +to do. Forecasting the next value is out of scope. With a *single* metric, `"timeseries"` has nothing to +correlate against and a threshold is the better tool. DQX also does not detect which profile you need — +[Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile) explains why the obvious +heuristic was measured and rejected. ## One model, not one per group From d286d0815de702081017b43145d9d47425903ba1 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 10:44:32 +0100 Subject: [PATCH 047/107] Measure the fix for trend that does work: persist a fitted trend and subtract it The previous commit concluded trend was not cheaply fixable after two fixes failed. That was one fix short. A third works, nearly perfectly, and this records it so the limitation reads as "not yet built" rather than "cannot be done". ## Why the first two failed points straight at it An elapsed-time *feature* fails because the model must learn the slope from data covering only the training range, then meet values outside it. A time-bucket `baseline_by` fails because a median **lookup table** has no entry for a future bucket. A *fitted* trend has neither problem: it is a function of time, so it extrapolates to any future t, and the model never sees time at all -- it sees the residual, which is stationary by construction. Which makes it the same shape as the `_rel_baseline` feature this PR already ships: observed value minus its expected level. Only the source of the expected level changes, from a per-group median to a fitted line. Same append-at-tail rule, same persistence path through `SparkFeatureMetadata`, whose from_json is already unknown-key tolerant, so slope/intercept/reference-epoch are additive. ## Measured trend over window raw detrended (correct answer ~2%) 10% 19.6% 2.8% 40% 80.2% 2.8% 100% 95.8% 2.8% 200% 99.2% 2.8% Flat at 2.8% however steep the trend. And it does not blunt real detection: on a batch carrying a 3x spike in 5% of rows both reach 100% recall, but the raw feature emits **96.4%** false positives against the detrended feature's **2.1%**. ## Four things a design has to answer, all measured rather than guessed **Extrapolation horizon.** Accuracy decays with distance past the training window -- 2.8% at the boundary, 3.4% one window out, 6.4% five out, 89.6% twenty-five out. So it needs a horizon cap and a warning past it, the same shape of contract as `is_new_baseline`. **Regime change.** When the trend itself changes, 70-85% of rows flag. Arguably correct, since growth stalling *is* an anomaly, but reporting a table-level event row by row is not useful. Worth noting detrending is the only one that notices at all: when growth *reverses*, the raw feature flags just 6.0%, because falling values look like a return to trained levels, while detrended flags 85.2%. **Functional form.** Exponential growth fitted with a straight line barely helps -- 95.8% against a raw 99.8%. Business metrics compound, so a log-scale fit is needed and the form becomes a choice, not a default. **API cost.** It needs a time column, and the current design deliberately requires none: `"timeseries"` models cross-metric correlation, not time. This would add the first temporal parameter to the public surface, which is a decision about the shape of the API rather than an implementation detail. ## Not implemented here, and why that is a scope call rather than a verdict This PR already carries a breaking removal, baseline conditioning, a second detector behind `profile`, the attribution rework, the invariant pins, docs and demos. A temporal transform brings a new public parameter, a new persisted field, an extrapolation contract, a functional-form choice and its own failure modes -- and it is a new capability, not a repair to anything this PR changed. The user-facing guidance is unaffected either way: model a quantity that does not trend, and where the level matters, retrain on a schedule. That advice costs nothing and remains correct whether or not the transform is built. No `src/` change. The docs are untouched: they describe what ships, and what ships has no trend handling. Co-authored-by: Isaac --- .../anomaly_conditioning/trend_limits.py | 100 ++++++++++++++++-- 1 file changed, 94 insertions(+), 6 deletions(-) diff --git a/benchmarks/anomaly_conditioning/trend_limits.py b/benchmarks/anomaly_conditioning/trend_limits.py index ab6d868bf..9ae7265bb 100644 --- a/benchmarks/anomaly_conditioning/trend_limits.py +++ b/benchmarks/anomaly_conditioning/trend_limits.py @@ -40,14 +40,56 @@ At 200% growth, 99.2% of a batch in which nothing is wrong is flagged, and drift detection stays silent. -## What the documentation says because of this +## Fix 3: persist a fitted trend and subtract it -- this one works -Not "enable drift_threshold and you will be warned" -- that would be a false safety guarantee, and a user -relying on it gets burned without notice. Instead: model a quantity that does not trend (a rate or a -ratio, not a running level), and where the level itself matters, retrain on a schedule. +Both failures above share a cause, and it points straight at the fix. An elapsed-time *feature* fails +because the model must learn the slope from data covering only the training range, then meet values outside +it. A time-bucket `baseline_by` fails because a median **lookup table** has no entry for a future bucket. A +fitted trend has neither problem: it is a *function* of time, so it extrapolates to any future t, and the +model never sees time at all -- it sees the residual, which is stationary by construction. -Fixing drift detection for this case is a separate change -- it needs a trend-aware statistic rather than -a z-score against an inflated baseline -- and is deliberately not attempted here. +That is exactly the shape of the existing `_rel_baseline` feature -- observed value minus its expected +level. Only the source of the expected level changes, from a per-group median to a fitted line. + + trend over window raw detrended (correct answer ~2%) + 10% 19.6% 2.8% + 40% 80.2% 2.8% + 100% 95.8% 2.8% + 200% 99.2% 2.8% + +Flat at 2.8% however steep the trend, and it still finds real anomalies -- on a batch carrying a 3x spike +in 5% of rows, both score 100% recall, but the raw feature emits 96.4% false positives against the +detrended feature's 2.1%. + +### Where it breaks, which is the part a design has to answer + +**Extrapolation horizon.** Accuracy decays with distance beyond the training window: 2.8% at the boundary, +3.4% one window out, 6.4% five windows out, 89.6% twenty-five windows out. Usable, but it needs a horizon +cap and a warning past it -- the same shape of contract as ``is_new_baseline``. + +**Regime change.** When the trend itself changes, residuals blow up and 70-85% of rows flag. Arguably +correct -- growth stalling *is* an anomaly -- but reporting a table-level event row by row is not useful. +Note that detrending is the only one of the two that notices: when growth *reverses*, the raw feature +flags just 6.0% because falling values look like a return to trained levels, while the detrended feature +flags 85.2%. + +**Functional form.** Exponential growth fitted with a straight line barely helps: 95.8% against a raw +99.8%. Business metrics compound, so a log-scale fit would be needed, which makes the form a choice rather +than a default. + +**API cost.** It needs a time column. The current design deliberately requires none -- `"timeseries"` +models cross-metric correlation, not time -- so this would add the first temporal parameter to the public +surface. + +## What the documentation says today + +The docs describe what ships, and what ships has no trend handling. So: model a quantity that does not +trend (a rate or a ratio, not a running level), and where the level itself matters, retrain on a schedule. +What they must *not* say is "enable drift_threshold and you will be warned", which was the earlier claim +and is measurably false. + +Neither the detrending transform nor a trend-aware drift statistic is attempted here. Both are real, +scoped follow-ups rather than impossibilities, and this module exists so that stays clear. Run: uv run python benchmarks/anomaly_conditioning/trend_limits.py """ @@ -103,6 +145,38 @@ def elapsed_time_feature_makes_it_worse() -> tuple[float, float]: return without, with_time +def fit_trend(t: np.ndarray, values: np.ndarray) -> tuple[float, float]: + """Least-squares slope and intercept -- what a training run would persist.""" + slope, intercept = np.polyfit(t, values, 1) + return float(slope), float(intercept) + + +def detrended(t: np.ndarray, values: np.ndarray, slope: float, intercept: float) -> np.ndarray: + """The proposed feature: observed value minus the trend's expectation at this row's time.""" + return values - (intercept + slope * t) + + +def detrending_fixes_it(slope: float, gap: int = 0) -> tuple[float, float]: + """Fix 3, measured: raw versus detrended false-flag rate on a batch where nothing is wrong. + + *gap* pushes the scored window further past the end of training, which is how the extrapolation + horizon is measured. + """ + rng = np.random.default_rng(42) + t_train = np.arange(N_TRAIN, dtype=float) + train = _series(slope, 0, N_TRAIN, rng) + t_score = np.arange(N_TRAIN + gap, N_TRAIN + gap + N_SCORE, dtype=float) + score = _series(slope, N_TRAIN + gap, N_SCORE, rng) + + raw = false_flag_rate(train, score) + fitted_slope, intercept = fit_trend(t_train, train) + residual_rate = false_flag_rate( + detrended(t_train, train, fitted_slope, intercept), + detrended(t_score, score, fitted_slope, intercept), + ) + return raw, residual_rate + + def main() -> None: print("Does a trend break scoring, and does drift detection warn?\n") print(f"{'trend over window':>18} {'false flags':>12} {'drift score':>12} warns?") @@ -133,6 +207,20 @@ def main() -> None: print(" severity and contributions are nulled. Such a model would score nothing. See the") print(" module docstring.") + print("\nFix 3 -- persist a fitted trend and subtract it (not implemented; this is the proposal):") + print(f" {'trend over window':>18} {'raw':>10} {'detrended':>12}") + for slope in (0.005, 0.02, 0.05, 0.1): + raw, residual_rate = detrending_fixes_it(slope) + print(f" {slope * N_TRAIN:>17.0f}% {raw:>10.1%} {residual_rate:>12.1%}") + + print("\n Extrapolation horizon, at 100% trend -- accuracy decays with distance past training:") + for gap in (0, N_TRAIN, N_TRAIN * 5, N_TRAIN * 25): + _, residual_rate = detrending_fixes_it(0.05, gap=gap) + windows = gap / N_TRAIN + print(f" {windows:>17.0f} windows out {residual_rate:>10.1%}") + print("\n So it works, and needs a horizon cap. See the module docstring for the other three") + print(" design questions it raises (regime change, functional form, and needing a time column).") + if __name__ == "__main__": main() From 16e94adc8939889f6f1cc01e1ebaed4558ed9c40 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 11:06:32 +0100 Subject: [PATCH 048/107] Move the limitations out of a warning block and into one FAQ entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `type="warning"` admonition on the quality reference page, plus a bulleted "What neither profile covers" section in the guide, gave the limitations more prominence than they earn. Most readers do not need them, and a warning block reads as an alarm about the feature rather than as guidance — the wrong weight for "here are the shapes of problem this is not for". Nothing factual is dropped or softened. The same content is now one FAQ entry, *Are there anomalies DQX will not find?*, in the section a reader reaches when they have exactly that question. Trend, non-calendar cycles, forecasting, single-metric series and already-labelled data are all still there, each with what to do instead. Discoverable without shouting. The reference page keeps two sentences pointing at the guide, since a reader of the quality page may want the caveats and should not have to guess where they went. Its `Admonition` import went with the block -- nothing else on that page used it. **"What is handled automatically" stays**, and is now the only section of its kind in that part of the guide. It is a positive statement -- calendar seasonality, group context, categorical and null handling -- and it is the correction from two commits ago, where the docs had wrongly claimed seasonality was *not* modelled. Keeping it while the limitations move is the right asymmetry: what DQX does needs a section, what it does not do needs an answer to a question. One dead cross-reference fixed as a consequence: the "Trend" bullet linked to `#what-neither-profile-covers` from the reference page, an anchor that no longer exists. Verified there are no remaining references to it. Verified: `make docs-build` SUCCESS after a full cache clear, 156 documents, no broken links; the FAQ entry renders and the warning block is absent from the built quality page. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 58 ++++++++++--------- .../reference/anomaly_detection_quality.mdx | 17 +----- 2 files changed, 34 insertions(+), 41 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 564ea99b0..d8ee6ff2e 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -419,36 +419,10 @@ Both profiles get the same feature engineering, so these need no configuration: sine/cosine pairs so that 23:00 and 00:00 are adjacent rather than maximally far apart. A quiet Sunday is therefore learned as normal *for a Sunday*, and the same volume on a Wednesday still stands out. - **Group context.** `baseline_by` judges each metric against its own group's baseline; see - [Group-aware anomaly detection](#group-aware-anomaly-detection). + [Group-aware anomaly detection](#group-aware-anomaly-detection) above. - **Categoricals, booleans and nulls.** One-hot or frequency encoding by cardinality, 0/1 mapping, and an explicit indicator for columns that contain nulls. -### What neither profile covers - -Being straight about the edges is more useful than a longer feature list: - -- **Trend.** Neither profile models "traffic has grown 40% this quarter". Nothing in the feature set - represents elapsed time — a datetime column contributes only *where in the cycle* a row sits, never - *how far along* it is — so a steadily growing metric eventually sits outside the range it was trained - on, and ordinary rows start being flagged. - - **What to do:** give the model a quantity that does not trend. `orders_per_customer` instead of - `daily_orders`, `revenue_per_order` instead of `cumulative_revenue` — a rate or a ratio stays - comparable as the business grows, and is usually the thing you actually wanted to watch. Where the - level itself matters, retrain on a schedule rather than waiting to be told: a gradual trend inflates - the spread of the training window it is measured against, which suppresses the drift signal, so - `drift_threshold` is not a reliable tripwire for this particular failure. For volume and freshness over - time, Databricks Data Quality Monitoring is the tool built for it. -- **Cycles that are not hourly, daily, weekly, or monthly.** The calendar features cover those (see - [What is handled automatically](#what-is-handled-automatically) below). A six-week promotional cycle, or - a fiscal quarter that does not align to calendar months, is not one of them — pass the cycle as a - column and use `baseline_by`, which judges each row against its own phase of the cycle. -- **Forecasting.** DQX judges rows against learned normal; it does not predict the next value and compare. -- **Single-series anomaly detection.** With one metric and no others to correlate against, `"timeseries"` - has nothing to model. A rule or a threshold is the right tool. -- **Labelled anomaly classification.** If you have labels, train a classifier — it will beat any - unsupervised detector on the pattern it was taught. - ## Upgrading and breaking changes Row anomaly detection was **Experimental** in earlier releases and is now **Beta**. This release replaces the old per-group model path (`segment_by`) with `baseline_by`, which measured better on detection, reliability, and cost. If you never used `segment_by`, nothing here affects you. @@ -578,6 +552,36 @@ Use row anomaly detection when you want to catch unusual combinations across col `segment_by` has been removed. Replace it with `baseline_by`, which compares each metric against its own group's baseline on a single model instead of training one model per group, and retrain (models from earlier releases no longer load). Row anomaly detection was Experimental in earlier releases, which is what allowed this break, and it is now Beta. See [Upgrading and breaking changes](#upgrading-and-breaking-changes).
+
+Q: Are there anomalies DQX will not find? + +Yes, and they are worth knowing before you rely on it. + +**Trend.** DQX learns what normal looks like from a training window. A metric that grows steadily +eventually sits outside that window, and ordinary rows start being flagged. Give the model a quantity +that does not trend — `orders_per_customer` rather than `daily_orders`, `revenue_per_order` rather than +`cumulative_revenue` — which stays comparable as the business grows and is usually what you wanted to +watch anyway. Where the level itself matters, retrain on a schedule; a gradual trend inflates the spread +of the window it is measured against, so `drift_threshold` will not reliably warn you about this +particular case. + +**Cycles other than hourly, daily, weekly, or monthly.** Those four are handled automatically from a +datetime column. A six-week promotional cycle, or a fiscal quarter that does not align to calendar +months, is not — pass the cycle as a column and use `baseline_by`, which then judges each row against +its own phase. + +**Forecasting.** DQX judges rows against learned normal. It does not predict the next value and compare. + +**A single metric with `profile="timeseries"`.** That detector models how metrics move *together*, so +with only one metric it has nothing to work with. Use a rule or a threshold. + +**Anything you already have labels for.** Train a classifier — it will beat any unsupervised detector on +the pattern it was taught. Anomaly detection is for the problems you cannot describe in advance. + +For volume, freshness and row counts over time, Databricks Data Quality Monitoring is the tool built for +that job, and the two work well together. +
+
Q: How much training data do I really need? diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index e4c3053ef..89ef91844 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -6,8 +6,6 @@ sidebar_position: 509 --- -import Admonition from '@theme/Admonition'; - # Anomaly Detection Quality How well does row anomaly detection actually detect? This page reports measured detection quality; @@ -103,18 +101,9 @@ adjustment credits an entire incident as detected from a single lucky row, and K ([AAAI 2022](https://arxiv.org/abs/2109.05257)) showed that random scores reach state-of-the-art under it. Published figures produced that way are not comparable with the ones above, in either direction. - -Calendar seasonality *is* handled, by both profiles: a datetime column becomes cyclical hour-of-day, -day-of-week and month features plus a weekend flag, so "quiet on Sundays" is learned rather than flagged. -Cycles whose period is not one of those three are not, and neither is **trend** — a steadily growing -metric leaves the range it was trained on, and DQX has no feature representing elapsed time to explain the -growth away. Feed it a quantity that does not trend (a rate or a ratio rather than a running level), and -retrain on a schedule; see [Trend](/docs/guide/row_anomaly_detection#what-neither-profile-covers) for what -to do. Forecasting the next value is out of scope. With a *single* metric, `"timeseries"` has nothing to -correlate against and a threshold is the better tool. DQX also does not detect which profile you need — -[Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile) explains why the obvious -heuristic was measured and rejected. - +DQX does not detect which profile you need, and calendar seasonality is handled automatically by both. +[Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile) covers both points, and the +[FAQ](/docs/guide/row_anomaly_detection#frequently-asked-questions) lists what neither detector finds. ## One model, not one per group From 7e3d0327b4f5e60746e0b4f3ff10830d53cff7a9 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 11:58:12 +0100 Subject: [PATCH 049/107] Tell the LLM what a contribution measures, so explanations stop misdescribing correlation breaks The AI explanation was **wrong** for the correlation-aware detector. Observed on a live workspace, on rows whose every metric sat inside its healthy range: "Abnormal coolant flow and bearing temperature may signal impending equipment failure" "Abnormal motor and spindle readings may signal equipment stress" "Multiple sensor deviations suggest equipment malfunction or calibration drift" action: "Inspect coolant system and bearing sensors" The same scoring run printed the ranges that contradict it: motor_current healthy 10.2-25.4 during incident 14.1-22.4 (inside) coolant_flow healthy 14.0-36.0 during incident 16.6-30.2 (inside) No metric was abnormal. Every value was ordinary and the *relationship between them* had broken -- which is the entire reason this detector exists. So the explanation asserted something the data does not support, and its action sent an operator to inspect a sensor that reads fine. That is worse than no explanation: a false lead is paid for twice, once in wasted time and once in trust. ## Cause The prompt never said which detector produced the numbers. It carries feature_contributions, group_size, severity_range, confidence, baseline_grouping, threshold and drift_summary -- and a per-feature importance from a correlation-aware detector is *identical in shape* to one from a tree. Given no way to tell them apart, the model used the only reading it knew: * tree / tabular -- a high contribution means this feature's own value was unusual (true) * correlation-aware -- a high contribution means this metric left its usual relationship with the others, and its value may sit mid-range (was not conveyed) Same class of error as the signed attribution decomposition rejected earlier in this work: a number that is technically derived being rendered to a user as a claim the data does not support. Caught by reading the generated text rather than by checking it was non-null -- the previous verification confirmed 41 of 41 narratives were present, which was true and insufficient. ## Fix A new `attribution_basis` field, placed **first** in the prompt so the semantics are read before the numbers, carrying a per-algorithm sentence rather than the raw algorithm name -- the model needs to know what a contribution *means*, not an implementation label it would only guess about. The correlation-aware text also carries the explicit prohibition on the observed failure: do not call an individual metric abnormal unless the contributions are concentrated in one. `record.identity.algorithm` was already available at the call site, beside the `is_ensemble` threaded through the same way, so this needed no new plumbing. `ExplanationContext.algorithm` is additive and defaults to None; matching is by prefix so `IsolationForest_Ensemble_3` resolves, and an unknown algorithm falls back to the value-based reading -- the conservative direction, since claiming an extreme value where a relationship broke understates the finding, whereas the reverse invents a relationship claim. ## Verified by re-running and reading the output, both ways Correlation-aware, after: "26 rows show broken relationships between coolant_flow and bearing_temp (42% and 32%), which no longer align as expected with each other or other metrics." impact: "Decoupled coolant and temperature signals may mask equipment stress..." action: "Verify sensor calibration and expected correlations between coolant_flow and bearing_temp." action elsewhere: "Investigate why throughput and spindle_load no longer correlate as expected." No metric is called abnormal, and the action points at the relationship, which is where the problem is. Tabular, unchanged and still correct -- the value framing is preserved exactly where it is true: "37 rows are flagged primarily by item_count (29%) and amount (26%), both unusual relative to their merchant_category baseline, with weekend timing contributing (11%)." The committed prompt snapshot test did its job and failed on the first run; regenerated per its own docstring, and the diff is one added line. Four unit tests cover the distinction, the prefix match for ensembles, and the fallback. Gates: unit 2554 passed, mypy clean, pylint 10.00/10. Co-authored-by: Isaac --- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 52 +++++++++++++++- .../labs/dqx/anomaly/scoring_run.py | 2 +- tests/resources/ai_query_prompt_header.txt | 1 + tests/unit/test_anomaly_llm_explainer.py | 60 +++++++++++++++++++ 4 files changed, 113 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index c3c191d59..804e9e910 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -41,7 +41,45 @@ "'might indicate'. Do not restate the input field names back to the user, and do not invent " "feature names, values, or baseline groups that are not present in the input." ) +# What a contribution means, per detector family. Keyed by the ``ModelIdentity.algorithm`` prefix that is +# persisted in the registry, so a model trained by any version resolves as long as that string is stable. +# The fallback is the value-based reading, which is what every algorithm before the correlation-aware one +# meant and is the safer default: it claims less about relationships than the other way round would. +_ATTRIBUTION_SEMANTICS: tuple[tuple[str, str], ...] = ( + ( + "Mahalanobis", + "relationships between metrics. A high contribution means this metric departed from its usual " + "relationship with the others -- its own value may sit well inside its normal range. Describe the " + "pattern as a broken relationship between metrics, and do NOT call an individual metric abnormal, " + "high, low, or deviating unless the contributions are concentrated in a single metric.", + ), + ( + "IsolationForest", + "individual feature values. A high contribution means this feature's own value was unusual for the " + "rows it was compared against.", + ), +) +_DEFAULT_ATTRIBUTION_SEMANTICS = _ATTRIBUTION_SEMANTICS[-1][1] + + +def attribution_semantics(algorithm: str | None) -> str: + """What a high contribution means for *algorithm*, as a sentence for the prompt. + + Falls back to the value-based reading when the algorithm is unknown or absent, which is both the + historical behaviour and the more conservative claim. + """ + for prefix, meaning in _ATTRIBUTION_SEMANTICS: + if algorithm and algorithm.startswith(prefix): + return meaning + return _DEFAULT_ATTRIBUTION_SEMANTICS + + _PROMPT_INPUT_FIELDS: tuple[tuple[str, str], ...] = ( + ( + "attribution_basis", + "What the feature_contributions below are measuring. Read them accordingly -- this decides " + "whether the pattern is 'these values were extreme' or 'these metrics stopped agreeing'.", + ), ( "feature_contributions", "Mean contributions across the group, already named for a reader, e.g. 'amount vs its " @@ -207,10 +245,18 @@ class ExplanationContext: # as human labels. Optional: a caller that builds the context directly without it falls back to # best-effort redaction of the baseline-relative feature only, and to raw engineered keys. feature_metadata: SparkFeatureMetadata | None = None + # The trained model's algorithm, from ``ModelIdentity.algorithm``. Decides how the prompt tells the + # model to read a contribution -- as an extreme value or as a broken relationship between metrics. + # Optional, and absent means the value-based reading, which is what every algorithm before the + # correlation-aware one meant. + algorithm: str | None = None @classmethod def from_scoring_config( - cls, config: "ScoringConfig", feature_metadata: SparkFeatureMetadata | None = None + cls, + config: "ScoringConfig", + feature_metadata: SparkFeatureMetadata | None = None, + algorithm: str | None = None, ) -> "ExplanationContext": return cls( severity_col=config.severity_col, @@ -224,6 +270,7 @@ def from_scoring_config( redact_columns=tuple(config.redact_columns or ()), pattern_col=config.pattern_col, feature_metadata=feature_metadata, + algorithm=algorithm, ) @@ -437,6 +484,9 @@ def _build_ai_query_prompt_column( group_size_expr = F.concat(F.col("group_size").cast(StringType()), F.lit(" rows")) return F.concat( F.lit(_AI_QUERY_PROMPT_HEADER), + F.lit("attribution_basis: "), + F.lit(attribution_semantics(ctx.algorithm)), + F.lit("\n"), F.lit("feature_contributions: "), F.col("feature_contributions"), F.lit("\n"), diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index d07ceb09c..7a8be5a54 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -236,7 +236,7 @@ def score_global_model( if config.enable_ai_explanation: scored_df = add_explanation_column( scored_df, - ExplanationContext.from_scoring_config(config, parsed_metadata), + ExplanationContext.from_scoring_config(config, parsed_metadata, record.identity.algorithm), is_ensemble=record.identity.is_ensemble, drift_summary=format_drift_summary(drift_result, config.redact_columns), ) diff --git a/tests/resources/ai_query_prompt_header.txt b/tests/resources/ai_query_prompt_header.txt index 700b15bed..b37631d76 100644 --- a/tests/resources/ai_query_prompt_header.txt +++ b/tests/resources/ai_query_prompt_header.txt @@ -2,6 +2,7 @@ You are a data quality analyst. Given aggregate metadata for a GROUP of anomalou Be direct and concrete. Avoid hedging phrases like 'The data shows', 'It appears that', or 'might indicate'. Do not restate the input field names back to the user, and do not invent feature names, values, or baseline groups that are not present in the input. Inputs: +- attribution_basis: What the feature_contributions below are measuring. Read them accordingly -- this decides whether the pattern is 'these values were extreme' or 'these metrics stopped agreeing'. - feature_contributions: Mean contributions across the group, already named for a reader, e.g. 'amount vs its group baseline (82%), quantity (11%), discount (5%)'. A phrase like 'X vs its group baseline' means X was unusual relative to its own baseline group, not in absolute terms. These are aggregated relative importances — not raw data values. - group_size: Number of rows in this group, e.g. '312 rows'. - severity_range: Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'. diff --git a/tests/unit/test_anomaly_llm_explainer.py b/tests/unit/test_anomaly_llm_explainer.py index b5265a96c..e940fc6d4 100644 --- a/tests/unit/test_anomaly_llm_explainer.py +++ b/tests/unit/test_anomaly_llm_explainer.py @@ -266,3 +266,63 @@ def test_probe_endpoint_reachable_rejects_non_databricks_provider(): with pytest.raises(InvalidParameterError, match="require a Databricks serving endpoint"): llm_explainer.probe_endpoint_reachable(spark, LLMModelConfig(model_name="openai/gpt-4")) assert not spark.queries + + +def test_attribution_semantics_distinguishes_correlation_from_value_anomalies(): + """The prompt must say what a contribution *measures*, because the two detectors differ. + + Measured on a live workspace before this existed: with the correlation-aware detector the LLM wrote + "Abnormal coolant flow and bearing temperature" and advised "Inspect coolant system and bearing + sensors" for rows whose every reading sat *inside* its healthy range. The values were normal; only the + relationship between them had broken. Given per-feature importances and nothing else, the model cannot + tell the two situations apart -- they look identical in shape -- so it defaults to the value reading + and asserts something the data does not support. + """ + correlation = llm_explainer.attribution_semantics("Mahalanobis") + assert "relationship" in correlation + assert "inside its normal range" in correlation + # The instruction that prevents the specific false claim observed. + assert "do NOT call an individual metric abnormal" in correlation + + value_based = llm_explainer.attribution_semantics("IsolationForest") + assert "own value was unusual" in value_based + assert "relationship" not in value_based + + assert correlation != value_based + + +def test_attribution_semantics_falls_back_to_the_value_reading(): + """Unknown or absent algorithms get the value-based reading. + + That is both the historical behaviour and the more conservative claim: describing an extreme value + where a relationship broke understates the finding, whereas the reverse invents a relationship claim. + """ + fallback = llm_explainer.attribution_semantics("IsolationForest") + for algorithm in (None, "", "SomeFutureAlgorithm"): + assert llm_explainer.attribution_semantics(algorithm) == fallback + + +def test_attribution_semantics_matches_ensemble_algorithm_strings(): + """Ensemble models persist as 'IsolationForest_Ensemble_3', so matching is by prefix. + + A registry value that failed to match would silently fall back, which is safe but would lose the + distinction for every ensemble model -- i.e. the default configuration. + """ + assert llm_explainer.attribution_semantics("IsolationForest_Ensemble_3") == llm_explainer.attribution_semantics( + "IsolationForest" + ) + + +def test_explanation_context_defaults_algorithm_to_none(): + """The field is additive: a caller building the context directly keeps working, and gets the + conservative value-based reading.""" + ctx = llm_explainer.ExplanationContext( + severity_col="s", + contributions_col="c", + score_std_col="std", + ai_explanation_col="ai", + threshold=95.0, + model_name="cat.sch.model", + ) + assert ctx.algorithm is None + assert llm_explainer.attribution_semantics(ctx.algorithm) == llm_explainer.attribution_semantics("IsolationForest") From 74c73d932fb2d4bd55f60ccef9fc04a67a5b8a28 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 11:58:54 +0100 Subject: [PATCH 050/107] Add two demos framed on the domain problem, one per profile The existing demo teaches the API: generate a frame, train, score, inspect. That is the right shape for a reference demo and the wrong shape for teaching `profile`, because the whole point of that argument is that **the user's data decides it** -- so a demo that does not start from a recognisable problem cannot teach the choice. One per profile, each named for its domain rather than the DQX feature it exercises. ## `dqx_demo_anomaly_tabular_transactions.py` -- "The transaction that passed every rule" Card transactions across twelve merchant categories, each with its own typical basket. Opens by applying the rules a payments team would already have -- amount in range, item count in range, amount not null -- and showing they catch **0 of 31** injected rows, because every individual value is ordinary and only the combination is not. Then the model catches 22 of 31 with no thresholds specified. Also gives `baseline_by` its first demo: the same amount is unremarkable for electronics and absurd for coffee, so the notebook trains a second model with `baseline_by=[]` and compares. Measured 22 against 18, and the takeaway says so plainly rather than overselling it -- some injected rows (a GBP 900 coffee) are extreme enough to stand out against the whole table too, and conditioning earns its keep on the ones that are not. An earlier draft of that bullet claimed more than the run supports; corrected against the number. ## `dqx_demo_anomaly_timeseries_fleet.py` -- "The machine where every gauge read normal" Eight machine metrics driven by two latent factors, so they move together the way telemetry does. The incident permutes three of them among a block of rows, which preserves each metric's own distribution exactly -- the same values, reordered -- so only the joint behaviour changes. The notebook then **verifies its own premise before modelling**, printing each metric's healthy range against its range during the incident and showing every incident reading falls inside. Without that, "no threshold could catch this" is an assertion; with it, it is demonstrated. Both profiles are then trained on identical data: profile incident rows caught total flagged false alarms tabular 0 of 60 71 71 timeseries 47 of 60 147 100 A fixed severity threshold is a percentile of the *training* score distribution, so the two need not flag the same number of rows -- which leaves the fair question of whether the second one simply alerted more. So a following cell repeats the comparison at an equal budget: rank by severity, take the same N from each, count. That is how the DQX benchmarks compare detectors and it closes the only real hole in the argument. Both demos are synthetic and self-contained (SMD is real telemetry and not shippable), quote the measured SMD figures without claiming state of the art, and state where DQX is weak. Registered in `demos.mdx`. ## Two bugs found only by running them **`.cache()` raises on serverless** -- `NOT_SUPPORTED_WITH_SERVERLESS: PERSIST TABLE is not supported`. Both demos died on their first data cell. Serverless is what a reader will reach for, so this was not a missed optimisation but a crash on cell one. **Removing the cache then emptied the AI-explanation display**, which is the more interesting failure. The scored frame became a lazy plan, and AI explanations call an LLM through `ai_query` *inside* that plan, so every action re-invoked the model -- roughly eight times across the fleet demo's cells, at eight times the cost, with a later re-execution returning nulls and blanking the display. Fixed by writing the scored frame to a Delta table once and reading it back, which is also the pattern a real pipeline should use. That diagnosis is what surfaced the misleading-narrative defect fixed in the previous commit. Verified by executing both on a live workspace and reading every cell's output, not the exit code: 34 and 19 cells, zero errors, no empty tables, and the numbers quoted above are the ones the run printed. Co-authored-by: Isaac --- .../dqx_demo_anomaly_tabular_transactions.py | 380 ++++++++++++++++++ demos/dqx_demo_anomaly_timeseries_fleet.py | 371 +++++++++++++++++ docs/dqx/docs/demos.mdx | 2 + 3 files changed, 753 insertions(+) create mode 100644 demos/dqx_demo_anomaly_tabular_transactions.py create mode 100644 demos/dqx_demo_anomaly_timeseries_fleet.py diff --git a/demos/dqx_demo_anomaly_tabular_transactions.py b/demos/dqx_demo_anomaly_tabular_transactions.py new file mode 100644 index 000000000..d9b435b99 --- /dev/null +++ b/demos/dqx_demo_anomaly_tabular_transactions.py @@ -0,0 +1,380 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC # The transaction that passed every rule +# MAGIC +# MAGIC A payments team has good rules. Amount is positive, under the card limit. Quantity is at least one. +# MAGIC The merchant category is one of the twelve they support. Every rule passes, every day, and the +# MAGIC dashboards are green. +# MAGIC +# MAGIC Then a reconciliation breaks, and someone finds a £4 grocery basket with 38 items in it, and a £900 +# MAGIC coffee. Both were inside every threshold. Neither was flagged. +# MAGIC +# MAGIC That is the gap this notebook is about, and it has two halves: +# MAGIC +# MAGIC 1. **A row can be wrong in the *combination* of its values** while every value is individually fine. +# MAGIC No single-column rule sees it, because there is no single column to write the rule against. +# MAGIC 2. **"Normal" depends on context.** £900 is unremarkable for electronics and absurd for coffee. A +# MAGIC threshold that catches the coffee rejects half the laptops. +# MAGIC +# MAGIC DQX row anomaly detection handles both, with no thresholds to pick. This is the **`tabular`** profile, +# MAGIC which is the default — see the companion notebook `dqx_demo_anomaly_timeseries_fleet.py` for the case +# MAGIC that needs the other one. + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Install + +# COMMAND ---------- + +dbutils.widgets.text("test_library_ref", "", "Test Library Ref") + +if dbutils.widgets.get("test_library_ref") != "": + %pip install 'databricks-labs-dqx[anomaly] @ {dbutils.widgets.get("test_library_ref")}' +else: + %pip install 'databricks-labs-dqx[anomaly]' + +%restart_python + +# COMMAND ---------- + +dbutils.widgets.text("demo_catalog", "main", "Catalog Name") +dbutils.widgets.text("demo_schema", "default", "Schema Name") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## The data +# MAGIC +# MAGIC Card transactions across twelve merchant categories. Each category has its own **typical basket** — +# MAGIC a coffee is a couple of pounds for one item, a laptop is several hundred for one item, a weekly +# MAGIC grocery shop is tens of pounds across dozens of items. That per-category structure is the whole +# MAGIC point: it is what makes a single global threshold useless. + +# COMMAND ---------- +# DBTITLE 1,Generate three months of clean history + +import numpy as np +import pyspark.sql.functions as F +from datetime import datetime, timedelta +from databricks.sdk import WorkspaceClient +from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine +from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies +from databricks.labs.dqx.engine import DQEngine +from databricks.labs.dqx.rule import DQDatasetRule, DQRowRule +from databricks.labs.dqx.check_funcs import is_in_range, is_not_null + +# Per-category basket shape: (typical unit price, typical item count). +# These are the patterns a model has to learn; nobody writes them down as rules. +CATEGORY_BASKETS = { + "coffee_shop": (3.20, 1.4), + "grocery": (2.10, 24.0), + "fuel": (68.00, 1.0), + "electronics": (420.00, 1.1), + "pharmacy": (8.50, 2.6), + "restaurant": (23.00, 2.2), + "clothing": (38.00, 2.4), + "transport": (2.80, 1.0), + "streaming": (9.99, 1.0), + "hardware": (14.00, 3.8), + "books": (11.00, 1.7), + "gym": (42.00, 1.0), +} +CHANNELS = ("chip_and_pin", "contactless", "online") +START = datetime(2024, 1, 1) + + +def generate_transactions(n_rows: int, seed: int, inject: bool = False): + """Card transactions whose amount and item count follow their category's basket shape. + + When *inject* is set, a small number of rows are made **jointly** implausible while every individual + value stays inside the range that category, or some other category, occupies normally. That is the + point: an injected row must not be catchable by a threshold on one column. + """ + rng = np.random.default_rng(seed) + categories = list(CATEGORY_BASKETS) + rows, labels = [], [] + + for i in range(n_rows): + category = categories[rng.integers(len(categories))] + unit_price, typical_items = CATEGORY_BASKETS[category] + + items = max(1, int(rng.normal(typical_items, max(0.4, typical_items * 0.25)))) + amount = round(items * unit_price * rng.uniform(0.82, 1.18), 2) + channel = CHANNELS[rng.integers(len(CHANNELS))] + is_anomaly = 0.0 + + if inject and rng.random() < 0.02: + kind = rng.integers(3) + if kind == 0: + # A grocery-sized basket at a coffee-shop price. £4 and 38 items are each ordinary + # somewhere in this table; together they are not. + items = int(rng.integers(30, 45)) + amount = round(rng.uniform(3.0, 6.0), 2) + elif kind == 1: + # An electronics-sized amount on a single coffee. Inside the global amount range. + category = "coffee_shop" + items = 1 + amount = round(rng.uniform(600.0, 950.0), 2) + else: + # A plausible amount and count, but for the wrong category: a £420 grocery single item. + category = "grocery" + items = 1 + amount = round(rng.uniform(380.0, 460.0), 2) + is_anomaly = 1.0 + + rows.append( + ( + f"TXN{i:06d}", + START + timedelta(days=int(rng.integers(0, 90)), hours=int(rng.integers(7, 22))), + amount, + items, + category, + channel, + is_anomaly, + ) + ) + labels.append(is_anomaly) + + schema = ( + "transaction_id string, transaction_time timestamp, amount double, " + "item_count int, merchant_category string, channel string, is_anomaly double" + ) + return spark.createDataFrame(rows, schema), int(sum(labels)) + + +history_df, _ = generate_transactions(6000, seed=11) +history_df.createOrReplaceTempView("history") +print(f"✅ {history_df.count():,} historical transactions across {len(CATEGORY_BASKETS)} categories") +display(history_df.limit(5)) + +# COMMAND ---------- +# DBTITLE 1,Why rules cannot catch this +# MAGIC %md +# MAGIC Before training anything, here is the honest version of the problem. These are sensible rules, and +# MAGIC they are the rules a good team would already have: + +# COMMAND ---------- + +catalog = dbutils.widgets.get("demo_catalog") +schema = dbutils.widgets.get("demo_schema") + +ws = WorkspaceClient() +dq_engine = DQEngine(ws) +anomaly_engine = AnomalyEngine(ws) + +# The rules a payments team would already have. Deliberately reasonable, not strawmen. +rules = [ + DQRowRule(check_func=is_not_null, column="amount", criticality="error"), + DQRowRule( + check_func=is_in_range, + column="amount", + check_func_kwargs={"min_limit": 0.01, "max_limit": 2000.0}, + criticality="error", + ), + DQRowRule( + check_func=is_in_range, + column="item_count", + check_func_kwargs={"min_limit": 1, "max_limit": 60}, + criticality="error", + ), +] + +# Nothing is persisted or cached in this notebook: PERSIST is unsupported on serverless compute, +# which is what most readers will run this on. These frames are small local relations built from +# seeded RNGs, so recomputation is both cheap and deterministic. +new_df, injected = generate_transactions(1500, seed=99, inject=True) +print(f"✅ {new_df.count():,} new transactions, {injected} of them jointly implausible\n") + +rule_results = dq_engine.apply_checks(new_df, rules) +caught_by_rules = rule_results.filter(F.col("_errors").isNotNull() & (F.col("is_anomaly") == 1.0)).count() + +print(f"Rules caught {caught_by_rules} of the {injected} implausible transactions.") +print("Every injected row sits inside every threshold — each value is ordinary on its own.") +print("Widening the rules cannot help; tightening them would reject legitimate transactions.") + +# COMMAND ---------- +# DBTITLE 1,Train: no thresholds, no labels +# MAGIC %md +# MAGIC Now the model. Note what is *not* being passed: no thresholds, no per-category limits, no labels. +# MAGIC `baseline_by=["merchant_category"]` is the one modelling decision, and it says the thing a payments +# MAGIC analyst already knows — **judge each transaction against its own category**, not against the table. + +# COMMAND ---------- + +model_name = f"{catalog}.{schema}.card_transactions_monitor" +registry_table = f"{catalog}.{schema}.dqx_anomaly_models" + +trained = anomaly_engine.train( + df=history_df, + model_name=model_name, + registry_table=registry_table, + columns=["amount", "item_count", "transaction_time"], + baseline_by=["merchant_category"], + # profile="tabular" is the default: independent records, anomalies are unusual values or + # unusual combinations of them. Stated here only to make the choice visible. + profile="tabular", +) +print(f"\n✅ trained: {trained}") + +# COMMAND ---------- +# DBTITLE 1,What conditioning bought +# MAGIC %md +# MAGIC `baseline_by` appends, for each metric, its deviation from **that category's own** median. So the +# MAGIC model sees both the raw amount and "how this amount compares to a typical basket in this category" — +# MAGIC which is how £900 can be normal for electronics and extreme for coffee inside one model. + +# COMMAND ---------- + +display( + spark.table(registry_table) + .filter(F.col("identity.model_name") == trained) + .selectExpr( + "identity.algorithm", + "training.columns", + "grouping.baseline_by", + "training.training_rows", + "from_json(features.feature_metadata, 'engineered_feature_names array')" + ".engineered_feature_names as features", + ) +) + +# COMMAND ---------- +# DBTITLE 1,Score, and read why +# MAGIC %md +# MAGIC One check. Contributions and AI explanations are on by default. + +# COMMAND ---------- + +anomaly_check = [ + DQDatasetRule( + criticality="error", + check_func=has_no_row_anomalies, + check_func_kwargs={ + "model_name": trained, + "registry_table": registry_table, + "threshold": 95.0, + }, + ) +] + +# Written to a table rather than left lazy: AI explanations call an LLM through ai_query inside the +# scoring plan, so each action on an unmaterialised result would call the model again -- paying repeatedly +# and getting a different answer each time. `.cache()` cannot be used (PERSIST is unsupported on +# serverless), so writing once is both the fix and the pattern to copy in a real pipeline. +scored_table = f"{catalog}.{schema}.transactions_scored" +dq_engine.apply_checks(new_df, anomaly_check).write.mode("overwrite").option( + "overwriteSchema", "true" +).saveAsTable(scored_table) +scored = spark.table(scored_table) +anomaly = F.element_at(F.col("_dq_info"), 1).getField("anomaly") + +flagged = scored.filter(anomaly.getField("is_anomaly")) +caught = scored.filter(anomaly.getField("is_anomaly") & (F.col("is_anomaly") == 1.0)).count() + +print(f"Rules caught {caught_by_rules:>3} of {injected}") +print(f"Anomaly detection caught {caught:>3} of {injected}") +print(f"\n{flagged.count()} rows flagged in total out of {new_df.count():,} ({flagged.count() / new_df.count():.1%})") + +# COMMAND ---------- +# DBTITLE 1,The explanations name the columns that combined badly + +display( + flagged.select( + "transaction_id", + "merchant_category", + "amount", + "item_count", + anomaly.getField("severity_percentile").alias("severity"), + anomaly.getField("contributions").alias("contributions"), + ) + .orderBy(F.desc("severity")) + .limit(10) +) + +# COMMAND ---------- +# DBTITLE 1,And in plain language, per group of similar anomalies + +display( + flagged.select( + "transaction_id", + "merchant_category", + "amount", + "item_count", + anomaly.getField("ai_explanation").getField("top_features").alias("pattern"), + anomaly.getField("ai_explanation").getField("narrative").alias("narrative"), + anomaly.getField("ai_explanation").getField("action").alias("action"), + ) + .filter(F.col("narrative").isNotNull()) + .orderBy("pattern") + .limit(10) +) + +# COMMAND ---------- +# DBTITLE 1,Does conditioning actually matter? Train without it and compare. +# MAGIC %md +# MAGIC The claim above was that comparing each transaction against its own category is what makes this +# MAGIC work. Worth testing rather than asserting: the same data, the same everything, `baseline_by=[]`. + +# COMMAND ---------- + +pooled = anomaly_engine.train( + df=history_df, + model_name=f"{catalog}.{schema}.card_transactions_pooled", + registry_table=registry_table, + columns=["amount", "item_count", "transaction_time"], + baseline_by=[], # compare against the whole table instead +) + +pooled_scored = dq_engine.apply_checks( + new_df, + [ + DQDatasetRule( + criticality="error", + check_func=has_no_row_anomalies, + check_func_kwargs={ + "model_name": pooled, + "registry_table": registry_table, + "threshold": 95.0, + # This model exists only to produce one comparison number, so skip the attribution and + # the LLM call. Both are on by default. + "enable_contributions": False, + "enable_ai_explanation": False, + }, + ) + ], +) +pooled_anomaly = F.element_at(F.col("_dq_info"), 1).getField("anomaly") +pooled_caught = pooled_scored.filter(pooled_anomaly.getField("is_anomaly") & (F.col("is_anomaly") == 1.0)).count() + +print(f"conditioned on merchant_category : {caught:>3} of {injected} caught") +print(f"compared against whole table : {pooled_caught:>3} of {injected} caught") +print("\nBoth are real models on identical data. The difference is the basis of comparison.") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## What to take away +# MAGIC +# MAGIC - Rules catch what you can **name in advance**. They caught none of these, and no threshold would +# MAGIC have, because every individual value was ordinary. +# MAGIC - Row anomaly detection catches **implausible combinations**, with no thresholds to choose. +# MAGIC - `baseline_by` is how "normal" becomes contextual, and it helps here rather than transforming the +# MAGIC result: the comparison above is a real but modest gain. That is honest and expected — some of these +# MAGIC injected rows (a £900 coffee) are extreme enough to stand out against the whole table too. +# MAGIC Conditioning earns its keep on the ones that are not, like a 40-item basket costing £4. +# MAGIC - Contributions tell you **which columns combined badly**, so a flagged row is actionable rather +# MAGIC than just suspicious. +# MAGIC +# MAGIC Use rules *and* this. Rules are cheaper, clearer and versioned; they should catch everything you can +# MAGIC describe. Anomaly detection is for what is left. +# MAGIC +# MAGIC ### Where this profile is the wrong tool +# MAGIC +# MAGIC These transactions are **independent records** — each row stands on its own. When rows are instead +# MAGIC repeated measurements of the same thing over time, and the failure is metrics that normally move +# MAGIC together drifting apart, the `tabular` profile is close to blind to it. That is +# MAGIC `dqx_demo_anomaly_timeseries_fleet.py`. +# MAGIC +# MAGIC Neither profile models trend or forecasts the next value. See +# MAGIC [Choosing a profile](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection#choosing-a-profile). diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py new file mode 100644 index 000000000..c81a3fc33 --- /dev/null +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -0,0 +1,371 @@ +# Databricks notebook source +# MAGIC %md +# MAGIC # The machine where every gauge read normal +# MAGIC +# MAGIC A plant records eight metrics per machine per minute: spindle load, motor current, coolant flow, +# MAGIC bearing temperature, vibration, hydraulic pressure, air pressure, throughput. Every metric has a safe +# MAGIC operating band, and there are alerts on all of them. +# MAGIC +# MAGIC A bearing fails. Afterwards the logs show every gauge sat inside its band for two hours beforehand. +# MAGIC No alert fired. Nothing to see. +# MAGIC +# MAGIC But something *was* visible: motor current stopped tracking spindle load. On a healthy machine those +# MAGIC rise and fall together, because cutting harder draws more current. For two hours load was high while +# MAGIC current sat mid-band. Both readings were ordinary. **Their relationship was not.** +# MAGIC +# MAGIC This is a different shape of anomaly from the one in `dqx_demo_anomaly_tabular_transactions.py`, and +# MAGIC it needs a different detector. It is what **`profile="timeseries"`** is for. +# MAGIC +# MAGIC We will train *both* profiles on the same data and compare, because a second algorithm is only worth +# MAGIC having if it earns its place. + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## Install + +# COMMAND ---------- + +dbutils.widgets.text("test_library_ref", "", "Test Library Ref") + +if dbutils.widgets.get("test_library_ref") != "": + %pip install 'databricks-labs-dqx[anomaly] @ {dbutils.widgets.get("test_library_ref")}' +else: + %pip install 'databricks-labs-dqx[anomaly]' + +%restart_python + +# COMMAND ---------- + +dbutils.widgets.text("demo_catalog", "main", "Catalog Name") +dbutils.widgets.text("demo_schema", "default", "Schema Name") + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## No timestamp column is required +# MAGIC +# MAGIC Worth saying immediately, because everyone expects otherwise. `profile="timeseries"` models +# MAGIC **correlation between metrics**, not behaviour over time. It never looks at row order. The name +# MAGIC describes the *data you have* — repeated multivariate measurements — not a temporal algorithm. +# MAGIC +# MAGIC A consequence worth knowing: it will not learn "Tuesdays are busy" from row order either. Include a +# MAGIC timestamp column and DQX derives calendar features from it (hour, day-of-week, month, weekend) exactly +# MAGIC as it does for the tabular profile. What neither profile models is **trend**. + +# COMMAND ---------- +# DBTITLE 1,Generate healthy machine telemetry + +import numpy as np +import pyspark.sql.functions as F +from databricks.sdk import WorkspaceClient +from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine +from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies +from databricks.labs.dqx.engine import DQEngine +from databricks.labs.dqx.rule import DQDatasetRule + +METRICS = [ + "spindle_load", + "motor_current", + "coolant_flow", + "bearing_temp", + "vibration", + "hydraulic_pressure", + "air_pressure", + "throughput", +] + +# Two latent drivers -- how hard the machine is working, and how hot it is running. Every metric is a +# mix of the two plus its own noise, which is why they all move together on a healthy machine. +LOADINGS = np.array( + [ + [0.95, 0.10], # spindle_load <- mostly work rate + [0.90, 0.15], # motor_current <- tracks spindle load closely + [0.35, 0.80], # coolant_flow <- mostly heat + [0.30, 0.90], # bearing_temp <- mostly heat + [0.70, 0.45], # vibration + [0.80, 0.20], # hydraulic_pressure + [0.25, 0.30], # air_pressure <- mostly independent + [0.85, 0.25], # throughput + ] +).T +BASELINES = np.array([62.0, 18.5, 24.0, 58.0, 2.4, 145.0, 6.2, 480.0]) +SCALES = np.array([9.0, 2.6, 3.4, 6.5, 0.45, 12.0, 0.35, 55.0]) + + +def generate_telemetry(n_rows: int, seed: int, break_correlation: bool = False): + """Machine telemetry driven by two shared latent factors. + + When *break_correlation* is set, a contiguous block of rows has three metrics decoupled from the rest + by **permuting their values among those rows**. That preserves every metric's own distribution exactly + -- the same values, reordered -- so no single-metric alert can possibly fire. Only the joint + behaviour changes, which is precisely the failure this profile exists to catch. + """ + rng = np.random.default_rng(seed) + factors = rng.normal(0.0, 1.0, size=(n_rows, 2)) + values = BASELINES + SCALES * (factors @ LOADINGS + rng.normal(0.0, 0.18, size=(n_rows, len(METRICS)))) + + labels = np.zeros(n_rows) + if break_correlation: + # A two-hour incident: spindle_load, motor_current and vibration stop tracking each other. + start, length = int(n_rows * 0.72), max(4, int(n_rows * 0.04)) + block = values[start : start + length, :3] + values[start : start + length, :3] = np.roll(block, shift=1, axis=0) + labels[start : start + length] = 1.0 + + rows = [ + (f"CNC-{(i % 4) + 1:02d}", i, *[float(v) for v in values[i]], float(labels[i])) for i in range(n_rows) + ] + schema = "machine_id string, reading_seq int, " + ", ".join(f"{m} double" for m in METRICS) + ", is_incident double" + return spark.createDataFrame(rows, schema), int(labels.sum()) + + +# Nothing is persisted or cached in this notebook: PERSIST is unsupported on serverless compute, +# which is what most readers will run this on. These frames are small local relations built from +# seeded RNGs, so recomputation is both cheap and deterministic. +healthy_df, _ = generate_telemetry(4000, seed=5) +print(f"✅ {healthy_df.count():,} healthy readings across 4 machines, {len(METRICS)} metrics each") +display(healthy_df.limit(5)) + +# COMMAND ---------- +# DBTITLE 1,The correlation this depends on +# MAGIC %md +# MAGIC Before modelling anything, confirm the premise: on healthy data these metrics really do move +# MAGIC together. `motor_current` against `spindle_load` is the pair from the story. + +# COMMAND ---------- + +display( + healthy_df.select( + F.round(F.corr("spindle_load", "motor_current"), 3).alias("load_vs_current"), + F.round(F.corr("bearing_temp", "coolant_flow"), 3).alias("temp_vs_coolant"), + F.round(F.corr("spindle_load", "air_pressure"), 3).alias("load_vs_air_pressure"), + ) +) +print("Strong pairs are the ones a correlation-aware detector can exploit.") +print("air_pressure is mostly independent by design — not everything correlates, and that is realistic.") + +# COMMAND ---------- +# DBTITLE 1,Now the incident, and proof it is invisible per-metric + +incident_df, injected = generate_telemetry(1500, seed=77, break_correlation=True) +print(f"✅ {incident_df.count():,} readings, {injected} of them during the incident\n") + +# Compare each metric's range during the incident against its range outside it. If a range check could +# catch this, we have not built the scenario we claimed to. +during = incident_df.filter(F.col("is_incident") == 1.0) +outside = incident_df.filter(F.col("is_incident") == 0.0) + +print(f"{'metric':<22}{'healthy range':>26}{'during incident':>26}") +for metric in METRICS[:3]: + o = outside.select(F.min(metric).alias("lo"), F.max(metric).alias("hi")).first() + d = during.select(F.min(metric).alias("lo"), F.max(metric).alias("hi")).first() + inside = "inside" if d["lo"] >= o["lo"] and d["hi"] <= o["hi"] else "OUTSIDE" + print(f"{metric:<22}{f'{o.lo:.1f} – {o.hi:.1f}':>26}{f'{d.lo:.1f} – {d.hi:.1f} ({inside})':>26}") + +print("\nEvery incident reading sits inside the healthy range for its own metric.") +print("No threshold, no range check and no per-metric z-score can separate these rows.") + +# COMMAND ---------- +# DBTITLE 1,Train both profiles on identical data +# MAGIC %md +# MAGIC The comparison that justifies a second algorithm. Same data, same columns, same everything except +# MAGIC one word. + +# COMMAND ---------- + +catalog = dbutils.widgets.get("demo_catalog") +schema = dbutils.widgets.get("demo_schema") +registry_table = f"{catalog}.{schema}.dqx_anomaly_models" + +ws = WorkspaceClient() +dq_engine = DQEngine(ws) +anomaly_engine = AnomalyEngine(ws) + +models = {} +for profile in ("tabular", "timeseries"): + models[profile] = anomaly_engine.train( + df=healthy_df, + model_name=f"{catalog}.{schema}.fleet_telemetry_{profile}", + registry_table=registry_table, + columns=METRICS, + # No grouping: these machines are interchangeable and share one operating envelope. Pass + # baseline_by=["machine_id"] instead if each machine has its own normal. + baseline_by=[], + profile=profile, + ) + print(f"✅ {profile:<11} -> {models[profile]}") + +# COMMAND ---------- +# DBTITLE 1,The registry records which detector each model uses + +display( + spark.table(registry_table) + .filter(F.col("identity.model_name").contains("fleet_telemetry")) + .selectExpr( + "identity.model_name", + "identity.algorithm", + "training.hyperparameters['covariance'] as covariance", + "training.training_rows", + ) + .orderBy("identity.model_name") +) +print("Scoring reads the algorithm back off the registry, so you never repeat the choice.") + +# COMMAND ---------- +# DBTITLE 1,Score both, and count incidents caught + +# Scoring is written to a table rather than held as a lazy plan, and that matters here rather than being +# housekeeping. AI explanations call an LLM through ai_query *inside* the scoring plan, so every action on +# an unmaterialised result calls the model again -- the cells below take several actions each, which would +# mean paying for the LLM repeatedly and getting a different answer each time. `.cache()` would normally +# prevent that, but PERSIST is unsupported on serverless compute. Writing once is the pattern to copy. +results = {} +for profile, model in models.items(): + scored = dq_engine.apply_checks( + incident_df, + [ + DQDatasetRule( + criticality="error", + check_func=has_no_row_anomalies, + check_func_kwargs={ + "model_name": model, + "registry_table": registry_table, + "threshold": 95.0, + }, + ) + ], + ) + scored_table = f"{catalog}.{schema}.fleet_scored_{profile}" + scored.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(scored_table) + results[profile] = spark.table(scored_table) + print(f"✅ {profile:<11} scored -> {scored_table}") + +def anomaly_of(df): + """The anomaly struct from the first _dq_info element (element_at is 1-based).""" + return F.element_at(F.col("_dq_info"), 1).getField("anomaly") + + +print(f"{'profile':<13}{'incident rows caught':>22}{'total flagged':>16}{'false alarms':>15}") +for profile, scored in results.items(): + a = anomaly_of(scored) + caught = scored.filter(a.getField("is_anomaly") & (F.col("is_incident") == 1.0)).count() + flagged = scored.filter(a.getField("is_anomaly")).count() + false_alarms = flagged - caught + print(f"{profile:<13}{f'{caught} of {injected}':>22}{flagged:>16}{false_alarms:>15}") + +print("\nSame data. Same columns. The tabular detector splits one metric at a time, so a broken") +print("relationship between two in-range values is close to invisible to it.") + +# COMMAND ---------- +# DBTITLE 1,The same comparison at an equal alert budget +# MAGIC %md +# MAGIC The two detectors above did not flag the same number of rows, and they need not: a severity +# MAGIC threshold is a percentile of the *training* score distribution, so how many new rows exceed it +# MAGIC depends on the detector. That leaves a fair question — did the second one just alert more? +# MAGIC +# MAGIC Settle it by giving both the same budget. Rank every reading by severity, take the top N from each +# MAGIC with N identical, and count how many incident rows each one bought. This is how the DQX benchmarks +# MAGIC compare detectors, and it is the honest way to read any such comparison. + +# COMMAND ---------- + +from pyspark.sql import Window + +BUDGET_FRACTION = 0.05 # alert on the 5% of readings each detector ranks most anomalous +budget = int(incident_df.count() * BUDGET_FRACTION) + +print(f"budget: the top {budget} readings by severity ({BUDGET_FRACTION:.0%} of {incident_df.count():,})\n") +print(f"{'profile':<13}{'incident rows caught':>22}{'precision':>12}") +for profile, scored in results.items(): + ranked = scored.select( + F.col("is_incident"), + anomaly_of(scored).getField("severity_percentile").alias("severity"), + ).withColumn("rank", F.row_number().over(Window.orderBy(F.col("severity").desc_nulls_last()))) + + top = ranked.filter(F.col("rank") <= budget) + caught = top.filter(F.col("is_incident") == 1.0).count() + print(f"{profile:<13}{f'{caught} of {injected}':>22}{caught / budget:>11.0%}") + +print("\nSame number of alerts for each. The difference is what those alerts are worth.") + +# COMMAND ---------- +# DBTITLE 1,Why the correlation-aware detector can see it +# MAGIC %md +# MAGIC It measures how far a reading sits from normal **once the relationships between metrics are +# MAGIC accounted for** — a distance in a space where the metrics have been decorrelated. A load/current +# MAGIC pair that never co-occurs on healthy data is far away in that space even though each value is +# MAGIC near the middle of its own range. +# MAGIC +# MAGIC It explains itself by leaving each metric out in turn and reporting how much of the anomaly +# MAGIC disappears — so contributions work here without SHAP. + +# COMMAND ---------- + +ts = results["timeseries"] +a = anomaly_of(ts) + +display( + ts.filter(a.getField("is_anomaly")) + .select( + "machine_id", + "reading_seq", + F.round("spindle_load", 1).alias("spindle_load"), + F.round("motor_current", 1).alias("motor_current"), + F.round("vibration", 2).alias("vibration"), + a.getField("severity_percentile").alias("severity"), + a.getField("contributions").alias("contributions"), + ) + .orderBy(F.desc("severity")) + .limit(10) +) + +# COMMAND ---------- +# DBTITLE 1,And in plain language + +display( + ts.filter(a.getField("is_anomaly")) + .select( + "machine_id", + "reading_seq", + a.getField("ai_explanation").getField("narrative").alias("narrative"), + a.getField("ai_explanation").getField("business_impact").alias("impact"), + a.getField("ai_explanation").getField("action").alias("action"), + ) + .filter(F.col("narrative").isNotNull()) + .limit(5) +) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC ## What to take away +# MAGIC +# MAGIC - Some failures are **broken relationships**, not extreme values. Every gauge reads normal and the +# MAGIC machine is still in trouble. +# MAGIC - Per-metric alerts cannot see those, however well tuned. In this notebook every incident reading +# MAGIC sits inside its own metric's healthy range — verified above, not asserted. +# MAGIC - `profile="timeseries"` switches to a detector that models how metrics move **together**. One word, +# MAGIC and everything else is unchanged: same feature engineering, same registry, same check, same +# MAGIC contributions and AI explanations. +# MAGIC - It needs **no timestamp column** and trains a single model rather than an ensemble, because it is +# MAGIC deterministic. +# MAGIC +# MAGIC ### Being straight about the limits +# MAGIC +# MAGIC On the **Server Machine Dataset** — 28 machines of real telemetry with labelled incidents — this +# MAGIC detector surfaces 79% of incidents inside an alert budget of 1% of rows, against 33% for the tabular +# MAGIC detector. That is a large gain on the data it is for, and it is *not* state of the art for +# MAGIC multivariate time-series anomaly detection. Published figures near 0.80 F1 are not a fair comparison: +# MAGIC they use point adjustment, which +# MAGIC [Kim et al. (AAAI 2022)](https://arxiv.org/abs/2109.05257) showed random scores also reach. +# MAGIC +# MAGIC What this does **not** do: +# MAGIC +# MAGIC - **Trend.** Sustained growth eventually drifts outside the trained range. Model a rate or a ratio +# MAGIC rather than a running level, and retrain on a schedule. +# MAGIC - **A single metric.** With nothing to correlate against, use a rule or a threshold. +# MAGIC - **Forecasting.** DQX judges readings against learned normal; it does not predict the next value. +# MAGIC - **Choosing the profile for you.** You pick it. The cheap heuristic for guessing was measured and +# MAGIC rejected because it fires on ordinary sorted tables — see +# MAGIC [Choosing a profile](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection#choosing-a-profile). diff --git a/docs/dqx/docs/demos.mdx b/docs/dqx/docs/demos.mdx index 7b7671b30..59f2a8b1d 100644 --- a/docs/dqx/docs/demos.mdx +++ b/docs/dqx/docs/demos.mdx @@ -15,6 +15,8 @@ Import the following notebooks in the Databricks workspace to try DQX out: * [DQX Demo Notebook for Actions and Alerting](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_alerting.py) - demonstrates how to react to data quality problems by firing alerts (driver log, with optional Slack) when summary metrics cross a threshold. * [DQX Demo Notebook for Profiling and Applying Checks at Scale on Multiple Tables](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_multi_table_demo.py) - demonstrates how to use DQX as a library at scale to apply checks on multiple tables. * [DQX Demo Notebook for Row Anomaly Detection](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_row_anomaly_detection_demo.py) - comprehensive demo showing how to use DQX Row Anomaly Detection to detect unusual patterns in your data. +* [DQX Demo Notebook for Row Anomaly Detection on Transactions](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_anomaly_tabular_transactions.py) - starts from a domain problem: card transactions that pass every rule but are jointly implausible. Shows why no threshold catches them, how `baseline_by` makes "normal" depend on the merchant category, and what the contributions tell you. Uses the default `tabular` profile. +* [DQX Demo Notebook for Row Anomaly Detection on Machine Telemetry](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_anomaly_timeseries_fleet.py) - starts from a domain problem: machine metrics that each stay inside their safe band while the *relationship* between them breaks. Verifies that no per-metric range check could catch it, then trains both profiles on identical data to show what `profile="timeseries"` buys. * [DQX Demo Notebook for AI-assisted checks generation](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_ai_assisted_checks_generation.py) - demonstrates how to generate DQX rules/checks with LLM using natural language. * [DQX Demo Notebook for Data Contract Integration (ODCS)](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_datacontract_odcs.py) - demonstrates how to generate DQX quality rules from ODCS (Open Data Contract Standard) data contracts, including predefined rules from schema constraints, explicit custom rules, and contract metadata tracking. * [DQX Demo Notebook for Spark Structured Streaming (Native End-to-End Approach)](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_streaming_demo_native.py) - demonstrates how to use DQX as a library with Spark Structured Streaming, using the built-in end-to-end method to handle both reading and writing. From 5d0b995f608d3c5bfdff0a729f946cae7e76d197 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 13:16:12 +0100 Subject: [PATCH 051/107] Fix two broken links and a parameter that never existed in the comprehensive demo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found while extracting the presentation style of `dqx_row_anomaly_detection_demo.py` to apply elsewhere. Three defects in its closing cells, all pre-existing: **Two 404 links.** Both omit the `/docs/` path segment: https://databrickslabs.github.io/dqx/guide/row_anomaly_detection https://databrickslabs.github.io/dqx/reference/quality_checks#has_no_row_anomalies Confirmed against the built site rather than assumed: `docs/dqx/build/docs/guide/…` exists and `docs/dqx/build/guide` does not. The anchor was wrong too — the reference page's heading renders as `#row-anomaly-detection`, not `#has_no_row_anomalies`. **A parameter that has never existed.** "Add group conditioning (`group_by` for training)" — the argument is `baseline_by`. Not even a stale rename: `group_by` was never accepted, so a reader following that line gets a TypeError. (`segment_by` was the old name, and it was removed earlier in this PR.) Nothing verifies demo links, so these could only be caught by looking. Checked the rest of `demos/` for the same malformed link shape and found none. Co-authored-by: Isaac --- demos/dqx_row_anomaly_detection_demo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/demos/dqx_row_anomaly_detection_demo.py b/demos/dqx_row_anomaly_detection_demo.py index acdb7d495..1f165e385 100644 --- a/demos/dqx_row_anomaly_detection_demo.py +++ b/demos/dqx_row_anomaly_detection_demo.py @@ -835,7 +835,7 @@ def inject_anomalies_and_dq_issues( # MAGIC ``` # MAGIC # MAGIC **Optional next steps:** -# MAGIC - Add group conditioning (`group_by` for training), drift detection, and scheduled scoring. +# MAGIC - Add baseline conditioning (`baseline_by` for training), drift detection, and scheduled scoring. # MAGIC - Automate retraining and alerting. # COMMAND ---------- @@ -845,8 +845,8 @@ def inject_anomalies_and_dq_issues( # MAGIC # MAGIC ### 📚 Resources # MAGIC -# MAGIC - [DQX Row Anomaly Detection Documentation](https://databrickslabs.github.io/dqx/guide/row_anomaly_detection) -# MAGIC - [API Reference](https://databrickslabs.github.io/dqx/reference/quality_checks#has_no_row_anomalies) +# MAGIC - [DQX Row Anomaly Detection Documentation](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection) +# MAGIC - [API Reference](https://databrickslabs.github.io/dqx/docs/reference/quality_checks#row-anomaly-detection) # MAGIC - [Data Quality Monitoring (DQM)](https://docs.databricks.com/aws/en/data-quality-monitoring/anomaly-detection/#-table-quality-details) # MAGIC # MAGIC ### 🎉 You're Ready! From dba3b05c8e1d01f8c4a4b2663365cebc041b971f Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 14:08:31 +0100 Subject: [PATCH 052/107] Rewrite the anomaly demos for teaching, retire the old one, and cover both in e2e MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two domain demos ran correctly but read like engineering write-ups: 19 cells each with a 76-line and a 55-line wall of code, and both built around A/B comparisons of DQX's own options — training two models and racing them. That was the right instinct for verification and the wrong shape for a demo, which should say what the tool is for and show it working. ## Restructured against the house benchmark `dqx_row_anomaly_detection_demo.py` was the reference (its style, extracted: emoji-led H1, `---` rules, `## Section N:`, DBTITLE on every code cell, emoji print narrative, print-before-display, f-string numeric tables, and a Summary / Resources / You're Ready close). Worth noting the other demos deliberately use no emoji and no rules — where they conflicted, the anomaly demo won as the closest sibling. largest code cell 76 → 21 and 55 → 20 lines cells 19 → 27 and 19 → 24 code cells > 25 lines 3 → 0 and 2 → 0 ## Comparisons removed, per the instruction to state rather than compare The fleet demo no longer trains both profiles or runs the equal-budget cell; a markdown table says which profile is for which data and cites the measured SMD figures (79% against 33%) as documented fact. The transactions demo no longer trains a pooled model to argue for `baseline_by`. Removing these deleted the three largest cells and about a third of the code in each. Kept: the rules-catch-nothing cell (rules versus ML is the value proposition, not a comparison of DQX options — the old demo taught it too) and the fleet range-proof cell, without which "no threshold could catch this" is an assertion rather than a demonstration. ## Unity Catalog, not hand-rolled write-then-read Scoring now goes through `apply_checks_and_save_in_table(input_config=…, output_config=…)` — name the input table, name the output table — instead of `apply_checks(...).write.saveAsTable(...)` followed by `spark.table(...)`. That is the shape UC users expect, and it still gives the materialisation the AI explanations need, since `ai_query` runs inside the scoring plan and every action on a lazy result would re-invoke the LLM. Also stopped re-reading frames already held in a variable. ## Old demo retired `dqx_row_anomaly_detection_demo.py` is deleted, with its `demos.mdx` entry, its published workspace copy, and its e2e test. The e2e test is replaced by a parametrised `test_run_dqx_anomaly_demo` over both new notebooks, following the shape of every other test in that file and keeping the 45-minute wait for the same reason: these train a model and score with contributions and AI explanations on by default. ## Three defects that only reading cell output could find **The registry display was showing stale, contradictory data.** Re-running accumulated rows, so the "registered model" cell printed 8 rows for transactions and 4 for fleet — and the transactions rows showed `columns=['amount','item_count','transaction_time']`, contradicting the notebook text that had just explained the timestamp is excluded. Fixed by dropping the registry during setup, which is what the old demo already did. Now one row, correct configuration. **A cell rendered nothing at all** — the fleet correlation `display()`, the only `display()` of a pure aggregate. Replaced with an f-string table, which is better style anyway and lets each pair carry its own explanation. **The model was being fed noise.** `transaction_time` was in `columns` while this generator draws hours uniformly over 90 uniform days, so seven of eleven engineered features carried no signal — diluting the two that did and manufacturing "unusual timing" alerts. Measured at threshold 98: 23/30 caught with the column, 30/30 without. Making the timestamps realistic did *not* help (20/30) — the model then spends budget on genuine timing outliers. Dropping it wins at every threshold. The column stays in the table, out of the model, and the notebook now teaches the rule: feed a column only if it relates to the anomalies you want, and note that auto-discovery would have included it. ## Threshold semantics, which are the most misread part of the feature `threshold=95` means "flag the top 5% by *training* severity" — roughly 75 alerts on 1,500 rows before any anomaly exists. It is an alert budget, not a confidence score, and that puts a hard ceiling on precision: ask for 75 alerts when 33 rows are bad and 44% is the best anyone could do. The tuning table now prints that ceiling beside the observed value, because precision read against 100% looks like failure and read against the budget looks like what it is. On the verified run the model sits *at* the ceiling with 100% recall at both threshold 90 and 95. One thing checked and deliberately not changed: the injected rows were suspected of being partly borderline, since injection kind 0 keeps whichever category was drawn. Measured over 24 of them, none were — the mildest sat 9.2x from its category's normal item count, the rest 14x–42x. The fixture is sound. Verified by executing both on a live workspace and reading every cell: 27 and 24 cells, zero errors, no empty outputs, registry showing one correct row, contributions naming only meaningful features (`amount 60.4, amount_rel_baseline 32.1` on an £893 coffee), and the fleet narratives still describing broken relationships rather than abnormal metrics. Recall at the default threshold varies run to run (76% then 100% observed) — expected, since training samples 30% of rows and the tabular path ensembles three differently-seeded forests. Co-authored-by: Isaac --- .../dqx_demo_anomaly_tabular_transactions.py | 541 +++++++---- demos/dqx_demo_anomaly_timeseries_fleet.py | 549 ++++++----- demos/dqx_row_anomaly_detection_demo.py | 861 ------------------ docs/dqx/docs/demos.mdx | 3 +- tests/e2e/test_run_demos.py | 23 +- 5 files changed, 707 insertions(+), 1270 deletions(-) delete mode 100644 demos/dqx_row_anomaly_detection_demo.py diff --git a/demos/dqx_demo_anomaly_tabular_transactions.py b/demos/dqx_demo_anomaly_tabular_transactions.py index d9b435b99..fbfde59bc 100644 --- a/demos/dqx_demo_anomaly_tabular_transactions.py +++ b/demos/dqx_demo_anomaly_tabular_transactions.py @@ -1,31 +1,61 @@ # Databricks notebook source # MAGIC %md -# MAGIC # The transaction that passed every rule +# MAGIC # 🔍 Finding the Transactions Your Rules Will Never Catch # MAGIC -# MAGIC A payments team has good rules. Amount is positive, under the card limit. Quantity is at least one. -# MAGIC The merchant category is one of the twelve they support. Every rule passes, every day, and the -# MAGIC dashboards are green. +# MAGIC ## Learn Row Anomaly Detection on Business Records in 10 Minutes # MAGIC -# MAGIC Then a reconciliation breaks, and someone finds a £4 grocery basket with 38 items in it, and a £900 -# MAGIC coffee. Both were inside every threshold. Neither was flagged. +# MAGIC **What you'll do:** +# MAGIC - Write the quality rules a good payments team would already have +# MAGIC - Watch them pass a batch that contains real problems +# MAGIC - Train a DQX anomaly model with no thresholds and no labels +# MAGIC - Read *why* each row was flagged, in plain language # MAGIC -# MAGIC That is the gap this notebook is about, and it has two halves: +# MAGIC **Dataset**: Card transactions across twelve merchant categories (no domain expertise required) # MAGIC -# MAGIC 1. **A row can be wrong in the *combination* of its values** while every value is individually fine. -# MAGIC No single-column rule sees it, because there is no single column to write the rule against. -# MAGIC 2. **"Normal" depends on context.** £900 is unremarkable for electronics and absurd for coffee. A -# MAGIC threshold that catches the coffee rejects half the laptops. + +# COMMAND ---------- + +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ## The problem: a row can be wrong without any value being wrong +# MAGIC +# MAGIC A payments team has good rules. Amount is positive and under the card limit. Quantity is at least +# MAGIC one. The merchant category is one they support. Every rule passes, every day, and the dashboards +# MAGIC are green. +# MAGIC +# MAGIC Then a reconciliation breaks, and someone finds a **£4 grocery basket with 38 items** in it, and a +# MAGIC **£900 coffee**. Both were inside every threshold. Neither was flagged. +# MAGIC +# MAGIC **Known vs unknown issues** +# MAGIC - **Known unknowns**: nulls, ranges, formats. Write a rule — it is cheap, clear and versioned. +# MAGIC - **Unknown unknowns**: a combination of values that is individually ordinary and jointly absurd. +# MAGIC There is no single column to write the rule against. +# MAGIC +# MAGIC **"Normal" also depends on context.** £900 is unremarkable for electronics and absurd for coffee. A +# MAGIC threshold that catches the coffee rejects half the laptops. DQX handles this with `baseline_by`, +# MAGIC which judges every row against **its own group's** normal rather than the whole table's. +# MAGIC +# MAGIC Use rules *and* anomaly detection. Rules catch what you can describe; anomaly detection covers +# MAGIC what is left. # MAGIC -# MAGIC DQX row anomaly detection handles both, with no thresholds to pick. This is the **`tabular`** profile, -# MAGIC which is the default — see the companion notebook `dqx_demo_anomaly_timeseries_fleet.py` for the case -# MAGIC that needs the other one. # COMMAND ---------- # MAGIC %md -# MAGIC ## Install +# MAGIC --- +# MAGIC +# MAGIC ## Prerequisites: Install DQX with Anomaly Support +# MAGIC +# MAGIC ```python +# MAGIC %pip install 'databricks-labs-dqx[anomaly]' +# MAGIC ``` +# MAGIC +# MAGIC **Note**: On ML Runtime or Serverless most dependencies are already present. +# MAGIC # COMMAND ---------- +# DBTITLE 1,Install DQX dbutils.widgets.text("test_library_ref", "", "Test Library Ref") @@ -37,6 +67,7 @@ %restart_python # COMMAND ---------- +# DBTITLE 1,Configure catalog and schema dbutils.widgets.text("demo_catalog", "main", "Catalog Name") dbutils.widgets.text("demo_schema", "default", "Schema Name") @@ -44,28 +75,67 @@ # COMMAND ---------- # MAGIC %md -# MAGIC ## The data +# MAGIC --- +# MAGIC +# MAGIC ## Section 1: Setup & Data Generation +# MAGIC +# MAGIC | Column | Type | Description | +# MAGIC |---|---|---| +# MAGIC | `transaction_id` | string | Unique transaction reference | +# MAGIC | `transaction_time` | timestamp | When the card was used | +# MAGIC | `amount` | double | Total basket value, GBP | +# MAGIC | `item_count` | int | Items in the basket | +# MAGIC | `merchant_category` | string | One of twelve categories — the **baseline group** | +# MAGIC | `channel` | string | `chip_and_pin`, `contactless` or `online` | +# MAGIC | `is_anomaly` | double | Ground truth, for this demo only — never given to the model | +# MAGIC +# MAGIC Each category has its own **typical basket**: a coffee is a couple of pounds for one item, a laptop +# MAGIC several hundred for one, a weekly shop tens of pounds across dozens. That structure is the point — +# MAGIC it is what makes a single global threshold useless. # MAGIC -# MAGIC Card transactions across twelve merchant categories. Each category has its own **typical basket** — -# MAGIC a coffee is a couple of pounds for one item, a laptop is several hundred for one item, a weekly -# MAGIC grocery shop is tens of pounds across dozens of items. That per-category structure is the whole -# MAGIC point: it is what makes a single global threshold useless. # COMMAND ---------- -# DBTITLE 1,Generate three months of clean history +# DBTITLE 1,Setup engines + +from datetime import datetime, timedelta import numpy as np import pyspark.sql.functions as F -from datetime import datetime, timedelta from databricks.sdk import WorkspaceClient + from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies +from databricks.labs.dqx.check_funcs import is_in_range, is_not_null +from databricks.labs.dqx.config import InputConfig, OutputConfig from databricks.labs.dqx.engine import DQEngine from databricks.labs.dqx.rule import DQDatasetRule, DQRowRule -from databricks.labs.dqx.check_funcs import is_in_range, is_not_null -# Per-category basket shape: (typical unit price, typical item count). -# These are the patterns a model has to learn; nobody writes them down as rules. +catalog = dbutils.widgets.get("demo_catalog") +schema = dbutils.widgets.get("demo_schema") + +ws = WorkspaceClient() +dq_engine = DQEngine(ws) +anomaly_engine = AnomalyEngine(ws) + +print(f"✅ Setup complete — writing to {catalog}.{schema}") + +# COMMAND ---------- +# DBTITLE 1,Prepare a clean model registry + +# Drop the registry so each run of this notebook starts from nothing. Without this, re-running leaves +# every previous run's rows behind and the "registered model" cell below shows a pile of stale +# configurations rather than the one just trained. +registry_table = f"{catalog}.{schema}.dqx_anomaly_models" +spark.sql(f"DROP TABLE IF EXISTS {registry_table}") + +print(f"📋 Model registry: {registry_table}") +print("✅ Registry reset — ready for this run's model") + +# COMMAND ---------- +# DBTITLE 1,Typical basket per merchant category + +# (typical unit price, typical item count). These are the patterns a model has to learn; +# nobody writes them down as rules. CATEGORY_BASKETS = { "coffee_shop": (3.20, 1.4), "grocery": (2.10, 24.0), @@ -82,88 +152,89 @@ } CHANNELS = ("chip_and_pin", "contactless", "online") START = datetime(2024, 1, 1) +SCHEMA = ( + "transaction_id string, transaction_time timestamp, amount double, " + "item_count int, merchant_category string, channel string, is_anomaly double" +) +print(f"📊 {len(CATEGORY_BASKETS)} merchant categories, each with its own basket shape") + +# COMMAND ---------- +# DBTITLE 1,The three shapes of implausible row + + +def make_implausible(rng, category: str, items: int): + """Return (category, item_count, amount) for a row that is ordinary per column and absurd overall.""" + kind = rng.integers(3) + if kind == 0: + # A grocery-sized basket at a coffee-shop price. £4 and 38 items are each ordinary + # somewhere in this table; together they are not. + return category, int(rng.integers(30, 45)), round(rng.uniform(3.0, 6.0), 2) + if kind == 1: + # An electronics-sized amount on a single coffee — still inside the global amount range. + return "coffee_shop", 1, round(rng.uniform(600.0, 950.0), 2) + # A plausible amount and count, for the wrong category: a £420 single grocery item. + return "grocery", 1, round(rng.uniform(380.0, 460.0), 2) -def generate_transactions(n_rows: int, seed: int, inject: bool = False): - """Card transactions whose amount and item count follow their category's basket shape. - When *inject* is set, a small number of rows are made **jointly** implausible while every individual - value stays inside the range that category, or some other category, occupies normally. That is the - point: an injected row must not be catchable by a threshold on one column. - """ +# COMMAND ---------- +# DBTITLE 1,Generate transactions + + +def generate_transactions(n_rows: int, seed: int, inject: bool = False): + """Transactions whose amount and item count follow their category's basket shape.""" rng = np.random.default_rng(seed) categories = list(CATEGORY_BASKETS) - rows, labels = [], [] + rows = [] for i in range(n_rows): category = categories[rng.integers(len(categories))] unit_price, typical_items = CATEGORY_BASKETS[category] - items = max(1, int(rng.normal(typical_items, max(0.4, typical_items * 0.25)))) amount = round(items * unit_price * rng.uniform(0.82, 1.18), 2) - channel = CHANNELS[rng.integers(len(CHANNELS))] is_anomaly = 0.0 if inject and rng.random() < 0.02: - kind = rng.integers(3) - if kind == 0: - # A grocery-sized basket at a coffee-shop price. £4 and 38 items are each ordinary - # somewhere in this table; together they are not. - items = int(rng.integers(30, 45)) - amount = round(rng.uniform(3.0, 6.0), 2) - elif kind == 1: - # An electronics-sized amount on a single coffee. Inside the global amount range. - category = "coffee_shop" - items = 1 - amount = round(rng.uniform(600.0, 950.0), 2) - else: - # A plausible amount and count, but for the wrong category: a £420 grocery single item. - category = "grocery" - items = 1 - amount = round(rng.uniform(380.0, 460.0), 2) + category, items, amount = make_implausible(rng, category, items) is_anomaly = 1.0 - rows.append( - ( - f"TXN{i:06d}", - START + timedelta(days=int(rng.integers(0, 90)), hours=int(rng.integers(7, 22))), - amount, - items, - category, - channel, - is_anomaly, - ) - ) - labels.append(is_anomaly) - - schema = ( - "transaction_id string, transaction_time timestamp, amount double, " - "item_count int, merchant_category string, channel string, is_anomaly double" - ) - return spark.createDataFrame(rows, schema), int(sum(labels)) + when = START + timedelta(days=int(rng.integers(0, 90)), hours=int(rng.integers(7, 22))) + channel = CHANNELS[rng.integers(len(CHANNELS))] + rows.append((f"TXN{i:06d}", when, amount, items, category, channel, is_anomaly)) + return spark.createDataFrame(rows, SCHEMA) -history_df, _ = generate_transactions(6000, seed=11) -history_df.createOrReplaceTempView("history") -print(f"✅ {history_df.count():,} historical transactions across {len(CATEGORY_BASKETS)} categories") -display(history_df.limit(5)) # COMMAND ---------- -# DBTITLE 1,Why rules cannot catch this -# MAGIC %md -# MAGIC Before training anything, here is the honest version of the problem. These are sensible rules, and -# MAGIC they are the rules a good team would already have: +# DBTITLE 1,Create the training table + +print("🔄 Generating three months of clean history...\n") + +history_df = generate_transactions(6000, seed=11) +history_table = f"{catalog}.{schema}.card_transactions_history" +history_df.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(history_table) + +print("📊 Sample of historical transactions:") +display(history_df.limit(10)) + +print(f"\n✅ {history_df.count():,} transactions saved to {history_table}") # COMMAND ---------- -catalog = dbutils.widgets.get("demo_catalog") -schema = dbutils.widgets.get("demo_schema") +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ## Section 2: The Rules a Good Team Already Has +# MAGIC +# MAGIC These are sensible rules, not strawmen — amount present, amount in range, item count in range. +# MAGIC They are exactly what you should write, and they will catch a great deal of real breakage. +# MAGIC +# MAGIC They will not catch what we are about to inject. +# MAGIC -ws = WorkspaceClient() -dq_engine = DQEngine(ws) -anomaly_engine = AnomalyEngine(ws) +# COMMAND ---------- +# DBTITLE 1,Define the rules -# The rules a payments team would already have. Deliberately reasonable, not strawmen. rules = [ DQRowRule(check_func=is_not_null, column="amount", criticality="error"), DQRowRule( @@ -180,51 +251,92 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): ), ] -# Nothing is persisted or cached in this notebook: PERSIST is unsupported on serverless compute, -# which is what most readers will run this on. These frames are small local relations built from -# seeded RNGs, so recomputation is both cheap and deterministic. -new_df, injected = generate_transactions(1500, seed=99, inject=True) -print(f"✅ {new_df.count():,} new transactions, {injected} of them jointly implausible\n") +print(f"✅ {len(rules)} rules defined") + +# COMMAND ---------- +# DBTITLE 1,Generate a new batch containing real problems + +# Nothing is cached in this notebook: PERSIST is unsupported on serverless compute, which is what most +# readers will run this on. These frames are small local relations built from seeded RNGs, so +# recomputation is both cheap and deterministic. +print("🔄 Generating a new batch with problems injected...\n") + +new_df = generate_transactions(1500, seed=99, inject=True) +new_table = f"{catalog}.{schema}.card_transactions_new" +new_df.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(new_table) + +total_new = new_df.count() +injected = new_df.filter(F.col("is_anomaly") == 1.0).count() + +print(f"✅ {total_new:,} new transactions saved to {new_table}") +print(f" {injected} of them jointly implausible") + +# COMMAND ---------- +# DBTITLE 1,Apply the rules + +print("🔍 Applying the rule-based checks...\n") rule_results = dq_engine.apply_checks(new_df, rules) caught_by_rules = rule_results.filter(F.col("_errors").isNotNull() & (F.col("is_anomaly") == 1.0)).count() -print(f"Rules caught {caught_by_rules} of the {injected} implausible transactions.") -print("Every injected row sits inside every threshold — each value is ordinary on its own.") -print("Widening the rules cannot help; tightening them would reject legitimate transactions.") +print(f"⚠️ Rules caught {caught_by_rules} of the {injected} implausible transactions.") +print(" Every injected row sits inside every threshold — each value is ordinary on its own.") +print(" Widening the rules cannot help; tightening them would reject legitimate transactions.") # COMMAND ---------- -# DBTITLE 1,Train: no thresholds, no labels + # MAGIC %md -# MAGIC Now the model. Note what is *not* being passed: no thresholds, no per-category limits, no labels. -# MAGIC `baseline_by=["merchant_category"]` is the one modelling decision, and it says the thing a payments -# MAGIC analyst already knows — **judge each transaction against its own category**, not against the table. +# MAGIC --- +# MAGIC +# MAGIC ## Section 3: Train the Anomaly Model +# MAGIC +# MAGIC Note what is **not** passed: no thresholds, no per-category limits, no labels. DQX learns the +# MAGIC patterns from the history table. +# MAGIC +# MAGIC `baseline_by=["merchant_category"]` is the one modelling decision, and it says what a payments +# MAGIC analyst already knows: **judge each transaction against its own category**. DQX then adds, for every +# MAGIC metric, its deviation from that category's own median — so one model can hold "£900 is normal for +# MAGIC electronics and extreme for coffee". +# MAGIC +# MAGIC `profile="tabular"` is the default and is right for independent records like these. Use +# MAGIC `profile="timeseries"` for repeated multivariate measurements such as machine telemetry — see the +# MAGIC companion notebook. +# MAGIC +# MAGIC **Why `transaction_time` is not in `columns`.** A datetime column becomes seven features (cyclical +# MAGIC hour, day of week and month, plus a weekend flag). That is valuable when *when* something happened +# MAGIC carries meaning — off-hours activity, weekend spikes. In this dataset it does not, so those seven +# MAGIC features would be noise, and noise costs you twice: it dilutes the columns that do carry signal, and +# MAGIC it manufactures "unusual timing" alerts that spend your alert budget. Measured on this data, +# MAGIC excluding it lifts recall at threshold 98 from **77% to 100%** and precision from 38% to 50%. +# MAGIC +# MAGIC The rule is general: **feed a column only if it relates to the anomalies you care about.** Every +# MAGIC extra column adds features, and features you do not need make the ones you do harder to see. Note +# MAGIC that auto-discovery — `train()` with no `columns` — would have included the timestamp here. +# MAGIC # COMMAND ---------- +# DBTITLE 1,Train the model + +print("🎯 Training the anomaly model...\n") model_name = f"{catalog}.{schema}.card_transactions_monitor" -registry_table = f"{catalog}.{schema}.dqx_anomaly_models" trained = anomaly_engine.train( - df=history_df, + df=spark.table(history_table), model_name=model_name, registry_table=registry_table, - columns=["amount", "item_count", "transaction_time"], + # transaction_time is deliberately excluded — see the note above. + columns=["amount", "item_count"], baseline_by=["merchant_category"], - # profile="tabular" is the default: independent records, anomalies are unusual values or - # unusual combinations of them. Stated here only to make the choice visible. profile="tabular", ) -print(f"\n✅ trained: {trained}") -# COMMAND ---------- -# DBTITLE 1,What conditioning bought -# MAGIC %md -# MAGIC `baseline_by` appends, for each metric, its deviation from **that category's own** median. So the -# MAGIC model sees both the raw amount and "how this amount compares to a typical basket in this category" — -# MAGIC which is how £900 can be normal for electronics and extreme for coffee inside one model. +print(f"\n✅ Model trained: {trained}") # COMMAND ---------- +# DBTITLE 1,What DQX engineered for you + +print("📋 Registered model and its engineered features:\n") display( spark.table(registry_table) @@ -235,16 +347,27 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): "grouping.baseline_by", "training.training_rows", "from_json(features.feature_metadata, 'engineered_feature_names array')" - ".engineered_feature_names as features", + ".engineered_feature_names as engineered_features", ) ) +print("💡 Note the `_rel_baseline` features — each metric's deviation from its own category's median.") +print(" Four features from two columns, and every one of them carries signal.") + # COMMAND ---------- -# DBTITLE 1,Score, and read why + # MAGIC %md -# MAGIC One check. Contributions and AI explanations are on by default. +# MAGIC --- +# MAGIC +# MAGIC ## Section 4: Score and Triage +# MAGIC +# MAGIC One check. Feature contributions and AI explanations are **on by default**. +# MAGIC # COMMAND ---------- +# DBTITLE 1,Apply the anomaly check + +print("🔍 Scoring the new batch...\n") anomaly_check = [ DQDatasetRule( @@ -258,26 +381,28 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): ) ] -# Written to a table rather than left lazy: AI explanations call an LLM through ai_query inside the -# scoring plan, so each action on an unmaterialised result would call the model again -- paying repeatedly -# and getting a different answer each time. `.cache()` cannot be used (PERSIST is unsupported on -# serverless), so writing once is both the fix and the pattern to copy in a real pipeline. +# One DQX call: name the input table, name the output table. Writing the result rather than keeping a +# lazy DataFrame also matters here — AI explanations call an LLM through ai_query *inside* the scoring +# plan, so each action on an unmaterialised result would call the model again. scored_table = f"{catalog}.{schema}.transactions_scored" -dq_engine.apply_checks(new_df, anomaly_check).write.mode("overwrite").option( - "overwriteSchema", "true" -).saveAsTable(scored_table) + +dq_engine.apply_checks_and_save_in_table( + input_config=InputConfig(location=new_table), + output_config=OutputConfig(location=scored_table, mode="overwrite", options={"overwriteSchema": "true"}), + checks=anomaly_check, +) + scored = spark.table(scored_table) anomaly = F.element_at(F.col("_dq_info"), 1).getField("anomaly") - flagged = scored.filter(anomaly.getField("is_anomaly")) -caught = scored.filter(anomaly.getField("is_anomaly") & (F.col("is_anomaly") == 1.0)).count() -print(f"Rules caught {caught_by_rules:>3} of {injected}") -print(f"Anomaly detection caught {caught:>3} of {injected}") -print(f"\n{flagged.count()} rows flagged in total out of {new_df.count():,} ({flagged.count() / new_df.count():.1%})") +print(f"✅ Scoring complete — {flagged.count()} of {total_new:,} rows flagged") # COMMAND ---------- -# DBTITLE 1,The explanations name the columns that combined badly +# DBTITLE 1,Which columns combined badly + +caught = flagged.filter(F.col("is_anomaly") == 1.0).count() +print(f"🔝 Anomaly detection caught {caught} of the {injected} implausible transactions.\n") display( flagged.select( @@ -293,7 +418,9 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): ) # COMMAND ---------- -# DBTITLE 1,And in plain language, per group of similar anomalies +# DBTITLE 1,Why each group was flagged, in plain language + +print("🤖 AI explanations, one per group of similar anomalies:\n") display( flagged.select( @@ -311,70 +438,124 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): ) # COMMAND ---------- -# DBTITLE 1,Does conditioning actually matter? Train without it and compare. + # MAGIC %md -# MAGIC The claim above was that comparing each transaction against its own category is what makes this -# MAGIC work. Worth testing rather than asserting: the same data, the same everything, `baseline_by=[]`. +# MAGIC --- +# MAGIC +# MAGIC ## Section 5: (Optional) Tune the Threshold +# MAGIC +# MAGIC **The threshold is an alert budget, not a confidence score.** `threshold=95` means "flag the rows +# MAGIC above the 95th percentile of *training* severity" — so on 1,500 rows it flags roughly 75 before a +# MAGIC single anomaly exists. A row at severity 97 is not "97% likely to be a problem"; it is in the top 3% +# MAGIC most unusual. This is the most commonly misread number in the feature. +# MAGIC +# MAGIC That also puts a hard ceiling on precision. Ask for the top 5% of 1,500 rows and you get 75 alerts; +# MAGIC if only 30 rows are genuinely bad, the best precision anyone could achieve is 30/75 = **40%**. The +# MAGIC table below prints that ceiling next to what the model actually achieved, which is the only fair way +# MAGIC to read the number. +# MAGIC +# MAGIC Severity is computed for **every** row, so you can count would-be anomalies at other thresholds +# MAGIC without rescoring. +# MAGIC # COMMAND ---------- +# DBTITLE 1,Threshold tradeoffs + +severity = anomaly.getField("severity_percentile") +truth = F.col("is_anomaly") == 1.0 + +print("🎚️ Testing different thresholds:\n") +print("Threshold | Alerts | Caught | Precision | Best possible | Recall") +print("-" * 68) + +for threshold in (90.0, 95.0, 98.0): + alerts = scored.filter(severity >= threshold) + n_alerts = alerts.count() + n_caught = alerts.filter(truth).count() + # The ceiling: you cannot be more precise than "every alert is a real anomaly". + ceiling = min(1.0, injected / n_alerts) if n_alerts else 0.0 + precision = n_caught / n_alerts if n_alerts else 0.0 + print( + f" {threshold:>5.0f} | {n_alerts:>6d} | {n_caught:>4d}/{injected:<3d}|" + f" {precision:>6.1%} | {ceiling:>7.1%} | {n_caught / injected:>5.1%}" + ) -pooled = anomaly_engine.train( - df=history_df, - model_name=f"{catalog}.{schema}.card_transactions_pooled", - registry_table=registry_table, - columns=["amount", "item_count", "transaction_time"], - baseline_by=[], # compare against the whole table instead -) +print("\n💡 Read precision against the ceiling, not against 100%. Where the two are equal, every") +print(" planted anomaly is inside the model's ranking and no alert is wasted — the ranking is") +print(" optimal for that budget. A tighter threshold then trades recall for precision; it does") +print(" not reveal a better model.") -pooled_scored = dq_engine.apply_checks( - new_df, - [ - DQDatasetRule( - criticality="error", - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": pooled, - "registry_table": registry_table, - "threshold": 95.0, - # This model exists only to produce one comparison number, so skip the attribution and - # the LLM call. Both are on by default. - "enable_contributions": False, - "enable_ai_explanation": False, - }, - ) - ], -) -pooled_anomaly = F.element_at(F.col("_dq_info"), 1).getField("anomaly") -pooled_caught = pooled_scored.filter(pooled_anomaly.getField("is_anomaly") & (F.col("is_anomaly") == 1.0)).count() +# COMMAND ---------- -print(f"conditioned on merchant_category : {caught:>3} of {injected} caught") -print(f"compared against whole table : {pooled_caught:>3} of {injected} caught") -print("\nBoth are real models on identical data. The difference is the basis of comparison.") +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ## Summary & Next Steps +# MAGIC +# MAGIC **Key takeaways:** +# MAGIC - Rules catch what you can name in advance. They caught **none** of these rows, and no threshold +# MAGIC would have, because every individual value was ordinary. +# MAGIC - Row anomaly detection finds implausible **combinations**, with no thresholds to choose. +# MAGIC - `baseline_by` makes "normal" contextual, so one model covers a coffee shop and an electronics store. +# MAGIC - Contributions and AI explanations tell you *which columns combined badly*, so a flagged row is +# MAGIC actionable rather than merely suspicious. +# MAGIC - The threshold is an **alert budget**, not a confidence score. Judge precision against the ceiling +# MAGIC that budget implies. +# MAGIC - Feed the model only columns that relate to what you are looking for. Here, excluding a timestamp +# MAGIC whose values carried no meaning took recall from 77% to 100%. +# MAGIC +# MAGIC **Apply to your data:** +# MAGIC ```python +# MAGIC model = anomaly_engine.train( +# MAGIC df=spark.table("your_catalog.your_schema.your_table"), +# MAGIC model_name="your_catalog.your_schema.your_model", +# MAGIC registry_table="your_catalog.your_schema.dqx_anomaly_models", +# MAGIC baseline_by=["your_grouping_column"], # judge each row against its own group +# MAGIC ) +# MAGIC +# MAGIC checks = [ +# MAGIC DQDatasetRule( +# MAGIC criticality="error", +# MAGIC check_func=has_no_row_anomalies, +# MAGIC check_func_kwargs={ +# MAGIC "model_name": model, +# MAGIC "registry_table": "your_catalog.your_schema.dqx_anomaly_models", +# MAGIC }, +# MAGIC ) +# MAGIC ] +# MAGIC +# MAGIC # Name the input table and the output table — DQX reads, scores and writes in one call. +# MAGIC dq_engine.apply_checks_and_save_in_table( +# MAGIC input_config=InputConfig(location="your_catalog.your_schema.new_data"), +# MAGIC output_config=OutputConfig(location="your_catalog.your_schema.scored"), +# MAGIC checks=checks, +# MAGIC ) +# MAGIC ``` +# MAGIC +# MAGIC **Optional next steps:** +# MAGIC - Add `drift_threshold=3.0` to be warned when the input distribution moves away from training. +# MAGIC - Quarantine flagged rows with `apply_checks_and_split` instead of tagging them in place. +# MAGIC - Schedule retraining as "normal" changes — new products, new pricing, new processes. +# MAGIC # COMMAND ---------- # MAGIC %md -# MAGIC ## What to take away -# MAGIC -# MAGIC - Rules catch what you can **name in advance**. They caught none of these, and no threshold would -# MAGIC have, because every individual value was ordinary. -# MAGIC - Row anomaly detection catches **implausible combinations**, with no thresholds to choose. -# MAGIC - `baseline_by` is how "normal" becomes contextual, and it helps here rather than transforming the -# MAGIC result: the comparison above is a real but modest gain. That is honest and expected — some of these -# MAGIC injected rows (a £900 coffee) are extreme enough to stand out against the whole table too. -# MAGIC Conditioning earns its keep on the ones that are not, like a 40-item basket costing £4. -# MAGIC - Contributions tell you **which columns combined badly**, so a flagged row is actionable rather -# MAGIC than just suspicious. -# MAGIC -# MAGIC Use rules *and* this. Rules are cheaper, clearer and versioned; they should catch everything you can -# MAGIC describe. Anomaly detection is for what is left. -# MAGIC -# MAGIC ### Where this profile is the wrong tool -# MAGIC -# MAGIC These transactions are **independent records** — each row stands on its own. When rows are instead -# MAGIC repeated measurements of the same thing over time, and the failure is metrics that normally move -# MAGIC together drifting apart, the `tabular` profile is close to blind to it. That is -# MAGIC `dqx_demo_anomaly_timeseries_fleet.py`. -# MAGIC -# MAGIC Neither profile models trend or forecasts the next value. See -# MAGIC [Choosing a profile](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection#choosing-a-profile). +# MAGIC --- +# MAGIC +# MAGIC ### 📚 Resources +# MAGIC +# MAGIC - [Row Anomaly Detection guide](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection) +# MAGIC - [`has_no_row_anomalies` reference](https://databrickslabs.github.io/dqx/docs/reference/quality_checks#row-anomaly-detection) +# MAGIC - [Choosing a profile](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection#choosing-a-profile) +# MAGIC +# MAGIC ### 🎉 You're Ready! +# MAGIC +# MAGIC You now understand: +# MAGIC - ✅ Why rule-based checks cannot catch implausible combinations +# MAGIC - ✅ How to train an anomaly model with no thresholds and no labels +# MAGIC - ✅ How `baseline_by` makes "normal" depend on context +# MAGIC - ✅ How to read contributions and AI explanations to triage a flagged row +# MAGIC +# MAGIC **Start finding the rows your rules miss!** 🚀 +# MAGIC diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py index c81a3fc33..7028cdf13 100644 --- a/demos/dqx_demo_anomaly_timeseries_fleet.py +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -1,30 +1,69 @@ # Databricks notebook source # MAGIC %md -# MAGIC # The machine where every gauge read normal +# MAGIC # ⚙️ When Every Gauge Reads Normal and the Machine Still Fails # MAGIC -# MAGIC A plant records eight metrics per machine per minute: spindle load, motor current, coolant flow, -# MAGIC bearing temperature, vibration, hydraulic pressure, air pressure, throughput. Every metric has a safe -# MAGIC operating band, and there are alerts on all of them. +# MAGIC ## Learn Row Anomaly Detection on Machine Telemetry in 10 Minutes # MAGIC -# MAGIC A bearing fails. Afterwards the logs show every gauge sat inside its band for two hours beforehand. -# MAGIC No alert fired. Nothing to see. +# MAGIC **What you'll do:** +# MAGIC - Generate healthy telemetry where the metrics move together, as real machines do +# MAGIC - Break the *relationship* between three of them while every reading stays in range +# MAGIC - Prove no threshold or range check could ever catch it +# MAGIC - Train with `profile="timeseries"` and read the explanation # MAGIC -# MAGIC But something *was* visible: motor current stopped tracking spindle load. On a healthy machine those -# MAGIC rise and fall together, because cutting harder draws more current. For two hours load was high while -# MAGIC current sat mid-band. Both readings were ordinary. **Their relationship was not.** +# MAGIC **Dataset**: Eight metrics per reading across four CNC machines (no domain expertise required) # MAGIC -# MAGIC This is a different shape of anomaly from the one in `dqx_demo_anomaly_tabular_transactions.py`, and -# MAGIC it needs a different detector. It is what **`profile="timeseries"`** is for. + +# COMMAND ---------- + +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ## The problem: a broken relationship is not an extreme value +# MAGIC +# MAGIC A plant records spindle load, motor current, coolant flow, bearing temperature, vibration, +# MAGIC hydraulic pressure, air pressure and throughput. Every metric has a safe band, and every band has +# MAGIC an alert. +# MAGIC +# MAGIC A bearing fails. Afterwards the logs show every gauge sat inside its band for two hours +# MAGIC beforehand. No alert fired. +# MAGIC +# MAGIC But something *was* visible: **motor current stopped tracking spindle load**. On a healthy machine +# MAGIC those rise and fall together, because cutting harder draws more current. For two hours load was +# MAGIC high while current sat mid-band. Both readings were ordinary. Their relationship was not. +# MAGIC +# MAGIC ## Two kinds of anomaly, two profiles +# MAGIC +# MAGIC | Your data | `profile` | An anomaly looks like | +# MAGIC |---|---|---| +# MAGIC | Independent records — transactions, orders, customers | `"tabular"` (default) | A row whose values, or combination of values, is unusual | +# MAGIC | Repeated multivariate measurements — machine or service metrics, sensors | `"timeseries"` | Metrics that normally move **together** stop doing so, each staying in its own range | +# MAGIC +# MAGIC Use `"timeseries"` for data like this. The default detector splits on one feature at a time, so a +# MAGIC broken relationship between two in-range values is close to invisible to it. On the **Server Machine +# MAGIC Dataset** — 28 machines of real telemetry with labelled incidents — the correlation-aware detector +# MAGIC surfaces **79%** of incidents inside an alert budget of 1% of rows, against **33%** for the default. +# MAGIC +# MAGIC **It needs no timestamp column.** It models correlation *between metrics*, not behaviour over time, +# MAGIC and never looks at row order. Include a timestamp anyway and DQX derives calendar features from it +# MAGIC (hour, day of week, month, weekend) exactly as it does for the tabular profile. # MAGIC -# MAGIC We will train *both* profiles on the same data and compare, because a second algorithm is only worth -# MAGIC having if it earns its place. # COMMAND ---------- # MAGIC %md -# MAGIC ## Install +# MAGIC --- +# MAGIC +# MAGIC ## Prerequisites: Install DQX with Anomaly Support +# MAGIC +# MAGIC ```python +# MAGIC %pip install 'databricks-labs-dqx[anomaly]' +# MAGIC ``` +# MAGIC +# MAGIC **Note**: On ML Runtime or Serverless most dependencies are already present. +# MAGIC # COMMAND ---------- +# DBTITLE 1,Install DQX dbutils.widgets.text("test_library_ref", "", "Test Library Ref") @@ -36,6 +75,7 @@ %restart_python # COMMAND ---------- +# DBTITLE 1,Configure catalog and schema dbutils.widgets.text("demo_catalog", "main", "Catalog Name") dbutils.widgets.text("demo_schema", "default", "Schema Name") @@ -43,27 +83,59 @@ # COMMAND ---------- # MAGIC %md -# MAGIC ## No timestamp column is required +# MAGIC --- +# MAGIC +# MAGIC ## Section 1: Setup & Healthy Telemetry # MAGIC -# MAGIC Worth saying immediately, because everyone expects otherwise. `profile="timeseries"` models -# MAGIC **correlation between metrics**, not behaviour over time. It never looks at row order. The name -# MAGIC describes the *data you have* — repeated multivariate measurements — not a temporal algorithm. +# MAGIC | Column | Type | Description | +# MAGIC |---|---|---| +# MAGIC | `machine_id` | string | `CNC-01` … `CNC-04` | +# MAGIC | `reading_seq` | int | Reading order — for reference only, the model never uses it | +# MAGIC | `spindle_load` … `throughput` | double | The eight metrics | +# MAGIC | `is_incident` | double | Ground truth, for this demo only — never given to the model | +# MAGIC +# MAGIC Healthy readings are driven by **two hidden factors**: how hard the machine is working, and how hot +# MAGIC it is running. Every metric is a mix of the two plus its own noise, which is exactly why they move +# MAGIC together — and what a correlation-aware detector learns. # MAGIC -# MAGIC A consequence worth knowing: it will not learn "Tuesdays are busy" from row order either. Include a -# MAGIC timestamp column and DQX derives calendar features from it (hour, day-of-week, month, weekend) exactly -# MAGIC as it does for the tabular profile. What neither profile models is **trend**. # COMMAND ---------- -# DBTITLE 1,Generate healthy machine telemetry +# DBTITLE 1,Setup engines import numpy as np import pyspark.sql.functions as F from databricks.sdk import WorkspaceClient + from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies +from databricks.labs.dqx.config import InputConfig, OutputConfig from databricks.labs.dqx.engine import DQEngine from databricks.labs.dqx.rule import DQDatasetRule +catalog = dbutils.widgets.get("demo_catalog") +schema = dbutils.widgets.get("demo_schema") + +ws = WorkspaceClient() +dq_engine = DQEngine(ws) +anomaly_engine = AnomalyEngine(ws) + +print(f"✅ Setup complete — writing to {catalog}.{schema}") + +# COMMAND ---------- +# DBTITLE 1,Prepare a clean model registry + +# Drop the registry so each run of this notebook starts from nothing. Without this, re-running leaves +# every previous run's rows behind and the "registered model" cell below shows a pile of stale +# configurations rather than the one just trained. +registry_table = f"{catalog}.{schema}.dqx_anomaly_models" +spark.sql(f"DROP TABLE IF EXISTS {registry_table}") + +print(f"📋 Model registry: {registry_table}") +print("✅ Registry reset — ready for this run's model") + +# COMMAND ---------- +# DBTITLE 1,How the metrics relate to each other + METRICS = [ "spindle_load", "motor_current", @@ -75,297 +147,334 @@ "throughput", ] -# Two latent drivers -- how hard the machine is working, and how hot it is running. Every metric is a -# mix of the two plus its own noise, which is why they all move together on a healthy machine. +# Each row is one metric's sensitivity to (work rate, heat). Note motor_current tracks spindle_load +# closely, and air_pressure is mostly independent — not everything correlates, which is realistic. LOADINGS = np.array( - [ - [0.95, 0.10], # spindle_load <- mostly work rate - [0.90, 0.15], # motor_current <- tracks spindle load closely - [0.35, 0.80], # coolant_flow <- mostly heat - [0.30, 0.90], # bearing_temp <- mostly heat - [0.70, 0.45], # vibration - [0.80, 0.20], # hydraulic_pressure - [0.25, 0.30], # air_pressure <- mostly independent - [0.85, 0.25], # throughput - ] + [[0.95, 0.10], [0.90, 0.15], [0.35, 0.80], [0.30, 0.90], [0.70, 0.45], [0.80, 0.20], [0.25, 0.30], [0.85, 0.25]] ).T BASELINES = np.array([62.0, 18.5, 24.0, 58.0, 2.4, 145.0, 6.2, 480.0]) SCALES = np.array([9.0, 2.6, 3.4, 6.5, 0.45, 12.0, 0.35, 55.0]) +SCHEMA = "machine_id string, reading_seq int, " + ", ".join(f"{m} double" for m in METRICS) + ", is_incident double" + +print(f"📊 {len(METRICS)} metrics driven by 2 hidden factors") + +# COMMAND ---------- +# DBTITLE 1,Generate telemetry def generate_telemetry(n_rows: int, seed: int, break_correlation: bool = False): - """Machine telemetry driven by two shared latent factors. + """Telemetry driven by two shared latent factors. - When *break_correlation* is set, a contiguous block of rows has three metrics decoupled from the rest - by **permuting their values among those rows**. That preserves every metric's own distribution exactly - -- the same values, reordered -- so no single-metric alert can possibly fire. Only the joint - behaviour changes, which is precisely the failure this profile exists to catch. + With *break_correlation*, a contiguous block has three metrics decoupled by **permuting their values + among those rows** — the same values, reordered. Every metric's own distribution is preserved + exactly, so only the joint behaviour changes. """ rng = np.random.default_rng(seed) factors = rng.normal(0.0, 1.0, size=(n_rows, 2)) values = BASELINES + SCALES * (factors @ LOADINGS + rng.normal(0.0, 0.18, size=(n_rows, len(METRICS)))) - labels = np.zeros(n_rows) + if break_correlation: - # A two-hour incident: spindle_load, motor_current and vibration stop tracking each other. start, length = int(n_rows * 0.72), max(4, int(n_rows * 0.04)) block = values[start : start + length, :3] values[start : start + length, :3] = np.roll(block, shift=1, axis=0) labels[start : start + length] = 1.0 - rows = [ - (f"CNC-{(i % 4) + 1:02d}", i, *[float(v) for v in values[i]], float(labels[i])) for i in range(n_rows) - ] - schema = "machine_id string, reading_seq int, " + ", ".join(f"{m} double" for m in METRICS) + ", is_incident double" - return spark.createDataFrame(rows, schema), int(labels.sum()) + rows = [(f"CNC-{(i % 4) + 1:02d}", i, *[float(v) for v in values[i]], float(labels[i])) for i in range(n_rows)] + return spark.createDataFrame(rows, SCHEMA) + + +# COMMAND ---------- +# DBTITLE 1,Create the training table + +# Nothing is cached in this notebook: PERSIST is unsupported on serverless compute, which is what most +# readers will run this on. These frames are small local relations built from seeded RNGs, so +# recomputation is both cheap and deterministic. +print("🔄 Generating healthy telemetry...\n") + +healthy_df = generate_telemetry(4000, seed=5) +healthy_table = f"{catalog}.{schema}.fleet_telemetry_healthy" +healthy_df.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(healthy_table) +print("📊 Sample of healthy readings:") +display(healthy_df.limit(10)) -# Nothing is persisted or cached in this notebook: PERSIST is unsupported on serverless compute, -# which is what most readers will run this on. These frames are small local relations built from -# seeded RNGs, so recomputation is both cheap and deterministic. -healthy_df, _ = generate_telemetry(4000, seed=5) -print(f"✅ {healthy_df.count():,} healthy readings across 4 machines, {len(METRICS)} metrics each") -display(healthy_df.limit(5)) +print(f"\n✅ {healthy_df.count():,} readings saved to {healthy_table}") # COMMAND ---------- -# DBTITLE 1,The correlation this depends on +# DBTITLE 1,Confirm the metrics really do move together + +PAIRS = [ + ("spindle_load", "motor_current", "cutting harder draws more current"), + ("bearing_temp", "coolant_flow", "coolant responds to heat"), + ("spindle_load", "air_pressure", "barely related — and that is realistic"), +] + +correlations = spark.table(healthy_table).select( + *[F.round(F.corr(left, right), 3).alias(f"{left}__{right}") for left, right, _ in PAIRS] +).first() + +print("🔍 Correlations on healthy data — the premise this detector relies on:\n") +print(f"{'metric pair':<34}{'correlation':>13} why") +print("-" * 78) + +for left, right, reason in PAIRS: + print(f"{left + ' vs ' + right:<34}{correlations[f'{left}__{right}']:>13.3f} {reason}") + +print("\n💡 Strong pairs are what the detector exploits. Not every metric relates to every other,") +print(" and real telemetry looks exactly like this.") + +# COMMAND ---------- + # MAGIC %md -# MAGIC Before modelling anything, confirm the premise: on healthy data these metrics really do move -# MAGIC together. `motor_current` against `spindle_load` is the pair from the story. +# MAGIC --- +# MAGIC +# MAGIC ## Section 2: The Incident +# MAGIC +# MAGIC Two hours in which spindle load, motor current and coolant flow stop tracking each other, while +# MAGIC every single reading stays inside the range it has always occupied. +# MAGIC # COMMAND ---------- +# DBTITLE 1,Inject the correlation break -display( - healthy_df.select( - F.round(F.corr("spindle_load", "motor_current"), 3).alias("load_vs_current"), - F.round(F.corr("bearing_temp", "coolant_flow"), 3).alias("temp_vs_coolant"), - F.round(F.corr("spindle_load", "air_pressure"), 3).alias("load_vs_air_pressure"), - ) -) -print("Strong pairs are the ones a correlation-aware detector can exploit.") -print("air_pressure is mostly independent by design — not everything correlates, and that is realistic.") +print("🔄 Generating a batch containing the incident...\n") + +incident_df = generate_telemetry(1500, seed=77, break_correlation=True) +incident_table = f"{catalog}.{schema}.fleet_telemetry_incident" +incident_df.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(incident_table) + +total_readings = incident_df.count() +injected = incident_df.filter(F.col("is_incident") == 1.0).count() + +print(f"✅ {total_readings:,} readings saved to {incident_table}") +print(f" {injected} of them during the incident") # COMMAND ---------- -# DBTITLE 1,Now the incident, and proof it is invisible per-metric +# DBTITLE 1,Prove no range check could catch it -incident_df, injected = generate_telemetry(1500, seed=77, break_correlation=True) -print(f"✅ {incident_df.count():,} readings, {injected} of them during the incident\n") +readings = spark.table(incident_table) +during = readings.filter(F.col("is_incident") == 1.0) +outside = readings.filter(F.col("is_incident") == 0.0) -# Compare each metric's range during the incident against its range outside it. If a range check could -# catch this, we have not built the scenario we claimed to. -during = incident_df.filter(F.col("is_incident") == 1.0) -outside = incident_df.filter(F.col("is_incident") == 0.0) +print("🔍 Each metric's healthy range vs its range during the incident:\n") +print(f"{'metric':<18}{'healthy range':>22}{'during incident':>22} verdict") +print("-" * 74) -print(f"{'metric':<22}{'healthy range':>26}{'during incident':>26}") for metric in METRICS[:3]: - o = outside.select(F.min(metric).alias("lo"), F.max(metric).alias("hi")).first() - d = during.select(F.min(metric).alias("lo"), F.max(metric).alias("hi")).first() - inside = "inside" if d["lo"] >= o["lo"] and d["hi"] <= o["hi"] else "OUTSIDE" - print(f"{metric:<22}{f'{o.lo:.1f} – {o.hi:.1f}':>26}{f'{d.lo:.1f} – {d.hi:.1f} ({inside})':>26}") + healthy = outside.select(F.min(metric).alias("lo"), F.max(metric).alias("hi")).first() + broken = during.select(F.min(metric).alias("lo"), F.max(metric).alias("hi")).first() + inside = broken["lo"] >= healthy["lo"] and broken["hi"] <= healthy["hi"] + healthy_range = f"{healthy.lo:.1f} – {healthy.hi:.1f}" + broken_range = f"{broken.lo:.1f} – {broken.hi:.1f}" + print(f"{metric:<18}{healthy_range:>22}{broken_range:>22} {'inside ✅' if inside else 'OUTSIDE'}") -print("\nEvery incident reading sits inside the healthy range for its own metric.") -print("No threshold, no range check and no per-metric z-score can separate these rows.") +print("\n⚠️ Every incident reading sits inside the healthy range for its own metric.") +print(" No threshold, no range check and no per-metric z-score can separate these rows.") # COMMAND ---------- -# DBTITLE 1,Train both profiles on identical data + # MAGIC %md -# MAGIC The comparison that justifies a second algorithm. Same data, same columns, same everything except -# MAGIC one word. +# MAGIC --- +# MAGIC +# MAGIC ## Section 3: Train with `profile="timeseries"` +# MAGIC +# MAGIC One word selects the correlation-aware detector. Everything else is unchanged: the same automatic +# MAGIC feature engineering, the same registry, the same `has_no_row_anomalies` check, the same +# MAGIC contributions and AI explanations. +# MAGIC +# MAGIC It measures how far a reading sits from normal **once the relationships between metrics are +# MAGIC accounted for**. A load/current pair that never co-occurs on healthy data is far away in that +# MAGIC space even though each value sits mid-range. +# MAGIC +# MAGIC `baseline_by=[]` keeps the comparison across the whole fleet, because these four machines are +# MAGIC interchangeable and share one operating envelope. Pass `baseline_by=["machine_id"]` instead when +# MAGIC each machine has its own normal. +# MAGIC # COMMAND ---------- +# DBTITLE 1,Train the model -catalog = dbutils.widgets.get("demo_catalog") -schema = dbutils.widgets.get("demo_schema") -registry_table = f"{catalog}.{schema}.dqx_anomaly_models" +print("🎯 Training the correlation-aware model...\n") -ws = WorkspaceClient() -dq_engine = DQEngine(ws) -anomaly_engine = AnomalyEngine(ws) +model_name = f"{catalog}.{schema}.fleet_telemetry_monitor" -models = {} -for profile in ("tabular", "timeseries"): - models[profile] = anomaly_engine.train( - df=healthy_df, - model_name=f"{catalog}.{schema}.fleet_telemetry_{profile}", - registry_table=registry_table, - columns=METRICS, - # No grouping: these machines are interchangeable and share one operating envelope. Pass - # baseline_by=["machine_id"] instead if each machine has its own normal. - baseline_by=[], - profile=profile, - ) - print(f"✅ {profile:<11} -> {models[profile]}") +trained = anomaly_engine.train( + df=spark.table(healthy_table), + model_name=model_name, + registry_table=registry_table, + columns=METRICS, + baseline_by=[], + profile="timeseries", +) + +print(f"\n✅ Model trained: {trained}") # COMMAND ---------- -# DBTITLE 1,The registry records which detector each model uses +# DBTITLE 1,The registry records which detector was used + +print("📋 Registered model:\n") display( spark.table(registry_table) - .filter(F.col("identity.model_name").contains("fleet_telemetry")) + .filter(F.col("identity.model_name") == trained) .selectExpr( "identity.model_name", "identity.algorithm", "training.hyperparameters['covariance'] as covariance", "training.training_rows", ) - .orderBy("identity.model_name") ) -print("Scoring reads the algorithm back off the registry, so you never repeat the choice.") - -# COMMAND ---------- -# DBTITLE 1,Score both, and count incidents caught - -# Scoring is written to a table rather than held as a lazy plan, and that matters here rather than being -# housekeeping. AI explanations call an LLM through ai_query *inside* the scoring plan, so every action on -# an unmaterialised result calls the model again -- the cells below take several actions each, which would -# mean paying for the LLM repeatedly and getting a different answer each time. `.cache()` would normally -# prevent that, but PERSIST is unsupported on serverless compute. Writing once is the pattern to copy. -results = {} -for profile, model in models.items(): - scored = dq_engine.apply_checks( - incident_df, - [ - DQDatasetRule( - criticality="error", - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": model, - "registry_table": registry_table, - "threshold": 95.0, - }, - ) - ], - ) - scored_table = f"{catalog}.{schema}.fleet_scored_{profile}" - scored.write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(scored_table) - results[profile] = spark.table(scored_table) - print(f"✅ {profile:<11} scored -> {scored_table}") - -def anomaly_of(df): - """The anomaly struct from the first _dq_info element (element_at is 1-based).""" - return F.element_at(F.col("_dq_info"), 1).getField("anomaly") - -print(f"{'profile':<13}{'incident rows caught':>22}{'total flagged':>16}{'false alarms':>15}") -for profile, scored in results.items(): - a = anomaly_of(scored) - caught = scored.filter(a.getField("is_anomaly") & (F.col("is_incident") == 1.0)).count() - flagged = scored.filter(a.getField("is_anomaly")).count() - false_alarms = flagged - caught - print(f"{profile:<13}{f'{caught} of {injected}':>22}{flagged:>16}{false_alarms:>15}") - -print("\nSame data. Same columns. The tabular detector splits one metric at a time, so a broken") -print("relationship between two in-range values is close to invisible to it.") +print("💡 Scoring reads the algorithm back off the registry, so you never repeat the choice.") +print(" It also trains a single model rather than an ensemble, because it is deterministic.") # COMMAND ---------- -# DBTITLE 1,The same comparison at an equal alert budget + # MAGIC %md -# MAGIC The two detectors above did not flag the same number of rows, and they need not: a severity -# MAGIC threshold is a percentile of the *training* score distribution, so how many new rows exceed it -# MAGIC depends on the detector. That leaves a fair question — did the second one just alert more? +# MAGIC --- +# MAGIC +# MAGIC ## Section 4: Score and Read the Explanation +# MAGIC +# MAGIC The detector explains itself by leaving each metric out in turn and reporting how much of the +# MAGIC anomaly disappears — so contributions work here with no SHAP involved. # MAGIC -# MAGIC Settle it by giving both the same budget. Rank every reading by severity, take the top N from each -# MAGIC with N identical, and count how many incident rows each one bought. This is how the DQX benchmarks -# MAGIC compare detectors, and it is the honest way to read any such comparison. # COMMAND ---------- +# DBTITLE 1,Apply the anomaly check + +print("🔍 Scoring the incident batch...\n") + +anomaly_check = [ + DQDatasetRule( + criticality="error", + check_func=has_no_row_anomalies, + check_func_kwargs={ + "model_name": trained, + "registry_table": registry_table, + "threshold": 95.0, + }, + ) +] -from pyspark.sql import Window - -BUDGET_FRACTION = 0.05 # alert on the 5% of readings each detector ranks most anomalous -budget = int(incident_df.count() * BUDGET_FRACTION) - -print(f"budget: the top {budget} readings by severity ({BUDGET_FRACTION:.0%} of {incident_df.count():,})\n") -print(f"{'profile':<13}{'incident rows caught':>22}{'precision':>12}") -for profile, scored in results.items(): - ranked = scored.select( - F.col("is_incident"), - anomaly_of(scored).getField("severity_percentile").alias("severity"), - ).withColumn("rank", F.row_number().over(Window.orderBy(F.col("severity").desc_nulls_last()))) +# One DQX call: name the input table, name the output table. Writing the result rather than keeping a +# lazy DataFrame also matters here — AI explanations call an LLM through ai_query *inside* the scoring +# plan, so each action on an unmaterialised result would call the model again. +scored_table = f"{catalog}.{schema}.fleet_scored" - top = ranked.filter(F.col("rank") <= budget) - caught = top.filter(F.col("is_incident") == 1.0).count() - print(f"{profile:<13}{f'{caught} of {injected}':>22}{caught / budget:>11.0%}") +dq_engine.apply_checks_and_save_in_table( + input_config=InputConfig(location=incident_table), + output_config=OutputConfig(location=scored_table, mode="overwrite", options={"overwriteSchema": "true"}), + checks=anomaly_check, +) -print("\nSame number of alerts for each. The difference is what those alerts are worth.") +scored = spark.table(scored_table) +anomaly = F.element_at(F.col("_dq_info"), 1).getField("anomaly") +flagged = scored.filter(anomaly.getField("is_anomaly")) -# COMMAND ---------- -# DBTITLE 1,Why the correlation-aware detector can see it -# MAGIC %md -# MAGIC It measures how far a reading sits from normal **once the relationships between metrics are -# MAGIC accounted for** — a distance in a space where the metrics have been decorrelated. A load/current -# MAGIC pair that never co-occurs on healthy data is far away in that space even though each value is -# MAGIC near the middle of its own range. -# MAGIC -# MAGIC It explains itself by leaving each metric out in turn and reporting how much of the anomaly -# MAGIC disappears — so contributions work here without SHAP. +print(f"✅ Scoring complete — {flagged.count()} of {total_readings:,} readings flagged") # COMMAND ---------- +# DBTITLE 1,Which relationships broke -ts = results["timeseries"] -a = anomaly_of(ts) +caught = flagged.filter(F.col("is_incident") == 1.0).count() +print(f"🔝 Caught {caught} of the {injected} incident readings — none of which any range check could see.\n") display( - ts.filter(a.getField("is_anomaly")) - .select( + flagged.select( "machine_id", "reading_seq", F.round("spindle_load", 1).alias("spindle_load"), F.round("motor_current", 1).alias("motor_current"), - F.round("vibration", 2).alias("vibration"), - a.getField("severity_percentile").alias("severity"), - a.getField("contributions").alias("contributions"), + anomaly.getField("severity_percentile").alias("severity"), + anomaly.getField("contributions").alias("contributions"), ) .orderBy(F.desc("severity")) .limit(10) ) # COMMAND ---------- -# DBTITLE 1,And in plain language +# DBTITLE 1,Why each group was flagged, in plain language + +print("🤖 AI explanations, one per group of similar anomalies:\n") display( - ts.filter(a.getField("is_anomaly")) - .select( + flagged.select( "machine_id", "reading_seq", - a.getField("ai_explanation").getField("narrative").alias("narrative"), - a.getField("ai_explanation").getField("business_impact").alias("impact"), - a.getField("ai_explanation").getField("action").alias("action"), + anomaly.getField("ai_explanation").getField("narrative").alias("narrative"), + anomaly.getField("ai_explanation").getField("business_impact").alias("impact"), + anomaly.getField("ai_explanation").getField("action").alias("action"), ) .filter(F.col("narrative").isNotNull()) - .limit(5) + .limit(6) ) +print("💡 Note the wording: broken *relationships*, not abnormal metrics. DQX tells the model which") +print(" detector produced the contributions, so the explanation describes the right kind of problem.") + # COMMAND ---------- # MAGIC %md -# MAGIC ## What to take away +# MAGIC --- +# MAGIC +# MAGIC ## Summary & Next Steps # MAGIC +# MAGIC **Key takeaways:** # MAGIC - Some failures are **broken relationships**, not extreme values. Every gauge reads normal and the # MAGIC machine is still in trouble. -# MAGIC - Per-metric alerts cannot see those, however well tuned. In this notebook every incident reading -# MAGIC sits inside its own metric's healthy range — verified above, not asserted. -# MAGIC - `profile="timeseries"` switches to a detector that models how metrics move **together**. One word, -# MAGIC and everything else is unchanged: same feature engineering, same registry, same check, same -# MAGIC contributions and AI explanations. -# MAGIC - It needs **no timestamp column** and trains a single model rather than an ensemble, because it is -# MAGIC deterministic. -# MAGIC -# MAGIC ### Being straight about the limits -# MAGIC -# MAGIC On the **Server Machine Dataset** — 28 machines of real telemetry with labelled incidents — this -# MAGIC detector surfaces 79% of incidents inside an alert budget of 1% of rows, against 33% for the tabular -# MAGIC detector. That is a large gain on the data it is for, and it is *not* state of the art for -# MAGIC multivariate time-series anomaly detection. Published figures near 0.80 F1 are not a fair comparison: -# MAGIC they use point adjustment, which -# MAGIC [Kim et al. (AAAI 2022)](https://arxiv.org/abs/2109.05257) showed random scores also reach. -# MAGIC -# MAGIC What this does **not** do: -# MAGIC -# MAGIC - **Trend.** Sustained growth eventually drifts outside the trained range. Model a rate or a ratio -# MAGIC rather than a running level, and retrain on a schedule. +# MAGIC - Per-metric alerts cannot see those, however well tuned — verified above, not asserted. +# MAGIC - `profile="timeseries"` switches to a detector that models how metrics move together. One word; +# MAGIC everything else is unchanged. +# MAGIC - It needs no timestamp column and trains a single model rather than an ensemble. +# MAGIC +# MAGIC **Apply to your data:** +# MAGIC ```python +# MAGIC model = anomaly_engine.train( +# MAGIC df=spark.table("your_catalog.your_schema.your_metrics"), +# MAGIC model_name="your_catalog.your_schema.your_model", +# MAGIC registry_table="your_catalog.your_schema.dqx_anomaly_models", +# MAGIC profile="timeseries", +# MAGIC ) +# MAGIC +# MAGIC # Then score straight from one table into another. +# MAGIC dq_engine.apply_checks_and_save_in_table( +# MAGIC input_config=InputConfig(location="your_catalog.your_schema.new_readings"), +# MAGIC output_config=OutputConfig(location="your_catalog.your_schema.scored"), +# MAGIC checks=checks, +# MAGIC ) +# MAGIC ``` +# MAGIC +# MAGIC **What this does not do**, so you can plan around it: +# MAGIC - **Trend.** A steadily growing metric eventually leaves the range it was trained on. Model a rate +# MAGIC or a ratio rather than a running level, and retrain on a schedule. # MAGIC - **A single metric.** With nothing to correlate against, use a rule or a threshold. # MAGIC - **Forecasting.** DQX judges readings against learned normal; it does not predict the next value. -# MAGIC - **Choosing the profile for you.** You pick it. The cheap heuristic for guessing was measured and -# MAGIC rejected because it fires on ordinary sorted tables — see -# MAGIC [Choosing a profile](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection#choosing-a-profile). +# MAGIC +# MAGIC The SMD figures quoted earlier are a large gain on the data this detector is for, and they are *not* +# MAGIC state of the art for multivariate time-series anomaly detection. Published figures near 0.80 F1 use +# MAGIC point adjustment, which [Kim et al. (AAAI 2022)](https://arxiv.org/abs/2109.05257) showed random +# MAGIC scores also reach, so they are not a fair comparison in either direction. +# MAGIC + +# COMMAND ---------- + +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ### 📚 Resources +# MAGIC +# MAGIC - [Choosing a profile](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection#choosing-a-profile) +# MAGIC - [Anomaly detection quality](https://databrickslabs.github.io/dqx/docs/reference/anomaly_detection_quality) — how the numbers above were measured +# MAGIC - [Row Anomaly Detection guide](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection) +# MAGIC +# MAGIC ### 🎉 You're Ready! +# MAGIC +# MAGIC You now understand: +# MAGIC - ✅ The difference between an extreme value and a broken relationship +# MAGIC - ✅ When to reach for `profile="timeseries"` instead of the default +# MAGIC - ✅ Why it needs no timestamp column +# MAGIC - ✅ How to read contributions and AI explanations for a correlation break +# MAGIC +# MAGIC **Start watching the relationships, not just the gauges!** 🚀 +# MAGIC diff --git a/demos/dqx_row_anomaly_detection_demo.py b/demos/dqx_row_anomaly_detection_demo.py deleted file mode 100644 index 1f165e385..000000000 --- a/demos/dqx_row_anomaly_detection_demo.py +++ /dev/null @@ -1,861 +0,0 @@ -# Databricks notebook source -# MAGIC %md -# MAGIC # 📊 Row Anomaly Detection Demo -# MAGIC -# MAGIC ## Learn Row Anomaly Detection in 15 Minutes -# MAGIC -# MAGIC **Quickstart (5–10 minutes):** -# MAGIC - Train an anomaly model on sample data using DQX Row Anomaly Detection Engine -# MAGIC - Apply checks and see flagged anomalies -# MAGIC - View severity percentiles and top contributors -# MAGIC -# MAGIC **Dataset**: Simple sales transactions (universally relatable, no domain expertise required) -# MAGIC - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ## What is Row Anomaly Detection? -# MAGIC -# MAGIC - Standard rule-based checks catch *known* issues (nulls, ranges, formats). -# MAGIC - Row anomaly detection finds *unknown* patterns in rows across multiple columns. -# MAGIC - Use both together for better coverage. -# MAGIC -# MAGIC **Why row anomaly detection** -# MAGIC - Learns "normal" from data -# MAGIC - Flags deviations without manual rules and thresholds -# MAGIC - Complements rule-based checks rather than replacing them -# MAGIC -# MAGIC **Known vs Unknown Issues** -# MAGIC - **Known unknowns**: rule‑based checks (nulls, ranges, formats). -# MAGIC - **Unknown unknowns**: multi‑column or subtle patterns you didn’t anticipate. -# MAGIC -# MAGIC **Data Quality Monitoring (DQM) vs DQX Row Anomaly detection** -# MAGIC - **[Data Quality Monitoring (DQM)](https://docs.databricks.com/aws/en/data-quality-monitoring/anomaly-detection)**: uses table‑level signals such as row counts and commit patterns. -# MAGIC - **DQX Anomaly**: look for row‑level patterns within the data (per‑record anomalies with explanations). -# MAGIC - DQM and DQX each provide distinct capabilities. Together, they complement one another to deliver comprehensive coverage across the full spectrum of data quality checks. -# MAGIC - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ## Prerequisites: Install DQX with Anomaly Support -# MAGIC -# MAGIC ```python -# MAGIC %pip install 'databricks-labs-dqx[anomaly]' -# MAGIC dbutils.library.restartPython() -# MAGIC ``` -# MAGIC -# MAGIC **What's included in `[anomaly]` extras:** -# MAGIC - `scikit-learn` - Machine learning algorithms used for row anomaly detection -# MAGIC - `mlflow` - Model tracking and registry -# MAGIC - `shap` - Feature contributions for tree-based models (the `timeseries` detector computes its -# MAGIC own contributions and needs no SHAP) -# MAGIC - `cloudpickle` - Model serialization -# MAGIC -# MAGIC **Note**: If you are using ML Runtime or Serverless compute, most dependencies are already pre-installed. -# MAGIC - -# COMMAND ---------- -# DBTITLE 1,Prerequisites: Install DQX with Anomaly Support - -dbutils.widgets.text("test_library_ref", "", "Test Library Ref") - -if dbutils.widgets.get("test_library_ref") != "": - %pip install 'databricks-labs-dqx[anomaly] @ {dbutils.widgets.get("test_library_ref")}' -else: - %pip install databricks-labs-dqx[anomaly] - -%restart_python - -# COMMAND ---------- -# DBTITLE 1,Prerequisites: Configure test catalog and schema - -default_catalog = "main" -default_schema = "default" - -# Configure widgets for catalog and schema -dbutils.widgets.text("demo_catalog", default_catalog, "Catalog Name") -dbutils.widgets.text("demo_schema", default_schema, "Schema Name") - -# COMMAND ---------- - -# MAGIC %md -# MAGIC ## Section 1: Setup & Data Generation -# MAGIC - -# COMMAND ---------- -# DBTITLE 1,Setup engines - -import pyspark.sql.functions as F -from pyspark.sql.types import * -from datetime import datetime, timedelta -import random -import numpy as np - -from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine -from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies -from databricks.labs.dqx.engine import DQEngine -from databricks.labs.dqx.rule import DQDatasetRule, DQRowRule -from databricks.labs.dqx.check_funcs import is_not_null, is_in_range -from databricks.sdk import WorkspaceClient - -# Initialize DQX engines -ws = WorkspaceClient() -anomaly_engine = AnomalyEngine(ws) -dq_engine = DQEngine(ws) - -# Set seeds for reproducibility for demo purposes -random.seed(42) -np.random.seed(42) - -print("✅ Setup complete!") - -# COMMAND ---------- -# DBTITLE 1,Data Generation - -# Generate historical (training) data -def generate_historical_sales_data( - num_rows: int = 1000, -): - """ - Generate historical sales data (no synthetic anomalies). - """ - data = [] - categories = ["Electronics", "Clothing", "Food", "Books", "Home"] - regions = ["North", "South", "East", "West"] - - # Regional pricing patterns (normal baseline) - region_patterns = { - "North": {"base_amount": 200, "quantity": 5}, - "South": {"base_amount": 150, "quantity": 4}, - "East": {"base_amount": 180, "quantity": 4}, - "West": {"base_amount": 220, "quantity": 6}, - } - - start_date = datetime(2024, 1, 1, 9, 0) # Jan 1, 2024, 9am - - for i in range(num_rows): - transaction_id = f"TXN{i:06d}" - category = random.choice(categories) - region = random.choice(regions) - pattern = region_patterns[region] - - # Generate timestamp (mostly business hours weekdays) - days_offset = random.randint(0, 90) # 3 months of data - hours_offset = random.randint(0, 9) # 9am-6pm = 9 hours - date = start_date + timedelta(days=days_offset, hours=hours_offset) - - # Skip weekends for normal transactions - if date.weekday() >= 5: # Saturday=5, Sunday=6 - date = date - timedelta(days=date.weekday() - 4) # Move to Friday - - # Normal transaction (tighter variance for more consistent patterns) - amount = round(pattern["base_amount"] * random.uniform(0.85, 1.15), 2) - quantity = max(1, int(np.random.normal(pattern["quantity"], 1))) - - # Ensure valid ranges (skip for injected nulls/negatives) - if amount is not None: - amount = max(10, min(10000, amount)) - if quantity is not None: - quantity = max(1, min(150, quantity)) # Allow bulk orders up to 150 - - data.append((transaction_id, date, amount, quantity, category, region)) - - return data - -# Generate historical data -print("🔄 Generating historical (training) data...\n") -train_rows = 5000 -historical_data = generate_historical_sales_data(num_rows=train_rows) - -schema = StructType([ - StructField("transaction_id", StringType(), False), - StructField("date", TimestampType(), False), - StructField("amount", DoubleType(), True), - StructField("quantity", IntegerType(), True), - StructField("category", StringType(), False), - StructField("region", StringType(), False), -]) - -df_train = spark.createDataFrame(historical_data, schema) - -print("📊 Sample of sales transactions:") -display(df_train.orderBy("date")) - -total_train = df_train.count() -print(f"\n✅ Generated {total_train} historical transactions (for training)") - -# COMMAND ---------- -# DBTITLE 1,Save Test Data - -# Get catalog and schema from widgets -catalog = dbutils.widgets.get("demo_catalog") -schema_name = dbutils.widgets.get("demo_schema") - -print(f"📂 Using catalog: {catalog}") -print(f"📂 Using schema: {schema_name}\n") - -train_table = f"{catalog}.{schema_name}.sales_transactions_train" -df_train.write.mode("overwrite").saveAsTable(train_table) - -print(f"✅ Training data saved to: {train_table}") - -# COMMAND ---------- - -# Set up registry table for tracking trained models (always use fully qualified table name) -registry_table = f"{catalog}.{schema_name}.anomaly_model_registry_101" -print(f"📋 Model registry table: {registry_table}") - -# Clean up any existing registry from previous runs -spark.sql(f"DROP TABLE IF EXISTS {registry_table}") -print(f"✅ Registry ready for new models") - - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ## Section 2: Train the Anomaly Model -# MAGIC -# MAGIC We’ll run: -# MAGIC - Simple rule checks (nulls, ranges) -# MAGIC - Row anomaly detection for unusual multi‑column patterns -# MAGIC -# MAGIC In DQX you can run all types of rules in the same run. - -# COMMAND ---------- -# DBTITLE 1,Train the Anomaly Model - -# Train row anomaly detection model with zero configuration -print("🎯 Training row anomaly detection model...") -print(" DQX will automatically discover patterns in your data\n") - -model_name_auto = f"{catalog}.{schema_name}.sales_auto" # stored in Unity Catalog and must be fully qualified name -model_uri_auto = anomaly_engine.train( - df=spark.table(train_table), - model_name=model_name_auto, - registry_table=registry_table # must be fully qualified table name: catalog.schema.table_name -) - -print(f"✅ Model trained successfully!") -print(f" Model URI: {model_uri_auto}") - -# View what DQX created for you -print(f"\n📋 Trained Models:\n") - -display( - spark.table(registry_table) - .filter(F.col("identity.model_name").contains(model_name_auto)) - .select( - "identity.model_name", - "training.columns", - "grouping.baseline_by", - "training.training_rows", - "training.training_time", - "identity.status" - ) - .orderBy("identity.model_name") -) - -print("\n💡 DQX auto-discovered patterns and registered a model for scoring.") - -# COMMAND ---------- - -# MAGIC %md -# MAGIC ### Optional: View Models in the UI -# MAGIC -# MAGIC Your models are stored in Unity Catalog and registered within MLflow. -# MAGIC If you want to inspect them, open **Catalog Explorer** or **Experiments**. -# MAGIC - -# COMMAND ---------- - -# MAGIC %md -# MAGIC ### Section 3: Generate new data containing some anomalies -# MAGIC - -# COMMAND ---------- -# DBTITLE 1,Generate new data containing Anomalies - -def inject_anomalies_and_dq_issues( - base_rows: list[tuple], - anomaly_rate: float = 0.02, - dq_null_amount_rate: float = 0.01, - dq_null_quantity_rate: float = 0.005, - dq_negative_amount_rate: float = 0.005, -): - """ - Take clean (normal) rows and inject anomalies + simple DQ issues. - """ - rows = [] - for idx, row in enumerate(base_rows): - transaction_id, date, amount, quantity, category, region = row - transaction_id = f"NEW{idx:06d}" - is_synthetic_anomaly = False - - dq_roll = random.random() - if dq_roll < dq_null_amount_rate: - amount = None - elif dq_roll < dq_null_amount_rate + dq_null_quantity_rate: - quantity = None - elif dq_roll < dq_null_amount_rate + dq_null_quantity_rate + dq_negative_amount_rate: - amount = -abs(amount) - elif random.random() < anomaly_rate: - is_synthetic_anomaly = True - anomaly_type = random.choices( - ["extreme_scale", "mismatch_pair", "timing_spike"], - weights=[3, 3, 2], - )[0] - - if anomaly_type == "extreme_scale": - amount = round(amount * random.uniform(15, 25), 2) - quantity = int(quantity * random.uniform(15, 25)) - elif anomaly_type == "mismatch_pair": - # Large amount with tiny quantity (or vice versa) - if random.random() < 0.5: - amount = round(amount * random.uniform(12, 20), 2) - quantity = max(1, int(quantity * random.uniform(0.05, 0.2))) - else: - amount = round(amount * random.uniform(0.05, 0.2), 2) - quantity = int(quantity * random.uniform(12, 20)) - else: - # Off-hours + large spike - amount = round(amount * random.uniform(10, 18), 2) - quantity = int(quantity * random.uniform(10, 18)) - date = date.replace(hour=random.choice([2, 3, 4, 22, 23])) - - if amount is not None: - amount = max(10, min(10000, amount)) - if quantity is not None: - quantity = max(1, min(150, quantity)) - - rows.append((transaction_id, date, amount, quantity, category, region, is_synthetic_anomaly)) - return rows - -print("🔄 Generating new data with injected anomalies...\n") - -new_rows = 1000 -anomaly_rate = 0.02 -dq_null_amount_rate = 0.01 -dq_null_quantity_rate = 0.005 -dq_negative_amount_rate = 0.005 -dq_issue_rate = dq_null_amount_rate + dq_null_quantity_rate + dq_negative_amount_rate - -new_data_base = generate_historical_sales_data(num_rows=new_rows) -new_data = inject_anomalies_and_dq_issues( - base_rows=new_data_base, - anomaly_rate=anomaly_rate, - dq_null_amount_rate=dq_null_amount_rate, - dq_null_quantity_rate=dq_null_quantity_rate, - dq_negative_amount_rate=dq_negative_amount_rate, -) - -new_schema = StructType([ - StructField("transaction_id", StringType(), False), - StructField("date", TimestampType(), False), - StructField("amount", DoubleType(), True), - StructField("quantity", IntegerType(), True), - StructField("category", StringType(), False), - StructField("region", StringType(), False), - StructField("is_synthetic_anomaly", BooleanType(), False), -]) - -df_new = spark.createDataFrame(new_data, new_schema) - -print("📊 Sample of new data:") -display(df_new.orderBy("date")) - -total_new = df_new.count() -print(f"\n✅ Generated {total_new} NEW transactions") -print(f" Injected anomalies: ~{int(total_new * anomaly_rate)} ({anomaly_rate*100:.0f}%)") -print(f" Injected rule issues: ~{int(total_new * dq_issue_rate)} ({dq_issue_rate*100:.1f}%)") - -new_table = f"{catalog}.{schema_name}.sales_transactions_new" -df_new.write.mode("overwrite").saveAsTable(new_table) -print(f"✅ New data saved to: {new_table}") - -# COMMAND ---------- - -# MAGIC %md -# MAGIC ### Section 4: Apply checks including row anomaly detection -# MAGIC -# MAGIC Now apply row anomaly detection + rule-based checks to the **new data**. - -# COMMAND ---------- -# DBTITLE 1,Apply quality checks - -print("🔍 Applying quality checks to new data...\n") - -# Define all quality checks, use default criticality="error" -checks_combined = [ - # Rule-based checks for known issues and thresholds - DQRowRule(check_func=is_not_null, check_func_kwargs={"column": "transaction_id"}), - DQRowRule(check_func=is_not_null, check_func_kwargs={"column": "amount"}), - DQRowRule(check_func=is_in_range, check_func_kwargs={"column": "amount", "min_limit": 0, "max_limit": 100000}), - DQRowRule(check_func=is_not_null, check_func_kwargs={"column": "quantity"}), - DQRowRule(check_func=is_in_range, check_func_kwargs={"column": "quantity", "min_limit": 1, "max_limit": 1000}), - - # Row anomaly detection for unusual patterns - DQDatasetRule( - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": model_name_auto, - "registry_table": registry_table - } - ) -] - -df_valid, df_quarantine = dq_engine.apply_checks_and_split(df_new, checks_combined) - -display(df_quarantine) - -print("\n💡 Summary:") -print(" • We trained on historical data and applied checks on new data.") -print(" • Default threshold 95 flags rows above the 95th percentile of the *training* data.") -print(" • So more than 5% of new rows can be flagged — that is the point: this data has") -print(" anomalies injected into it, and the training data did not.") -print(" • Threshold is a percentile cutoff — tune it based on your data and alert tolerance.") - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ## Section 5a: (Optional) Review Results to understand why some records are anomalous -# MAGIC -# MAGIC You’ll see flagged anomalies, severity percentiles, and top contributors. -# MAGIC -# MAGIC This section is optional. Skip if you only want the quickstart. -# MAGIC -# MAGIC In the quarantine dataset we can find the regular `_error` and `_warnings` reporting columns, and `_dq_info` column (array of structs). The first check's info is at `_dq_info[0]`; `_dq_info[0].anomaly` includes: -# MAGIC - `severity_percentile` (0–100): percentile of anomaly severity -# MAGIC - `score`: raw model score (diagnostic only) -# MAGIC - `contributions`: feature-level explanations - -# COMMAND ---------- -# DBTITLE 1,Review Results - -df_quarantine = df_quarantine.filter( - F.col("_dq_info").getItem(0).getField("anomaly").getField("is_anomaly") == True -) -score_col = F.col("_dq_info").getItem(0).getField("anomaly").getField("score") -severity_col = F.col("_dq_info").getItem(0).getField("anomaly").getField("severity_percentile") -percentile_band = ( - F.when(severity_col >= 98, F.lit("p98+ (top 2%)")) - .when(severity_col >= 95, F.lit("p95-98 (top 5%)")) - .when(severity_col >= 90, F.lit("p90-95 (top 10%)")) - .otherwise(F.lit(" 0: - recall = synthetic_caught / synthetic_total * 100 - print(f"\n✅ Synthetic anomalies injected: {synthetic_total}") - print(f" Synthetic anomalies caught: {synthetic_caught} ({recall:.1f}% recall)") -print(f"\n🔝 Top 10 anomalies:\n") - -display(df_quarantine.orderBy(severity_col.desc()).select( - "transaction_id", "date", "amount", "quantity", "category", "region", - F.round(severity_col, 1).alias("severity_percentile"), - F.round(score_col, 3).alias("anomaly_score"), - percentile_band.alias("severity_band"), -).limit(10)) - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC ## Section 5b: AI explanations -# MAGIC -# MAGIC AI explanations are **on by default** — the checks in Section 4 already produced -# MAGIC `_dq_info[0].anomaly.ai_explanation` (a plain-language `narrative`, `business_impact`, -# MAGIC `action`, and the deterministic `top_features` pattern). The LLM call runs **inside Spark** -# MAGIC via the SQL `ai_query` function against a Databricks Model Serving endpoint — no extra -# MAGIC dependency, and rows are grouped so the model is called **once per group** (capped by -# MAGIC `max_groups`). This requires **Databricks serverless compute or Databricks Runtime 15.4 LTS -# MAGIC or above** (where `ai_query` is available). If `ai_query` is unavailable or no endpoint is -# MAGIC reachable, explanations are skipped with a warning and scoring still completes. -# MAGIC -# MAGIC This cell just shows how to override the endpoint or turn explanations off — none of these -# MAGIC kwargs are required. - -# COMMAND ---------- -# DBTITLE 1,Apply checks (AI explanations are on by default) - -checks_with_ai = [ - DQDatasetRule( - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": model_name_auto, - "registry_table": registry_table, - # All optional — explanations + contributions are on by default: - # "enable_ai_explanation": False, # turn explanations off - # "ai_explanation_llm_model_config": {"model_name": "databricks-claude-sonnet-4-5"}, # override endpoint - # "redact_columns": ["region"], # keep sensitive names out of the prompt - # "max_groups": 500, # cap on LLM calls per run - }, - ) -] - -df_ai = dq_engine.apply_checks(df_new, checks_with_ai) - -anomaly = F.col("_dq_info").getItem(0).getField("anomaly") -explanation = anomaly.getField("ai_explanation") -print("🤖 Top anomalies with AI explanations:\n") -display( - df_ai.filter(anomaly.getField("is_anomaly") == True) - .orderBy(anomaly.getField("severity_percentile").desc()) - .select( - "transaction_id", - "amount", - "quantity", - F.round(anomaly.getField("severity_percentile"), 1).alias("severity_percentile"), - explanation.getField("top_features").alias("top_features"), - explanation.getField("narrative").alias("why_flagged"), - explanation.getField("business_impact").alias("business_impact"), - explanation.getField("action").alias("suggested_action"), - ) - .limit(10) -) - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC ## Section 6: (Optional) Threshold Tradeoffs -# MAGIC -# MAGIC This section is optional. Skip if you only want the quickstart. - -# COMMAND ---------- -# DBTITLE 1,Threshold Tradeoffs - -print("📌 Summary:") -print(" • Default threshold = 95 (top 5%).") -print(" • Raise it to reduce alerts; lower it to catch more.") -print(" • The right setting depends on your data distribution and risk tolerance.") - -# (Optional) Quick normal vs anomaly sanity check -print("🔍 Sanity check (severity < 95 vs ≥ 95):\n") -normal_count = df_scored.filter(severity_col < 95).count() -anomaly_count = df_scored.filter(severity_col >= 95).count() -print(f" Normal: {normal_count} ({normal_count/total_scored*100:.1f}%)") -print(f" Anomaly: {anomaly_count} ({anomaly_count/total_scored*100:.1f}%)") - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ### Tuning the Threshold -# MAGIC -# MAGIC Threshold is a percentile cutoff: -# MAGIC - Lower (e.g., 90) = more alerts -# MAGIC - Higher (e.g., 98) = fewer alerts -# MAGIC -# MAGIC We already scored all records, so you can change thresholds without re‑scoring. -# MAGIC - -# COMMAND ---------- -# DBTITLE 1,Tuning the Threshold - -# Try different thresholds -print("🎚️ Testing Different Thresholds:\n") -print("Threshold | Anomalies | % of Data | Interpretation") -print("-" * 70) - -thresholds = [90, 95, 98] -total_count = total_scored - -for threshold in thresholds: - anomaly_count = df_scored.filter(severity_col >= threshold).count() - percentage = (anomaly_count / total_count) * 100 - - if threshold < 95: - interpretation = "Sensitive (more alerts)" - elif threshold == 95: - interpretation = "Balanced (default)" - else: - interpretation = "Strict (fewer alerts)" - - print(f" {threshold:>3d} | {anomaly_count:4d} | {percentage:5.1f}% | {interpretation}") - -print("\n💡 Start at 95, then explore thresholds on your data to balance noise vs. missed anomalies.") - -# COMMAND ---------- -# DBTITLE 1,Tuning the Threshold - -# Borderline slice (optional) -borderline = df_scored.filter((severity_col >= 90) & (severity_col < 95)).orderBy(severity_col.desc()) -print(f"\nBorderline (90-<95) examples: {borderline.count()}") -display(borderline.select( - "transaction_id", "amount", "quantity", - F.round(severity_col, 1).alias("severity_percentile"), - F.round(score_col, 3).alias("score"), -).limit(5)) - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ## Section 7: (Optional) Manual Column Selection -# MAGIC -# MAGIC Skip this if you only want the quickstart. -# MAGIC -# MAGIC We will train a model with specific columns. While applying the row anomaly detection check, only the columns the model was trained on will be used. -# MAGIC -# MAGIC This is important in production when you need strict feature control. By default, all supported columns are used. - -# COMMAND ---------- -# DBTITLE 1,Training with Manual Column Selection - -print("🎯 Training model with manual column selection...\n") -model_name_manual = f"{catalog}.{schema_name}.sales_manual" # stored in Unity Catalog and must be fully qualified name -model_uri_manual = anomaly_engine.train( - df=spark.table(train_table), - columns=["amount", "quantity"], # Explicitly specify numeric columns only - model_name=model_name_manual, - registry_table=registry_table -) - -print(f"✅ Manual model trained!") -print(f" Model URI: {model_uri_manual}") -print(f"\n💡 Manual selection is useful in production when you want strict feature control.") - -# COMMAND ---------- -# DBTITLE 1,Manual Column Selection - -# Score with manual model -print("🔍 Scoring with manual model...\n") - -checks_manual = [ - DQDatasetRule( - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": model_name_manual, - "threshold": 95.0, - "registry_table": registry_table - # we don't specify which column to apply the anomaly check on; the same columns that were selected for the training are used - } - ) -] - -df_valid, df_quarantine_manual = dq_engine.apply_checks_and_split(df_new, checks_manual) - -print(f"⚠️ Manual model found {df_quarantine_manual.count()} anomalies") -print(f" (Auto model found {df_quarantine.count()} anomalies)") -print(f"\n🔝 Top 5 anomalies from manual model:\n") - - -_dq_info_severity = F.col("_dq_info").getItem(0).getField("anomaly").getField("severity_percentile") -_dq_info_score = F.col("_dq_info").getItem(0).getField("anomaly").getField("score") -display(df_quarantine_manual.orderBy(_dq_info_severity.desc()).select( - "transaction_id", "amount", "quantity", "date", - F.round(_dq_info_severity, 1).alias("severity_percentile"), - F.round(_dq_info_score, 3).alias("score") -).limit(5)) - -print("\n💡 Different features → different anomalies. That’s expected.") - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ## Section 8: (Optional) Feature Contributions -# MAGIC -# MAGIC Skip this if you only want the quickstart. -# MAGIC -# MAGIC ### Advanced Options (Reference) -# MAGIC -# MAGIC **Scoring options (`has_no_row_anomalies`):** -# MAGIC - `threshold` (float, 0–100): percentile cutoff (default 95) -# MAGIC - `enable_contributions` (bool): feature contributions in `_dq_info[0].anomaly` -# MAGIC - `enable_confidence_std` (bool): confidence estimate (std dev across ensemble) -# MAGIC - `drift_threshold` (float): drift detection sensitivity -# MAGIC - `row_filter` (str): SQL filter applied before scoring -# MAGIC -# MAGIC **Training options (`AnomalyEngine.train` / `AnomalyParams`):** -# MAGIC - `columns` (list[str]): explicit feature list (disables auto‑discovery) -# MAGIC - `baseline_by` (list[str]): columns identifying a row's group, so each metric is judged -# MAGIC against its own group's baseline rather than the whole table — catches values that are -# MAGIC ordinary globally but wrong in context. One model, whatever the group count. -# MAGIC - `profile` (str): which detector to train — `"tabular"` (the default: Isolation Forest) or -# MAGIC `"timeseries"` for multivariate metrics whose anomalies are broken correlations rather than -# MAGIC extreme single values. No timestamp column needed. DQX never picks this for you. -# MAGIC - `sample_fraction`, `max_rows`: training sample controls -# MAGIC - `ensemble_size`: number of models in the ensemble -# MAGIC - `expected_anomaly_rate`: expected anomaly rate for calibration -# MAGIC -# MAGIC These are optional — the demo uses defaults for simplicity. -# MAGIC - -# COMMAND ---------- -# DBTITLE 1,Advanced Options (Reference) - -# Score with feature contributions -print("🔍 Scoring with feature contributions (explainability)...\n") - -checks_with_contrib = [ - DQDatasetRule( - check_func=has_no_row_anomalies, - check_func_kwargs={ - "model_name": model_name_manual, - "threshold": 95.0, - "enable_contributions": True, # on by default; shown here for clarity - "registry_table": registry_table - } - ) -] - -df_with_contrib = dq_engine.apply_checks(df_new, checks_with_contrib) - -print("✅ Scored with feature contributions!") -print("\n🎯 Top Anomalies with Explanations:\n") - -# Filter by _errors column (standard DQX pattern) to get flagged anomalies -anomalies_explained = df_with_contrib.filter( - F.size(F.col("_errors")) > 0 -).orderBy(F.col("_dq_info").getItem(0).getField("anomaly").getField("severity_percentile").desc()).limit(5) - -_dq_severity = F.col("_dq_info").getItem(0).getField("anomaly").getField("severity_percentile") -_dq_score = F.col("_dq_info").getItem(0).getField("anomaly").getField("score") -_dq_contrib = F.col("_dq_info").getItem(0).getField("anomaly").getField("contributions") -display(anomalies_explained.select( - "transaction_id", - "amount", - "quantity", - F.date_format("date", "yyyy-MM-dd HH:mm").alias("date"), - F.round(_dq_severity, 1).alias("severity_percentile"), - F.round(_dq_score, 3).alias("score"), - _dq_contrib.alias("contributions").alias("why_anomalous") -)) - -print("\n💡 Contributions show which features most influenced the anomaly.") -print(" Focus on features with the highest % contribution.") - -# COMMAND ---------- -# DBTITLE 1,Advanced Options (Reference) - -# Show one detailed example -print("🔎 Detailed Example - Top Anomaly:\n") - -# Extract the columns for easier access -anomalies_flattened = anomalies_explained.select( - "transaction_id", - "amount", - "quantity", - "date", - F.col("_dq_info").getItem(0).getField("anomaly").getField("severity_percentile").alias("severity_percentile"), - F.col("_dq_info").getItem(0).getField("anomaly").getField("score").alias("score"), - F.col("_dq_info").getItem(0).getField("anomaly").getField("contributions").alias("contributions"), -) - -top_anomaly = anomalies_flattened.first() - -print(f"Transaction ID: {top_anomaly['transaction_id']}") -print(f"Severity Percentile: {top_anomaly['severity_percentile']:.1f}") -print(f"Anomaly Score (raw): {top_anomaly['score']:.3f}") -print(f"\nTransaction Details:") -print(f" Amount: ${top_anomaly['amount']:.2f}") -print(f" Quantity: {top_anomaly['quantity']}") -print(f" Date: {top_anomaly['date']}") -print(f"\nFeature Contributions:") - -contributions = top_anomaly['contributions'] -if contributions: - # Sort by contribution value - sorted_contrib = sorted(contributions.items(), key=lambda x: abs(x[1]), reverse=True) - for feature, value in sorted_contrib[:3]: # Top 3 - print(f" {feature}: {abs(value):.1f}% contribution") - - print(f"\n🎯 Investigation Tip:") - top_feature = sorted_contrib[0][0] - if "amount" in top_feature: - print(f" → Check for pricing errors or incorrect price feeds") - elif "quantity" in top_feature: - print(f" → Investigate bulk order or inventory issue") - elif "date" in top_feature or "hour" in top_feature: - print(f" → Review transaction timing - off-hours activity?") -else: - print(" (No detailed contributions available)") - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ## Summary & Next Steps -# MAGIC -# MAGIC **Key takeaways:** -# MAGIC - You can apply row anomaly detection and rule-based checks together. -# MAGIC - Start with threshold 95 (default), tune as needed. -# MAGIC - Use contributions to triage anomalies faster. -# MAGIC -# MAGIC **Apply to your data:** -# MAGIC ```python -# MAGIC # Replace with your table -# MAGIC model = anomaly_engine.train( -# MAGIC df=spark.table("your_catalog.your_schema.your_table"), -# MAGIC model_name="your_catalog.your_schema.your_model_name", -# MAGIC registry_table="your_catalog.your_schema.dqx_anomaly_models", -# MAGIC ) -# MAGIC -# MAGIC checks = [ -# MAGIC has_no_row_anomalies( -# MAGIC model_name="your_catalog.your_schema.your_model_name", -# MAGIC registry_table="your_catalog.your_schema.dqx_anomaly_models", -# MAGIC ) -# MAGIC ] -# MAGIC df_scored = dq_engine.apply_checks(your_df, checks) -# MAGIC ``` -# MAGIC -# MAGIC **Optional next steps:** -# MAGIC - Add baseline conditioning (`baseline_by` for training), drift detection, and scheduled scoring. -# MAGIC - Automate retraining and alerting. - -# COMMAND ---------- - -# MAGIC %md -# MAGIC --- -# MAGIC -# MAGIC ### 📚 Resources -# MAGIC -# MAGIC - [DQX Row Anomaly Detection Documentation](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection) -# MAGIC - [API Reference](https://databrickslabs.github.io/dqx/docs/reference/quality_checks#row-anomaly-detection) -# MAGIC - [Data Quality Monitoring (DQM)](https://docs.databricks.com/aws/en/data-quality-monitoring/anomaly-detection/#-table-quality-details) -# MAGIC -# MAGIC ### 🎉 You're Ready! -# MAGIC -# MAGIC You now understand: -# MAGIC - ✅ What row anomaly detection is and when to use it -# MAGIC - ✅ How to implement it with minimal configuration -# MAGIC - ✅ How to interpret and tune results -# MAGIC - ✅ How to integrate it into production -# MAGIC -# MAGIC **Start detecting anomalies in your data today!** 🚀 -# MAGIC diff --git a/docs/dqx/docs/demos.mdx b/docs/dqx/docs/demos.mdx index 59f2a8b1d..a591d6ba4 100644 --- a/docs/dqx/docs/demos.mdx +++ b/docs/dqx/docs/demos.mdx @@ -14,9 +14,8 @@ Import the following notebooks in the Databricks workspace to try DQX out: * [DQX Demo Notebook for Data Quality Summary Metrics](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_summary_metrics.py) - demonstrates how to generate summary-level data quality metrics when validating data with DQX. * [DQX Demo Notebook for Actions and Alerting](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_alerting.py) - demonstrates how to react to data quality problems by firing alerts (driver log, with optional Slack) when summary metrics cross a threshold. * [DQX Demo Notebook for Profiling and Applying Checks at Scale on Multiple Tables](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_multi_table_demo.py) - demonstrates how to use DQX as a library at scale to apply checks on multiple tables. -* [DQX Demo Notebook for Row Anomaly Detection](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_row_anomaly_detection_demo.py) - comprehensive demo showing how to use DQX Row Anomaly Detection to detect unusual patterns in your data. * [DQX Demo Notebook for Row Anomaly Detection on Transactions](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_anomaly_tabular_transactions.py) - starts from a domain problem: card transactions that pass every rule but are jointly implausible. Shows why no threshold catches them, how `baseline_by` makes "normal" depend on the merchant category, and what the contributions tell you. Uses the default `tabular` profile. -* [DQX Demo Notebook for Row Anomaly Detection on Machine Telemetry](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_anomaly_timeseries_fleet.py) - starts from a domain problem: machine metrics that each stay inside their safe band while the *relationship* between them breaks. Verifies that no per-metric range check could catch it, then trains both profiles on identical data to show what `profile="timeseries"` buys. +* [DQX Demo Notebook for Row Anomaly Detection on Machine Telemetry](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_anomaly_timeseries_fleet.py) - starts from a domain problem: machine metrics that each stay inside their safe band while the *relationship* between them breaks. Verifies that no per-metric range check could catch it, then trains with `profile="timeseries"` and reads the explanation. * [DQX Demo Notebook for AI-assisted checks generation](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_ai_assisted_checks_generation.py) - demonstrates how to generate DQX rules/checks with LLM using natural language. * [DQX Demo Notebook for Data Contract Integration (ODCS)](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_demo_datacontract_odcs.py) - demonstrates how to generate DQX quality rules from ODCS (Open Data Contract Standard) data contracts, including predefined rules from schema constraints, explicit custom rules, and contract metadata tracking. * [DQX Demo Notebook for Spark Structured Streaming (Native End-to-End Approach)](https://github.com/databrickslabs/dqx/blob/v0.16.0/demos/dqx_streaming_demo_native.py) - demonstrates how to use DQX as a library with Spark Structured Streaming, using the built-in end-to-end method to handle both reading and writing. diff --git a/tests/e2e/test_run_demos.py b/tests/e2e/test_run_demos.py index 5070f5a98..f0a58dca3 100644 --- a/tests/e2e/test_run_demos.py +++ b/tests/e2e/test_run_demos.py @@ -533,10 +533,18 @@ def test_run_dqx_demo_llm_pk_detection(ws, make_notebook, make_job, library_ref) logging.info(f"Job run {run.run_id} completed successfully for dqx_demo_llm_pk_detection") -def test_run_dqx_row_anomaly_detection_demo(ws, make_notebook, make_schema, make_job, library_ref): +@pytest.mark.parametrize( + "demo_notebook", + [ + "dqx_demo_anomaly_tabular_transactions.py", + "dqx_demo_anomaly_timeseries_fleet.py", + ], +) +def test_run_dqx_anomaly_demo(ws, make_notebook, make_schema, make_job, library_ref, demo_notebook): + """Run the row anomaly detection demos: the tabular profile and the timeseries profile.""" catalog = TEST_CATALOG schema = make_schema(catalog_name=catalog).name - path = Path(__file__).parent.parent.parent / "demos" / "dqx_row_anomaly_detection_demo.py" + path = Path(__file__).parent.parent.parent / "demos" / demo_notebook with open(path, "rb") as f: notebook = make_notebook(content=f, format=ImportFormat.SOURCE) @@ -549,18 +557,19 @@ def test_run_dqx_row_anomaly_detection_demo(ws, make_notebook, make_schema, make "test_library_ref": library_ref, }, ) - job = make_job(tasks=[Task(task_key="dqx_row_anomaly_detection_demo", notebook_task=notebook_task)]) + task_key = demo_notebook.removesuffix(".py") + job = make_job(tasks=[Task(task_key=task_key, notebook_task=notebook_task)]) - # This demo trains two IsolationForest models and scores with contributions + AI explanations - # (on by default), so it is the slowest e2e demo and can exceed 30 minutes on a cold serverless - # start. Use a 45-minute wait (still well within the e2e CI job's 2h wrapper). + # These train a model and score with contributions + AI explanations (both on by default), which + # makes them among the slowest e2e demos; a cold serverless start can exceed 30 minutes. Use a + # 45-minute wait, still well within the e2e CI job's 2h wrapper. waiter = ws.jobs.run_now_and_wait(job.job_id, timeout=timedelta(minutes=45)) run = ws.jobs.wait_get_run_job_terminated_or_skipped( run_id=waiter.run_id, timeout=timedelta(minutes=45), callback=lambda r: validate_run_status(r, ws), ) - logging.info(f"Job run {run.run_id} completed successfully for dqx_row_anomaly_detection_demo") + logging.info(f"Job run {run.run_id} completed successfully for {task_key}") def test_dbt_demo(make_schema, library_ref, debug_env, ws): From dc14ef6a8f4d9dd96d74c71f056b1b6f6daed689 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 21:40:28 +0100 Subject: [PATCH 053/107] Replace the correlation slide with an inspection line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The slide had to land one thing: there is an anomaly the forest cannot isolate, which is why a second detector exists. Six mockup rounds got there; two of the wrong turns are worth recording because both were found by looking at the rendered result rather than by reasoning about it. **Colour must not vary.** Early versions scattered ripeness filters so the belt would not look monotonous. Colour is the most salient channel on a screen, so the eye nominates the green fruit — and when the gate then flags an ordinary yellow one the verdict looks arbitrary. It also broke the argument being made: "each value is normal, only the pair is impossible" needs the culprit to be genuinely indistinguishable, which it cannot be if its neighbours all look wildly different. So the crate is one ripeness. What varies visibly is length, one of the two numbers the check reads. Weight — the number that is actually wrong — is invisible, which is precisely why a check is needed and the eye is not enough. **No instrument panel.** A readout grew to six rows, collided with its own nameplate, and turned the argument into something read rather than seen. One pill now carries the verdict; the reasoning lives in the slide body, where there is room for it. What ships: a conveyor with a scanner gantry, fruit of varying length at one ripeness, and a pill reading `19cm · 38g pair ✗`. On a failure the whole line halts — belt, slats and rollers all stop — and the culprit drifts *up* off the belt, because being hollow is the one thing about it that is visible. The packing line also continues the deck's existing factory story: trucks deliver, DQM watches the trucks, DQX inspects what comes off them one row at a time. Left-to-right motion supplies the sequence feel honestly, as throughput rather than trend, which matters because this profile does not model time. Three CSS bugs found along the way, all of the same shape — changing how something was positioned or animated without rechecking what its units referred to: * `bvPopIn` animates `transform: scale()`, which silently replaced the `translate(-50%,-50%)` a plotted element relied on for centring, leaving every point half a glyph off its coordinate. * Rebuilding the strip with `innerHTML` each tick meant every element was new, so there was no previous position to animate from and the transition never fired. Riders are now created once and only moved. * `translateX(105%)` is 105% of *the element*, not the track. With a 27px glyph that is 28px of travel, so all eight riders piled up at the left edge. Positions are pixels now; the `-50%` stays only for centring. Verified in the minified output rather than the source, because the CSS minifier renames keyframes: the slats animation resolves to `@keyframes n{to{transform:translateX(-15px)}}` and the paused-state override lands on the same class. Full cache clear before building, since an incremental docs build has served stale CSS from this file before. `make docs-build` SUCCESS, 156 documents, no broken links; `tsc --noEmit` clean apart from the four pre-existing `FeatureTags` errors. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 12 +- docs/dqx/src/components/SlideVisuals.tsx | 196 ++++++++++++++---- docs/dqx/src/css/custom.css | 185 +++++++++++++---- 3 files changed, 305 insertions(+), 88 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index d8ee6ff2e..0c58ecbff 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -37,17 +37,17 @@ Use row anomaly detection to automatically find unusual rows in your data using DQX learns what "normal" looks like from your good data, then checks every single row. No labels needed; it figures out what's unusual on its own. *"Is this banana weird?"* - - A 30cm plantain is perfectly normal; a 30cm lady finger does not exist. `baseline_by` judges each row against **its own group's** normal instead of the whole table's, so a value that is unremarkable overall can still be flagged where it does not belong. - - + Each banana becomes a set of numbers: size, colour, spots, bend. An Isolation Forest then tries to separate each point from the rest. If a banana is easy to isolate, it's probably odd. An unusual banana gets separated in just a few steps, so it sticks out. A normal banana is buried in the crowd and takes many steps to single out. - - Some problems are not extreme values at all — they are a *relationship* breaking. Count and weight always move together, until a crate holds 33 bananas and weighs 1.6kg. Splitting one number at a time cannot see that, so `profile="timeseries"` switches to a detector that models how metrics move together. + + A 30cm plantain is perfectly normal; a 30cm lady finger does not exist. `baseline_by` judges each row against **its own group's** normal instead of the whole table's, so a value that is unremarkable overall can still be flagged where it does not belong. + + + A banana packing line. Every piece of fruit passes the same gate and DQX scores it as it goes by. On this one, length and weight are each perfectly ordinary — and impossible together, because a banana that long has never weighed that little. Splitting one feature at a time never sees it, since neither value is out of range. `profile="timeseries"` reads the two **as a pair**, which is why it gets pulled while every single-column check waves it through. It needs no timestamp: it models how metrics relate to each other, not how they change over time. A score alone isn't enough. You want to know *why*. DQX breaks the score down per column: "too brown", "wrong size". So you can act on the insight straight away. diff --git a/docs/dqx/src/components/SlideVisuals.tsx b/docs/dqx/src/components/SlideVisuals.tsx index e3f285b08..4fa9c04e4 100644 --- a/docs/dqx/src/components/SlideVisuals.tsx +++ b/docs/dqx/src/components/SlideVisuals.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { Highlight, themes } from 'prism-react-renderer'; import type { Token, RenderProps } from 'prism-react-renderer'; @@ -59,18 +59,39 @@ const BANANA_VARIETIES: Array<{ name: string; emoji: string; scale: number; low: { name: 'Lady finger', emoji: '🍌', scale: 0.72, low: 10, high: 14 }, ]; -// Correlation break: crate weight normally tracks the banana count, because bananas have a weight. -// The last crate keeps both numbers inside their usual ranges and breaks the relationship between them. -const CRATE_READINGS: Array<{ count: number; weight: number; broken?: boolean }> = [ - { count: 30, weight: 3.6 }, - { count: 34, weight: 4.1 }, - { count: 28, weight: 3.4 }, - { count: 36, weight: 4.3 }, - { count: 31, weight: 3.7 }, - { count: 35, weight: 4.2 }, - { count: 29, weight: 3.5 }, - { count: 33, weight: 1.6, broken: true }, +// Correlation break, as a packing line. One crate at one ripeness -- colour deliberately does NOT vary, +// because it is the most salient channel on screen and varying it makes the eye nominate the wrong fruit, +// which then makes the gate's verdict look arbitrary. What varies visibly is LENGTH, one of the two numbers +// the check reads. WEIGHT is invisible, which is the whole reason a check is needed. +const CRATE_RIPENESS = [ + 'sepia(.08) saturate(1.04)', + 'sepia(.13) saturate(1.02)', + 'sepia(.05) saturate(1.06)', + 'sepia(.1) saturate(1.03)', ]; +const CRATE_NORMAL: Array<{ len: number; wt: number }> = [ + { len: 15, wt: 83 }, + { len: 16, wt: 90 }, + { len: 17, wt: 97 }, + { len: 18, wt: 104 }, + { len: 20, wt: 117 }, + { len: 21, wt: 124 }, + { len: 22, wt: 131 }, + { len: 23, wt: 138 }, +]; +// 19cm sits mid-pack, so nothing about it looks wrong. 19cm of banana weighs ~110g; this one is 38g. +const CRATE_ODD = { len: 19, wt: 38, bad: true }; + +// Longer fruit is drawn larger, so the visible variation is a variable the check actually reads. +const fruitSize = (len: number) => `${(1.3 + ((len - 15) / 8) * 0.8).toFixed(2)}rem`; +const expectedWeight = (len: number) => Math.round(len * 5.8); + +// Belt geometry, in percent of track width per second. +const LINE_GATE_X = 50; +const LINE_SPEED = 4.6; // a banana crosses the gate about every 4.3s -- slow enough to read +const LINE_SPACING = 20; +const LINE_COUNT = 8; +const LINE_HOLD_MS = 4200; // the line halts this long on a failure const DQX_RULES_CODE = `from databricks.labs.dqx.rule import DQRowRule from databricks.labs.dqx.check_funcs import is_not_null, is_in_range, is_in_list @@ -108,11 +129,11 @@ const SUMMARY_BULLETS: Array<{ title: string; detail?: string; code?: string }> }, { title: 'No ML expertise needed', - detail: 'DQX auto-discovers which columns to use, engineers features, and even segments your data when it makes sense. You just point it at a table.', + detail: 'DQX auto-discovers which columns to use, engineers features, and even finds a grouping to judge each row against when one makes sense. You just point it at a table.', }, { title: 'Explainable results', - detail: 'Every flagged row comes with a breakdown of why — which columns drove the score. Powered by SHAP, so you can act on it, not just stare at a number.', + detail: 'Every flagged row comes with a breakdown of why — which columns drove the score. So you can act on it, not just stare at a number.', }, { title: 'Works with batch and streaming', @@ -498,7 +519,7 @@ function ShapCarousel() { return (
-

Why was this banana flagged? SHAP top contributors

+

Why was this banana flagged? Top contributing features

{[-1, 0, 1].map(off => { const i = (center + off + n) % n; @@ -629,41 +650,138 @@ function BaselineGroups() { ); } +type LineItem = { len: number; wt: number; bad?: boolean; filter: string }; +type Rider = { item: LineItem; x: number; lift: number }; + function CorrelationBreak() { - const [revealed, setRevealed] = useState(false); + const trackRef = useRef(null); + const elsRef = useRef>([]); + const [reading, setReading] = useState(null); + useEffect(() => { - const id = setInterval(() => setRevealed(r => !r), 2600); - return () => clearInterval(id); + const seqRef = { n: 0 }; + const pick = (): LineItem => { + seqRef.n += 1; + const base = seqRef.n % 4 === 0 ? CRATE_ODD : CRATE_NORMAL[seqRef.n % CRATE_NORMAL.length]; + return { ...base, filter: CRATE_RIPENESS[seqRef.n % CRATE_RIPENESS.length] }; + }; + + const riders: Rider[] = Array.from({ length: LINE_COUNT }, (_, i) => ({ + item: pick(), + x: 105 - i * LINE_SPACING, + lift: 0, + })); + + // A percentage in translateX resolves against the ELEMENT's width, not the track's, so positions are + // converted to pixels here. The trailing -50% is kept because centring the glyph on its position is the + // one place percent-of-self is what we want. + const place = (r: Rider, el: HTMLSpanElement | null, trackWidth: number) => { + if (!el) return; + el.style.transform = + `translateX(${((r.x / 100) * trackWidth).toFixed(1)}px) translateX(-50%) translateY(${-r.lift.toFixed(1)}px)`; + el.style.filter = r.item.filter; + el.style.fontSize = fruitSize(r.item.len); + }; + + let raf = 0; + let last = performance.now(); + let holdUntil = 0; + let culprit: number | null = null; + + const frame = (now: number) => { + const dt = Math.min((now - last) / 1000, 0.05); + last = now; + const width = trackRef.current?.clientWidth || 1; + + if (now < holdUntil) { + // Hollow, so it drifts up off the belt rather than dropping. That is the only visible tell. + if (culprit !== null) { + riders[culprit].lift = Math.min(riders[culprit].lift + 16 * dt, 26); + place(riders[culprit], elsRef.current[culprit], width); + } + raf = requestAnimationFrame(frame); + return; + } + + if (culprit !== null) { + const r = riders[culprit]; + r.x = 105; + r.lift = 0; + r.item = pick(); + culprit = null; + setReading(null); + } + + riders.forEach((r, i) => { + const was = r.x; + r.x -= LINE_SPEED * dt; + if (was > LINE_GATE_X && r.x <= LINE_GATE_X) { + setReading(r.item); + if (r.item.bad) { + culprit = i; + holdUntil = now + LINE_HOLD_MS; + } + } + if (r.x < -8) { + r.x = 105; + r.item = pick(); + } + place(r, elsRef.current[i], width); + }); + + raf = requestAnimationFrame(frame); + }; + + raf = requestAnimationFrame(frame); + return () => cancelAnimationFrame(raf); }, []); - const maxCount = Math.max(...CRATE_READINGS.map(r => r.count)); - const maxWeight = Math.max(...CRATE_READINGS.map(r => r.weight)); + const fail = !!reading?.bad; return ( -
-
- {CRATE_READINGS.map((reading, i) => ( -
- - -
+
+ + {reading + ? <>{reading.len}cm · {reading.wt}g  pair {fail ? '✗' : '✓'} + : 'waiting…'} + + + + + + + +
+ {Array.from({ length: LINE_COUNT }, (_, i) => ( + { elsRef.current[i] = el; }} + >🍌 ))}
-
- bananas counted - crate weight -
-

- {revealed ? ( + + + + + + + + {Array.from({ length: 9 }, (_, i) => )} + + + {['0%', '33%', '66%', '100%'].map(left => )} + + + +

+ {fail ? ( <> - The last crate: 33 bananas, 1.6kg. Both numbers are ordinary on their own — and - 33 bananas have never weighed 1.6kg. + 19 cm, 38 g. Both mid-range, so every one-column rule says pass — but a banana + that long weighs about {expectedWeight(CRATE_ODD.len)} g. It is hollow, and only the pair gives it away. ) : ( - <>Count and weight always rise and fall together. Until one crate stops. + <>Two numbers read off each banana. Both ordinary, and consistent with each other. Pass. )}

diff --git a/docs/dqx/src/css/custom.css b/docs/dqx/src/css/custom.css index 528bcc7ee..dbeb830d7 100644 --- a/docs/dqx/src/css/custom.css +++ b/docs/dqx/src/css/custom.css @@ -739,56 +739,155 @@ button { text-align: center; } -/* ── Correlation break (`profile="timeseries"`) ──────────────────── */ -.bv-corr { margin-top: 0.75rem; } -.bv-corr__chart { - display: flex; - align-items: flex-end; - gap: 0.5rem; - height: 7rem; - padding: 0 0.25rem; - /* A narrow viewport must scroll the chart, not the page. */ - overflow-x: auto; +/* ── Correlation break: the inspection line (`profile="timeseries"`) ─ */ +.bv-line { position: relative; height: 10.6rem; margin: 0.75rem 0.25rem 0; overflow: hidden; } + +/* One pill carries the verdict. An earlier multi-row readout grew until it was more complicated than the + idea it was explaining, and the argument ended up being read rather than seen. */ +.bv-line__badge { + position: absolute; + left: 50%; + top: 0.35rem; + transform: translateX(-50%); + z-index: 6; + white-space: nowrap; + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 0.68rem; + font-weight: 700; + color: #44403c; + background: #fff; + border: 1px solid #d6d3d1; + border-radius: 999px; + padding: 0.22rem 0.6rem; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.08); + transition: color 0.2s ease, background 0.2s ease, border-color 0.2s ease; +} +.bv-line__badge b { color: #15803d; } +.bv-line__badge--fail { border-color: #ef4444; background: #fef2f2; color: #b91c1c; } +.bv-line__badge--fail b { color: #b91c1c; } + +/* the scanner the fruit passes through */ +.bv-line__gantry { + position: absolute; + left: 50%; + top: 2.35rem; + transform: translateX(-50%); + width: 6rem; + height: 3.5rem; + z-index: 4; + border: 3px solid #78716c; + border-top: 0; + border-radius: 0 0 0.4rem 0.4rem; + background: linear-gradient(180deg, rgba(120, 113, 108, 0.05), rgba(120, 113, 108, 0.13)); + transition: border-color 0.2s ease, background 0.2s ease; +} +.bv-line__gantry--fail { border-color: #ef4444; background: rgba(239, 68, 68, 0.09); } +.bv-line__scan { + position: absolute; + left: 8%; + right: 8%; + height: 2px; + border-radius: 2px; + background: linear-gradient(90deg, transparent, #fbbf24, transparent); + animation: bvLineScan 1.4s ease-in-out infinite; } -.bv-corr__col { - display: flex; - align-items: flex-end; - gap: 2px; - flex: 1 1 0; - min-width: 1.75rem; - height: 100%; - padding: 0.15rem; - border-radius: 0.25rem; - transition: background 0.4s ease; +.bv-line__gantry--fail .bv-line__scan { background: linear-gradient(90deg, transparent, #ef4444, transparent); } +@keyframes bvLineScan { 0%, 100% { top: 10%; } 50% { top: 80%; } } + +.bv-line__chute { + position: absolute; + left: 50%; + top: 7.15rem; + transform: translateX(-50%); + width: 6rem; + height: 2.2rem; + z-index: 1; + border: 2px dashed #d6d3d1; + border-top: 0; + border-radius: 0 0 0.45rem 0.45rem; + transition: border-color 0.2s ease; +} +.bv-line__chute--on { border-color: #ef4444; } + +/* the fruit rides here; the animation loop writes transform directly onto these */ +.bv-line__track { position: absolute; left: 0; right: 0; top: 3.85rem; height: 2rem; z-index: 3; } +.bv-line__fruit { position: absolute; left: 0; bottom: 0; line-height: 1; will-change: transform; } + +/* belt assembly — slats, lit top lip, spinning rollers, legs, floor */ +.bv-line__belt { + position: absolute; + left: -1%; + right: -1%; + top: 5.85rem; + height: 1.3rem; + z-index: 2; + background: #57534e; + border-radius: 2px; + overflow: hidden; + box-shadow: 0 3px 6px rgba(0, 0, 0, 0.18); } -.bv-corr__col--flagged { background: rgba(254, 243, 199, 0.9); outline: 2px solid rgba(251, 191, 36, 0.6); } -.bv-corr__bar { flex: 1; border-radius: 2px 2px 0 0; transition: height 0.4s ease; } -.bv-corr__bar--count { background: #a8a29e; } -.bv-corr__bar--weight { background: #fbbf24; } -.bv-corr__legend { +.bv-line__slats { + position: absolute; + inset: 0; + background: repeating-linear-gradient(90deg, #3f3b38 0 3px, #57534e 3px 15px); + animation: bvLineSlats 0.5s linear infinite; +} +@keyframes bvLineSlats { to { transform: translateX(-15px); } } +.bv-line__lip { position: absolute; left: 0; right: 0; height: 3px; } +.bv-line__lip--top { top: 0; background: linear-gradient(180deg, #e7e5e4, #a8a29e); } +.bv-line__lip--bottom { bottom: 0; background: #78716c; } + +.bv-line__rollers { + position: absolute; + left: -1%; + right: -1%; + top: 7rem; display: flex; - gap: 1rem; - justify-content: center; - margin-top: 0.5rem; - font-size: 0.75rem; - color: #57534e; + justify-content: space-between; + z-index: 1; } -.bv-corr__key { display: inline-flex; align-items: center; gap: 0.3rem; } -.bv-corr__key::before { - content: ''; - width: 0.6rem; - height: 0.6rem; +.bv-line__roller { + width: 1rem; + height: 1rem; + border-radius: 50%; + background: conic-gradient(from 0deg, #d6d3d1 0 25%, #8b8684 25% 50%, #d6d3d1 50% 75%, #8b8684 75%); + box-shadow: inset 0 0 0 2px #6b6664, 0 1px 2px rgba(0, 0, 0, 0.25); + animation: bvLineSpin 0.5s linear infinite; +} +@keyframes bvLineSpin { to { transform: rotate(360deg); } } + +.bv-line__legs { position: absolute; left: 8%; right: 8%; top: 7.9rem; height: 1.6rem; z-index: 0; } +.bv-line__leg { + position: absolute; + width: 0.36rem; + height: 100%; + border-radius: 1px; + background: linear-gradient(90deg, #d6d3d1, #a8a29e); +} +.bv-line__floor { + position: absolute; + left: 3%; + right: 3%; + top: 9.45rem; + height: 3px; + background: #d6d3d1; border-radius: 2px; } -.bv-corr__key--count::before { background: #a8a29e; } -.bv-corr__key--weight::before { background: #fbbf24; } -.bv-corr__caption { - margin-top: 0.6rem; - font-size: 0.82rem; - color: #44403c; + +/* The whole line halts while a failure is on screen, so the stop is itself the signal. */ +.bv-line--hold .bv-line__slats, +.bv-line--hold .bv-line__roller { animation-play-state: paused; } + +.bv-line__caption { + position: absolute; + left: 0; + right: 0; + bottom: -0.35rem; + margin: 0; text-align: center; - /* Both captions are close in length; a fixed floor stops the slide jumping as they swap. */ - min-height: 2.6em; + font-size: 0.8rem; + line-height: 1.5; + color: #44403c; } /* ── SHAP carousel ────────────────────────────────────────────────── */ From 3e81084bd742e3b08f8228c0b654de8419da8bec Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 22:01:05 +0100 Subject: [PATCH 054/107] Correct the FAQ on single-metric series: a timestamp is context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FAQ said that with only one metric `profile="timeseries"` "has nothing to work with". The first half of that is right and the conclusion is wrong, because DQX derives seven cyclical features from a datetime column — so one metric plus a timestamp is not one feature, it is eight, and the metric can be judged against where it sits in the week. Measured on a series that is busy on weekdays and quiet at weekends, then given a weekday-sized value landing on a Saturday: the anomalous rows score a median 177.4 against 3.2 for ordinary weekend rows, a 55x separation, while the value's own z-score is 0.92 — nowhere near extreme. So the contextual case genuinely works, and the earlier wording talked a user out of it. Verified separately that with a *bare* single feature the detector does reduce to a z-score, matching (x-mu)^2/sigma^2 to within 0.05% — the residual being the ridge floor on the covariance. So the advice to reach for a range check or `has_no_outliers` is correct for that case, and now stated as being about that case rather than about single metrics in general. Same failure as the seasonality claim corrected earlier in this PR, from the same cause: having established that the profile does not model *time*, I over-corrected into implying the calendar features do not count. They do. The boundary that actually holds is contextual versus sequential — "unusual for a Tuesday morning" is in reach, "unusual compared to five minutes ago" is not, because there are no lag features and no memory of the previous row. Kept deliberately non-technical, per review: no mention of covariance or z-scores in the FAQ text, just what to compare against and why. `make docs-build` SUCCESS, 156 documents, no broken links — which also confirms the new `has_no_outliers` reference resolves. Co-authored-by: Isaac --- docs/dqx/docs/guide/row_anomaly_detection/index.mdx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 0c58ecbff..2c62b6bdb 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -572,8 +572,12 @@ its own phase. **Forecasting.** DQX judges rows against learned normal. It does not predict the next value and compare. -**A single metric with `profile="timeseries"`.** That detector models how metrics move *together*, so -with only one metric it has nothing to work with. Use a rule or a threshold. +**A single metric on its own.** With nothing to compare it against, `profile="timeseries"` can only ask +"is this value unusually high or low?" — which a simple range check already does, and +[`has_no_outliers`](/docs/reference/quality_checks) does better. Give it something to compare against and +that changes: include the timestamp and it can spot a value that is fine in general but wrong *for a +Saturday*; add `baseline_by` and it can spot one that is wrong *for that region*. Context is what it needs, +and a timestamp or a grouping column both count. **Anything you already have labels for.** Train a classifier — it will beat any unsupervised detector on the pattern it was taught. Anomaly detection is for the problems you cannot describe in advance. From c76eccfa7eea5aac3c329d593be3da5aec8367c8 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 22:37:40 +0100 Subject: [PATCH 055/107] Advise rather than apply a discovered grouping on explicit columns Naming `columns` keeps the whole-table comparison, as it did before `baseline_by` existed. Discovery still runs, but only to name the grouping it found so the caller can opt in. Reverting the coupling change: the defect was the silence, not the pooling. An explicit column list is a decision, and a discovered grouping is data-dependent, so applying one makes the feature set shift when a cardinality crosses a threshold between retrains, moving every score and drifting a calibrated alert budget. The advisory is silent when the data supports no grouping, which is what keeps it worth reading, and `suggest_baseline_columns` scans only what the grouping decision needs rather than computing a full profile and discarding the numeric half. Also removes the anomaly migration guide and heuristic map. The migration steps now sit in the user guide beside the feature; the heuristic map was a second copy of the control flow that had to be updated alongside it. --- docs/dqx/docs/dev/anomaly_compatibility.mdx | 77 ------------- docs/dqx/docs/dev/anomaly_heuristic_map.mdx | 106 ------------------ .../guide/row_anomaly_detection/index.mdx | 25 ++++- .../reference/anomaly_detection_quality.mdx | 4 +- docs/dqx/docs/reference/quality_checks.mdx | 2 +- .../labs/dqx/anomaly/anomaly_engine.py | 10 +- src/databricks/labs/dqx/anomaly/profiler.py | 72 +++++++++++- .../labs/dqx/anomaly/training_service.py | 57 ++++++---- .../test_anomaly_autodiscovery.py | 100 ++++++++++++++--- 9 files changed, 222 insertions(+), 231 deletions(-) delete mode 100644 docs/dqx/docs/dev/anomaly_compatibility.mdx delete mode 100644 docs/dqx/docs/dev/anomaly_heuristic_map.mdx diff --git a/docs/dqx/docs/dev/anomaly_compatibility.mdx b/docs/dqx/docs/dev/anomaly_compatibility.mdx deleted file mode 100644 index a7289c5d3..000000000 --- a/docs/dqx/docs/dev/anomaly_compatibility.mdx +++ /dev/null @@ -1,77 +0,0 @@ ---- - -title: Anomaly migration guide - -sidebar_position: 640 - ---- - -# Anomaly Detection Migration Guide - -Baseline conditioning ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) replaced the -previous per-group modelling with a single conditioned model, and the compatibility affordances that -once kept the old path and old models alive have been removed. Row anomaly detection was Experimental -through 0.16.0 — its on-disk and API formats were explicitly allowed to change without a migration -path — so this break is taken deliberately now that the feature is Beta, rather than carried forever. - -If you never used row anomaly detection, nothing here affects you. If you did, the four steps under -[What you have to do](#what-you-have-to-do) are the whole migration. - -## What changed - -- **`segment_by` is gone.** It trained one model per group. `baseline_by` trains one model whatever - the group count, judging each metric against its own group's baseline. This is not a rename — the - semantics and the scores differ. On the Server Machine Dataset the old per-group configuration was - the *worst* of three measured, with one entity emitting 15,963 false positives across 28,392 normal - rows, so nothing is lost by dropping it. -- **The registry `segmentation` struct is now `grouping`.** It holds `baseline_by`, `sklearn_version`, - and `config_hash` — the per-segment fields (`segment_values`, `is_global_model`) are gone, because - every model is now single and conditioned. This is a persisted Delta schema change; the registry - write uses `mergeSchema`, so retraining into an existing table adds the new column in place. -- **`_dq_info.anomaly.segment` was removed.** It was permanently null once segmentation was gone. -- **`compute_config_hash` now includes `baseline_by`.** The hash is `(columns, baseline_by)`. A model - retrained under the same name with a different grouping now hashes differently, so the collision - detection that guards "same name, different configuration" finally sees a grouping change. A - scoring-time hash mismatch **raises**, which is what forces the retrain below. -- **Grouping auto-discovery is decoupled from column discovery.** Passing explicit `columns` no longer - silently gives up conditioning, and a discovered grouping is used as `baseline_by` rather than - turning into one model per group. Auto-discovered models may therefore score differently even with - no configuration change. - -## What you have to do - -1. **Replace `segment_by` with `baseline_by`.** Not a rename — `segment_by` partitioned into N models; - `baseline_by` judges each metric against its own group's baseline on one model. Expect different - scores. -2. **Retrain every model.** Metadata from before #1484 is no longer loadable, and the config hash - changes even where the configuration did not, so old models raise at scoring rather than scoring - wrong. -3. **Migrate or recreate registry tables.** Retraining into an existing table works — the write merges - the renamed `grouping` schema — but a table you never retrain into keeps the old `segmentation` - column and will not be read. -4. **Expect different scores on auto-discovered groupings** even without any config change, since - discovery now selects a finer grouping routed to `baseline_by`. - -## What did *not* change - -These are permanent runtime behaviours, not compatibility shims, and they behave exactly as before: - -- **Unseen baseline group → global median.** A group absent at training gets the global baseline, so - the row reads as ordinary rather than extreme, and `is_new_baseline` reports it. New groups appear in - production forever. -- **Missing per-group quantiles → global calibration.** A group without a full quantile set falls back - to table-wide calibration rather than being left half-calibrated. -- **The ungrouped path.** A model trained with no `baseline_by` engineers exactly the features it - always did; conditioning adds nothing when there is no grouping. -- **Python/Spark baseline-key agreement.** `build_baseline_key` and `baseline_key_column` must keep - matching for as long as both exist; the notes there are a contract, not debt. - -## Still carried - -One affordance from the pre-conditioning code has *not* been removed and is not part of this break: - -- **`RobustScaler` residue** in `core.py` and `explainability.py`. The scaler is an affine per-feature - transform and Isolation Forest splits on per-feature thresholds, so the model is invariant to it - (measured identical to four decimal places across five ADBench datasets). New models are fitted - without it, but the load and explain paths still tolerate a scaler pickled inside older pipelines. - Removing it is a separate, non-breaking cleanup. diff --git a/docs/dqx/docs/dev/anomaly_heuristic_map.mdx b/docs/dqx/docs/dev/anomaly_heuristic_map.mdx deleted file mode 100644 index 9daea8d11..000000000 --- a/docs/dqx/docs/dev/anomaly_heuristic_map.mdx +++ /dev/null @@ -1,106 +0,0 @@ ---- - -title: Anomaly heuristic map - -sidebar_position: 645 - ---- - -# Anomaly Detection Heuristic Map - -Every decision row anomaly detection makes on the way from a DataFrame to a flagged row, with the -default that governs it. It exists to make a bad result diagnosable by working down a path, rather -than by reading the module. - -Training and scoring are two passes over the same feature-engineering code: training persists what -scoring later reads back. Diamonds are branches. - -```mermaid -flowchart TD - A["train(df)"] --> B{"columns given?"} - - B -- no --> C["1 pick metrics
numeric, stddev>0, nulls<50%
low-card categoricals also eligible"] - B -- yes --> D["use the caller's columns"] - C --> E["2 pick a grouping
nulls<10%, not id-like
≥30 rows/group, ≤5000 groups"] - D --> E - - E --> H["3 one model,
grouping becomes baseline_by"] - - H --> I["4 sample 30%, split 80/20"] - I --> J["5 feature engineering
onehot card≤20 else frequency
+ metric_rel_baseline per metric"] - J --> K["6 IsolationForest
200 trees, 256 rows/tree,
contamination 0.02, no scaling"] - K --> L["7 quantiles: global
+ per baseline group"] - L --> M["register: MLflow + registry row
+ feature_metadata JSON"] - - M -.->|"persisted baselines, medians, quantiles"| N - - N["score(df)"] --> O["8 re-engineer from metadata
baselines broadcast-joined
unseen group → global median"] - O --> P["pandas UDF: anomaly score"] - P --> Q["9 mark unseen baselines
isin if ≤200 keys, else join"] - Q --> R["10 severity percentile
per-group quantiles if present,
else global"] - R --> S{"baseline unseen?"} - S -- yes --> T["null score, not flagged
is_new_baseline = true"] - S -- no --> U{"severity ≥ threshold 95?"} - U -- yes --> V["flagged"] - U -- no --> W["passes"] -``` - -There is one model whatever the grouping. `baseline_by` conditions each metric against its own -group's baseline rather than training a model per group; if it is not declared, stage 2 may discover -one. A column named both as a metric and as a baseline is rejected, as are floating-point and decimal -baseline columns (Spark and Python format them differently, which would break the key lookup). - -## Reading a bad result - -Working down the path, three questions separate most failures. - -**What was chosen?** Stages 1 and 2 decide silently, and both defects found while building this map -were there. - -```sql -SELECT identity.model_name, training.columns, grouping.baseline_by -FROM -WHERE identity.status = 'active' -``` - -**Did conditioning engage, and on what?** Look for `_rel_baseline` in -`engineered_feature_names`, and count the entries in `baseline_medians`. Three groups where the data -has ninety means stage 2 chose too coarsely — a model can be conditioned and still be conditioned on -almost nothing. - -**Is the comparison even valid?** Training samples 30% of rows by default via `.sample`, and splits -80/20 via `.randomSplit`; under Spark Connect both depend on partition ordering. Two runs of identical -code can therefore train on different rows. If you are comparing two builds, set -`sample_fraction=1.0` first — otherwise you are measuring the sampler. Suspect stage 4 before -suspecting the design. - -## Defaults in one place - -| Default | Value | Governs | -|---|---|---| -| `MIN_ROWS_PER_BASELINE_GROUP` | 30 | rows/group needed to trust a baseline median | -| `MAX_BASELINE_GROUPS` | 5000 | ceiling on total baseline groups | -| `MAX_BASELINE_COLUMN_CARDINALITY` | 50 | per-column ceiling for a baseline column | -| `DEFAULT_SAMPLE_FRACTION` | 0.3 | share of rows used for training | -| `DEFAULT_TRAIN_RATIO` | 0.8 | train/validation split | -| `categorical_cardinality_threshold` | 20 | one-hot below, frequency-encode above | -| `num_trees` | 200 | Isolation Forest size | -| `max_samples` | 256 | sklearn "auto"; rows per tree | -| `expected_anomaly_rate` | 0.02 | becomes contamination when unset | -| `threshold` | 95.0 | severity percentile that flags a row | - -## Known weak points - -Measured, with the evidence in `benchmarks/anomaly_conditioning/`: - -- **Stage 1 can pick dimensions as metrics.** Low-cardinality categoricals qualify as features, so a - table keyed by several dimensions can produce a dozen one-hot columns beside a single real metric, - diluting the signal. Largely avoided when those columns are recognised as a grouping instead, since - the profiler removes the grouping from the feature list — but a categorical that is *not* selected - as a grouping can still land in features. -- **Stage 6 has a blind spot.** Isolation Forest loses to a max-abs-z baseline where anomalies are - single-feature extremes in few dimensions, and in high dimension. Not fixable by tuning, and adding - a complementary scorer measured worse on average — see `complementary_detector.py`. -- **Stage 8's fallback is silent by design.** A group absent at training falls back to the global - baseline, which makes the row read as ordinary rather than extreme. `is_new_baseline` is how that - case is detected; it is the conservative direction, not a bug. diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 2c62b6bdb..a615c8225 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -427,11 +427,13 @@ Both profiles get the same feature engineering, so these need no configuration: Row anomaly detection was **Experimental** in earlier releases and is now **Beta**. This release replaces the old per-group model path (`segment_by`) with `baseline_by`, which measured better on detection, reliability, and cost. If you never used `segment_by`, nothing here affects you. -If you did, three things change. The full detail is in the [migration guide](/docs/dev/anomaly_compatibility). +If you did, here is the whole migration. -- **Replace `segment_by` with `baseline_by`.** They are not the same. `segment_by` trained one model per group; `baseline_by` judges each metric against its own group's baseline on a single model. Your scores will change. -- **Retrain your models.** A model trained before this release raises an error at scoring time until you retrain it. -- **Your registry table is handled for you.** Retraining writes the new schema into your existing table in place. +- **Replace `segment_by` with `baseline_by`.** They are not the same. `segment_by` trained one model per group; `baseline_by` judges each metric against its own group's baseline on a single model. Your scores will change. `AnomalyParams.max_segment_models` is gone too, since there is only ever one model now. +- **Retrain your models.** A model trained before this release raises an error at scoring time until you retrain it, rather than scoring against a feature list that no longer matches. +- **Your registry table is handled for you.** Retraining writes the new schema into your existing table in place. The `segmentation` column becomes `grouping`, and a table you never retrain into keeps the old column and will not be read. +- **`_dq_info[].anomaly` changes shape.** The `segment` field is gone (it was always null once per-group models were), and `is_new_baseline` and `new_baseline_key` are added. Named-field queries keep working, but appending to a table that already holds `_dq_info` needs `mergeSchema`. +- **Auto-discovered groupings may score differently** even with no configuration change, because a discovered grouping is now used as `baseline_by` and the policy that picks it is finer than the segmented one it replaced. ## How it works under the hood @@ -443,7 +445,7 @@ For full parameter and schema details, see [Row Anomaly Detection in Quality Che 2. **Smart sampling and training**: DQX samples your data (default 30%, capped at 1M rows), trains the detector your `profile` selects — an ensemble of Isolation Forest models by default, or a single correlation-aware model for `profile="timeseries"` — and captures baseline statistics for drift detection. 3. **Model registry**: Models and metadata live in MLflow and a Delta table; each `train()` call registers one model under its name and archives the previously active version. 4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. Feature contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. How they are computed depends on the detector — SHAP for Isolation Forest, an exact leave-one-out decomposition for the correlation-aware detector — but the output is the same map either way. -5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping (categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group, for example region or product category), and this happens whether or not you passed `columns`, since what to measure and what to compare it against are independent questions. A discovered grouping is used as `baseline_by` (see [Group-aware anomaly detection](#group-aware-anomaly-detection)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. +5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping (categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group, for example region or product category). A discovered grouping is used as `baseline_by` (see [Group-aware anomaly detection](#group-aware-anomaly-detection)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. When you *do* pass `columns`, you have decided what to measure, so DQX leaves the comparison pooled rather than adding a grouping you did not ask for. If your data looks grouped it says so in a warning naming the grouping to pass. ### Which algorithm, and why @@ -610,7 +612,18 @@ See the **Training data requirements** tip under Quick start. In short: 1,000+ r
Q: Why did auto-training choose a grouping? -When you train without specifying a grouping, DQX looks for one: columns that look like good dimensions (for example region or category) with low cardinality (2–50 distinct values), low null rate, and enough rows per group. That is intentional, because behaviour often does differ by group, and the user least likely to pass `baseline_by` explicitly is the one most likely to miss a contextual anomaly. A discovered grouping is used as `baseline_by`: each metric gains its deviation from that group's baseline, on a **single** model, so the discovery cannot turn into an hours-long run however many groups it finds. The grouping it chose is logged. To compare against the whole table instead, pass `baseline_by=[]`. +When you let DQX pick the columns as well, it looks for a grouping too: columns that look like good dimensions (for example region or category) with low cardinality (2–50 distinct values), low null rate, and enough rows per group. That is intentional, because behaviour often does differ by group, and the user least likely to pass `baseline_by` explicitly is the one most likely to miss a contextual anomaly. A discovered grouping is used as `baseline_by`: each metric gains its deviation from that group's baseline, on a **single** model, so the discovery cannot turn into an hours-long run however many groups it finds. The grouping it chose is logged. To compare against the whole table instead, pass `baseline_by=[]`. + +Naming `columns` yourself changes this. Then DQX keeps the whole-table comparison and only *tells* you what it found: + +``` +WARNING ['region'] looks like a grouping (3 groups, ~200 rows/group), but metrics are being + compared against the whole table, so a value that is ordinary overall yet wrong for its + own group will not be flagged. Pass baseline_by=['region'] to compare each row against + its own group, or baseline_by=[] to keep the whole-table comparison and silence this. +``` + +Two reasons it advises rather than acts. Your explicit column list is a decision, and adding engineered features on top of it would change the model you asked for. And a discovered grouping depends on the data, so if a column's cardinality shifts between retrains the feature set would change with it and move every score, drifting the threshold you calibrated. The warning is silent when the data has no usable grouping, so it only speaks when there is something to do.
diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx index 89ef91844..8990583b6 100644 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ b/docs/dqx/docs/reference/anomaly_detection_quality.mdx @@ -51,7 +51,9 @@ group's daily volume collapses by 80% while the total across all groups stays fl that shape, a model comparing against the whole table scored PR-AUC 0.0028 against a 0.0026 base rate — chance. Conditioned on the group, 0.6962. -To compare against the whole table anyway, pass `baseline_by=[]`. +A grouping is discovered for you only when you let DQX pick the feature columns too. Name `columns` yourself +and the comparison stays pooled, with a warning naming the grouping to pass if the data looks grouped. To +compare against the whole table and silence that warning, pass `baseline_by=[]`. ## Which detector to use diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index e923b7a13..6863fad55 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -3496,7 +3496,7 @@ Required arguments: `df`, `model_name`, `registry_table`. Optional parameters: | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `columns` | list[str] | None | Columns to use for row anomaly detection (auto-discovered if omitted) | -| `baseline_by` | list[str] | None | Columns identifying the group a row belongs to, so each metric is judged against its own group's baseline rather than against the whole table. One model whatever the group count. Auto-discovered when both `columns` and `baseline_by` are omitted; pass `baseline_by=[]` to suppress discovery and compare against the whole table. | +| `baseline_by` | list[str] | None | Columns identifying the group a row belongs to, so each metric is judged against its own group's baseline rather than against the whole table. One model whatever the group count. Auto-discovered when both `columns` and `baseline_by` are omitted; pass `baseline_by=[]` to suppress discovery and compare against the whole table. When you name `columns` but not `baseline_by`, the comparison stays pooled and a warning names the grouping to pass if the data looks grouped. | | `profile` | str | `"tabular"` | Which detector to train. `"tabular"` is Isolation Forest — the behaviour that predates this option. `"timeseries"` selects a correlation-aware detector for multivariate metrics whose anomalies are broken *correlations* rather than extreme single values; it needs no timestamp column and trains a single model rather than an ensemble. There is no automatic option: DQX never changes the algorithm on your behalf, because the choice cannot be verified without labels. See [Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile). | | `exclude_columns` | list[str] | None | Columns to exclude from training (e.g., IDs, labels, ground truth) | | `expected_anomaly_rate` | float | 0.02 | Expected fraction of anomalies in your data (0.02 = 2%). Sets model contamination parameter. | diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index 7d778f6bb..be7e8ce84 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -74,7 +74,11 @@ def train( Auto-discovery behavior: - columns=None, baseline_by=None: Auto-discovers both the feature columns and a grouping - - columns specified, baseline_by=None: Uses the columns, still discovers a grouping + - columns specified, baseline_by=None: Uses the columns and compares against the whole + table. Naming the columns means you decided what to measure, so no grouping is added + on your behalf. If the data looks grouped, a warning names the grouping to pass. + - baseline_by=[]: Compares against the whole table, and suppresses both the grouping + discovery above and that warning - baseline_by specified: Conditions on that grouping Args: @@ -99,7 +103,9 @@ def train( numeric metric gains its deviation from that baseline as an extra feature on a single pooled model, so the cost does not grow with the group count. This is what catches a value that is unremarkable across the table but wrong for - its own group. Auto-discovered when omitted. + its own group. Auto-discovered only when *columns* is also omitted; pass + ``baseline_by=[]`` to compare against the whole table and suppress both that + discovery and the advisory warning. params: Optional anomaly parameters for tuning training behavior. exclude_columns: Columns to exclude from training (e.g., IDs, labels, ground truth). Exclusions always take precedence over `columns` if both are provided. diff --git a/src/databricks/labs/dqx/anomaly/profiler.py b/src/databricks/labs/dqx/anomaly/profiler.py index f318509cb..594226338 100644 --- a/src/databricks/labs/dqx/anomaly/profiler.py +++ b/src/databricks/labs/dqx/anomaly/profiler.py @@ -278,8 +278,10 @@ def select_baseline_columns(candidates: list[tuple[str, int, float]], total_coun groups = prospective if selected: + # Neutral wording on purpose: this policy is shared by the path that applies a discovered + # grouping and the advisory path that only reports one, so it must not claim application. logger.info( - f"Auto-detected baseline grouping {selected}: {groups} groups, " + f"Baseline grouping {selected}: {groups} groups, " f"~{int(total_count / groups)} rows/group (one model regardless of group count)" ) skipped = [c[0] for c in candidates if c[0] not in selected] @@ -292,6 +294,74 @@ def select_baseline_columns(candidates: list[tuple[str, int, float]], total_coun return selected +@dataclass(frozen=True) +class BaselineSuggestion: + """A grouping the data would support, for advising a caller who did not ask for one.""" + + columns: list[str] + group_count: int + rows_per_group: int + + +def suggest_baseline_columns(df: DataFrame, exclude: list[str]) -> BaselineSuggestion | None: + """Find a grouping the data would support, without selecting feature columns. + + Deliberately narrower than :func:`auto_discover_columns`: the grouping decision needs null rates + and distinct counts on the categorical columns only, so this skips the numeric mean/stddev + aggregation a full profile computes and would then throw away. + + Used to *advise*, never to apply. A caller who named *columns* has decided what to measure, and + DQX does not add engineered features they did not ask for. + + Args: + df: DataFrame to scan. + exclude: Columns the caller already named as features. A column cannot be both what is + measured and what it is measured against. + + Returns: + The grouping and its shape, or None when the data supports none. + """ + excluded = set(exclude) + categorical_types = (StringType, IntegerType) + categorical = [ + f.name for f in df.schema.fields if isinstance(f.dataType, categorical_types) and f.name not in excluded + ] + if not categorical: + return None + + total_count = df.count() + if total_count == 0: + return None + null_counts, distinct_counts = compute_null_and_distinct_counts(df, categorical, categorical, approx=True, rsd=0.05) + distinct_counts.update(compute_exact_distinct_counts(df, categorical)) + + id_pattern = re.compile(r"(?i)(id|key)$") + candidates = [] + for name in categorical: + distinct_count = distinct_counts.get(name) + if distinct_count is None: + continue + if _is_grouping_candidate( + distinct_count, + null_rate=null_counts.get(name, 0) / total_count, + is_id_column=id_pattern.search(name) is not None, + total_count=total_count, + ): + candidates.append((name, distinct_count, total_count / distinct_count)) + + candidates.sort(key=lambda candidate: candidate[1]) + selected = select_baseline_columns(candidates, total_count) + if not selected: + return None + + group_count = _count_group_combinations(selected, distinct_counts) + return BaselineSuggestion( + columns=selected, + group_count=group_count, + rows_per_group=int(total_count / group_count) if group_count else total_count, + ) + + def _select_segment_columns( df: DataFrame, recommended_columns: list[str], diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index e10cde75d..14b03e63c 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -28,7 +28,7 @@ ModelIdentity, TrainingMetadata, ) -from databricks.labs.dqx.anomaly.profiler import auto_discover_columns +from databricks.labs.dqx.anomaly.profiler import auto_discover_columns, suggest_baseline_columns from databricks.labs.dqx.anomaly.training_strategies import ( DEFAULT_PROFILE, AnomalyTrainingStrategy, @@ -48,6 +48,7 @@ ) from databricks.labs.dqx.config import AnomalyParams from databricks.labs.dqx.errors import InvalidParameterError +from databricks.labs.dqx.utils import sanitize_for_logging logger = logging.getLogger(__name__) @@ -156,9 +157,12 @@ def _discover_columns_and_grouping( ) -> tuple[list[str], list[str] | None]: """Fill in whichever of the feature columns and the grouping the caller left unspecified. - Returns ``(columns, baseline_by)``. A discovered grouping always becomes ``baseline_by``: - there is one model regardless of group count, so discovery cannot turn into an hours-long - run however many groups it finds. + Returns ``(columns, baseline_by)``. + + Discovery of a grouping is reachable only when the feature columns were discovered too, so + naming *columns* keeps the whole-table comparison. A discovered grouping then becomes + ``baseline_by``: there is one model regardless of group count, so discovery cannot turn into + an hours-long run however many groups it finds. """ if columns is None: columns, discovered = self._perform_auto_discovery(df_filtered) @@ -170,32 +174,39 @@ def _discover_columns_and_grouping( return columns, discovered if declared_baseline_by is None: - # Grouping discovery used to be reachable only when the columns were discovered too, so - # naming your feature columns silently gave up any chance of conditioning. Those are - # independent questions. Costs one extra profiling pass for callers who pass explicit - # columns and no grouping. - return columns, self._discover_baseline_columns(df_filtered, columns) + # Naming *columns* keeps the pooled comparison, as it did before baseline_by existed. + # Explicit configuration wins: adding engineered baseline features the caller never + # asked for would also make the feature set data-dependent, so a retrain after a + # cardinality shift would silently move every score and drift their threshold. + # Discovery still runs here, but only to advise. + self._advise_baseline_columns(df_filtered, columns) + return columns, None return columns, declared_baseline_by @staticmethod - def _discover_baseline_columns(df_filtered: DataFrame, columns: list[str]) -> list[str] | None: - """Discover a baseline grouping when the caller named feature columns but no grouping. + def _advise_baseline_columns(df_filtered: DataFrame, columns: list[str]) -> None: + """Warn when the data looks grouped but the caller left the comparison pooled. - Kept separate from ``_perform_auto_discovery`` so that discovering a grouping does not - require also discovering the feature columns. + Advisory only, and silent when there is nothing to act on, which is what keeps it worth + reading: a warning that fires on every explicit-columns call is one nobody looks at. - Anything the caller named as a feature is excluded. When discovery picks the columns itself - the profiler already keeps the two lists disjoint, but here the feature list came from the - caller: a column they asked to have measured must not silently become the basis it is - measured against, which ``validate_baseline_columns`` would reject anyway. + Anything the caller named as a feature is excluded from the suggestion. A column they asked + to have measured must not be offered as the basis it is measured against, which + ``validate_baseline_columns`` would reject anyway. """ - profile = auto_discover_columns(df_filtered) - discovered = [c for c in profile.recommended_segments if c not in set(columns)] - if not discovered: - return None - logger.info(f"Auto-detected {len(discovered)} baseline columns: {discovered}") - return discovered + suggestion = suggest_baseline_columns(df_filtered, exclude=columns) + if suggestion is None: + return + # Column names come from the caller's schema, so they are untrusted for logging (CWE-117). + safe_columns = [sanitize_for_logging(name) for name in suggestion.columns] + logger.warning( + f"{safe_columns} looks like a grouping ({suggestion.group_count} groups, " + f"~{suggestion.rows_per_group} rows/group), but metrics are being compared against the " + f"whole table, so a value that is ordinary overall yet wrong for its own group will not " + f"be flagged. Pass baseline_by={safe_columns} to compare each row against its own group, " + f"or baseline_by=[] to keep the whole-table comparison and silence this." + ) def build_context( self, diff --git a/tests/integration_anomaly/test_anomaly_autodiscovery.py b/tests/integration_anomaly/test_anomaly_autodiscovery.py index 12f1eb556..71a430544 100644 --- a/tests/integration_anomaly/test_anomaly_autodiscovery.py +++ b/tests/integration_anomaly/test_anomaly_autodiscovery.py @@ -1,9 +1,11 @@ """Integration tests for auto-discovery of anomaly detection columns and segments.""" +import logging + from pyspark.sql import SparkSession from pyspark.sql import functions as F -from databricks.labs.dqx.anomaly.profiler import auto_discover_columns +from databricks.labs.dqx.anomaly.profiler import auto_discover_columns, suggest_baseline_columns from tests.constants import TEST_CATALOG from tests.integration_anomaly.constants import SEGMENT_REGIONS from tests.integration_anomaly.conftest import qualify_model_name @@ -125,12 +127,15 @@ def test_zero_config_training(spark: SparkSession, make_schema, make_random, ano assert set(model.training.columns) == {"amount", "discount"} -def test_explicit_columns_still_get_baseline_discovery(spark: SparkSession, make_schema, make_random, anomaly_engine): - """Naming the feature columns does not turn off baseline discovery — they are independent choices. +def test_explicit_columns_keep_the_pooled_comparison_but_warn( + spark: SparkSession, make_schema, make_random, anomaly_engine, caplog +): + """Naming the feature columns keeps the whole-table comparison, and says so. - Under the old segmented policy, passing explicit columns suppressed auto-segmentation. Baseline - conditioning has no reason to: there is one model regardless of group count, so a caller who - names the metrics still gets a grouping discovered for them. See databrickslabs/dqx#1484. + Explicit configuration wins: DQX does not add engineered baseline features the caller never + asked for, because a discovered grouping is data-dependent and a retrain after a cardinality + shift would silently move every score. What it does instead is name the grouping it found, so + the caller can opt in. Silence was the actual defect here, not the pooling. """ # Create unique schema for test isolation schema = make_schema(catalog_name=TEST_CATALOG) @@ -146,20 +151,87 @@ def test_explicit_columns_still_get_baseline_discovery(spark: SparkSession, make table_name = f"{TEST_CATALOG}.{schema.name}.explicit_cols_test_{suffix}" df.write.saveAsTable(table_name) - # Explicit feature column, no explicit grouping: the grouping is still discovered. + # Explicit feature column, no explicit grouping: pooled, with an advisory naming what it found. registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" - anomaly_engine.train( - df=spark.table(table_name), - columns=["amount"], - model_name=qualify_model_name(f"test_explicit_{suffix}", registry_table), - registry_table=registry_table, - ) + with caplog.at_level(logging.WARNING, logger="databricks.labs.dqx.anomaly.training_service"): + anomaly_engine.train( + df=spark.table(table_name), + columns=["amount"], + model_name=qualify_model_name(f"test_explicit_{suffix}", registry_table), + registry_table=registry_table, + ) registry = spark.table(registry_table) models = registry.filter("identity.status = 'active'").collect() assert len(models) == 1 assert models[0].training.columns == ["amount"] - assert models[0].grouping.baseline_by == ["region"] + assert not models[0].grouping.baseline_by + + advisories = [r.message for r in caplog.records if "looks like a grouping" in r.message] + assert advisories, "an explicit-columns train over groupable data should name the grouping" + # The message has to carry the fix, not just the diagnosis, or the caller has to go and read docs. + assert "region" in advisories[0] + assert "baseline_by=['region']" in advisories[0] + assert "baseline_by=[]" in advisories[0] + + +def test_no_advisory_when_the_data_has_no_grouping( + spark: SparkSession, make_schema, make_random, anomaly_engine, caplog +): + """The advisory stays quiet when there is nothing to act on. + + This is what keeps it worth reading: a warning that fires on every explicit-columns call is one + callers learn to filter out, and then it is worth nothing when it does matter. + """ + schema = make_schema(catalog_name=TEST_CATALOG) + suffix = make_random(8).lower() + + # Numeric metrics only, so there is no categorical column to group on. + df = spark.createDataFrame([(100.0 + i, 5.0 + i % 7) for i in range(400)], "amount double, discount double") + table_name = f"{TEST_CATALOG}.{schema.name}.no_grouping_{suffix}" + df.write.saveAsTable(table_name) + + registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" + with caplog.at_level(logging.WARNING, logger="databricks.labs.dqx.anomaly.training_service"): + anomaly_engine.train( + df=spark.table(table_name), + columns=["amount", "discount"], + model_name=qualify_model_name(f"test_nogroup_{suffix}", registry_table), + registry_table=registry_table, + ) + + assert not [r for r in caplog.records if "looks like a grouping" in r.message] + + +def test_suggestion_never_offers_a_column_the_caller_measures(spark: SparkSession): + """A column cannot be both what is measured and what it is measured against. + + ``validate_baseline_columns`` would reject the overlap anyway, so suggesting it would be advice + that fails if taken. + """ + df = spark.createDataFrame( + [("US", "retail", 100.0 + i) for i in range(300)] + [("EU", "retail", 100.0 + i) for i in range(300)], + "region string, channel string, amount double", + ) + + suggestion = suggest_baseline_columns(df, exclude=["region"]) + + assert suggestion is None or "region" not in suggestion.columns + + +def test_suggestion_reports_the_shape_the_warning_quotes(spark: SparkSession): + """The advisory quotes a group count and rows per group, so those have to be right.""" + df = spark.createDataFrame( + [(region, 100.0 + i) for region in ("US", "EU", "APAC") for i in range(200)], + "region string, amount double", + ) + + suggestion = suggest_baseline_columns(df, exclude=["amount"]) + + assert suggestion is not None + assert suggestion.columns == ["region"] + assert suggestion.group_count == 3 + assert suggestion.rows_per_group == 200 def test_autodiscovery_excludes_high_null_numeric_columns(spark: SparkSession): From edfb069a39975d2bb57e70a7dd093e1000607ed8 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 22:41:31 +0100 Subject: [PATCH 056/107] Leave CHANGELOG.md to the maintainers The release notes are theirs to write and are generated at release time; a feature branch editing them just creates a conflict for whoever cuts the release. Everything these entries said is in the PR description and the user guide. --- CHANGELOG.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec755b00..0461d983a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,6 @@ ## 0.16.0 -* Added baseline conditioning to row anomaly detection ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). Anomaly detection could not detect a **contextual** anomaly — a value that is unremarkable across the table but wrong for its own group. On the measurements in the issue, one group's volume dropping 80% behind a flat daily total scored 45.1, the 45th percentile, so no threshold recovered it. `AnomalyEngine.train()` now takes `baseline_by`: each numeric metric gains its deviation from that metric's own baseline within the row's group, as a signed log-ratio, on a **single** pooled model — so the cost does not grow with the group count, and the same collapse scores above 95. Measured offline in the unit suite, a contextual collapse goes from PR-AUC 0.0028 (chance) to 0.6962, while an anomaly that was already globally extreme is unchanged at 1.0000, so conditioning costs nothing measurable when there is nothing to gain. Baseline columns must be string, integral, boolean or date; floating-point and decimal types are rejected because Spark and Python format them differently, which would silently break the key lookup that matches persisted baselines to rows. Grouping auto-discovery is no longer coupled to column discovery, so passing explicit `columns` no longer silently gives up conditioning, and a discovered grouping is used as `baseline_by` rather than turning into one model per group. Measured against v0.16.0 on identical tables through the real pipeline, a contextual collapse goes from PR-AUC 0.0376 to 0.5703, and the untouched ungrouped path scores identically on both builds. Across a wider sweep — 1,545 configurations over synthetic data, the Server Machine Dataset, NSL-KDD and ten classical tabular benchmarks — conditioning is worth a median +0.0742 PR-AUC where anomalies are contextual and nothing measurable where they are not, and beats one-model-per-group in every case measured; see [Anomaly detection quality](https://databrickslabs.github.io/dqx/docs/reference/anomaly_detection_quality) for the full results, the licences, and what these numbers do not mean. -* Added a `profile` argument to row anomaly training, and a second detector behind it ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). Row anomaly detection had one algorithm, scikit-learn's Isolation Forest, which splits on one feature at a time. That is why it is strong on tabular data and weak when the anomaly *is* a broken relationship between metrics that each stay inside their own range: on the Server Machine Dataset (28 machines, 38 metrics, chronological split, unadjusted metrics) it surfaced only 33% of labelled incidents inside an alert budget of 1% of rows. `AnomalyEngine.train()` now takes `profile`, which describes the data you have rather than the algorithm: `"tabular"` — the default, and exactly the behaviour that predates this option — keeps Isolation Forest, while `"timeseries"` selects a correlation-aware detector (Mahalanobis distance with Ledoit–Wolf shrinkage where the sample warrants it, standardised internally) that surfaces **79%** of the same incidents. (Those figures are measured with anomalies present in the training data, which is what DQX does since it fits a sample of your table; on a curated clean training split the same comparison is 36% against 82%.) It needs no timestamp column, because it models correlation between metrics rather than behaviour over time, and it trains a single model rather than an ensemble because it is deterministic, so `ensemble_size` is ignored and `confidence_std` is unavailable for it. Feature engineering, the registry, `has_no_row_anomalies`, contributions and AI explanations are all unchanged; the algorithm is persisted in the registry so scoring inherits the choice with no scoring-side API change. Contributions for the new detector are an exact leave-one-out decomposition (`aᵢ = zᵢ²/(Σ⁻¹)ᵢᵢ`, the Schur-complement drop from marginalising feature *i*) rather than SHAP, which needs no SHAP dependency and is non-negative by construction — the naive signed decomposition was rejected because its terms can go negative and every downstream consumer takes `abs()`, so a distance-*reducing* feature would have been rendered to the LLM as a driver. There is deliberately **no automatic option**: choosing correctly cannot be verified without labels, and the one cheap signal for "this looks temporal" (lag-1 autocorrelation) was measured and rejected because it is confounded by any ordering correlated with the values, which sorted warehouse storage produces routinely — three of ten classical tabular benchmarks scored above the weakest SMD entity. The resolved profile is logged on every training run. Published SMD figures near 0.80 F1 are **not** a comparable target: they use point adjustment, which Kim et al. (AAAI 2022) showed random scores also reach. * Added a pluggable actions and alerting subsystem ([#1289](https://github.com/databrickslabs/dqx/issues/1289)). DQX now supports extensible *actions* that run when checked data violates an optional condition evaluated against the summary metrics produced by `DQMetricsObserver`. The built-in `DQAlert` action can send notifications to Slack, Microsoft Teams, a generic HTTPS webhook, or the log, so pipelines can react to data quality regressions without custom plumbing. You can create your own custom actions as well, and custom alerting is possible via the callback destination, which invokes an in-process Python callable for each alert. * Added an MCP (Model Context Protocol) server for DQX ([#1252](https://github.com/databrickslabs/dqx/issues/1252)). The server exposes DQX's data quality capabilities as tools that any MCP-compatible AI agent (Claude, Genie Code, Cursor, Mosaic AI) can discover and orchestrate. It runs as a Databricks App with on-behalf-of (OBO) authentication, so all data access is governed by the calling user's Unity Catalog permissions. * Added support for summary metrics in Lakeflow Declarative Pipelines (LDP/DLT) ([#1301](https://github.com/databrickslabs/dqx/issues/1301)). A new `DQEngine.compute_summary_metrics(...)` produces the same row counts, per-check breakdown, and custom observer metrics as a lazy aggregation over the results DataFrame, so metrics can be computed inside Spark Declarative Pipelines where the observer- and streaming-listener-based paths cannot be used. @@ -50,11 +48,6 @@ BREAKING CHANGES! -* Removed the legacy `segment_by` path from row anomaly detection ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). `segment_by` trained one model per group; it is gone, and `baseline_by` — one conditioned model whatever the group count — is the only grouping mechanism. This is not a rename: `segment_by` partitioned into N models, `baseline_by` judges each metric against its own group's baseline on a single model, and the scores differ. On the Server Machine Dataset per-group models were the worst of three configurations measured (PR-AUC 0.1416 against 0.1499 for plain pooling and 0.1536 with baseline conditioning), with one entity producing 15,963 false positives across 28,392 normal rows. Row anomaly detection is not GA and its formats were allowed to change without a migration path, so the break is taken now rather than carried: replace `segment_by=[...]` with `baseline_by=[...]`, and note that `segment_by` and `AnomalyParams.max_segment_models` are no longer accepted. Auto-discovery no longer trains one model per discovered group either — a discovered grouping is routed to `baseline_by`, so zero-config runs train a single conditioned model and produce different scores. -* Row anomaly models trained before this release must be retrained ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). `compute_config_hash` now includes `baseline_by`, so a model's stored hash changes and a scoring-time mismatch **raises** rather than silently scoring against a different feature list; metadata from before baseline conditioning is also no longer loadable. The registry's `segmentation` struct is renamed `grouping` and holds `baseline_by`, `sklearn_version`, and `config_hash` — the per-segment fields (`segment_values`, `is_global_model`) are gone because every model is now single and conditioned. The registry write uses `mergeSchema`, so retraining into an existing table adds the renamed column in place; a table you never retrain into keeps the old `segmentation` column and is not read. -* Removed the permanently-null `segment` field from the `_dq_info[].anomaly` struct ([#1484](https://github.com/databrickslabs/dqx/issues/1484)). It carried the segment identity of a per-segment model; with segmentation gone it was always null. Queries against the other anomaly fields are unaffected; a query that selected `.anomaly.segment` must drop it. -* Rows whose group was absent from training now return a **null** score and severity instead of a number, and are not flagged as violations. Previously they were scored 0.0 — the most normal-looking value in the table — because one-hot encoding emits all zeros for an unseen category, which resembles the majority on every axis; frequency encoding has the same defect with the opposite sign, coalescing the miss to a frequency below anything seen in training. Neither is a signal a caller can act on. The new `_dq_info[].anomaly.is_new_baseline` and `.new_baseline_key` fields report the fact. To fail on unrecognised group values, use `foreign_key` or `is_in_list` on the baseline column, which is the check built for that question. ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) -* The anomaly struct inside `_dq_info` gains `is_new_baseline` (boolean) and `new_baseline_key` (string). Existing named-field queries such as `_dq_info[0].anomaly.score` keep working, but the struct is wider, so **appending to a Delta table that already holds `_dq_info` requires `mergeSchema`** (`.option("mergeSchema", "true")` or `spark.databricks.delta.schema.autoMerge.enabled`). ([#1484](https://github.com/databrickslabs/dqx/issues/1484)) * `is_in_list`, `is_not_in_list`, and `is_not_null_and_is_in_list` now resolve their `allowed` / `forbidden` string values as **column expressions** (consistent with the comparison checks), not string literals. A bare string is interpreted as a column reference, a numeric string (e.g. `"3"`) is parsed as a number, and an ISO-date string (e.g. `"2024-01-01"`) as a date. To match a string literal, single-quote the value (e.g. `'value'`) or wrap it in `F.lit("value")`. Existing checks that relied on bare strings being treated as literals must quote them. ([#1419](https://github.com/databrickslabs/dqx/issues/1419)) * `user_metadata` saved through the **Delta** table storage backend is now JSON-encoded at rest to preserve non-string types through the `MAP` column. Save→load via DQX is transparent (you get the original typed value back), but the stored representation changes: direct SQL/dashboard consumers now read JSON-encoded values (decode with `from_json`), existing tables are not migrated, and legacy string values that look like JSON atoms (`"true"`, `"1"`, `"null"`) read back as typed values (`True` / `1` / `None`) — re-save affected rule sets after upgrading to normalize. The File/Volume (YAML/JSON) and Lakebase (JSONB) backends are unaffected. ([#1319](https://github.com/databrickslabs/dqx/issues/1319)) From 4717277241e75c04eb3bc52734f8bed66a72a63b Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 28 Aug 2026 22:59:55 +0100 Subject: [PATCH 057/107] Publish anomaly quality in the existing benchmarks report Removes the separate anomaly_conditioning harness and its dedicated quality page. Detection quality now rides in the benchmarks report alongside DQX core's timings, fed by tests/perf/test_anomaly_benchmark.py. Two perf tests are added for the features this branch introduces: one scores profile="timeseries" on correlated metrics whose relationship breaks, and one scores baseline_by on a contextual collapse. Both fixtures are generated in-repo from a fixed seed, so nothing is downloaded and no third-party dataset is redistributed, and the published numbers carry no licence conditions. The anomaly table gains fixture, row-count, feature-count and anomaly-rate columns, because the rows are now different problems and comparing quality across them would otherwise look meaningful. The nightly step that regenerated the removed page goes with it. The harness is kept outside the repo for future reference rather than carried here: it downloads ~250MB of third-party data and committed 27k lines of results for numbers the repo cannot verify on its own. --- .github/workflows/nightly.yml | 23 +- benchmarks/anomaly_conditioning/README.md | 116 - .../complementary_detector.py | 158 - .../anomaly_conditioning/conditioning.py | 147 - .../anomaly_conditioning/datasets/__init__.py | 10 - .../anomaly_conditioning/datasets/real.py | 151 - .../datasets/synthetic.py | 104 - .../anomaly_conditioning/datasets/tabular.py | 99 - benchmarks/anomaly_conditioning/emit_docs.py | 265 - benchmarks/anomaly_conditioning/metrics.py | 161 - .../profile_advisory_gate.py | 119 - .../results/2026-08-25-f9c703a1.json | 26301 ---------------- .../results/2026-08-25-f9c703a1.md | 99 - .../results/complementary-detector.json | 102 - .../smd-bakeoff-contaminated-0.033.json | 126 - .../results/smd-bakeoff.json | 103 - .../anomaly_conditioning/run_experiment.py | 567 - .../anomaly_conditioning/smd_bakeoff.py | 428 - .../anomaly_conditioning/trend_limits.py | 226 - demos/dqx_demo_anomaly_timeseries_fleet.py | 2 +- .../guide/row_anomaly_detection/index.mdx | 13 +- .../reference/anomaly_detection_quality.mdx | 214 - .../labs/dqx/anomaly/timeseries_detector.py | 5 +- .../labs/dqx/anomaly/training_strategies.py | 4 +- tests/perf/generate_md_report.py | 39 +- tests/perf/test_anomaly_benchmark.py | 120 +- 26 files changed, 150 insertions(+), 29552 deletions(-) delete mode 100644 benchmarks/anomaly_conditioning/README.md delete mode 100644 benchmarks/anomaly_conditioning/complementary_detector.py delete mode 100644 benchmarks/anomaly_conditioning/conditioning.py delete mode 100644 benchmarks/anomaly_conditioning/datasets/__init__.py delete mode 100644 benchmarks/anomaly_conditioning/datasets/real.py delete mode 100644 benchmarks/anomaly_conditioning/datasets/synthetic.py delete mode 100644 benchmarks/anomaly_conditioning/datasets/tabular.py delete mode 100644 benchmarks/anomaly_conditioning/emit_docs.py delete mode 100644 benchmarks/anomaly_conditioning/metrics.py delete mode 100644 benchmarks/anomaly_conditioning/profile_advisory_gate.py delete mode 100644 benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.json delete mode 100644 benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.md delete mode 100644 benchmarks/anomaly_conditioning/results/complementary-detector.json delete mode 100644 benchmarks/anomaly_conditioning/results/smd-bakeoff-contaminated-0.033.json delete mode 100644 benchmarks/anomaly_conditioning/results/smd-bakeoff.json delete mode 100644 benchmarks/anomaly_conditioning/run_experiment.py delete mode 100644 benchmarks/anomaly_conditioning/smd_bakeoff.py delete mode 100644 benchmarks/anomaly_conditioning/trend_limits.py delete mode 100644 docs/dqx/docs/reference/anomaly_detection_quality.mdx diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e639e810e..16c0b5c47 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -555,23 +555,6 @@ jobs: ${{ env.NEW_BASELINE }} ${{ env.UPDATED_BASELINE }} - - name: Refresh anomaly detection quality benchmarks - timeout-minutes: 90 - continue-on-error: true - run: | - # Detection quality, not timing, so it does not go through pytest-benchmark. The baseline - # merge below is keep-old-on-conflict, which would freeze an extra_info payload at its first - # observation and publish a fossil for ever; this writes the numbers straight into the page - # instead, between markers, so the hand-written guidance around them survives. - # - # Scoped for CI: 5 seeds rather than 15, and the real datasets are ~250MB fetched from - # GitHub on a cold cache. continue-on-error because a third-party download failing should - # not fail the timing benchmarks that ran before it. - UV_FROZEN=1 uv run --all-extras python benchmarks/anomaly_conditioning/run_experiment.py \ - --seeds 5 --datasets synthetic smd nslkdd tabular - LATEST=$(ls -t benchmarks/anomaly_conditioning/results/*.json | head -1) - UV_FROZEN=1 uv run --all-extras python benchmarks/anomaly_conditioning/emit_docs.py "$LATEST" - - name: Create PR with updated baseline if changed env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -582,8 +565,6 @@ jobs: # Stage baseline and report git add $FINAL_BASELINE || true git add $BENCHMARK_REPORT || true - git add docs/dqx/docs/reference/anomaly_detection_quality.mdx || true - git add benchmarks/anomaly_conditioning/results || true # Check if there are actual changes if git diff --cached --quiet; then @@ -604,13 +585,11 @@ jobs: EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --state open --json number --jq '.[0].number') if [ -z "$EXISTING_PR" ]; then gh pr create \ - --title "Update performance and detection quality benchmarks" \ + --title "Update performance benchmark baseline" \ --body "$(cat <<'EOF' ## Summary - Updated `tests/perf/.benchmarks/baseline.json` with latest nightly benchmark results - Regenerated `docs/dqx/docs/reference/benchmarks.mdx` report - - Refreshed the generated tables in `docs/dqx/docs/reference/anomaly_detection_quality.mdx` - and added a dated results file under `benchmarks/anomaly_conditioning/results/` ## Action required This commit is **not GPG-signed** (created by GitHub Actions). diff --git a/benchmarks/anomaly_conditioning/README.md b/benchmarks/anomaly_conditioning/README.md deleted file mode 100644 index 32625c63b..000000000 --- a/benchmarks/anomaly_conditioning/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# Anomaly conditioning experiment - -Measures whether conditioning row anomaly detection on a group actually detects better, and which -mechanism to use. This is the evidence behind [#1484](https://github.com/databrickslabs/dqx/issues/1484) -and behind the decision to remove the heterogeneity gate rather than keep it. - -Not part of `make test`, `make integration`, or the nightly. It is a manual harness: it downloads -third-party datasets, takes minutes to hours, and produces a correlation rather than a pass/fail. -The assertions that guard the same claims in CI live in -`tests/unit/test_anomaly_relative_feature_separability.py` (mechanism, numpy, under five seconds) -and `tests/integration_anomaly/test_anomaly_quality.py` (the real DQX pipeline through Spark and -MLflow). - -## Running it - -```shell -# Synthetic sweep only: no network, no Databricks workspace, a few minutes. -uv run python benchmarks/anomaly_conditioning/run_experiment.py --seeds 5 - -# Add the real datasets. Downloads ~250 MB on first run, cached in ~/.cache/dqx-benchmarks. -uv run python benchmarks/anomaly_conditioning/run_experiment.py --seeds 5 \ - --datasets synthetic smd nslkdd -``` - -Results are written to `results/YYYY-MM-DD-.md` and `.json`, stamped with the DQX commit they -came from. The markdown is meant to be readable on its own; the JSON holds every cell for reanalysis. - -## What it compares - -Three configurations, each fitted and scored on the same rows: - -| config | what it is | in DQX | -|---|---|---| -| `pooled` | one model over the raw metrics | DQX before #1484 | -| `relative` | one model, raw metrics **plus** each metric's deviation from its own group's baseline | `baseline_by` | -| `per_group` | one model per group | the legacy `segment_by` | - -The comparison is **paired by seed** and tested with Wilcoxon signed-rank. The seed drives both the -data draw and the forest and dominates the between-cell variance, so unpaired means would mostly -measure the seed. - -## Why the metrics are what they are - -**PR-AUC is primary.** These datasets are heavily imbalanced and ROC-AUC flatters a detector that -merely ranks the majority class well. - -**No point-adjusted F1, anywhere.** Under the point-adjust protocol — crediting a whole labelled -anomaly segment when any single point inside it is detected — a *random* score achieves -state-of-the-art F1 (Kim et al., *Towards a Rigorous Evaluation of Time-series Anomaly Detection*, -AAAI 2022). Numbers produced that way are uninterpretable, so none are computed. If you are about to -add one, read that paper first. - -**Worst-group false-positive rate, not the average.** A per-group model's failure mode is one -group's calibration collapsing while the rest look fine, and an average over groups is exactly what -hides it. - -**Trivial baselines are reported.** Kim et al.'s other recommendation: state the improvement over -doing almost nothing, not an absolute number. - -## Reading the numbers honestly - -DQX scores **rows independently**. It is not a sequence model. PR-AUC around 0.15 on SMD against -published sequence-model results above 0.80 is a *different task*, not a worse implementation — -those models consume a window of history per prediction, and DQX deliberately does not. Any table -lifted out of here needs that caveat attached, or it reads as a failure. - -The harness also reimplements the relative transform in numpy rather than calling DQX. That is -deliberate: it makes a sweep of a few thousand fits possible without a Spark session, at the cost of -measuring the *mechanism* rather than DQX's implementation of it. Pipeline fidelity is a separate -question, asserted in `tests/integration_anomaly/test_anomaly_quality.py`. - -## Datasets, licences, citations - -Everything is downloaded at run time and cached. **Nothing is vendored into this repository**, which -is what keeps the licensing position simple — DQX redistributes none of it. - -| dataset | licence | grouping candidates | citation | -|---|---|---|---| -| Server Machine Dataset (SMD) | MIT, via [`NetManAIOps/OmniAnomaly`](https://github.com/NetManAIOps/OmniAnomaly) | server entity (28), machine family (3) | Su et al., *Robust Anomaly Detection for Multivariate Time Series through Stochastic Recurrent Neural Networks*, KDD 2019 | -| NSL-KDD | redistributable with citation | `service`, `protocol_type`, `flag` | Tavallaee et al., *A detailed analysis of the KDD CUP 99 data set*, CISDA 2009 | - -**SMAP and MSL are deliberately excluded.** Their data files carry "© Original Authors" with no -permissive licence, so they cannot be used even under download-at-runtime. - -**ADBench is not used** despite being the obvious benchmark suite: it ships pre-processed `.npz` -numeric matrices, so categorical column identity is gone and there is no group left to condition on. -Only raw datasets retain what this experiment measures. - -Two adjustments are made to the real data, both documented at their definitions in `datasets/real.py`: -SMD is capped at 4,000 rows per entity so a run takes minutes rather than hours, and NSL-KDD's -attacks are downsampled from ~46% to 2% so the task is anomaly detection rather than classification. -Every configuration sees identical rows, so neither affects the comparison. - -## The heterogeneity gate question - -DQX briefly had `MIN_GROUP_HETEROGENEITY = 0.10`: conditioning was skipped when eta-squared — the -share of variance explained by the grouping — fell below it, on the theory that a grouping which -explains little contributes noise. - -The counter-argument, and the reason it was removed rather than kept pending: at low heterogeneity -every group median approaches the global median, so `signed_log(x) - signed_log(median)` becomes a -monotone transform of `x` — a near-duplicate of an informative column, not noise. Redundancy, not -misdirection. - -That is a falsifiable prediction, and the decision rules are fixed in `run_experiment.py` **before** -any run, so a reader can check the conclusion against the criterion rather than against a narrative -written afterwards: - -- **No gate needed** if the worst delta below eta-squared 0.10 stays above −0.01. -- **Gate needed** if any such cell reaches −0.02, in which case refit the threshold from the sweep - rather than reinstating 0.10 by inheritance. - -The harness also reports Spearman ρ(eta-squared, delta) with a bootstrap CI, and regresses -`delta ~ eta_squared + is_contextual`, because the gate's premise requires eta-squared to carry -signal *after* controlling for which mechanism produced the anomaly. A correlation that disappears -under that control was the mechanism all along. diff --git a/benchmarks/anomaly_conditioning/complementary_detector.py b/benchmarks/anomaly_conditioning/complementary_detector.py deleted file mode 100644 index d1b746177..000000000 --- a/benchmarks/anomaly_conditioning/complementary_detector.py +++ /dev/null @@ -1,158 +0,0 @@ -"""Does a second detector alongside Isolation Forest help? Measured answer: not as a default. - -Isolation Forest loses to a four-line z-score on two of ten classical benchmarks, and loses under -every hyperparameter setting tried -- more trees, more samples per tree, all samples. So the gap is -inductive bias, not tuning: axis-parallel random splits dilute when an anomaly is one extreme feature -among few dimensions, and degrade in high dimension. - -The obvious remedy is to add a cheap complementary scorer to the ensemble. This module exists so that -proposal stays measured rather than re-argued. **It is not wired into DQX**, and the numbers below are -why. - -## What was measured - -Held-out splits (70/30), 5 seeds, the shipped forest configuration. Each member's scores are mapped to -a percentile against its own *training* distribution before combining -- the calibration a real -implementation would have to persist, since DQX averages member scores raw and the two scales are -incomparable (Isolation Forest sits around 0.4-0.7, max-abs-z is unbounded). Ranking inside the -scoring UDF would be wrong: the UDF sees a pandas batch, not the frame, so ranks would depend on -partitioning. - -## Result - - mean-of-percentiles wins 5/10 median -0.0028 worst -0.1409 (thyroid) - max-of-percentiles wins 3/10 median -0.0253 worst -0.0690 - -Per dataset, where the two disagree most: - - cover IF 0.0509 z 0.1009 mean +0.0121 max +0.0295 - mnist IF 0.2609 z 0.3641 mean +0.0475 max +0.0724 - thyroid IF 0.5446 z 0.2939 mean -0.1409 max -0.0690 - fraud IF 0.2607 z 0.1249 mean -0.0371 max -0.0598 - shuttle IF 0.9764 z 0.8948 mean -0.0176 max -0.0556 - -It helps exactly where predicted and hurts where Isolation Forest is genuinely stronger. A coin flip -on average, with a 0.14 worst case, is not a default. - -## Why this cannot be fixed by choosing per dataset - -Picking the better detector requires knowing which regime you are in, and that means labels. DQX is -unsupervised: there are none. So the honest options are to expose the choice to a user who does know -their anomalies are single-feature extremes, or to document the limitation. Guessing is not among -them. - -Run it with: - - uv run python benchmarks/anomaly_conditioning/complementary_detector.py -""" - -import json -import pathlib -import sys - -import numpy as np -from sklearn.ensemble import IsolationForest -from sklearn.metrics import average_precision_score - -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) - -from conditioning import CONTAMINATION, N_TREES # noqa: E402 -from datasets import tabular # noqa: E402 - -SEEDS = 5 -TRAIN_FRACTION = 0.7 -RESULTS = pathlib.Path(__file__).resolve().parent / "results" - - -def percentile_of(reference: np.ndarray, values: np.ndarray) -> np.ndarray: - """Map *values* onto their percentile within a *reference* distribution. - - This is the calibration a real implementation would persist: the reference is the member's - training scores, computed once at training. Partition-independent by construction. - """ - order = np.sort(reference) - return np.searchsorted(order, values, side="right") / max(1, len(order)) - - -def max_abs_z(train: np.ndarray, values: np.ndarray) -> np.ndarray: - """Largest absolute z-score across features, standardised on the training split.""" - means, stds = np.nanmean(train, axis=0), np.nanstd(train, axis=0) - stds = np.where(stds == 0, 1.0, stds) - return np.nanmax(np.abs((values - means) / stds), axis=1) - - -def measure_dataset(values: np.ndarray, labels: np.ndarray) -> dict[str, float] | None: - """Median PR-AUC per strategy over *SEEDS* held-out splits, or None if unusable.""" - per_strategy: dict[str, list[float]] = {k: [] for k in ("iforest", "zscore", "mean_pct", "max_pct")} - for seed in range(SEEDS): - rng = np.random.default_rng(seed) - idx = rng.permutation(len(values)) - cut = int(TRAIN_FRACTION * len(values)) - train_idx, test_idx = idx[:cut], idx[cut:] - if len(np.unique(labels[test_idx])) < 2: - continue - - forest = IsolationForest(n_estimators=N_TREES, contamination=CONTAMINATION, random_state=seed, n_jobs=-1).fit( - values[train_idx] - ) - if_train = -forest.score_samples(values[train_idx]) - if_test = -forest.score_samples(values[test_idx]) - z_train = max_abs_z(values[train_idx], values[train_idx]) - z_test = max_abs_z(values[train_idx], values[test_idx]) - - if_pct, z_pct = percentile_of(if_train, if_test), percentile_of(z_train, z_test) - held_out = labels[test_idx] - per_strategy["iforest"].append(average_precision_score(held_out, if_test)) - per_strategy["zscore"].append(average_precision_score(held_out, z_test)) - per_strategy["mean_pct"].append(average_precision_score(held_out, (if_pct + z_pct) / 2)) - per_strategy["max_pct"].append(average_precision_score(held_out, np.maximum(if_pct, z_pct))) - - if not per_strategy["iforest"]: - return None - out = {k: float(np.median(v)) for k, v in per_strategy.items()} - out["mean_vs_iforest"] = out["mean_pct"] - out["iforest"] - out["max_vs_iforest"] = out["max_pct"] - out["iforest"] - return out - - -def main() -> int: - results: list[dict[str, object]] = [] - # Deltas are accumulated as floats alongside the rows rather than read back out of them: the rows - # mix a dataset name with numbers, so indexing them yields object and the statistics below would - # need casts to satisfy the type checker. - deltas: dict[str, list[float]] = {"mean": [], "max": []} - for name, values, labels in tabular.iter_datasets(): - measured = measure_dataset(values, labels) - if measured is None: - continue - row: dict[str, object] = {"dataset": name, "base_rate": float(labels.mean()), **measured} - results.append(row) - deltas["mean"].append(measured["mean_vs_iforest"]) - deltas["max"].append(measured["max_vs_iforest"]) - print( - f"{name:<12} base {float(labels.mean()):>7.3%} IF {measured['iforest']:.4f} " - f"z {measured['zscore']:.4f} " - f"mean {measured['mean_pct']:.4f} ({measured['mean_vs_iforest']:+.4f}) " - f"max {measured['max_pct']:.4f} ({measured['max_vs_iforest']:+.4f})", - flush=True, - ) - - print("\n--- verdict ---") - for strategy, values_for_strategy in deltas.items(): - print( - f"{strategy}-of-percentiles: " - f"wins {sum(d > 0 for d in values_for_strategy)}/{len(values_for_strategy)}, " - f"median {np.median(values_for_strategy):+.4f}, " - f"worst {min(values_for_strategy):+.4f}, best {max(values_for_strategy):+.4f}" - ) - print("\nNot a default. See the module docstring.") - - RESULTS.mkdir(parents=True, exist_ok=True) - path = RESULTS / "complementary-detector.json" - path.write_text(json.dumps(results, indent=2), encoding="utf-8") - print(f"wrote {path}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/anomaly_conditioning/conditioning.py b/benchmarks/anomaly_conditioning/conditioning.py deleted file mode 100644 index 468ab5e2e..000000000 --- a/benchmarks/anomaly_conditioning/conditioning.py +++ /dev/null @@ -1,147 +0,0 @@ -"""The three ways to condition an anomaly model on a group, reimplemented offline. - -Mirrors ``databricks.labs.dqx.anomaly.transformers`` closely enough to measure the *mechanism*, -while running in numpy and sklearn so a sweep of a few thousand fits needs no Spark session, no -Databricks workspace, and no MLflow registry. Pipeline equivalence is a separate question and is -asserted in ``tests/integration_anomaly/test_anomaly_quality.py``; this harness is about which -mechanism detects better, not about whether DQX implements it faithfully. - -The three configurations: - -``pooled`` - One model over the raw metrics. No notion of a group at all. This is DQX before #1484. - -Note the estimator is configured to match the shipped defaults (see :data:`N_TREES`), so the absolute -figures are comparable with the product rather than with a lighter stand-in. - -``relative`` - One model over the raw metrics *plus* each metric's deviation from its own group's baseline. - This is what ``baseline_by`` does. - -``per_group`` - One model per group, each trained only on that group's rows. This is what the legacy - ``segment_by`` does, and what auto-discovery used to select. -""" - -import time -from dataclasses import dataclass - -import numpy as np -from sklearn.ensemble import IsolationForest - - -def signed_log1p(values: np.ndarray) -> np.ndarray: - """``signum(x) * log1p(|x|)``, matching ``transformers._signed_log1p``. - - Defined for negative input, unlike a bare log, and symmetric about zero so halving and - doubling move the same distance in opposite directions. - """ - return np.sign(values) * np.log1p(np.abs(values)) - - -def relative_to_group_baseline(values: np.ndarray, groups: np.ndarray) -> np.ndarray: - """Each column's deviation from its own group's median, in signed-log space. - - The medians come from the data being transformed, which is correct here because every - configuration is fitted and scored on the same split; DQX instead persists the training - medians and reuses them at scoring, falling back to a global median for unseen groups. - """ - out = np.zeros_like(values, dtype=float) - for group in np.unique(groups): - mask = groups == group - medians = np.median(values[mask], axis=0) - out[mask] = signed_log1p(values[mask]) - signed_log1p(medians) - return out - - -def eta_squared(values: np.ndarray, groups: np.ndarray) -> float: - """Share of total variance that lies *between* groups: ``SS_between / SS_total``. - - Removed from DQX itself — it gated whether conditioning was applied at all, and cost a full - Spark aggregation to compute a number whose predictive value had never been measured. It lives - here because measuring that value is precisely this harness's job. Averaged over columns so a - multi-metric dataset gets one number. - """ - per_column = [] - for column in range(values.shape[1]): - series = values[:, column] - grand_mean = series.mean() - ss_total = float(((series - grand_mean) ** 2).sum()) - if ss_total == 0: - continue - ss_between = 0.0 - for group in np.unique(groups): - member = series[groups == group] - ss_between += len(member) * (member.mean() - grand_mean) ** 2 - per_column.append(float(ss_between) / ss_total) - return float(np.mean(per_column)) if per_column else 0.0 - - -@dataclass -class FitResult: - """Scores for every row, plus what it cost to produce them.""" - - scores: np.ndarray - n_models: int - seconds: float - - -# Mirrors what DQX ships: IsolationForestConfig(num_trees=200), and contamination taken from -# expected_anomaly_rate (default 0.02) rather than sklearn's "auto". An earlier version of this -# harness used 100 trees and "auto", which understated the product -- contamination only moves the -# predict/offset_ threshold and cannot change score_samples ranking, so PR-AUC was unaffected by that -# half, but tree count is not neutral. -N_TREES = 200 -CONTAMINATION = 0.02 - - -def _forest(seed: int) -> IsolationForest: - """One estimator configuration for every cell, so comparisons are not confounded by tuning.""" - return IsolationForest(n_estimators=N_TREES, contamination=CONTAMINATION, random_state=seed, n_jobs=-1) - - -def fit_pooled(values: np.ndarray, groups: np.ndarray, seed: int) -> FitResult: - """One model over raw metrics. *groups* is accepted and ignored, to keep one call signature.""" - del groups - started = time.perf_counter() - model = _forest(seed).fit(values) - scores = -model.score_samples(values) - return FitResult(scores=scores, n_models=1, seconds=time.perf_counter() - started) - - -def fit_relative(values: np.ndarray, groups: np.ndarray, seed: int) -> FitResult: - """One model over raw metrics plus baseline-relative ones.""" - started = time.perf_counter() - features = np.hstack([values, relative_to_group_baseline(values, groups)]) - model = _forest(seed).fit(features) - scores = -model.score_samples(features) - return FitResult(scores=scores, n_models=1, seconds=time.perf_counter() - started) - - -def fit_per_group(values: np.ndarray, groups: np.ndarray, seed: int) -> FitResult: - """One model per group. - - A group too small to fit falls back to a score of zero rather than being dropped, so every - configuration returns a score for every row and the metrics stay comparable. Note what this - costs even when it works: each model calibrates its own contamination on its own group, so a - group whose rows are all normal still has its most-unusual few percent scored as extreme. That - is the mechanism behind the 56% false-alarm entity in the SMD result. - """ - started = time.perf_counter() - scores = np.zeros(len(values), dtype=float) - n_models = 0 - for group in np.unique(groups): - mask = groups == group - if mask.sum() < 10: - continue - model = _forest(seed).fit(values[mask]) - scores[mask] = -model.score_samples(values[mask]) - n_models += 1 - return FitResult(scores=scores, n_models=n_models, seconds=time.perf_counter() - started) - - -CONFIGS = { - "pooled": fit_pooled, - "relative": fit_relative, - "per_group": fit_per_group, -} diff --git a/benchmarks/anomaly_conditioning/datasets/__init__.py b/benchmarks/anomaly_conditioning/datasets/__init__.py deleted file mode 100644 index 4c2aa56ea..000000000 --- a/benchmarks/anomaly_conditioning/datasets/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Dataset loaders for the conditioning harness. - -Every real dataset is downloaded at run time and cached under ``~/.cache/dqx-benchmarks``. Nothing -is vendored into this repository, which is what keeps the licensing position simple: DQX -redistributes none of it. Citations and licences are recorded in the harness README and reproduced -in any published results table. - -SMAP and MSL are deliberately absent. Their data files carry "(c) Original Authors" with no -permissive licence, so they cannot be used even under download-at-runtime. -""" diff --git a/benchmarks/anomaly_conditioning/datasets/real.py b/benchmarks/anomaly_conditioning/datasets/real.py deleted file mode 100644 index 565061997..000000000 --- a/benchmarks/anomaly_conditioning/datasets/real.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Real datasets with genuine categorical groupings, downloaded at run time. - -Both are used because they fail differently. SMD's grouping (server entity) explains a great deal of -the variance and is the case conditioning is meant for. NSL-KDD's groupings (network service and -protocol) are the awkward case: hundreds of services, many tiny, which is where per-group models -become both expensive and badly calibrated. - -ADBench is deliberately not used despite being the obvious choice: it ships pre-processed ``.npz`` -numeric matrices, so categorical column identity is gone and there is no group left to condition -on. Only raw datasets retain what this experiment measures. - -Licences, for the results page: - -* **SMD** (Server Machine Dataset) — MIT, from ``NetManAIOps/OmniAnomaly``. Cite Su et al., KDD - 2019, "Robust Anomaly Detection for Multivariate Time Series through Stochastic Recurrent Neural - Networks". -* **NSL-KDD** — redistributable with citation. Cite Tavallaee et al., CISDA 2009, "A detailed - analysis of the KDD CUP 99 data set". - -Nothing is committed to this repository; files are cached under ``~/.cache/dqx-benchmarks``. -""" - -import pathlib -import urllib.request -from collections.abc import Iterator - -import numpy as np - -CACHE = pathlib.Path.home() / ".cache" / "dqx-benchmarks" - -SMD_BASE = "https://raw.githubusercontent.com/NetManAIOps/OmniAnomaly/master/ServerMachineDataset" -# 28 entities: verified against the repository, where machine-3-12 is a 404. -SMD_ENTITIES = ( - [f"machine-1-{i}" for i in range(1, 9)] - + [f"machine-2-{i}" for i in range(1, 10)] - + [f"machine-3-{i}" for i in range(1, 12)] -) -NSLKDD_URL = "https://raw.githubusercontent.com/defcom17/NSL_KDD/master/KDDTrain%2B.txt" - -# Rows per SMD entity. The full test set is ~708k rows over 38 features, and the sweep fits every -# configuration five times; capping keeps a run to minutes without changing the comparison, since -# every configuration sees exactly the same rows. -SMD_MAX_ROWS_PER_ENTITY = 4000 - -# NSL-KDD is ~46% attacks, which is a classification problem rather than an anomaly-detection one. -# Attacks are downsampled to this rate so the task matches what DQX actually does, and so PR-AUC -# means what it means everywhere else in this harness. -NSLKDD_ANOMALY_RATE = 0.02 - - -def _fetch(url: str, name: str) -> pathlib.Path: - """Download *url* into the cache once, and return the local path.""" - CACHE.mkdir(parents=True, exist_ok=True) - path = CACHE / name - if path.exists() and path.stat().st_size > 0: - return path - path.parent.mkdir(parents=True, exist_ok=True) - with urllib.request.urlopen(url, timeout=120) as resp: # noqa: S310 - fixed https literals above - path.write_bytes(resp.read()) - return path - - -def _load_smd() -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Concatenate every entity's labelled test split. - - SMD ships train (unlabelled) and test (labelled) per entity. Only the test split carries labels, - so that is what is measured; a configuration is fitted and scored on it, exactly as in the - synthetic sweep, which keeps the two comparable. - """ - values_list, labels_list, entity_list = [], [], [] - for entity in SMD_ENTITIES: - test = np.loadtxt(_fetch(f"{SMD_BASE}/test/{entity}.txt", f"smd/test-{entity}.txt"), delimiter=",") - labels = np.loadtxt(_fetch(f"{SMD_BASE}/test_label/{entity}.txt", f"smd/label-{entity}.txt"), delimiter=",") - take = min(len(test), len(labels), SMD_MAX_ROWS_PER_ENTITY) - values_list.append(test[:take]) - labels_list.append(labels[:take]) - entity_list.append(np.full(take, entity)) - return np.vstack(values_list), np.concatenate(labels_list), np.concatenate(entity_list) - - -def load_smd_split() -> tuple[dict[str, np.ndarray], dict[str, np.ndarray], dict[str, np.ndarray]]: - """Return ``(train values, test values, test labels)`` keyed by entity, in file (time) order. - - The counterpart to :func:`_load_smd`, and the honest one for any claim about generalisation. - ``_load_smd`` fits and scores the labelled *test* split, which measures separability; this loads - the **unlabelled train split as well**, so a model can be fitted on train and scored on test. - Train precedes test in time, so that is a chronological protocol for free. - - Rows stay in file order, which for SMD is time order. Both splits are capped at the same - per-entity row limit the rest of the harness uses, so a run stays in minutes. The train split - carries no labels, which is exactly the semi-supervised setup DQX targets: learn what normal looks - like, then score unseen rows. - """ - train, test, labels = {}, {}, {} - for entity in SMD_ENTITIES: - tr = np.loadtxt(_fetch(f"{SMD_BASE}/train/{entity}.txt", f"smd/train-{entity}.txt"), delimiter=",") - te = np.loadtxt(_fetch(f"{SMD_BASE}/test/{entity}.txt", f"smd/test-{entity}.txt"), delimiter=",") - lb = np.loadtxt(_fetch(f"{SMD_BASE}/test_label/{entity}.txt", f"smd/label-{entity}.txt"), delimiter=",") - take_test = min(len(te), len(lb), SMD_MAX_ROWS_PER_ENTITY) - train[entity] = tr[:SMD_MAX_ROWS_PER_ENTITY] - test[entity] = te[:take_test] - labels[entity] = lb[:take_test] - return train, test, labels - - -def _load_nslkdd() -> tuple[np.ndarray, np.ndarray, dict[str, np.ndarray]]: - """Return ``(numeric values, labels, {grouping name: group labels})``. - - Columns 1-3 are ``protocol_type``, ``service`` and ``flag``; column 41 is the attack name, where - ``normal`` is the only non-attack value. Everything else numeric becomes a feature. - """ - path = _fetch(NSLKDD_URL, "nslkdd/KDDTrain+.txt") - rows = [line.split(",") for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] - - categorical_idx = {1: "protocol_type", 2: "service", 3: "flag"} - label_idx = 41 - numeric_idx = [i for i in range(len(rows[0])) if i not in categorical_idx and i not in (label_idx, label_idx + 1)] - - values = np.array([[float(row[i]) for i in numeric_idx] for row in rows]) - labels = np.array([0.0 if row[label_idx] == "normal" else 1.0 for row in rows]) - groupings = {name: np.array([row[idx] for row in rows]) for idx, name in categorical_idx.items()} - - # Downsample attacks to a realistic anomaly rate. Seeded so the frame is identical across - # configurations and seeds -- the forest seed varies, the data does not. - rng = np.random.default_rng(0) - normal_idx = np.flatnonzero(labels == 0) - attack_idx = np.flatnonzero(labels == 1) - n_keep = max(1, int(len(normal_idx) * NSLKDD_ANOMALY_RATE / (1 - NSLKDD_ANOMALY_RATE))) - keep = np.sort(np.concatenate([normal_idx, rng.choice(attack_idx, min(n_keep, len(attack_idx)), replace=False)])) - - return values[keep], labels[keep], {name: groups[keep] for name, groups in groupings.items()} - - -def load_groupings(name: str) -> Iterator[tuple[str, np.ndarray, np.ndarray, np.ndarray]]: - """Yield ``(grouping label, values, labels, groups)`` for each candidate grouping of *name*.""" - if name == "smd": - values, labels, entities = _load_smd() - yield "entity", values, labels, entities - # The machine family prefix: a coarser grouping over the same data, which is the kind of - # choice a user actually faces. Included so the sweep has a within-dataset contrast between - # a fine and a coarse grouping rather than one point per dataset. - yield "machine_family", values, labels, np.array([e.rsplit("-", 1)[0] for e in entities]) - return - - if name == "nslkdd": - values, labels, groupings = _load_nslkdd() - for grouping_name, groups in groupings.items(): - yield grouping_name, values, labels, groups - return - - raise ValueError(f"unknown dataset {name!r}") diff --git a/benchmarks/anomaly_conditioning/datasets/synthetic.py b/benchmarks/anomaly_conditioning/datasets/synthetic.py deleted file mode 100644 index b72e87c5f..000000000 --- a/benchmarks/anomaly_conditioning/datasets/synthetic.py +++ /dev/null @@ -1,104 +0,0 @@ -"""The two-factor synthetic sweep that decides whether a heterogeneity gate is needed. - -Real datasets contribute roughly ten grouping candidates in total, which is far too few to -establish a correlation between heterogeneity and the benefit of conditioning. Synthetic data has -no licensing constraints and lets both factors be set directly, so the sweep carries the breadth -and the real datasets check that its conclusion survives contact with real data. - -**Factor 1 — level spread.** How far apart the per-group baseline levels sit. At spread 0 every -group has the same level and eta-squared is ~0; at the top of the range levels differ by an order -of magnitude and eta-squared approaches 0.9. This moves the gate's input continuously across its -whole range, including the region below the 0.10 threshold that was removed. - -**Factor 2 — anomaly mechanism.** What actually makes a row anomalous: - -``global`` - The value is extreme for the table as a whole. A pooled model should find these, and the - baseline-relative feature should be redundant rather than harmful — the prediction that - justified removing the gate rather than keeping it pending. - -``contextual`` - The value is ordinary for the table but wrong for its own group. Only a comparison against the - group's own baseline can see these. This is #1484. - -The two mechanisms are the confound that matters: without splitting on it, any correlation between -eta-squared and the benefit of conditioning could be entirely an artefact of contextual anomalies -being more common at high spread. -""" - -import numpy as np - -MECHANISMS = ("global", "contextual") - - -def generate( - *, - seed: int, - level_spread: float, - mechanism: str, - n_groups: int = 12, - n_rows_per_group: int = 400, - anomaly_rate: float = 0.02, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """Return ``(values, labels, groups)`` for one cell of the sweep. - - Args: - seed: Controls the data draw. Paired with the forest seed by the caller. - level_spread: 0.0 gives every group the same baseline level; 1.0 spans an order of - magnitude. This is the axis eta-squared tracks. - mechanism: ``"global"`` or ``"contextual"`` — see the module docstring. - n_groups: Number of groups. - n_rows_per_group: Rows per group. - anomaly_rate: Share of rows labelled anomalous. - """ - if mechanism not in MECHANISMS: - raise ValueError(f"mechanism must be one of {MECHANISMS}, got {mechanism!r}") - - rng = np.random.default_rng(seed) - base_level = 1000.0 - # Levels are spread multiplicatively and symmetrically about base_level, so the *mean* level is - # roughly constant across the sweep. Otherwise raising the spread would also raise the overall - # scale, and the two effects would be inseparable. - factors = np.linspace(1.0 - 0.9 * level_spread, 1.0 + 0.9 * level_spread, n_groups) - levels = base_level * np.clip(factors, 0.05, None) - - values_list, labels_list, groups_list = [], [], [] - for index in range(n_groups): - level = levels[index] - rows = rng.normal(level, level * 0.08, size=(n_rows_per_group, 2)) - rows[:, 1] = rng.normal(level * 0.5, level * 0.05, size=n_rows_per_group) - labels = np.zeros(n_rows_per_group) - - n_anomalies = max(1, int(n_rows_per_group * anomaly_rate)) - picks = rng.choice(n_rows_per_group, n_anomalies, replace=False) - if mechanism == "global": - # Extreme against the whole table: far above the largest group's normal range. - rows[picks, 0] = base_level * (1.0 + 0.9 * level_spread) * rng.uniform(4.0, 6.0, n_anomalies) - else: - # Ordinary globally, wrong for this group: collapse to a fifth of the group's own - # level, which lands inside the range other groups occupy normally. - rows[picks, 0] = level * 0.2 - labels[picks] = 1.0 - - values_list.append(rows) - labels_list.append(labels) - groups_list.append(np.full(n_rows_per_group, f"g{index:02d}")) - - return ( - np.vstack(values_list), - np.concatenate(labels_list), - np.concatenate(groups_list), - ) - - -def sweep_points(n_spreads: int = 9) -> list[tuple[float, str]]: - """The (level_spread, mechanism) grid. - - Spreads are dense at the low end because that is where the removed gate would have fired, and - where the "monotone duplicate rather than noise" prediction has to hold for its removal to be - defensible. - """ - spreads = sorted( - {round(v, 3) for v in np.concatenate([np.linspace(0.0, 0.2, 5), np.linspace(0.2, 1.0, n_spreads)])} - ) - return [(spread, mechanism) for spread in spreads for mechanism in MECHANISMS] diff --git a/benchmarks/anomaly_conditioning/datasets/tabular.py b/benchmarks/anomaly_conditioning/datasets/tabular.py deleted file mode 100644 index d24b99592..000000000 --- a/benchmarks/anomaly_conditioning/datasets/tabular.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Classical tabular anomaly benchmarks, for the regime that has no grouping at all. - -These answer a different question from the rest of the harness. SMD and NSL-KDD exist to test whether -conditioning on a group helps; these exist to characterise how DQX's mechanism performs on the -bread-and-butter case the field actually benchmarks on, and to confirm that adding baseline -conditioning did not disturb it. - -Source: [ADBench](https://github.com/Minqi824/ADBench) (BSD-2-Clause), which redistributes the ODDS / -UCI / Kaggle collections as `.npz` matrices of `X` and `y`. Downloaded at run time and cached; never -vendored. - -An earlier draft of this harness dismissed ADBench outright because its pre-processing discards -categorical column identity. That is a real limitation, but only for the grouping question — a -plain-tabular regime does not need a grouping, so excluding it here was too broad a call. - -The selection spans three axes deliberately, because a single dataset tells you almost nothing about -a detector: - -* **Scale**: 1.8k rows (cardio) to 285k (fraud). -* **Dimensionality**: 9 features (shuttle) to 100 (mnist). -* **Anomaly rate**: 0.17% (fraud) to ~32% (campaign-adjacent), which matters enormously — PR-AUC is - not comparable across base rates, so each dataset is reported against its own base rate and its own - trivial baselines rather than pooled into one headline number. - -`13_fraud` is the Kaggle credit-card competition set; `10_cover` is Covertype; `32_shuttle`, -`30_satellite`, `23_mammography`, `38_thyroid` and `6_cardio` are the long-standing ODDS benchmarks -most papers report. -""" - -import io -import pathlib -import urllib.request -from collections.abc import Iterator - -import numpy as np - -CACHE = pathlib.Path.home() / ".cache" / "dqx-benchmarks" / "adbench" -BASE = "https://raw.githubusercontent.com/Minqi824/ADBench/main/adbench/datasets/Classical" - -# name -> ADBench file. Kept explicit rather than globbed so a run is reproducible and a new upstream -# dataset cannot silently change the published table. -DATASETS = { - "cardio": "6_cardio.npz", - "thyroid": "38_thyroid.npz", - "mammography": "23_mammography.npz", - "satellite": "30_satellite.npz", - "shuttle": "32_shuttle.npz", - "covertype": "10_cover.npz", - "spambase": "35_SpamBase.npz", - "campaign": "5_campaign.npz", - "mnist": "24_mnist.npz", - "fraud": "13_fraud.npz", -} - -# Cap for the largest sets so a full sweep stays in minutes. Stratified so the anomaly rate -- the -# thing PR-AUC is most sensitive to -- is preserved rather than resampled away. -MAX_ROWS = 30000 - - -def _fetch(filename: str) -> pathlib.Path: - CACHE.mkdir(parents=True, exist_ok=True) - path = CACHE / filename - if path.exists() and path.stat().st_size > 0: - return path - with urllib.request.urlopen(f"{BASE}/{filename}", timeout=300) as resp: # noqa: S310 - fixed https literal - path.write_bytes(resp.read()) - return path - - -def _stratified_cap(values: np.ndarray, labels: np.ndarray, seed: int) -> tuple[np.ndarray, np.ndarray]: - """Cap row count while preserving the anomaly rate.""" - if len(values) <= MAX_ROWS: - return values, labels - rng = np.random.default_rng(seed) - keep_fraction = MAX_ROWS / len(values) - keep = [] - for label in (0.0, 1.0): - idx = np.flatnonzero(labels == label) - n = max(1, int(round(len(idx) * keep_fraction))) - keep.append(rng.choice(idx, size=min(n, len(idx)), replace=False)) - selected = np.sort(np.concatenate(keep)) - return values[selected], labels[selected] - - -def load(name: str, *, seed: int = 0) -> tuple[np.ndarray, np.ndarray]: - """Return ``(values, labels)`` for one dataset.""" - if name not in DATASETS: - raise ValueError(f"unknown tabular dataset {name!r}; known: {sorted(DATASETS)}") - with np.load(io.BytesIO(_fetch(DATASETS[name]).read_bytes())) as data: - values = np.asarray(data["X"], dtype=float) - labels = np.asarray(data["y"], dtype=float).ravel() - return _stratified_cap(values, labels, seed) - - -def iter_datasets(names: list[str] | None = None) -> Iterator[tuple[str, np.ndarray, np.ndarray]]: - """Yield ``(name, values, labels)`` for the requested datasets, or all of them.""" - for name in names or sorted(DATASETS): - values, labels = load(name) - yield name, values, labels diff --git a/benchmarks/anomaly_conditioning/emit_docs.py b/benchmarks/anomaly_conditioning/emit_docs.py deleted file mode 100644 index 67989cc76..000000000 --- a/benchmarks/anomaly_conditioning/emit_docs.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Refresh the generated tables in the user-facing quality page from a results JSON. - - python benchmarks/anomaly_conditioning/emit_docs.py results/2026-08-25-abc1234.json \ - [--bakeoff results/smd-bakeoff.json results/smd-bakeoff-contaminated-0.033.json] - -The conditioning sweep and the estimator bake-off are separate runs over different datasets, so the -profile table is fed from its own file(s) rather than from the conditioning results. Pass the clean -run first and the contaminated run second: contaminated training is what DQX actually does (it fits a -random sample of the user's table, anomalies included), so publishing only the clean number would -overstate what a user gets. - -Only the regions between marker comments are replaced, so the hand-written prose around them -- -which is most of the page, and the part that tells a reader what to do about a number -- survives -regeneration. That is the difference from ``docs/dqx/docs/reference/benchmarks.mdx``, which -``tests/perf/generate_md_report.py`` rewrites wholesale and where narrative therefore cannot live. - -Deliberately not routed through pytest-benchmark's ``extra_info``. The nightly merges baseline.json -with keep-old-on-conflict semantics, so a benchmark that already exists keeps its previous entry and -its extra_info is frozen at first observation. Quality published that way would never refresh. -""" - -import json -import pathlib -import sys - -MARKERS = { - "conditioning": ("", ""), - "per_group": ("", ""), - "tabular": ("", ""), - "cost": ("", ""), - "profile": ("", ""), -} - -# The estimator each profile ships. `maha_ridge` is the bake-off name for the configuration in -# ``anomaly/timeseries_detector.py``: standardised internally, with a small ridge floor on the -# covariance. Kept as a mapping rather than inlined so the table cannot silently start reporting a -# configuration DQX does not ship. -PROFILE_ESTIMATORS = {"tabular": "iforest", "timeseries": "maha_ridge"} - -_DOCS = pathlib.Path(__file__).resolve().parents[2] / "docs" / "dqx" / "docs" -PAGE = _DOCS / "reference" / "anomaly_detection_quality.mdx" -# The user guide carries the same profile table, in a shorter form. Generated rather than copied: a -# hand-typed duplicate of a measured number drifts the first time the measurement is refreshed. -GUIDE = _DOCS / "guide" / "row_anomaly_detection" / "index.mdx" - -MECHANISM_LABELS = { - "contextual": "**contextual** — ordinary for the table, wrong for their group", - "global": "**globally extreme** — unusual against the whole table", - "real": "mixed or unknown (real-world datasets)", -} -ADVICE = { - "contextual": "use `baseline_by`", - "global": "costs nothing; leave it on", - "real": "leave it on", -} - - -def _median(values: list[float]) -> float: - ordered = sorted(values) - if not ordered: - return float("nan") - mid = len(ordered) // 2 - return ordered[mid] if len(ordered) % 2 else (ordered[mid - 1] + ordered[mid]) / 2 - - -def _paired_medians(cells: list[dict], left: str, right: str) -> dict[str, float]: - """Median per-mechanism delta between two configurations, paired on the cell they share.""" - index: dict[tuple, dict[str, dict]] = {} - for cell in cells: - key = (cell["dataset"], cell["grouping"], cell["mechanism"], cell["seed"]) - index.setdefault(key, {})[cell["config"]] = cell - - by_mechanism: dict[str, list[float]] = {} - for (_dataset, _grouping, mechanism, _seed), configs in index.items(): - if left not in configs or right not in configs: - continue - delta = configs[left]["pr_auc"] - configs[right]["pr_auc"] - if delta == delta: # skip NaN - by_mechanism.setdefault(mechanism, []).append(delta) - return {m: _median(v) for m, v in by_mechanism.items()} - - -def conditioning_table(cells: list[dict]) -> str: - medians = _paired_medians(cells, "relative", "pooled") - rows = [ - "| your anomalies are... | median ΔPR-AUC with `baseline_by` | what to do |", - "|---|---|---|", - ] - for mechanism in ("contextual", "global", "real"): - if mechanism not in medians: - continue - value = medians[mechanism] - emphasis = f"**{value:+.4f}**" if value > 0.01 else f"{value:+.4f}" - rows.append(f"| {MECHANISM_LABELS[mechanism]} | {emphasis} | {ADVICE[mechanism]} |") - return "\n".join(rows) - - -def per_group_table(cells: list[dict]) -> str: - medians = _paired_medians(cells, "relative", "per_group") - rows = ["| your anomalies are... | median ΔPR-AUC vs one-model-per-group |", "|---|---|"] - labels = {"contextual": "contextual", "global": "globally extreme", "real": "mixed or unknown"} - for mechanism in ("contextual", "global", "real"): - if mechanism not in medians: - continue - value = medians[mechanism] - emphasis = f"**{value:+.4f}**" if value > 0.01 else f"{value:+.4f}" - rows.append(f"| {labels[mechanism]} | {emphasis} |") - return "\n".join(rows) - - -def cost_table(cells: list[dict]) -> str: - per_config: dict[str, tuple[list[int], list[float]]] = {} - for cell in cells: - models, seconds = per_config.setdefault(cell["config"], ([], [])) - models.append(cell["n_models"]) - seconds.append(cell["seconds"]) - - rows = ["| approach | models trained | median fit time |", "|---|---|---|"] - if "relative" in per_config: - models, seconds = per_config["relative"] - rows.append(f"| `baseline_by` | 1 | {_median(seconds):.2f}s |") - if "per_group" in per_config: - models, seconds = per_config["per_group"] - rows.append( - f"| `segment_by` | one per group ({int(_median([float(m) for m in models]))} in this sweep) " - f"| {_median(seconds):.2f}s |" - ) - return "\n".join(rows) - - -def tabular_table(baselines: list[dict]) -> str: - rows = [ - "| dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random " - "| beats max-abs-z |", - "|---|---|---|---|---|---|---|---|---|", - ] - # Best first: a reader scanning for "is this any good" should meet the strong cases before the weak. - for row in sorted(baselines, key=lambda r: -r["dqx_pr_auc"]): - lift = row["dqx_pr_auc"] / row["random"] if row["random"] else float("nan") - beats = "yes" if row["dqx_pr_auc"] > row["max_abs_z"] else "**no**" - score = f"**{row['dqx_pr_auc']:.4f}**" if lift > 10 else f"{row['dqx_pr_auc']:.4f}" - rows.append( - f"| {row['dataset']} | {row['n_rows']} | {row['n_features']} | {row['base_rate']:.2%} | " - f"{score} | {row['random']:.4f} | {row['max_abs_z']:.4f} | {lift:.1f}x | {beats} |" - ) - return "\n".join(rows) - - -def profile_table(runs: list[tuple[str, list[dict]]]) -> str: - """Incident coverage per profile, on raw features and pooled scope, across the runs given. - - Reports **event recall at a 1%-of-rows alert budget**, not PR-AUC, and not point-adjusted F1. - SMD's 3,732 anomalous rows sit in 39 incidents of median length 6 and maximum length 1041, so - point-wise PR-AUC is dominated by "did you find the one huge incident" -- the best-PR-AUC - configuration in this sweep covers 2 of 39 incidents. Counting incidents inside a fixed budget - keeps precision point-wise and answers the operational question instead. Point adjustment is - excluded on purpose: Kim et al. (AAAI 2022) showed random scores reach state-of-the-art under it, - which is why published SMD figures near 0.80 F1 are not a comparable target. - """ - header = ["| `profile` | detector |"] - divider = ["|---|---|"] - for label, _ in runs: - header.append(f" incidents surfaced ({label}) |") - divider.append("---|") - rows = ["".join(header), "".join(divider)] - - for profile, estimator in PROFILE_ESTIMATORS.items(): - detector = "Isolation Forest" if profile == "tabular" else "correlation-aware" - cells = [f"| `\"{profile}\"` | {detector} |"] - for _, results in runs: - match = [ - r - for r in results - if r["estimator"] == estimator and r["featuriser"] == "raw" and r["scope"] == "pooled" - ] - if not match: - cells.append(" n/a |") - continue - recall = match[0]["event_recall_at_1pct"] - emphasis = f"**{recall:.0%}**" if profile == "timeseries" else f"{recall:.0%}" - cells.append(f" {emphasis} |") - rows.append("".join(cells)) - return "\n".join(rows) - - -def guide_profile_table(runs: list[tuple[str, list[dict]]]) -> str: - """The same measurement, trimmed for the page where a user chooses. - - Shows only the contaminated-training column -- the number a user actually gets, since DQX fits a - random sample of their table with anomalies still in it. The clean-split upper bound and the - methodology stay on the reference page, where a reader has come to interrogate the numbers rather - than to make a choice. - """ - preferred = [r for label, r in runs if "contaminated" in label] or [runs[-1][1]] - results = preferred[0] - rows = ["| `profile` | Incidents surfaced |", "|---|---|"] - for profile, estimator in PROFILE_ESTIMATORS.items(): - match = [ - r for r in results if r["estimator"] == estimator and r["featuriser"] == "raw" and r["scope"] == "pooled" - ] - recall = f"{match[0]['event_recall_at_1pct']:.0%}" if match else "n/a" - emphasis = f"**{recall}**" if profile == "timeseries" else recall - rows.append(f'| `"{profile}"` | {emphasis} |') - return "\n".join(rows) - - -def replace_region(text: str, name: str, body: str, page: pathlib.Path = PAGE) -> str: - start, end = MARKERS[name] - if start not in text or end not in text: - raise SystemExit(f"marker pair for {name!r} missing from {page.name}; add {start} / {end}") - head = text[: text.index(start) + len(start)] - tail = text[text.index(end) :] - return f"{head}\n{body}\n{tail}" - - -def main() -> int: - argv = sys.argv[1:] - bakeoff_paths: list[str] = [] - if "--bakeoff" in argv: - cut = argv.index("--bakeoff") - bakeoff_paths = argv[cut + 1 :] - argv = argv[:cut] - if not argv: - raise SystemExit(f"usage: {pathlib.Path(sys.argv[0]).name} [--bakeoff ...]") - data = json.loads(pathlib.Path(argv[0]).read_text(encoding="utf-8")) - cells = data["cells"] - baselines = data.get("tabular_baselines") or [] - - text = PAGE.read_text(encoding="utf-8") - text = replace_region(text, "conditioning", conditioning_table(cells)) - text = replace_region(text, "per_group", per_group_table(cells)) - text = replace_region(text, "cost", cost_table(cells)) - if bakeoff_paths: - runs = [] - for raw_path in bakeoff_paths: - bakeoff = json.loads(pathlib.Path(raw_path).read_text(encoding="utf-8")) - # `contaminate` is null in a run that trained on data containing anomalies, and 0.0 in one - # that trained on the clean split. Label from the file rather than from argument order, so a - # mislabelled column cannot outlive a typo on the command line. - contaminate = bakeoff.get("contaminate") - label = "clean training split" if contaminate == 0.0 else "contaminated training" - runs.append((label, bakeoff["results"])) - text = replace_region(text, "profile", profile_table(runs)) - - guide_text = GUIDE.read_text(encoding="utf-8") - guide_text = replace_region(guide_text, "profile", guide_profile_table(runs), GUIDE) - GUIDE.write_text(guide_text, encoding="utf-8") - print(f"refreshed the profile table in {GUIDE}") - if baselines: - text = replace_region(text, "tabular", tabular_table(baselines)) - else: - # Loud, not silent: a run without --datasets tabular leaves that table at whatever the last - # run published, and a stale table that looks fresh is worse than an obvious gap. - print( - "warning: results file carries no tabular_baselines, so the plain-tabular table was left " - "untouched. Re-run with '--datasets ... tabular' to refresh it.", - file=sys.stderr, - ) - PAGE.write_text(text, encoding="utf-8") - print(f"refreshed generated tables in {PAGE}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/anomaly_conditioning/metrics.py b/benchmarks/anomaly_conditioning/metrics.py deleted file mode 100644 index f4b252084..000000000 --- a/benchmarks/anomaly_conditioning/metrics.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Detection-quality metrics and the paired statistics used to compare configurations. - -Deliberately **no point-adjustment**, for the reason given at length in -``tests/integration_anomaly/quality_metrics.py``: under the point-adjust protocol a random score -reaches state-of-the-art F1 (Kim et al., AAAI 2022), so any number produced that way is -uninterpretable. This module and that one are kept separate rather than shared because the test -tree is not importable from a top-level script directory, and because the paired statistics below -have no place in a test. -""" - -from dataclasses import asdict, dataclass, field - -import numpy as np -from scipy import stats -from sklearn.metrics import average_precision_score, roc_auc_score - - -@dataclass -class CellMetrics: - """Every number recorded for one (dataset, grouping, config, seed) cell.""" - - pr_auc: float - roc_auc: float - precision_at_n: float - macro_pr_auc: float - worst_group_fpr: float - n_models: int - seconds: float - eta_squared: float = 0.0 - warnings: list[str] = field(default_factory=list) - - def as_dict(self) -> dict: - return asdict(self) - - -def pr_auc(labels: np.ndarray, scores: np.ndarray) -> float: - """Average precision. NaN when a split holds only one class.""" - if len(np.unique(labels)) < 2: - return float("nan") - return float(average_precision_score(labels, scores)) - - -def roc_auc(labels: np.ndarray, scores: np.ndarray) -> float: - """Reported alongside PR-AUC, never instead of it: these datasets are heavily imbalanced.""" - if len(np.unique(labels)) < 2: - return float("nan") - return float(roc_auc_score(labels, scores)) - - -def precision_at_n(labels: np.ndarray, scores: np.ndarray) -> float: - """Precision over the top-*n* scored rows, where *n* is the true number of anomalies. - - The operational metric: an analyst reviews a fixed-size queue, so what matters is how much of - that queue is worth reviewing. - """ - n_anomalies = int(labels.sum()) - if n_anomalies == 0: - return float("nan") - top = np.argsort(scores)[::-1][:n_anomalies] - return float(labels[top].sum() / n_anomalies) - - -def per_group_pr_auc(labels: np.ndarray, scores: np.ndarray, groups: np.ndarray) -> dict[str, float]: - """PR-AUC within each group, skipping groups that hold only one class.""" - out = {} - for group in np.unique(groups): - mask = groups == group - if len(np.unique(labels[mask])) < 2: - continue - out[str(group)] = pr_auc(labels[mask], scores[mask]) - return out - - -def worst_group_false_positive_rate( - labels: np.ndarray, scores: np.ndarray, groups: np.ndarray, *, quantile: float = 0.95 -) -> float: - """Highest per-group false-positive rate at a global score threshold. - - The threshold is the *global* score quantile, which is the point: a per-group model's scores - are calibrated within its own group, so a group of entirely normal rows still emits a full - complement of extreme scores. Averaging over groups hides that; taking the worst group does - not, and it is the statistic that exposed the SMD failure. - """ - if len(np.unique(labels)) < 2: - return float("nan") - threshold = float(np.quantile(scores, quantile)) - rates = [] - for group in np.unique(groups): - mask = (groups == group) & (labels == 0) - if mask.sum() == 0: - continue - rates.append(float((scores[mask] > threshold).mean())) - return max(rates) if rates else float("nan") - - -def macro_average(values: dict[str, float]) -> float: - """Unweighted mean across groups. - - Unweighted on purpose: weighting by group size reproduces the aggregate metric and re-hides - the small-group failures this exists to surface. - """ - finite = [v for v in values.values() if np.isfinite(v)] - return float(np.mean(finite)) if finite else float("nan") - - -def trivial_baselines(values: np.ndarray, labels: np.ndarray, *, seed: int = 42) -> dict[str, float]: - """PR-AUC of scores that required no model, as the floor a result has to clear. - - Absolute PR-AUC is uninterpretable on its own because it moves with the base rate: 0.30 is - excellent at a 0.17% anomaly rate and poor at 30%. ``random`` fixes the base rate to compare - against, and ``max_abs_z`` -- the largest absolute z-score across features -- is the cheapest - defensible detector, so beating it is what shows a model earns its cost. - """ - rng = np.random.default_rng(seed) - means, stds = np.nanmean(values, axis=0), np.nanstd(values, axis=0) - stds = np.where(stds == 0, 1.0, stds) - max_abs_z = np.nanmax(np.abs((values - means) / stds), axis=1) - return { - "random": pr_auc(labels, rng.random(len(labels))), - "max_abs_z": pr_auc(labels, max_abs_z), - } - - -def wilcoxon_paired(deltas: list[float]) -> tuple[float, float]: - """Wilcoxon signed-rank on per-seed differences: returns ``(statistic, p_value)``. - - Paired by seed rather than comparing unpaired means, because the seed controls both the data - draw and the forest, and is by far the largest source of variance between cells. - """ - finite = [d for d in deltas if np.isfinite(d)] - if len(finite) < 2 or all(d == 0 for d in finite): - return float("nan"), float("nan") - result = stats.wilcoxon(finite) - return float(result.statistic), float(result.pvalue) - - -def spearman_with_bootstrap_ci( - xs: list[float], ys: list[float], *, n_boot: int = 2000, seed: int = 0 -) -> tuple[float, float, float]: - """Spearman ρ plus a percentile bootstrap 95% CI: ``(rho, lo, hi)``. - - A point correlation over a few dozen configurations is not evidence on its own; the interval is - what says whether the sign is even determined. - """ - x_arr, y_arr = np.asarray(xs, dtype=float), np.asarray(ys, dtype=float) - keep = np.isfinite(x_arr) & np.isfinite(y_arr) - x_arr, y_arr = x_arr[keep], y_arr[keep] - if len(x_arr) < 4: - return float("nan"), float("nan"), float("nan") - - rho = float(stats.spearmanr(x_arr, y_arr).statistic) - rng = np.random.default_rng(seed) - boots = [] - for _ in range(n_boot): - idx = rng.integers(0, len(x_arr), len(x_arr)) - if len(np.unique(x_arr[idx])) < 3: - continue - boots.append(stats.spearmanr(x_arr[idx], y_arr[idx]).statistic) - if not boots: - return rho, float("nan"), float("nan") - return rho, float(np.percentile(boots, 2.5)), float(np.percentile(boots, 97.5)) diff --git a/benchmarks/anomaly_conditioning/profile_advisory_gate.py b/benchmarks/anomaly_conditioning/profile_advisory_gate.py deleted file mode 100644 index e07a49101..000000000 --- a/benchmarks/anomaly_conditioning/profile_advisory_gate.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Should DQX warn a user that their data looks temporal? Measured answer: not from this signal. - -`profile="timeseries"` selects a correlation-aware detector that measured far better on multivariate -metrics (incident coverage 0.82 against 0.36 on SMD). The obvious next step is for DQX to notice when a -table looks temporal and say so. This module exists so that proposal stays measured rather than -re-argued each time. **It is not wired into DQX**, and the numbers below are why. - -## What was rejected first, and why it was never viable - -Detecting a timestamp column. Nearly every Delta table has `created_at`, `updated_at` or `ingested_at`, -so "a timestamp exists, therefore this is a time series" fires on almost everything. Presence of a -column is not evidence of intent. - -## What was actually measured - -Mean absolute lag-1 autocorrelation of the numeric columns, under the row ordering given. The -hypothesis: genuine time series have serially correlated metrics, i.i.d. tabular rows do not, so the -statistic should separate the two even though timestamp presence cannot. - - TEMPORAL SMD, 28 entities in time order mean 0.660 min 0.426 max 0.791 - - TABULAR satellite 0.823 - cardio 0.728 - covertype 0.573 - spambase 0.104 - mnist 0.086 - thyroid 0.068 - fraud 0.065 - mammography 0.056 - shuttle 0.005 - campaign 0.004 - -## Result: it does not separate - -Three of ten tabular datasets score above the weakest SMD entity, and two score above SMD's *mean*. -There is no threshold that admits every time series and rejects every tabular table, so a warning built -on this would fire on ordinary tabular data. - -## Why it fails, which is the part worth remembering - -The statistic is confounded by **any ordering correlated with the values**, not just a temporal one. -These datasets are stored sorted -- `satellite` at 0.823 is almost certainly ordered by class -- and -sorted storage produces serial correlation without any time series underneath. - -That is not an artefact of the benchmark. It is the common case in a warehouse: batch loads, sorted ETL -output and backfills all leave `created_at` correlated with the values beside it. So the failure mode -transfers directly to the data a user would actually run this on, and it fails in the expensive -direction -- advising a change of algorithm on data that does not need one. - -## What DQX does instead - -Nothing automatic, and nothing silent. The resolved profile is logged at INFO on every training run, so -`auto` is visible rather than invisible, and the documentation states the choice and the measured -difference so a user who knows their data can make it. Choosing the algorithm for them would need to be -verified, and verifying it needs labels DQX does not have. - -A better signal may exist -- regular cadence per entity, or autocorrelation compared against a -permutation baseline that controls for the ordering itself. Neither is attempted here: the point of this -module is that the cheap version was tried, measured, and declined. - -Run: uv run python benchmarks/anomaly_conditioning/profile_advisory_gate.py -""" - -import numpy as np - -from datasets.real import SMD_ENTITIES, load_smd_split -from datasets.tabular import DATASETS, load - - -def mean_abs_lag1_autocorr(values: np.ndarray) -> float: - """Mean absolute lag-1 autocorrelation across columns, in the row order given. - - Constant columns are skipped: they have no correlation to measure, and including them as zeros - would dilute the statistic by however many one-hot or indicator columns a frame happens to carry. - """ - scores = [] - for column in range(values.shape[1]): - series = values[:, column] - if series.std() <= 1e-12 or len(series) < 3: - continue - current, previous = series[1:], series[:-1] - if current.std() <= 1e-12 or previous.std() <= 1e-12: - continue - scores.append(abs(float(np.corrcoef(current, previous)[0, 1]))) - return float(np.mean(scores)) if scores else 0.0 - - -def main() -> None: - print("Temporal reference: SMD, per entity, in file (time) order") - train, _, _ = load_smd_split() - temporal = [mean_abs_lag1_autocorr(train[entity]) for entity in SMD_ENTITIES] - print( - f" {len(SMD_ENTITIES)} entities | mean {np.mean(temporal):.3f} " - f"| min {min(temporal):.3f} | max {max(temporal):.3f}\n" - ) - - print("Tabular comparison: classical benchmarks, in stored order") - tabular: dict[str, float] = {} - for name in DATASETS: - values, _ = load(name) - tabular[name] = mean_abs_lag1_autocorr(values) - for name, score in sorted(tabular.items(), key=lambda item: -item[1]): - print(f" {name:14s} {score:.3f}") - - weakest_temporal = min(temporal) - strongest_tabular = max(tabular.values()) - print("\nSeparation") - print(f" weakest temporal {weakest_temporal:.3f}") - print(f" strongest tabular {strongest_tabular:.3f} ({max(tabular, key=lambda k: tabular[k])})") - if weakest_temporal > strongest_tabular: - print(f" SEPARATES: any threshold in ({strongest_tabular:.3f}, {weakest_temporal:.3f}) works") - else: - above = sorted(name for name, score in tabular.items() if score > weakest_temporal) - print(f" DOES NOT SEPARATE: {len(above)} tabular datasets score above the weakest time series") - print(f" overlapping: {', '.join(above)}") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.json b/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.json deleted file mode 100644 index 8e4d17b25..000000000 --- a/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.json +++ /dev/null @@ -1,26301 +0,0 @@ -{ - "generated": "2026-08-25", - "git_sha": "f9c703a1", - "seeds": 15, - "datasets": [ - "synthetic", - "smd", - "nslkdd", - "tabular" - ], - "verdict_pre_registered": "gate needed; refit the threshold from this sweep", - "verdict_robust": "no gate needed", - "evidence": { - "n_low_eta_cells": 90.0, - "n_low_eta_groupings": 4.0, - "worst_delta_single_cell": -0.049771945237090054, - "n_harmful_cells": 1.0, - "worst_median_delta_per_grouping": 0.0, - "n_harmful_groupings": 0.0, - "median_delta_below_threshold": 2.220446049250313e-16 - }, - "spearman": { - "rho": 0.5705144506753026, - "ci_low": 0.4897270783992741, - "ci_high": 0.6453979051966328 - }, - "regression": { - "intercept": -0.10148377259307864, - "eta_squared": 0.2807518647753969, - "is_contextual": 0.0849219328806492, - "r_squared": 0.578282695858033, - "n": 390.0 - }, - "cells": [ - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.28654276534927003, - "roc_auc": 0.712279662486274, - "precision_at_n": 0.31863905325443787, - "macro_pr_auc": 0.28654276534927003, - "worst_group_fpr": 0.0318557475582269, - "n_models": 1, - "seconds": 0.45140175000415184, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2731203426957805, - "roc_auc": 0.7090630348672307, - "precision_at_n": 0.3168639053254438, - "macro_pr_auc": 0.2731203426957805, - "worst_group_fpr": 0.0325694966190834, - "n_models": 1, - "seconds": 0.43827241599501576, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.28308659403148606, - "roc_auc": 0.7062419033604665, - "precision_at_n": 0.32337278106508877, - "macro_pr_auc": 0.28308659403148606, - "worst_group_fpr": 0.03102930127723516, - "n_models": 1, - "seconds": 0.46153479199711, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.29427162993120104, - "roc_auc": 0.7042722526996208, - "precision_at_n": 0.3289940828402367, - "macro_pr_auc": 0.29427162993120104, - "worst_group_fpr": 0.030315552216378664, - "n_models": 1, - "seconds": 0.444575542001985, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2963478420199089, - "roc_auc": 0.7207483362155962, - "precision_at_n": 0.32781065088757394, - "macro_pr_auc": 0.2963478420199089, - "worst_group_fpr": 0.030277986476333583, - "n_models": 1, - "seconds": 0.4534223330047098, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2967663272625093, - "roc_auc": 0.711231883977434, - "precision_at_n": 0.32751479289940827, - "macro_pr_auc": 0.2967663272625093, - "worst_group_fpr": 0.030015026296018033, - "n_models": 1, - "seconds": 0.4450537090015132, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2985561324179786, - "roc_auc": 0.7120676994651883, - "precision_at_n": 0.32662721893491126, - "macro_pr_auc": 0.2985561324179786, - "worst_group_fpr": 0.029827197595792637, - "n_models": 1, - "seconds": 0.47069399999600137, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2900148776746608, - "roc_auc": 0.7138835639884591, - "precision_at_n": 0.32514792899408285, - "macro_pr_auc": 0.2900148776746608, - "worst_group_fpr": 0.030991735537190084, - "n_models": 1, - "seconds": 0.49100608300068416, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.30683358269183825, - "roc_auc": 0.7184997766061021, - "precision_at_n": 0.3375739644970414, - "macro_pr_auc": 0.30683358269183825, - "worst_group_fpr": 0.029601803155522164, - "n_models": 1, - "seconds": 0.4592341250026948, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.31648638158245596, - "roc_auc": 0.7182507146381909, - "precision_at_n": 0.33579881656804733, - "macro_pr_auc": 0.31648638158245596, - "worst_group_fpr": 0.028437265214124718, - "n_models": 1, - "seconds": 0.4374390829980257, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2837749575539722, - "roc_auc": 0.7155108607222402, - "precision_at_n": 0.32751479289940827, - "macro_pr_auc": 0.2837749575539722, - "worst_group_fpr": 0.032719759579263714, - "n_models": 1, - "seconds": 0.4527861250026035, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2801629860263534, - "roc_auc": 0.7052293788538226, - "precision_at_n": 0.3210059171597633, - "macro_pr_auc": 0.2801629860263534, - "worst_group_fpr": 0.031893313298271976, - "n_models": 1, - "seconds": 0.4540731669985689, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.27636889431427625, - "roc_auc": 0.7038439865919206, - "precision_at_n": 0.3103550295857988, - "macro_pr_auc": 0.27636889431427625, - "worst_group_fpr": 0.03155522163786627, - "n_models": 1, - "seconds": 0.46590912499959813, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2794685845810904, - "roc_auc": 0.7071127727961801, - "precision_at_n": 0.3210059171597633, - "macro_pr_auc": 0.2794685845810904, - "worst_group_fpr": 0.032607062359128476, - "n_models": 1, - "seconds": 0.46779604200128233, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:campaign", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.28674470272020847, - "roc_auc": 0.7181384619830264, - "precision_at_n": 0.3224852071005917, - "macro_pr_auc": 0.28674470272020847, - "worst_group_fpr": 0.03204357625845229, - "n_models": 1, - "seconds": 0.4490214580000611, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5747040318516179, - "roc_auc": 0.9381694589398516, - "precision_at_n": 0.5227272727272727, - "macro_pr_auc": 0.5747040318516179, - "worst_group_fpr": 0.025377643504531724, - "n_models": 1, - "seconds": 0.12044458300078986, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5817186829361747, - "roc_auc": 0.9349491897830265, - "precision_at_n": 0.5340909090909091, - "macro_pr_auc": 0.5817186829361747, - "worst_group_fpr": 0.02175226586102719, - "n_models": 1, - "seconds": 0.12850574999902165, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5160671100710018, - "roc_auc": 0.9197370227959352, - "precision_at_n": 0.4772727272727273, - "macro_pr_auc": 0.5160671100710018, - "worst_group_fpr": 0.025377643504531724, - "n_models": 1, - "seconds": 0.1251544160040794, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5991840858390056, - "roc_auc": 0.9371017577588575, - "precision_at_n": 0.5227272727272727, - "macro_pr_auc": 0.5991840858390056, - "worst_group_fpr": 0.022356495468277945, - "n_models": 1, - "seconds": 0.12043650000123307, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5970158568934664, - "roc_auc": 0.9373146113705026, - "precision_at_n": 0.5625, - "macro_pr_auc": 0.5970158568934664, - "worst_group_fpr": 0.02175226586102719, - "n_models": 1, - "seconds": 0.12548333399900002, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5599964596064102, - "roc_auc": 0.9303041746772864, - "precision_at_n": 0.48863636363636365, - "macro_pr_auc": 0.5599964596064102, - "worst_group_fpr": 0.022356495468277945, - "n_models": 1, - "seconds": 0.11849504099518526, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5637472053356579, - "roc_auc": 0.9244129360065916, - "precision_at_n": 0.48863636363636365, - "macro_pr_auc": 0.5637472053356579, - "worst_group_fpr": 0.023564954682779457, - "n_models": 1, - "seconds": 0.11911787500139326, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6131736550587313, - "roc_auc": 0.9400302114803626, - "precision_at_n": 0.5284090909090909, - "macro_pr_auc": 0.6131736550587313, - "worst_group_fpr": 0.02175226586102719, - "n_models": 1, - "seconds": 0.1293290829999023, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6008414383264314, - "roc_auc": 0.9334695138698159, - "precision_at_n": 0.5454545454545454, - "macro_pr_auc": 0.6008414383264314, - "worst_group_fpr": 0.02054380664652568, - "n_models": 1, - "seconds": 0.15402783400350017, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6082662112640778, - "roc_auc": 0.9389178797033781, - "precision_at_n": 0.5625, - "macro_pr_auc": 0.6082662112640778, - "worst_group_fpr": 0.022356495468277945, - "n_models": 1, - "seconds": 0.14646341700427, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5694346979625783, - "roc_auc": 0.9276435045317221, - "precision_at_n": 0.5284090909090909, - "macro_pr_auc": 0.5694346979625783, - "worst_group_fpr": 0.022356495468277945, - "n_models": 1, - "seconds": 0.12798587499855785, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5843174027183013, - "roc_auc": 0.9226826421312827, - "precision_at_n": 0.5340909090909091, - "macro_pr_auc": 0.5843174027183013, - "worst_group_fpr": 0.02054380664652568, - "n_models": 1, - "seconds": 0.13026841700047953, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5242593935607399, - "roc_auc": 0.925233452348256, - "precision_at_n": 0.4943181818181818, - "macro_pr_auc": 0.5242593935607399, - "worst_group_fpr": 0.024773413897280966, - "n_models": 1, - "seconds": 0.1208520000000135, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5800825625288935, - "roc_auc": 0.928906893710519, - "precision_at_n": 0.5340909090909091, - "macro_pr_auc": 0.5800825625288935, - "worst_group_fpr": 0.021148036253776436, - "n_models": 1, - "seconds": 0.1326180409960216, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:cardio", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5810785900141355, - "roc_auc": 0.9325048063718758, - "precision_at_n": 0.5227272727272727, - "macro_pr_auc": 0.5810785900141355, - "worst_group_fpr": 0.021148036253776436, - "n_models": 1, - "seconds": 0.13559600000007777, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.05863077331032433, - "roc_auc": 0.8966191176030635, - "precision_at_n": 0.08680555555555555, - "macro_pr_auc": 0.05863077331032433, - "worst_group_fpr": 0.047489229940764675, - "n_models": 1, - "seconds": 0.4108022920045187, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.05197638212386508, - "roc_auc": 0.8538906371537127, - "precision_at_n": 0.09027777777777778, - "macro_pr_auc": 0.05197638212386508, - "worst_group_fpr": 0.04796042003231018, - "n_models": 1, - "seconds": 0.41273645799810765, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.07405783756103791, - "roc_auc": 0.8923926640190271, - "precision_at_n": 0.14583333333333334, - "macro_pr_auc": 0.07405783756103791, - "worst_group_fpr": 0.047287291330102316, - "n_models": 1, - "seconds": 0.43597833400417585, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.04102903280600602, - "roc_auc": 0.8618620703195117, - "precision_at_n": 0.059027777777777776, - "macro_pr_auc": 0.04102903280600602, - "worst_group_fpr": 0.048263327948303715, - "n_models": 1, - "seconds": 0.4380415420018835, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.06665877389720262, - "roc_auc": 0.8846589294261952, - "precision_at_n": 0.10069444444444445, - "macro_pr_auc": 0.06665877389720262, - "worst_group_fpr": 0.0473546042003231, - "n_models": 1, - "seconds": 0.4297401669973624, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.04091324210776709, - "roc_auc": 0.8409143284793275, - "precision_at_n": 0.08333333333333333, - "macro_pr_auc": 0.04091324210776709, - "worst_group_fpr": 0.0483642972536349, - "n_models": 1, - "seconds": 0.4420878340024501, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.06156281724480017, - "roc_auc": 0.8851713720232753, - "precision_at_n": 0.10069444444444445, - "macro_pr_auc": 0.06156281724480017, - "worst_group_fpr": 0.04752288637587507, - "n_models": 1, - "seconds": 0.4307920840001316, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.057937136441490775, - "roc_auc": 0.8834884334051336, - "precision_at_n": 0.09027777777777778, - "macro_pr_auc": 0.057937136441490775, - "worst_group_fpr": 0.04775848142164782, - "n_models": 1, - "seconds": 0.4298634170045261, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.07550091529958756, - "roc_auc": 0.9120703428842816, - "precision_at_n": 0.10416666666666667, - "macro_pr_auc": 0.07550091529958756, - "worst_group_fpr": 0.04691707054388799, - "n_models": 1, - "seconds": 0.4283183329971507, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.062231373450948146, - "roc_auc": 0.9062794493807216, - "precision_at_n": 0.08680555555555555, - "macro_pr_auc": 0.062231373450948146, - "worst_group_fpr": 0.04752288637587507, - "n_models": 1, - "seconds": 0.4181232920018374, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.04935501629482199, - "roc_auc": 0.8635736402800216, - "precision_at_n": 0.08333333333333333, - "macro_pr_auc": 0.04935501629482199, - "worst_group_fpr": 0.047893107162089395, - "n_models": 1, - "seconds": 0.4182043330001761, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.050823837097595845, - "roc_auc": 0.8820772003829355, - "precision_at_n": 0.0798611111111111, - "macro_pr_auc": 0.050823837097595845, - "worst_group_fpr": 0.047859450726978996, - "n_models": 1, - "seconds": 0.42386550000082934, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.0529526540163899, - "roc_auc": 0.8707557832974332, - "precision_at_n": 0.09027777777777778, - "macro_pr_auc": 0.0529526540163899, - "worst_group_fpr": 0.047691168551427035, - "n_models": 1, - "seconds": 0.42577320800046436, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.05343059863532222, - "roc_auc": 0.876983392419075, - "precision_at_n": 0.08680555555555555, - "macro_pr_auc": 0.05343059863532222, - "worst_group_fpr": 0.04802773290253096, - "n_models": 1, - "seconds": 0.4220345419962541, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:covertype", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.04440980646344468, - "roc_auc": 0.8639258642224615, - "precision_at_n": 0.08680555555555555, - "macro_pr_auc": 0.04440980646344468, - "worst_group_fpr": 0.04802773290253096, - "n_models": 1, - "seconds": 0.42304295899521094, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2465692235779982, - "roc_auc": 0.9758299000318501, - "precision_at_n": 0.34615384615384615, - "macro_pr_auc": 0.2465692235779982, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4254823329974897, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.25947397991604965, - "roc_auc": 0.9779560212059878, - "precision_at_n": 0.36538461538461536, - "macro_pr_auc": 0.25947397991604965, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4116599170010886, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.18388773085249868, - "roc_auc": 0.9758241207837174, - "precision_at_n": 0.28846153846153844, - "macro_pr_auc": 0.18388773085249868, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.41671454099559924, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.1795807148953323, - "roc_auc": 0.9794400036987189, - "precision_at_n": 0.3269230769230769, - "macro_pr_auc": 0.1795807148953323, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4084197500051232, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.16856047663142879, - "roc_auc": 0.9700095550235793, - "precision_at_n": 0.3076923076923077, - "macro_pr_auc": 0.16856047663142879, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.39896358300029533, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.24094286403702606, - "roc_auc": 0.975582034500827, - "precision_at_n": 0.36538461538461536, - "macro_pr_auc": 0.24094286403702606, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.3959473330032779, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.1635245883381066, - "roc_auc": 0.9734578397427336, - "precision_at_n": 0.3269230769230769, - "macro_pr_auc": 0.1635245883381066, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4198383749971981, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.23708364780494726, - "roc_auc": 0.9753919614511306, - "precision_at_n": 0.36538461538461536, - "macro_pr_auc": 0.23708364780494726, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4060974160020123, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.18755996109452192, - "roc_auc": 0.9757496326966743, - "precision_at_n": 0.3269230769230769, - "macro_pr_auc": 0.18755996109452192, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4110126249943278, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.24933591270925248, - "roc_auc": 0.9780427099279777, - "precision_at_n": 0.36538461538461536, - "macro_pr_auc": 0.24933591270925248, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4057716250026715, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.1931317946202666, - "roc_auc": 0.9727046110694435, - "precision_at_n": 0.3076923076923077, - "macro_pr_auc": 0.1931317946202666, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.3979578749931534, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.24973837478550712, - "roc_auc": 0.975131895285161, - "precision_at_n": 0.34615384615384615, - "macro_pr_auc": 0.24973837478550712, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4164667919976637, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.22402651624566144, - "roc_auc": 0.976911261571339, - "precision_at_n": 0.34615384615384615, - "macro_pr_auc": 0.22402651624566144, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4011265419976553, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.20220615974700726, - "roc_auc": 0.9764277311442398, - "precision_at_n": 0.34615384615384615, - "macro_pr_auc": 0.20220615974700726, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.40274466700066114, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:fraud", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.24498820618018466, - "roc_auc": 0.9742598709558106, - "precision_at_n": 0.34615384615384615, - "macro_pr_auc": 0.24498820618018466, - "worst_group_fpr": 0.04855082142380126, - "n_models": 1, - "seconds": 0.4057962079969002, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2238807780411677, - "roc_auc": 0.8635803773266009, - "precision_at_n": 0.25769230769230766, - "macro_pr_auc": 0.2238807780411677, - "worst_group_fpr": 0.04321157191247826, - "n_models": 1, - "seconds": 0.24048349999793572, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.21930579778341633, - "roc_auc": 0.8656585609757814, - "precision_at_n": 0.23076923076923078, - "macro_pr_auc": 0.21930579778341633, - "worst_group_fpr": 0.04266227226952302, - "n_models": 1, - "seconds": 0.2333686670026509, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.182872933288566, - "roc_auc": 0.8549151754589821, - "precision_at_n": 0.18461538461538463, - "macro_pr_auc": 0.182872933288566, - "worst_group_fpr": 0.04403552137691111, - "n_models": 1, - "seconds": 0.2408585410012165, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.17941433606707546, - "roc_auc": 0.8555120106479622, - "precision_at_n": 0.19230769230769232, - "macro_pr_auc": 0.17941433606707546, - "worst_group_fpr": 0.04366932161494095, - "n_models": 1, - "seconds": 0.23315041699970607, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.21586806123972352, - "roc_auc": 0.8570884302002126, - "precision_at_n": 0.24615384615384617, - "macro_pr_auc": 0.21586806123972352, - "worst_group_fpr": 0.042845372150508106, - "n_models": 1, - "seconds": 0.22282941699813819, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.26213192835196936, - "roc_auc": 0.8787290755568701, - "precision_at_n": 0.27307692307692305, - "macro_pr_auc": 0.26213192835196936, - "worst_group_fpr": 0.041289023162134945, - "n_models": 1, - "seconds": 0.22768337500019697, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.21348555051172324, - "roc_auc": 0.859291614729681, - "precision_at_n": 0.23846153846153847, - "macro_pr_auc": 0.21348555051172324, - "worst_group_fpr": 0.0433031218529708, - "n_models": 1, - "seconds": 0.24882616600370966, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2697302415338744, - "roc_auc": 0.8680913245867928, - "precision_at_n": 0.26153846153846155, - "macro_pr_auc": 0.2697302415338744, - "worst_group_fpr": 0.04220452256706033, - "n_models": 1, - "seconds": 0.2434860830035177, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.22921292704887455, - "roc_auc": 0.8652096141522123, - "precision_at_n": 0.24615384615384617, - "macro_pr_auc": 0.22921292704887455, - "worst_group_fpr": 0.04257072232903049, - "n_models": 1, - "seconds": 0.2308954170002835, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.24865535675164857, - "roc_auc": 0.8644905950041901, - "precision_at_n": 0.2692307692307692, - "macro_pr_auc": 0.24865535675164857, - "worst_group_fpr": 0.041929872745582714, - "n_models": 1, - "seconds": 0.24952804199710954, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.17923859209213988, - "roc_auc": 0.8493904886654131, - "precision_at_n": 0.18461538461538463, - "macro_pr_auc": 0.17923859209213988, - "worst_group_fpr": 0.04412707131740364, - "n_models": 1, - "seconds": 0.22745279200171353, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.23452842739895857, - "roc_auc": 0.8661916633215727, - "precision_at_n": 0.2692307692307692, - "macro_pr_auc": 0.23452842739895857, - "worst_group_fpr": 0.042021422686075255, - "n_models": 1, - "seconds": 0.243230667001626, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.23844682989086965, - "roc_auc": 0.8721990999936617, - "precision_at_n": 0.25384615384615383, - "macro_pr_auc": 0.23844682989086965, - "worst_group_fpr": 0.042753822210015564, - "n_models": 1, - "seconds": 0.23289158300030977, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.21194472435609962, - "roc_auc": 0.8622377622377622, - "precision_at_n": 0.23461538461538461, - "macro_pr_auc": 0.21194472435609962, - "worst_group_fpr": 0.04257072232903049, - "n_models": 1, - "seconds": 0.2331420420014183, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mammography", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2014456994766733, - "roc_auc": 0.8547007373291361, - "precision_at_n": 0.21153846153846154, - "macro_pr_auc": 0.2014456994766733, - "worst_group_fpr": 0.04412707131740364, - "n_models": 1, - "seconds": 0.23307858299813233, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.26341171282325326, - "roc_auc": 0.801063098859709, - "precision_at_n": 0.2842857142857143, - "macro_pr_auc": 0.26341171282325326, - "worst_group_fpr": 0.03766478342749529, - "n_models": 1, - "seconds": 0.20396720899589127, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2942781064028406, - "roc_auc": 0.8192200078640757, - "precision_at_n": 0.3442857142857143, - "macro_pr_auc": 0.2942781064028406, - "worst_group_fpr": 0.035057221497899464, - "n_models": 1, - "seconds": 0.21104345899948385, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.28390131830542825, - "roc_auc": 0.813244759007471, - "precision_at_n": 0.32142857142857145, - "macro_pr_auc": 0.28390131830542825, - "worst_group_fpr": 0.035781544256120526, - "n_models": 1, - "seconds": 0.21832416699908208, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2766453432404812, - "roc_auc": 0.8005322737526128, - "precision_at_n": 0.31, - "macro_pr_auc": 0.2766453432404812, - "worst_group_fpr": 0.036216137911053166, - "n_models": 1, - "seconds": 0.2121904590021586, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.24913424002114803, - "roc_auc": 0.7965526375695867, - "precision_at_n": 0.2757142857142857, - "macro_pr_auc": 0.24913424002114803, - "worst_group_fpr": 0.03969288715051427, - "n_models": 1, - "seconds": 0.20452533399657113, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.25279398706118017, - "roc_auc": 0.7974207901326544, - "precision_at_n": 0.28285714285714286, - "macro_pr_auc": 0.25279398706118017, - "worst_group_fpr": 0.03795451253078372, - "n_models": 1, - "seconds": 0.21117137499823002, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.28021063871464763, - "roc_auc": 0.8046619482212702, - "precision_at_n": 0.2985714285714286, - "macro_pr_auc": 0.28021063871464763, - "worst_group_fpr": 0.03491235694625525, - "n_models": 1, - "seconds": 0.22707304199866485, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2627035589597312, - "roc_auc": 0.7974106496140394, - "precision_at_n": 0.2842857142857143, - "macro_pr_auc": 0.2627035589597312, - "worst_group_fpr": 0.03751991887585108, - "n_models": 1, - "seconds": 0.21374645800096914, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.28187582019142704, - "roc_auc": 0.8174472796506694, - "precision_at_n": 0.3142857142857143, - "macro_pr_auc": 0.28187582019142704, - "worst_group_fpr": 0.03679559611763002, - "n_models": 1, - "seconds": 0.23416804099542787, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2979922289822242, - "roc_auc": 0.8243186192338734, - "precision_at_n": 0.33, - "macro_pr_auc": 0.2979922289822242, - "worst_group_fpr": 0.035781544256120526, - "n_models": 1, - "seconds": 0.22203474999696482, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2530813480265501, - "roc_auc": 0.7928681111731958, - "precision_at_n": 0.27285714285714285, - "macro_pr_auc": 0.2530813480265501, - "worst_group_fpr": 0.03983775170215848, - "n_models": 1, - "seconds": 0.2217996250037686, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2917199103505091, - "roc_auc": 0.8195225678276524, - "precision_at_n": 0.31857142857142856, - "macro_pr_auc": 0.2917199103505091, - "worst_group_fpr": 0.034767492394611035, - "n_models": 1, - "seconds": 0.22975045799830696, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.29507231326733696, - "roc_auc": 0.8193398315432214, - "precision_at_n": 0.3171428571428571, - "macro_pr_auc": 0.29507231326733696, - "worst_group_fpr": 0.035202086049543675, - "n_models": 1, - "seconds": 0.22104950000357348, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.2743140461522794, - "roc_auc": 0.8030489849133916, - "precision_at_n": 0.3157142857142857, - "macro_pr_auc": 0.2743140461522794, - "worst_group_fpr": 0.035346950601187886, - "n_models": 1, - "seconds": 0.20849816699774237, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:mnist", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.26139822373735183, - "roc_auc": 0.8049419507046626, - "precision_at_n": 0.30428571428571427, - "macro_pr_auc": 0.26139822373735183, - "worst_group_fpr": 0.03694046066927423, - "n_models": 1, - "seconds": 0.22115241600113222, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6403791801793419, - "roc_auc": 0.668364193326667, - "precision_at_n": 0.5545186640471512, - "macro_pr_auc": 0.6403791801793419, - "worst_group_fpr": 0.003864514662423278, - "n_models": 1, - "seconds": 0.179841375000251, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6623979377499488, - "roc_auc": 0.6969444296815092, - "precision_at_n": 0.5697445972495089, - "macro_pr_auc": 0.6623979377499488, - "worst_group_fpr": 0.0040918390543305296, - "n_models": 1, - "seconds": 0.18470820799848298, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6644205260710849, - "roc_auc": 0.7087140495853005, - "precision_at_n": 0.5805500982318271, - "macro_pr_auc": 0.6644205260710849, - "worst_group_fpr": 0.003864514662423278, - "n_models": 1, - "seconds": 0.17875091599853477, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.654998919438973, - "roc_auc": 0.6876682323317811, - "precision_at_n": 0.5667976424361493, - "macro_pr_auc": 0.654998919438973, - "worst_group_fpr": 0.0036371902705160265, - "n_models": 1, - "seconds": 0.19897937499627005, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6763875719164962, - "roc_auc": 0.711991942265857, - "precision_at_n": 0.5756385068762279, - "macro_pr_auc": 0.6763875719164962, - "worst_group_fpr": 0.0036371902705160265, - "n_models": 1, - "seconds": 0.21004141600133153, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.64718089577694, - "roc_auc": 0.6859499010982582, - "precision_at_n": 0.5491159135559921, - "macro_pr_auc": 0.64718089577694, - "worst_group_fpr": 0.004319163446237781, - "n_models": 1, - "seconds": 0.20022020900069037, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6915081633260632, - "roc_auc": 0.7071973626797661, - "precision_at_n": 0.587426326129666, - "macro_pr_auc": 0.6915081633260632, - "worst_group_fpr": 0.0015912707433507615, - "n_models": 1, - "seconds": 0.20368483300262596, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6533914685598003, - "roc_auc": 0.6998173589193115, - "precision_at_n": 0.5741650294695482, - "macro_pr_auc": 0.6533914685598003, - "worst_group_fpr": 0.004546487838145033, - "n_models": 1, - "seconds": 0.20090770899696508, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6668336717138339, - "roc_auc": 0.7130578882233907, - "precision_at_n": 0.5697445972495089, - "macro_pr_auc": 0.6668336717138339, - "worst_group_fpr": 0.003864514662423278, - "n_models": 1, - "seconds": 0.20171112500247546, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6792339193334375, - "roc_auc": 0.7134395163037143, - "precision_at_n": 0.5957760314341847, - "macro_pr_auc": 0.6792339193334375, - "worst_group_fpr": 0.0040918390543305296, - "n_models": 1, - "seconds": 0.18887791700399248, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6979689965372294, - "roc_auc": 0.7070788994283841, - "precision_at_n": 0.6051080550098232, - "macro_pr_auc": 0.6979689965372294, - "worst_group_fpr": 0.00136394635144351, - "n_models": 1, - "seconds": 0.1881111660040915, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.652471383397541, - "roc_auc": 0.6978401056500161, - "precision_at_n": 0.5677799607072691, - "macro_pr_auc": 0.652471383397541, - "worst_group_fpr": 0.004773812230052284, - "n_models": 1, - "seconds": 0.18804724999790778, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6697610641381991, - "roc_auc": 0.6952732157826547, - "precision_at_n": 0.5830058939096268, - "macro_pr_auc": 0.6697610641381991, - "worst_group_fpr": 0.003864514662423278, - "n_models": 1, - "seconds": 0.18602595900301822, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6670674623851404, - "roc_auc": 0.6951316404737458, - "precision_at_n": 0.5677799607072691, - "macro_pr_auc": 0.6670674623851404, - "worst_group_fpr": 0.0036371902705160265, - "n_models": 1, - "seconds": 0.19453533400519518, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:satellite", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6772328057647212, - "roc_auc": 0.7089098879857942, - "precision_at_n": 0.5849705304518664, - "macro_pr_auc": 0.6772328057647212, - "worst_group_fpr": 0.003409865878608775, - "n_models": 1, - "seconds": 0.20645341600175016, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9811631815989674, - "roc_auc": 0.9970048858578746, - "precision_at_n": 0.9655011655011655, - "macro_pr_auc": 0.9811631815989674, - "worst_group_fpr": 0.0003590019745108598, - "n_models": 1, - "seconds": 0.4364367080052034, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9788869653101445, - "roc_auc": 0.997158361294064, - "precision_at_n": 0.958041958041958, - "macro_pr_auc": 0.9788869653101445, - "worst_group_fpr": 0.0003949021719619458, - "n_models": 1, - "seconds": 0.4340969580007368, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9817910384582027, - "roc_auc": 0.9976532986549141, - "precision_at_n": 0.9599067599067599, - "macro_pr_auc": 0.9817910384582027, - "worst_group_fpr": 0.0003590019745108598, - "n_models": 1, - "seconds": 0.4223874160015839, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9819669316060807, - "roc_auc": 0.9971915166745539, - "precision_at_n": 0.965034965034965, - "macro_pr_auc": 0.9819669316060807, - "worst_group_fpr": 0.00032310177705977385, - "n_models": 1, - "seconds": 0.44787437500053784, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9818810429358318, - "roc_auc": 0.9975047772786061, - "precision_at_n": 0.9645687645687646, - "macro_pr_auc": 0.9818810429358318, - "worst_group_fpr": 0.00028720157960868787, - "n_models": 1, - "seconds": 0.4450239169964334, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9837926373440424, - "roc_auc": 0.9978865411498691, - "precision_at_n": 0.9687645687645687, - "macro_pr_auc": 0.9837926373440424, - "worst_group_fpr": 0.00028720157960868787, - "n_models": 1, - "seconds": 0.42829704099858645, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9842447280553007, - "roc_auc": 0.9979000978677877, - "precision_at_n": 0.9696969696969697, - "macro_pr_auc": 0.9842447280553007, - "worst_group_fpr": 0.00028720157960868787, - "n_models": 1, - "seconds": 0.4368969589995686, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9729041160727758, - "roc_auc": 0.9967768484731327, - "precision_at_n": 0.9333333333333333, - "macro_pr_auc": 0.9729041160727758, - "worst_group_fpr": 0.0007180039490217197, - "n_models": 1, - "seconds": 0.4372796669995296, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9779109874894807, - "roc_auc": 0.9961613065328736, - "precision_at_n": 0.9622377622377623, - "macro_pr_auc": 0.9779109874894807, - "worst_group_fpr": 0.00028720157960868787, - "n_models": 1, - "seconds": 0.4329318750023958, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9759659554541371, - "roc_auc": 0.9964731277816901, - "precision_at_n": 0.9505827505827505, - "macro_pr_auc": 0.9759659554541371, - "worst_group_fpr": 0.0006103033566684617, - "n_models": 1, - "seconds": 0.42724345899478067, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9687688224385602, - "roc_auc": 0.9967668231965485, - "precision_at_n": 0.9212121212121213, - "macro_pr_auc": 0.9687688224385602, - "worst_group_fpr": 0.0007898043439238916, - "n_models": 1, - "seconds": 0.4360776250032359, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9778553282619138, - "roc_auc": 0.996779442659895, - "precision_at_n": 0.9603729603729604, - "macro_pr_auc": 0.9778553282619138, - "worst_group_fpr": 0.0002154011847065159, - "n_models": 1, - "seconds": 0.4356054999952903, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9748345309853084, - "roc_auc": 0.9971416915520308, - "precision_at_n": 0.9445221445221446, - "macro_pr_auc": 0.9748345309853084, - "worst_group_fpr": 0.0006821037515706336, - "n_models": 1, - "seconds": 0.42429037500551203, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9788403825355294, - "roc_auc": 0.9969967350904346, - "precision_at_n": 0.9627039627039627, - "macro_pr_auc": 0.9788403825355294, - "worst_group_fpr": 0.00028720157960868787, - "n_models": 1, - "seconds": 0.42620766699837986, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:shuttle", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.9775212081798447, - "roc_auc": 0.9969381064696089, - "precision_at_n": 0.9473193473193473, - "macro_pr_auc": 0.9775212081798447, - "worst_group_fpr": 0.0005744031592173757, - "n_models": 1, - "seconds": 0.4447026660054689, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4996823139165014, - "roc_auc": 0.6505501692538507, - "precision_at_n": 0.5247170935080405, - "macro_pr_auc": 0.4996823139165014, - "worst_group_fpr": 0.04865506329113924, - "n_models": 1, - "seconds": 0.16026191700075287, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4637123206381993, - "roc_auc": 0.6162852172405214, - "precision_at_n": 0.50565812983919, - "macro_pr_auc": 0.4637123206381993, - "worst_group_fpr": 0.06131329113924051, - "n_models": 1, - "seconds": 0.1401472920042579, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4635364518748895, - "roc_auc": 0.616683378442563, - "precision_at_n": 0.4973198332340679, - "macro_pr_auc": 0.4635364518748895, - "worst_group_fpr": 0.05617088607594937, - "n_models": 1, - "seconds": 0.14207695799996145, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4773874757151399, - "roc_auc": 0.6245720356073914, - "precision_at_n": 0.5080405002977963, - "macro_pr_auc": 0.4773874757151399, - "worst_group_fpr": 0.049841772151898736, - "n_models": 1, - "seconds": 0.13609958300366998, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5115427344895851, - "roc_auc": 0.65513067226574, - "precision_at_n": 0.5306730196545563, - "macro_pr_auc": 0.5115427344895851, - "worst_group_fpr": 0.0446993670886076, - "n_models": 1, - "seconds": 0.13952529199741548, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4677266779695209, - "roc_auc": 0.62460513717478, - "precision_at_n": 0.5020845741512805, - "macro_pr_auc": 0.4677266779695209, - "worst_group_fpr": 0.06091772151898734, - "n_models": 1, - "seconds": 0.145223999999871, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.469784426449968, - "roc_auc": 0.6214238527303021, - "precision_at_n": 0.4973198332340679, - "macro_pr_auc": 0.469784426449968, - "worst_group_fpr": 0.0557753164556962, - "n_models": 1, - "seconds": 0.1338729169947328, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4818956661574424, - "roc_auc": 0.6314401985434368, - "precision_at_n": 0.5116140559857058, - "macro_pr_auc": 0.4818956661574424, - "worst_group_fpr": 0.049841772151898736, - "n_models": 1, - "seconds": 0.14036466600373387, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4798119762992656, - "roc_auc": 0.6289304871796805, - "precision_at_n": 0.5068493150684932, - "macro_pr_auc": 0.4798119762992656, - "worst_group_fpr": 0.048259493670886076, - "n_models": 1, - "seconds": 0.14640716700523626, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.47533003058254375, - "roc_auc": 0.6280062348745863, - "precision_at_n": 0.509827278141751, - "macro_pr_auc": 0.47533003058254375, - "worst_group_fpr": 0.05814873417721519, - "n_models": 1, - "seconds": 0.13803812499827472, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.48377517349008564, - "roc_auc": 0.6226054962266568, - "precision_at_n": 0.5074449076831448, - "macro_pr_auc": 0.48377517349008564, - "worst_group_fpr": 0.0446993670886076, - "n_models": 1, - "seconds": 0.13034129200241296, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.47324342257073737, - "roc_auc": 0.6320707775122323, - "precision_at_n": 0.5080405002977963, - "macro_pr_auc": 0.47324342257073737, - "worst_group_fpr": 0.06131329113924051, - "n_models": 1, - "seconds": 0.1449920410013874, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4758315595303919, - "roc_auc": 0.6310042238071185, - "precision_at_n": 0.521143537820131, - "macro_pr_auc": 0.4758315595303919, - "worst_group_fpr": 0.05617088607594937, - "n_models": 1, - "seconds": 0.14752429100190056, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4179513915431128, - "roc_auc": 0.557428981235063, - "precision_at_n": 0.44371649791542583, - "macro_pr_auc": 0.4179513915431128, - "worst_group_fpr": 0.0668512658227848, - "n_models": 1, - "seconds": 0.1595240000024205, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:spambase", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4824987826933403, - "roc_auc": 0.6376413825664763, - "precision_at_n": 0.5181655747468732, - "macro_pr_auc": 0.4824987826933403, - "worst_group_fpr": 0.053401898734177215, - "n_models": 1, - "seconds": 0.15749891699670115, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 0, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.49972632444908194, - "roc_auc": 0.9777230254831988, - "precision_at_n": 0.5483870967741935, - "macro_pr_auc": 0.49972632444908194, - "worst_group_fpr": 0.031530307148681706, - "n_models": 1, - "seconds": 0.15497462500206893, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 1, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5444216351253587, - "roc_auc": 0.979564339304451, - "precision_at_n": 0.5806451612903226, - "macro_pr_auc": 0.5444216351253587, - "worst_group_fpr": 0.03098668116335961, - "n_models": 1, - "seconds": 0.14914366700395476, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 2, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5070328987360753, - "roc_auc": 0.9773109219136805, - "precision_at_n": 0.5376344086021505, - "macro_pr_auc": 0.5070328987360753, - "worst_group_fpr": 0.03180212014134275, - "n_models": 1, - "seconds": 0.15282645899424097, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 3, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5017291279412852, - "roc_auc": 0.9771969358199838, - "precision_at_n": 0.5913978494623656, - "macro_pr_auc": 0.5017291279412852, - "worst_group_fpr": 0.03180212014134275, - "n_models": 1, - "seconds": 0.15333070900669554, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 4, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5177825020575512, - "roc_auc": 0.9788395046573548, - "precision_at_n": 0.6021505376344086, - "macro_pr_auc": 0.5177825020575512, - "worst_group_fpr": 0.031530307148681706, - "n_models": 1, - "seconds": 0.1534228329983307, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 5, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4980836054374582, - "roc_auc": 0.9761330656121491, - "precision_at_n": 0.5483870967741935, - "macro_pr_auc": 0.4980836054374582, - "worst_group_fpr": 0.032345746126664854, - "n_models": 1, - "seconds": 0.14515966599719832, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 6, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6499322129066327, - "roc_auc": 0.9818294475766263, - "precision_at_n": 0.6236559139784946, - "macro_pr_auc": 0.6499322129066327, - "worst_group_fpr": 0.03071486817069856, - "n_models": 1, - "seconds": 0.1430729589992552, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 7, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.4889107086450289, - "roc_auc": 0.9777844026105738, - "precision_at_n": 0.5161290322580645, - "macro_pr_auc": 0.4889107086450289, - "worst_group_fpr": 0.03207393313400381, - "n_models": 1, - "seconds": 0.14029387499613222, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 8, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.43467165817772785, - "roc_auc": 0.9722341566636562, - "precision_at_n": 0.5053763440860215, - "macro_pr_auc": 0.43467165817772785, - "worst_group_fpr": 0.032889372111986954, - "n_models": 1, - "seconds": 0.1374471250019269, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 9, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.6443810112816749, - "roc_auc": 0.9829108541065682, - "precision_at_n": 0.6344086021505376, - "macro_pr_auc": 0.6443810112816749, - "worst_group_fpr": 0.02989942919271541, - "n_models": 1, - "seconds": 0.14336575000197627, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 10, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5769111665754879, - "roc_auc": 0.9802365649852257, - "precision_at_n": 0.5913978494623656, - "macro_pr_auc": 0.5769111665754879, - "worst_group_fpr": 0.031530307148681706, - "n_models": 1, - "seconds": 0.14559333299985155, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 11, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5389354598908428, - "roc_auc": 0.9779889930351574, - "precision_at_n": 0.5483870967741935, - "macro_pr_auc": 0.5389354598908428, - "worst_group_fpr": 0.03180212014134275, - "n_models": 1, - "seconds": 0.13968100000056438, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 12, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.5466419419531122, - "roc_auc": 0.9763201197146256, - "precision_at_n": 0.5483870967741935, - "macro_pr_auc": 0.5466419419531122, - "worst_group_fpr": 0.03207393313400381, - "n_models": 1, - "seconds": 0.14631679200101644, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 13, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.586136736438214, - "roc_auc": 0.9802394877055769, - "precision_at_n": 0.6021505376344086, - "macro_pr_auc": 0.586136736438214, - "worst_group_fpr": 0.03098668116335961, - "n_models": 1, - "seconds": 0.14696887500031153, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "tabular:thyroid", - "grouping": "none", - "config": "pooled", - "seed": 14, - "mechanism": "tabular", - "level_spread": NaN, - "pr_auc": 0.575796106634768, - "roc_auc": 0.9809321724288098, - "precision_at_n": 0.5913978494623656, - "macro_pr_auc": 0.575796106634768, - "worst_group_fpr": 0.03098668116335961, - "n_models": 1, - "seconds": 0.14394399999582674, - "eta_squared": 0.0, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 1, - "seconds": 0.18409758299821988, - "eta_squared": 0.0008371029862412371, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.18579829199734377, - "eta_squared": 0.0008371029862412371, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9997841047394043, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.288182541000424, - "eta_squared": 0.0008371029862412371, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.18637058299646014, - "eta_squared": 0.0012807601285772677, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.1840459169980022, - "eta_squared": 0.0012807601285772677, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9975862799450697, - "roc_auc": 0.9999512825963719, - "precision_at_n": 0.96875, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3135257909962093, - "eta_squared": 0.0012807601285772677, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.18191958300303668, - "eta_squared": 0.00097235471194274, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9988727861917877, - "roc_auc": 0.9999778557256236, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.19064775000151712, - "eta_squared": 0.00097235471194274, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9997874149659862, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3351120419974905, - "eta_squared": 0.00097235471194274, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.18484791700029746, - "eta_squared": 0.00043327407293021155, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.1640156669964199, - "eta_squared": 0.00043327407293021155, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3733154159999685, - "eta_squared": 0.00043327407293021155, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.18438108299596934, - "eta_squared": 0.0011831306529618937, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.17161412499990547, - "eta_squared": 0.0011831306529618937, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9986319303600137, - "roc_auc": 0.9999734268707483, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3089136660055374, - "eta_squared": 0.0011831306529618937, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9997841047394043, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.17085050000605406, - "eta_squared": 0.001278796513683895, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.18761091699707322, - "eta_squared": 0.001278796513683895, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.996576575419816, - "roc_auc": 0.9999357816043083, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3391146669964655, - "eta_squared": 0.001278796513683895, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.1871012499977951, - "eta_squared": 0.0013720012875030802, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.18300454199925298, - "eta_squared": 0.0013720012875030802, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9959089137856371, - "roc_auc": 0.9999291383219955, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9931588955026455, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3320764579984825, - "eta_squared": 0.0013720012875030802, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.17152695799450157, - "eta_squared": 0.0017954572306766912, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.1670167499978561, - "eta_squared": 0.0017954572306766912, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.998003812170808, - "roc_auc": 0.9999601403061225, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3436600409986568, - "eta_squared": 0.0017954572306766912, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9996744556165973, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.1912815000032424, - "eta_squared": 0.00084717659914683, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.16627637499914272, - "eta_squared": 0.00084717659914683, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9960141105034992, - "roc_auc": 0.9999313527494331, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3255488340000738, - "eta_squared": 0.00084717659914683, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.16260399999737274, - "eta_squared": 0.0012853540416852677, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.1862242499992135, - "eta_squared": 0.0012853540416852677, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3023840000023483, - "eta_squared": 0.0012853540416852677, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.1731478750007227, - "eta_squared": 0.0012397194238675945, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.18117912500019884, - "eta_squared": 0.0012397194238675945, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3403592499962542, - "eta_squared": 0.0012397194238675945, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.18963520800025435, - "eta_squared": 0.0003383620246718858, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.17530620799516328, - "eta_squared": 0.0003383620246718858, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9940263209142927, - "roc_auc": 0.9998959219104309, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9897734788359788, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3515108339997823, - "eta_squared": 0.0003383620246718858, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 1, - "seconds": 0.1935919579991605, - "eta_squared": 0.0010503666837600957, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.18339049999485724, - "eta_squared": 0.0010503666837600957, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3044878329965286, - "eta_squared": 0.0010503666837600957, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.18961091699748067, - "eta_squared": 0.002415635561632408, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.16607283300254494, - "eta_squared": 0.002415635561632408, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9985267336421512, - "roc_auc": 0.9999712124433106, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3830415830016136, - "eta_squared": 0.002415635561632408, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.19411041699640919, - "eta_squared": 0.0009561098242806195, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 1, - "seconds": 0.182575042003009, - "eta_squared": 0.0009561098242806195, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.0, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3324455840047449, - "eta_squared": 0.0009561098242806195, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.18907358399883378, - "eta_squared": 0.0007681373849954036, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.1886240000021644, - "eta_squared": 0.0007681373849954036, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3481295829988085, - "eta_squared": 0.0007681373849954036, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9989935202589898, - "roc_auc": 0.9999800701530612, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.18981487499695504, - "eta_squared": 0.0023345860714670385, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.1884202910005115, - "eta_squared": 0.0023345860714670385, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4001940000016475, - "eta_squared": 0.0023345860714670385, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.18420899999910034, - "eta_squared": 0.0013075922239467576, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.1899541250022594, - "eta_squared": 0.0013075922239467576, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9767568942402173, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.337920917001611, - "eta_squared": 0.0013075922239467576, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.991638404762116, - "roc_auc": 0.9998693487811791, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9856812169312169, - "worst_group_fpr": 0.03826530612244898, - "n_models": 1, - "seconds": 0.19650370800081873, - "eta_squared": 0.0010029218229787487, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.1894969589993707, - "eta_squared": 0.0010029218229787487, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3478906250020373, - "eta_squared": 0.0010029218229787487, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.17953670800488908, - "eta_squared": 0.0017217081640612903, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.18294712500210153, - "eta_squared": 0.0017217081640612903, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.3613609160020133, - "eta_squared": 0.0017217081640612903, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9958642320763798, - "roc_auc": 0.9999291383219954, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.19453262500610435, - "eta_squared": 0.0012261319468726715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9931509781675918, - "roc_auc": 0.9998959219104309, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.1858277499995893, - "eta_squared": 0.0012261319468726715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.372646875002829, - "eta_squared": 0.0012261319468726715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.18463604200223926, - "eta_squared": 0.0012204410626207138, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9978764404259796, - "roc_auc": 0.9999601403061225, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.18741529199905926, - "eta_squared": 0.0012204410626207138, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.346244165993994, - "eta_squared": 0.0012204410626207138, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9906045148405054, - "roc_auc": 0.9998715632086168, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9896288029100528, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.20375691699882736, - "eta_squared": 0.0019679556564418787, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9994516328453016, - "roc_auc": 0.9999889278628119, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.2048590830017929, - "eta_squared": 0.0019679556564418787, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3410939999957918, - "eta_squared": 0.0019679556564418787, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.17705245799879776, - "eta_squared": 0.001183606140824988, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9993384082076205, - "roc_auc": 0.9999867134353742, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.1958594579991768, - "eta_squared": 0.001183606140824988, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3489050830030465, - "eta_squared": 0.001183606140824988, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.18971562499791617, - "eta_squared": 0.0010477149599897315, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.19377075000375044, - "eta_squared": 0.0010477149599897315, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3623629170033382, - "eta_squared": 0.0010477149599897315, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.19366850000369595, - "eta_squared": 0.0015314052300759431, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.18420812499971362, - "eta_squared": 0.0015314052300759431, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3384155000021565, - "eta_squared": 0.0015314052300759431, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9658179986738644, - "roc_auc": 0.9996545493197279, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9711557539682539, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.190427916997578, - "eta_squared": 0.0008561578874620307, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9956960258104145, - "roc_auc": 0.9999269238945578, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.19694179199723294, - "eta_squared": 0.0008561578874620307, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3296376250000321, - "eta_squared": 0.0008561578874620307, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.19709537499875296, - "eta_squared": 0.0009042790866601086, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.19135083400033182, - "eta_squared": 0.0009042790866601086, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3695947080050246, - "eta_squared": 0.0009042790866601086, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9924349918271719, - "roc_auc": 0.9998870642006803, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9896288029100528, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.18747533400164684, - "eta_squared": 0.0023824674825981534, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.18687129199679475, - "eta_squared": 0.0023824674825981534, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9082358809082258, - "roc_auc": 0.9985362634637187, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9244201689514191, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.35008045900031, - "eta_squared": 0.0023824674825981534, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.19990237500314834, - "eta_squared": 0.0017212676917411547, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.18206070899759652, - "eta_squared": 0.0017212676917411547, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.000", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.0, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.309598540996376, - "eta_squared": 0.0017212676917411547, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.17353662499954225, - "eta_squared": 0.03955842865464311, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.17513391700049397, - "eta_squared": 0.03955842865464311, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9997841047394043, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.339590750001662, - "eta_squared": 0.03955842865464311, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.17290741600299953, - "eta_squared": 0.035766956730376886, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.18505200000072364, - "eta_squared": 0.035766956730376886, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9982768210701308, - "roc_auc": 0.9999645691609977, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3701164169979165, - "eta_squared": 0.035766956730376886, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.20308208300411934, - "eta_squared": 0.0395464466000243, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9997841047394043, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.19559391700022388, - "eta_squared": 0.0395464466000243, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9994661873670918, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3472885000010137, - "eta_squared": 0.0395464466000243, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.18855170800088672, - "eta_squared": 0.038621152227959823, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.16623616699507693, - "eta_squared": 0.038621152227959823, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4932873750003637, - "eta_squared": 0.038621152227959823, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.18449270899873227, - "eta_squared": 0.0391862568727475, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.21453958300116938, - "eta_squared": 0.0391862568727475, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.998991157403909, - "roc_auc": 0.9999800701530612, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4944277079994208, - "eta_squared": 0.0391862568727475, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.1983925839958829, - "eta_squared": 0.03294941366121079, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.19444470799498959, - "eta_squared": 0.03294941366121079, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9961542267012345, - "roc_auc": 0.9999291383219954, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3954163340022205, - "eta_squared": 0.03294941366121079, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9970566287270859, - "roc_auc": 0.9999468537414966, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.19073270899389172, - "eta_squared": 0.03415590740792779, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.19973162499809405, - "eta_squared": 0.03415590740792779, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9969139346631589, - "roc_auc": 0.999944639314059, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.407847500006028, - "eta_squared": 0.03415590740792779, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.18568058300297707, - "eta_squared": 0.04191038320105364, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.1967608749982901, - "eta_squared": 0.04191038320105364, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9980038121708081, - "roc_auc": 0.9999601403061225, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.671937874998548, - "eta_squared": 0.04191038320105364, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, - "n_models": 1, - "seconds": 0.29679591600142885, - "eta_squared": 0.04475252345121021, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.3207631660043262, - "eta_squared": 0.04475252345121021, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9967692587372329, - "roc_auc": 0.9999424248866213, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 2.5013937080002506, - "eta_squared": 0.04475252345121021, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.23139412500313483, - "eta_squared": 0.03474882738233468, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.3954302500060294, - "eta_squared": 0.03474882738233468, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 2.4514795420036535, - "eta_squared": 0.03474882738233468, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.25115487500443123, - "eta_squared": 0.04213584867477234, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.2550350419987808, - "eta_squared": 0.04213584867477234, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.995787833002396, - "eta_squared": 0.04213584867477234, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.24527554200176382, - "eta_squared": 0.03654307103775154, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.22836079199623782, - "eta_squared": 0.03654307103775154, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9949478368548235, - "roc_auc": 0.9999092084750567, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.991075562169312, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 2.451732458001061, - "eta_squared": 0.03654307103775154, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.22093716700328514, - "eta_squared": 0.03988094020033876, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 1, - "seconds": 0.24318262500310084, - "eta_squared": 0.03988094020033876, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 2.112241249997169, - "eta_squared": 0.03988094020033876, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.29302350000216393, - "eta_squared": 0.038582085800609046, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9998926116838486, - "roc_auc": 0.9999977855725622, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.27036920899990946, - "eta_squared": 0.038582085800609046, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.998526733642151, - "roc_auc": 0.9999712124433106, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 2.3249247089988785, - "eta_squared": 0.038582085800609046, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.3159535419981694, - "eta_squared": 0.036627617689678975, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.3032592090021353, - "eta_squared": 0.036627617689678975, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.05, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 2.4489135420008097, - "eta_squared": 0.036627617689678975, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.3446114580001449, - "eta_squared": 0.059293082499552834, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.37284204199386295, - "eta_squared": 0.059293082499552834, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 2.9106780420042924, - "eta_squared": 0.059293082499552834, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9987530543910216, - "roc_auc": 0.999975641298186, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.32288629099639365, - "eta_squared": 0.05670231707537825, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.38123066700063646, - "eta_squared": 0.05670231707537825, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 3.0213677079955232, - "eta_squared": 0.05670231707537825, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.997231086663878, - "roc_auc": 0.9999490681689343, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9948950066137566, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.3023284579976462, - "eta_squared": 0.05921841304093976, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.3231065829968429, - "eta_squared": 0.05921841304093976, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 2.9427785000007134, - "eta_squared": 0.05921841304093976, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.3356336669967277, - "eta_squared": 0.058057615078346766, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.3980491250040359, - "eta_squared": 0.058057615078346766, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 2.9782467920013005, - "eta_squared": 0.058057615078346766, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9958588297035126, - "roc_auc": 0.9999291383219955, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.3495809579981142, - "eta_squared": 0.05544497562800828, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.31225608300155727, - "eta_squared": 0.05544497562800828, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 2.862608999996155, - "eta_squared": 0.05544497562800828, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9969838574473947, - "roc_auc": 0.9999424248866213, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.3957704999993439, - "eta_squared": 0.051197551161426436, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9974733447852491, - "roc_auc": 0.9999534970238095, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.34860474999732105, - "eta_squared": 0.051197551161426436, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 2.8150589580036467, - "eta_squared": 0.051197551161426436, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.28609125000366475, - "eta_squared": 0.050492980765247095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.38692079199972795, - "eta_squared": 0.050492980765247095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 3.080347499999334, - "eta_squared": 0.050492980765247095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.304620499999146, - "eta_squared": 0.05910198864179911, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.34869220899417996, - "eta_squared": 0.05910198864179911, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 2.949169083003653, - "eta_squared": 0.05910198864179911, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9974764602423118, - "roc_auc": 0.9999534970238096, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.31692675000522286, - "eta_squared": 0.06438107327632228, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9993384082076204, - "roc_auc": 0.999986713435374, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.27923370899952715, - "eta_squared": 0.06438107327632228, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 2.7171240000025136, - "eta_squared": 0.06438107327632228, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.3716905830005999, - "eta_squared": 0.054805584022504586, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 1, - "seconds": 0.34220654100499814, - "eta_squared": 0.054805584022504586, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 2.906894082996587, - "eta_squared": 0.054805584022504586, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.33631362500455, - "eta_squared": 0.06303098656943196, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.377403124999546, - "eta_squared": 0.06303098656943196, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 3.042988957997295, - "eta_squared": 0.06303098656943196, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9763155400681044, - "roc_auc": 0.9996634070294784, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9861565806878306, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.33773637500416953, - "eta_squared": 0.05142686270297512, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9985093813404057, - "roc_auc": 0.9999712124433107, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.3668642499978887, - "eta_squared": 0.05142686270297512, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 2.82656241600489, - "eta_squared": 0.05142686270297512, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.32261537500016857, - "eta_squared": 0.05895449280709131, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.3600661249947734, - "eta_squared": 0.05895449280709131, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 3.1006452910005464, - "eta_squared": 0.05895449280709131, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9866811716792745, - "roc_auc": 0.9998139880952381, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9903687169312169, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.26628616700327257, - "eta_squared": 0.052277832661798654, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05102040816326531, - "n_models": 1, - "seconds": 0.3878202920022886, - "eta_squared": 0.052277832661798654, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 2.709965332993306, - "eta_squared": 0.052277832661798654, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.2931164580004406, - "eta_squared": 0.05617290915117752, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.29841250000026776, - "eta_squared": 0.05617290915117752, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.050", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.05, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.633176374998584, - "eta_squared": 0.05617290915117752, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.2699645419997978, - "eta_squared": 0.12687309144122869, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.3061248330050148, - "eta_squared": 0.12687309144122869, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9997841047394043, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 3.4195997919960064, - "eta_squared": 0.12687309144122869, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.3291228750022128, - "eta_squared": 0.12119391519465993, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.32965295799658634, - "eta_squared": 0.12119391519465993, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9988154954031649, - "roc_auc": 0.999975641298186, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 3.1566792910016375, - "eta_squared": 0.12119391519465993, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9986723262321067, - "roc_auc": 0.9999734268707482, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.39594216600380605, - "eta_squared": 0.12540942706236655, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.35103045800497057, - "eta_squared": 0.12540942706236655, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.999787414965986, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.784541499997431, - "eta_squared": 0.12540942706236655, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.17943770899728406, - "eta_squared": 0.12527474321356308, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 1, - "seconds": 0.21587554100551642, - "eta_squared": 0.12527474321356308, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4354623749968596, - "eta_squared": 0.12527474321356308, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.17310749999887776, - "eta_squared": 0.12512147981398453, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, - "n_models": 1, - "seconds": 0.17312124999443768, - "eta_squared": 0.12512147981398453, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9992239393431517, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4705428340021172, - "eta_squared": 0.12512147981398453, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.17702162500063423, - "eta_squared": 0.1151837730978297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.1687370839936193, - "eta_squared": 0.1151837730978297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9967136368233248, - "roc_auc": 0.9999379960317459, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4160712910015718, - "eta_squared": 0.1151837730978297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9958581590975477, - "roc_auc": 0.9999291383219955, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.2092109170043841, - "eta_squared": 0.11684378654258522, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.18265700000483776, - "eta_squared": 0.11684378654258522, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9965173484043472, - "roc_auc": 0.999937996031746, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9948950066137566, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.4119265420013107, - "eta_squared": 0.11684378654258522, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, - "n_models": 1, - "seconds": 0.17269949999899836, - "eta_squared": 0.12961477677268454, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.17104045800078893, - "eta_squared": 0.12961477677268454, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9980038121708081, - "roc_auc": 0.9999601403061225, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4036386660009157, - "eta_squared": 0.12961477677268454, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.1794141250065877, - "eta_squared": 0.13340934644786664, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.18372316699969815, - "eta_squared": 0.13340934644786664, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9970566287270857, - "roc_auc": 0.9999468537414966, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4117635419970611, - "eta_squared": 0.13340934644786664, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.17737312499957625, - "eta_squared": 0.11917540465192976, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.18114274999970803, - "eta_squared": 0.11917540465192976, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4187604999970063, - "eta_squared": 0.11917540465192976, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.17882358300266787, - "eta_squared": 0.12876131345125413, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.17535625000164146, - "eta_squared": 0.12876131345125413, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.4296091249998426, - "eta_squared": 0.12876131345125413, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9981739069981774, - "roc_auc": 0.9999645691609979, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9976851851851851, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.17627454199828207, - "eta_squared": 0.1235703476453848, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.17232879099901766, - "eta_squared": 0.1235703476453848, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9940096746750579, - "roc_auc": 0.9998981363378685, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.991075562169312, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4056731669988949, - "eta_squared": 0.1235703476453848, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.17102750000049127, - "eta_squared": 0.12854427437882665, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 1, - "seconds": 0.19475837500067428, - "eta_squared": 0.12854427437882665, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.4142136660011602, - "eta_squared": 0.12854427437882665, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.17834583300282247, - "eta_squared": 0.1246847039613907, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9992332114897581, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.1776951249994454, - "eta_squared": 0.1246847039613907, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9985267336421512, - "roc_auc": 0.9999712124433106, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4102792910052813, - "eta_squared": 0.1246847039613907, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.182436791001237, - "eta_squared": 0.12175002598109345, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.17618145800224738, - "eta_squared": 0.12175002598109345, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.1, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.42044850000093, - "eta_squared": 0.12175002598109345, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9992239393431517, - "roc_auc": 0.9999844990079364, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.17877445799967973, - "eta_squared": 0.19451621854380768, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.17992270800459664, - "eta_squared": 0.19451621854380768, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4257903329998953, - "eta_squared": 0.19451621854380768, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9821091676743208, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9861565806878306, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.1845532090010238, - "eta_squared": 0.1898993994842026, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.17791879199648974, - "eta_squared": 0.1898993994842026, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.460088291001739, - "eta_squared": 0.1898993994842026, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9963227694148251, - "roc_auc": 0.9999357816043084, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.1762644589980482, - "eta_squared": 0.1936752703132149, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.17437245800101664, - "eta_squared": 0.1936752703132149, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4415905409987317, - "eta_squared": 0.1936752703132149, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.18049354200047674, - "eta_squared": 0.1934640118312397, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04846938775510204, - "n_models": 1, - "seconds": 0.17590970799938077, - "eta_squared": 0.1934640118312397, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.423768916996778, - "eta_squared": 0.1934640118312397, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9939120074223009, - "roc_auc": 0.9998937074829932, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9948950066137566, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.1685598750045756, - "eta_squared": 0.18861888416601494, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, - "n_models": 1, - "seconds": 0.17661041599785676, - "eta_squared": 0.18861888416601494, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.3883059590007178, - "eta_squared": 0.18861888416601494, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9973980281707969, - "roc_auc": 0.9999490681689343, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.1700266250045388, - "eta_squared": 0.1805813085726683, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.18030274999910034, - "eta_squared": 0.1805813085726683, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4230401249951683, - "eta_squared": 0.1805813085726683, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.17946370899881003, - "eta_squared": 0.17980556561098363, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.17470899999898393, - "eta_squared": 0.17980556561098363, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4238765409972984, - "eta_squared": 0.17980556561098363, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.17319483399478486, - "eta_squared": 0.1935769586972524, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.16585537500213832, - "eta_squared": 0.1935769586972524, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3955539160015178, - "eta_squared": 0.1935769586972524, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9959283238067923, - "roc_auc": 0.9999247094671202, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.17348287499771686, - "eta_squared": 0.20068556264939091, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.17678399999567773, - "eta_squared": 0.20068556264939091, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3915143749982235, - "eta_squared": 0.20068556264939091, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.999118742625289, - "roc_auc": 0.9999822845804989, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.17100987500452902, - "eta_squared": 0.1874671200172165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.18211825000616955, - "eta_squared": 0.1874671200172165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.4138579169957666, - "eta_squared": 0.1874671200172165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9956974507990147, - "roc_auc": 0.9999269238945578, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.1727192499965895, - "eta_squared": 0.19900625679340145, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.16367533300217474, - "eta_squared": 0.19900625679340145, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3640864999979385, - "eta_squared": 0.19900625679340145, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9555740461597554, - "roc_auc": 0.9993533871882085, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9783966901154401, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.1687240000028396, - "eta_squared": 0.1831366419552775, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9960141105034992, - "roc_auc": 0.9999313527494331, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.16969283299840754, - "eta_squared": 0.1831366419552775, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.37561216600443, - "eta_squared": 0.1831366419552775, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9971987550814965, - "roc_auc": 0.9999490681689341, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.17607562500052154, - "eta_squared": 0.1962296761690674, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.17710829200223088, - "eta_squared": 0.1962296761690674, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.382764624999254, - "eta_squared": 0.1962296761690674, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9720807360651503, - "roc_auc": 0.9996789080215419, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9882853835978835, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.18297341699508252, - "eta_squared": 0.18278781206386457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.1765655830022297, - "eta_squared": 0.18278781206386457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3746274999939487, - "eta_squared": 0.18278781206386457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.99846047105138, - "roc_auc": 0.999968998015873, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.17902783400495537, - "eta_squared": 0.18987545660900912, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.20602816700557014, - "eta_squared": 0.18987545660900912, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.100", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.1, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3872172090050299, - "eta_squared": 0.18987545660900912, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.16560841599857667, - "eta_squared": 0.21842060879328182, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.16980620899994392, - "eta_squared": 0.21842060879328182, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9997841047394043, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3945893329946557, - "eta_squared": 0.21842060879328182, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.18620050000026822, - "eta_squared": 0.2129441367707082, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.17440516699571162, - "eta_squared": 0.2129441367707082, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9980209770081038, - "roc_auc": 0.9999601403061225, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4028902919963002, - "eta_squared": 0.2129441367707082, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.17391474999749335, - "eta_squared": 0.21607807398685436, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.17599245800374774, - "eta_squared": 0.21607807398685436, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.386537333994056, - "eta_squared": 0.21607807398685436, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, - "n_models": 1, - "seconds": 0.17551283400098328, - "eta_squared": 0.2167824088169093, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05357142857142857, - "n_models": 1, - "seconds": 0.17358104099548655, - "eta_squared": 0.2167824088169093, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4019418329990003, - "eta_squared": 0.2167824088169093, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.17289695799991023, - "eta_squared": 0.2158502792872289, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.17523695799900452, - "eta_squared": 0.2158502792872289, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9994516328453017, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3947917919940664, - "eta_squared": 0.2158502792872289, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.17122699999890756, - "eta_squared": 0.20579994667793172, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.16683429099794012, - "eta_squared": 0.20579994667793172, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9973565105014646, - "roc_auc": 0.9999490681689343, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3734197079975274, - "eta_squared": 0.20579994667793172, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.1721250830014469, - "eta_squared": 0.20697832970174135, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, - "n_models": 1, - "seconds": 0.1647393750026822, - "eta_squared": 0.20697832970174135, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.996769258737233, - "roc_auc": 0.9999424248866213, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.36176987500221, - "eta_squared": 0.20697832970174135, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.16589779199421173, - "eta_squared": 0.2209510705543195, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, - "n_models": 1, - "seconds": 0.1698608750011772, - "eta_squared": 0.2209510705543195, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9985762117921207, - "roc_auc": 0.9999712124433107, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3871745409996947, - "eta_squared": 0.2209510705543195, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.16837633400427876, - "eta_squared": 0.223914333840144, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.1724463749997085, - "eta_squared": 0.223914333840144, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9977421731790774, - "roc_auc": 0.9999579258786848, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3297875420030323, - "eta_squared": 0.223914333840144, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, - "n_models": 1, - "seconds": 0.1701794160035206, - "eta_squared": 0.21066498526646607, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.17412170799798332, - "eta_squared": 0.21066498526646607, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3639297500048997, - "eta_squared": 0.21066498526646607, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, - "n_models": 1, - "seconds": 0.16910462499799905, - "eta_squared": 0.21894049322652165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.16824579200329026, - "eta_squared": 0.21894049322652165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3689260000028298, - "eta_squared": 0.21894049322652165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9993384082076204, - "roc_auc": 0.9999867134353742, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.16919154200149933, - "eta_squared": 0.2159597863844126, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.17326112500450108, - "eta_squared": 0.2159597863844126, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9940096746750579, - "roc_auc": 0.9998981363378685, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.991075562169312, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3610072499941452, - "eta_squared": 0.2159597863844126, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.1723165000003064, - "eta_squared": 0.22105758156006608, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.17333112499909475, - "eta_squared": 0.22105758156006608, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3661107500010985, - "eta_squared": 0.22105758156006608, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.16165904099761974, - "eta_squared": 0.21621224561046687, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06887755102040816, - "n_models": 1, - "seconds": 0.1595212089960114, - "eta_squared": 0.21621224561046687, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9987675894739253, - "roc_auc": 0.9999756412981858, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3435670410035527, - "eta_squared": 0.21621224561046687, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.1719130419951398, - "eta_squared": 0.21276020362000367, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.16653300000325544, - "eta_squared": 0.21276020362000367, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.15, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3313044579990674, - "eta_squared": 0.21276020362000367, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9975142413573642, - "roc_auc": 0.9999490681689343, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.16716983399965102, - "eta_squared": 0.3454157417517599, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.16568862499843817, - "eta_squared": 0.3454157417517599, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3630879589982214, - "eta_squared": 0.3454157417517599, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9780919278280775, - "roc_auc": 0.9997541985544217, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9896288029100528, - "worst_group_fpr": 0.1326530612244898, - "n_models": 1, - "seconds": 0.163802625000244, - "eta_squared": 0.34102334572506343, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.16768716699880315, - "eta_squared": 0.34102334572506343, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3686439580051228, - "eta_squared": 0.34102334572506343, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.16696858299837913, - "eta_squared": 0.34453696882915624, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, - "n_models": 1, - "seconds": 0.16658858400478493, - "eta_squared": 0.34453696882915624, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3819815000024391, - "eta_squared": 0.34453696882915624, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725625, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.17264137500023935, - "eta_squared": 0.3456480927214205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.16875304200220853, - "eta_squared": 0.3456480927214205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3209401249987422, - "eta_squared": 0.3456480927214205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.975828562183688, - "roc_auc": 0.9997054811507937, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9818617724867723, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.1670529590046499, - "eta_squared": 0.34002683577528714, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.1904448749992298, - "eta_squared": 0.34002683577528714, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.3737939999991795, - "eta_squared": 0.34002683577528714, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9719844664016125, - "roc_auc": 0.9995460423752835, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.985890903078403, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.16656937500374625, - "eta_squared": 0.3305219088439165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.1735560000015539, - "eta_squared": 0.3305219088439165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3466816660002223, - "eta_squared": 0.3305219088439165, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.16413079199992353, - "eta_squared": 0.32973445787453604, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.1655842920008581, - "eta_squared": 0.32973445787453604, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3275695420015836, - "eta_squared": 0.32973445787453604, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9971973944928514, - "roc_auc": 0.9999490681689343, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, - "n_models": 1, - "seconds": 0.17034458300622646, - "eta_squared": 0.3446151723429949, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.16753458400489762, - "eta_squared": 0.3446151723429949, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3903519999948912, - "eta_squared": 0.3446151723429949, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9997841047394043, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.1745359999986249, - "eta_squared": 0.3507338333119392, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.05612244897959184, - "n_models": 1, - "seconds": 0.1656812919973163, - "eta_squared": 0.3507338333119392, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.354504041999462, - "eta_squared": 0.3507338333119392, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9987530543910215, - "roc_auc": 0.999975641298186, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.17035816599673126, - "eta_squared": 0.33856980964700606, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.16907495800114702, - "eta_squared": 0.33856980964700606, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3467641250026645, - "eta_squared": 0.33856980964700606, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9981328388755406, - "roc_auc": 0.9999645691609976, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.17213825000362704, - "eta_squared": 0.3494306148914419, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.1726834580040304, - "eta_squared": 0.3494306148914419, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3647090419981396, - "eta_squared": 0.3494306148914419, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9503620548777874, - "roc_auc": 0.999320170776644, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9767184493746993, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.16600250000192318, - "eta_squared": 0.33458322121718237, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9991081986024108, - "roc_auc": 0.9999822845804989, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.16288820800400572, - "eta_squared": 0.33458322121718237, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.5557989579974674, - "eta_squared": 0.33458322121718237, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9857423520840152, - "roc_auc": 0.9997962726757371, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9918568121693121, - "worst_group_fpr": 0.13010204081632654, - "n_models": 1, - "seconds": 0.19819208300032187, - "eta_squared": 0.3490755282832429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.1862407090011402, - "eta_squared": 0.3490755282832429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4572139169977163, - "eta_squared": 0.3490755282832429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9274545831517811, - "roc_auc": 0.9992714533730159, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9602749969937469, - "worst_group_fpr": 0.1096938775510204, - "n_models": 1, - "seconds": 0.16133904100570362, - "eta_squared": 0.3333216444268115, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.1783835839960375, - "eta_squared": 0.3333216444268115, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9082358809082258, - "roc_auc": 0.9985362634637187, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9244201689514191, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.348618667005212, - "eta_squared": 0.3333216444268115, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.986146811966324, - "roc_auc": 0.999796272675737, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.16729599999962375, - "eta_squared": 0.34143017232311446, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.16671287499775644, - "eta_squared": 0.34143017232311446, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.150", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.15, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4168364169963752, - "eta_squared": 0.34143017232311446, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, - "n_models": 1, - "seconds": 0.16023679100180743, - "eta_squared": 0.2935663417355178, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.1728855420005857, - "eta_squared": 0.2935663417355178, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.361629166000057, - "eta_squared": 0.2935663417355178, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.16881350000039674, - "eta_squared": 0.288953232932902, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.17742975000146544, - "eta_squared": 0.288953232932902, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9988020366397061, - "roc_auc": 0.999975641298186, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4125322500040056, - "eta_squared": 0.288953232932902, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9997874149659862, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.18944229099724907, - "eta_squared": 0.29120655697879977, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.1854238749947399, - "eta_squared": 0.29120655697879977, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4347965830020257, - "eta_squared": 0.29120655697879977, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.17028995799773838, - "eta_squared": 0.2923445749968756, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.17227354199712863, - "eta_squared": 0.2923445749968756, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3645374170009745, - "eta_squared": 0.2923445749968756, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.1613175830061664, - "eta_squared": 0.29078029700670166, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.16172791699500522, - "eta_squared": 0.29078029700670166, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9994516328453015, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.345396291995712, - "eta_squared": 0.29078029700670166, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.17803733300388558, - "eta_squared": 0.282343864590447, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.16395645899319788, - "eta_squared": 0.282343864590447, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9968256172342184, - "roc_auc": 0.9999402104591837, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3976950829965062, - "eta_squared": 0.282343864590447, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9980276421576781, - "roc_auc": 0.9999623547335601, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.17328049999923678, - "eta_squared": 0.2827224241416107, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.1606982919984148, - "eta_squared": 0.2827224241416107, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9970566287270858, - "roc_auc": 0.9999468537414966, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3746588750000228, - "eta_squared": 0.2827224241416107, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.16199141600372968, - "eta_squared": 0.2957817245569126, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.1711746669971035, - "eta_squared": 0.2957817245569126, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9984642313812274, - "roc_auc": 0.999968998015873, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3538777079957072, - "eta_squared": 0.2957817245569126, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.16894041700288653, - "eta_squared": 0.2975501660008078, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.058673469387755105, - "n_models": 1, - "seconds": 0.1791822500017588, - "eta_squared": 0.2975501660008078, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9977421731790774, - "roc_auc": 0.9999579258786848, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3465448749993811, - "eta_squared": 0.2975501660008078, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.1691537079968839, - "eta_squared": 0.28690079185598694, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.16600350000226172, - "eta_squared": 0.28690079185598694, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4231247080024332, - "eta_squared": 0.28690079185598694, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.19377870799507946, - "eta_squared": 0.2932065449666956, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, - "n_models": 1, - "seconds": 0.19929208300163737, - "eta_squared": 0.2932065449666956, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.414609458995983, - "eta_squared": 0.2932065449666956, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.1728825419995701, - "eta_squared": 0.2919163375746864, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07142857142857142, - "n_models": 1, - "seconds": 0.17778083399753086, - "eta_squared": 0.2919163375746864, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9947936280692428, - "roc_auc": 0.9999092084750567, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4157218749969616, - "eta_squared": 0.2919163375746864, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9995636400137603, - "roc_auc": 0.9999911422902494, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.18477799999527633, - "eta_squared": 0.2964580999866939, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.061224489795918366, - "n_models": 1, - "seconds": 0.1686459999982617, - "eta_squared": 0.2964580999866939, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3955319169981522, - "eta_squared": 0.2964580999866939, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.1688257080022595, - "eta_squared": 0.29175866652881005, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.16431325000303332, - "eta_squared": 0.29175866652881005, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9986478576731588, - "roc_auc": 0.9999734268707482, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4059872499929043, - "eta_squared": 0.29175866652881005, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.18456200000218814, - "eta_squared": 0.28829596838598986, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.06377551020408163, - "n_models": 1, - "seconds": 0.17934149999928195, - "eta_squared": 0.28829596838598986, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.2, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3846435000014026, - "eta_squared": 0.28829596838598986, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9651287441700821, - "roc_auc": 0.999399890164399, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9892361111111111, - "worst_group_fpr": 0.16581632653061223, - "n_models": 1, - "seconds": 0.17997529199783457, - "eta_squared": 0.4778476892757978, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.16583841700048652, - "eta_squared": 0.4778476892757978, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3948089169934974, - "eta_squared": 0.4778476892757978, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9690309915092666, - "roc_auc": 0.9996988378684808, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9844730790043291, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.17768600000272272, - "eta_squared": 0.4744098790454379, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.17603745899396017, - "eta_squared": 0.4744098790454379, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3801205000054324, - "eta_squared": 0.4744098790454379, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9818643368287174, - "roc_auc": 0.9996855513038548, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9876602564102565, - "worst_group_fpr": 0.16071428571428573, - "n_models": 1, - "seconds": 0.19598387500445824, - "eta_squared": 0.4772875504667477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.16780179199849954, - "eta_squared": 0.4772875504667477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3479372919973684, - "eta_squared": 0.4772875504667477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9974745112652791, - "roc_auc": 0.9999534970238095, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.16843795799650252, - "eta_squared": 0.47920852546400317, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.1729825840011472, - "eta_squared": 0.47920852546400317, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3807125000021188, - "eta_squared": 0.47920852546400317, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9652492766036631, - "roc_auc": 0.9996501204648526, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.967104828042328, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.16445862500404473, - "eta_squared": 0.47375497337598166, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.16723541599640157, - "eta_squared": 0.47375497337598166, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.3959956249964307, - "eta_squared": 0.47375497337598166, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9246003273961511, - "roc_auc": 0.9992116638321995, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9771111411736411, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.17050491699774284, - "eta_squared": 0.4645660409992461, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.17489291699894238, - "eta_squared": 0.4645660409992461, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.431837334006559, - "eta_squared": 0.4645660409992461, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.17707912499463418, - "eta_squared": 0.46373875898462746, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, - "n_models": 1, - "seconds": 0.16968387499946402, - "eta_squared": 0.46373875898462746, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4110775420049322, - "eta_squared": 0.46373875898462746, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9835976221998108, - "roc_auc": 0.9997519841269842, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9922401094276094, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.16894850000244332, - "eta_squared": 0.47745063229238155, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, - "n_models": 1, - "seconds": 0.157957333001832, - "eta_squared": 0.47745063229238155, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.378067874997214, - "eta_squared": 0.47745063229238155, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9768890335189591, - "roc_auc": 0.9996501204648526, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9937375992063492, - "worst_group_fpr": 0.15306122448979592, - "n_models": 1, - "seconds": 0.16433320800570073, - "eta_squared": 0.48200866700409517, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.1653154169980553, - "eta_squared": 0.48200866700409517, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3574518329987768, - "eta_squared": 0.48200866700409517, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9893053104296298, - "roc_auc": 0.9998117736678005, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.16071428571428573, - "n_models": 1, - "seconds": 0.16279591700003948, - "eta_squared": 0.4722715287711093, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.17586354200466303, - "eta_squared": 0.4722715287711093, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.394391750000068, - "eta_squared": 0.4722715287711093, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9855322722251981, - "roc_auc": 0.9998361323696145, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.17189950000465615, - "eta_squared": 0.4809671284581783, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.17399262500111945, - "eta_squared": 0.4809671284581783, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3464244999995572, - "eta_squared": 0.4809671284581783, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9115082120378382, - "roc_auc": 0.999085441468254, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9604564995189994, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.17778216599981533, - "eta_squared": 0.469061169841381, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.0663265306122449, - "n_models": 1, - "seconds": 0.17343408299348084, - "eta_squared": 0.469061169841381, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.38340474999859, - "eta_squared": 0.469061169841381, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.992342976560897, - "roc_auc": 0.9998671343537415, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.16625833300349768, - "eta_squared": 0.4822963153336835, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.16770204199565342, - "eta_squared": 0.4822963153336835, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3579438329979894, - "eta_squared": 0.4822963153336835, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9224361945178026, - "roc_auc": 0.9990876558956916, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9789299242424242, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.16845174999616574, - "eta_squared": 0.4673396279875538, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.16399837499920977, - "eta_squared": 0.4673396279875538, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9088206216751, - "roc_auc": 0.9985429067460317, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.363436792002176, - "eta_squared": 0.4673396279875538, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.974059049230118, - "roc_auc": 0.9996833368764173, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.17070308300026227, - "eta_squared": 0.4750062627828297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, - "n_models": 1, - "seconds": 0.16764712500298629, - "eta_squared": 0.4750062627828297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.200", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.2, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3769224999996368, - "eta_squared": 0.4750062627828297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18877551020408162, - "n_models": 1, - "seconds": 0.16629520800051978, - "eta_squared": 0.3927094336787097, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.16802812500100117, - "eta_squared": 0.3927094336787097, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.365570124995429, - "eta_squared": 0.3927094336787097, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.17283054199651815, - "eta_squared": 0.38945558020781024, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, - "n_models": 1, - "seconds": 0.1684383340034401, - "eta_squared": 0.38945558020781024, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.999137094907938, - "roc_auc": 0.9999822845804989, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3717942499933997, - "eta_squared": 0.38945558020781024, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9994629892108767, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, - "n_models": 1, - "seconds": 0.1766787500018836, - "eta_squared": 0.391417879351872, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.1647379580026609, - "eta_squared": 0.391417879351872, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9996843434343431, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3938766670034966, - "eta_squared": 0.391417879351872, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18112244897959184, - "n_models": 1, - "seconds": 0.17747354199673282, - "eta_squared": 0.3925992743462009, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, - "n_models": 1, - "seconds": 0.1662358329995186, - "eta_squared": 0.3925992743462009, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3559329999989131, - "eta_squared": 0.3925992743462009, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, - "n_models": 1, - "seconds": 0.172857166005997, - "eta_squared": 0.39013809068873967, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.17408391600474715, - "eta_squared": 0.39013809068873967, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9994516328453015, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3925372079975205, - "eta_squared": 0.39013809068873967, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.16980783300095936, - "eta_squared": 0.385337437465984, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.17061158300202806, - "eta_squared": 0.385337437465984, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9973971597648099, - "roc_auc": 0.9999512825963718, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.390012999996543, - "eta_squared": 0.385337437465984, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9991081986024108, - "roc_auc": 0.9999822845804989, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, - "n_models": 1, - "seconds": 0.17900604200258385, - "eta_squared": 0.38448012991671376, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.16348675000335788, - "eta_squared": 0.38448012991671376, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9978740297191621, - "roc_auc": 0.9999601403061225, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.386373208995792, - "eta_squared": 0.38448012991671376, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17346938775510204, - "n_models": 1, - "seconds": 0.16611620799812954, - "eta_squared": 0.3945500497199951, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.17003979199944297, - "eta_squared": 0.3945500497199951, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9984642313812273, - "roc_auc": 0.9999689980158729, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4195754999964265, - "eta_squared": 0.3945500497199951, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16326530612244897, - "n_models": 1, - "seconds": 0.17566954100038856, - "eta_squared": 0.39452480740459633, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, - "n_models": 1, - "seconds": 0.16447487500408897, - "eta_squared": 0.39452480740459633, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9977421731790774, - "roc_auc": 0.9999579258786848, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3837716659982107, - "eta_squared": 0.39452480740459633, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, - "n_models": 1, - "seconds": 0.16298408299917355, - "eta_squared": 0.3881032222309504, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.1777255420020083, - "eta_squared": 0.3881032222309504, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3902658750012051, - "eta_squared": 0.3881032222309504, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.17622975000267616, - "eta_squared": 0.39197938133375715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.15857129199866904, - "eta_squared": 0.39197938133375715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3659195410000393, - "eta_squared": 0.39197938133375715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.1675723329972243, - "eta_squared": 0.39171866046305087, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.1658329169949866, - "eta_squared": 0.39171866046305087, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9953829558217713, - "roc_auc": 0.9999180661848073, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3530025000000023, - "eta_squared": 0.39171866046305087, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16326530612244897, - "n_models": 1, - "seconds": 0.1654347920048167, - "eta_squared": 0.39513960651559954, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.17483304099732777, - "eta_squared": 0.39513960651559954, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4074117500058492, - "eta_squared": 0.39513960651559954, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.16447770800004946, - "eta_squared": 0.39167748869152313, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.1615245409993804, - "eta_squared": 0.39167748869152313, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9990030018845482, - "roc_auc": 0.9999800701530611, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.376486749999458, - "eta_squared": 0.39167748869152313, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17346938775510204, - "n_models": 1, - "seconds": 0.170403834003082, - "eta_squared": 0.38870468849861617, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.16837604100146564, - "eta_squared": 0.38870468849861617, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.3, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.379614165998646, - "eta_squared": 0.38870468849861617, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.8685479265604363, - "roc_auc": 0.9984078266723357, - "precision_at_n": 0.875, - "macro_pr_auc": 0.9758969907407407, - "worst_group_fpr": 0.1989795918367347, - "n_models": 1, - "seconds": 0.16000670899666147, - "eta_squared": 0.6640971283653139, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.17192883300594985, - "eta_squared": 0.6640971283653139, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.367972500003816, - "eta_squared": 0.6640971283653139, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9435913175655869, - "roc_auc": 0.9994109623015873, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9786458333333333, - "worst_group_fpr": 0.18622448979591838, - "n_models": 1, - "seconds": 0.17479149999417132, - "eta_squared": 0.6621686277077374, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.17065937499864958, - "eta_squared": 0.6621686277077374, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3877288329967996, - "eta_squared": 0.6621686277077374, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9567273961682181, - "roc_auc": 0.9993378861961452, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9937375992063492, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.16829045800113818, - "eta_squared": 0.6640597584313623, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.16952591599692823, - "eta_squared": 0.6640597584313623, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3698817499971483, - "eta_squared": 0.6640597584313623, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9937583637293692, - "roc_auc": 0.999895921910431, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.18112244897959184, - "n_models": 1, - "seconds": 0.1770604170014849, - "eta_squared": 0.6662381870514966, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.1717158749961527, - "eta_squared": 0.6662381870514966, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3958179999972344, - "eta_squared": 0.6662381870514966, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9587057434347515, - "roc_auc": 0.9995150403911565, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9827824374699374, - "worst_group_fpr": 0.16326530612244897, - "n_models": 1, - "seconds": 0.17357199999969453, - "eta_squared": 0.6620305481663469, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, - "n_models": 1, - "seconds": 0.17166254200128606, - "eta_squared": 0.6620305481663469, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.3588842079989263, - "eta_squared": 0.6620305481663469, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9309252336755022, - "roc_auc": 0.9992913832199548, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9934027777777779, - "worst_group_fpr": 0.1989795918367347, - "n_models": 1, - "seconds": 0.1715607500009355, - "eta_squared": 0.6550141848809792, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.1753054999935557, - "eta_squared": 0.6550141848809792, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4161360419966513, - "eta_squared": 0.6550141848809792, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9959788857666181, - "roc_auc": 0.9999224950396824, - "precision_at_n": 0.96875, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17346938775510204, - "n_models": 1, - "seconds": 0.17546308400051203, - "eta_squared": 0.6541683708984072, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.17512133400305174, - "eta_squared": 0.6541683708984072, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.368081250002433, - "eta_squared": 0.6541683708984072, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9710655795109718, - "roc_auc": 0.9996014030612246, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9951264880952381, - "worst_group_fpr": 0.19642857142857142, - "n_models": 1, - "seconds": 0.16607929199381033, - "eta_squared": 0.6641774551533273, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.17151229199953377, - "eta_squared": 0.6641774551533273, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3807262499976787, - "eta_squared": 0.6641774551533273, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.983198469390041, - "roc_auc": 0.9997896293934241, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.19387755102040816, - "n_models": 1, - "seconds": 0.17668508300266694, - "eta_squared": 0.666609652326715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, - "n_models": 1, - "seconds": 0.17483558299863944, - "eta_squared": 0.666609652326715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3628304160010885, - "eta_squared": 0.666609652326715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9881947804279476, - "roc_auc": 0.9998339179421769, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.17091836734693877, - "n_models": 1, - "seconds": 0.17356795799423708, - "eta_squared": 0.6607993707927986, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, - "n_models": 1, - "seconds": 0.17469100000016624, - "eta_squared": 0.6607993707927986, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3728247500039288, - "eta_squared": 0.6607993707927986, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.959482208444036, - "roc_auc": 0.9995283269557823, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9926669973544974, - "worst_group_fpr": 0.15816326530612246, - "n_models": 1, - "seconds": 0.16373237499647075, - "eta_squared": 0.6655080804629401, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.17313954099518014, - "eta_squared": 0.6655080804629401, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3540781249976135, - "eta_squared": 0.6655080804629401, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.8950772036231616, - "roc_auc": 0.9989415036848073, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9716874849687348, - "worst_group_fpr": 0.16071428571428573, - "n_models": 1, - "seconds": 0.17875016600009985, - "eta_squared": 0.6590680406076661, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.16972233300475636, - "eta_squared": 0.6590680406076661, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.350839624996297, - "eta_squared": 0.6590680406076661, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9697032509277987, - "roc_auc": 0.9995991886337869, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9950810185185185, - "worst_group_fpr": 0.18877551020408162, - "n_models": 1, - "seconds": 0.1595385000036913, - "eta_squared": 0.6680621080435837, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.16372979099833174, - "eta_squared": 0.6680621080435837, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3488865839972277, - "eta_squared": 0.6680621080435837, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.8854033842732889, - "roc_auc": 0.9987665639172335, - "precision_at_n": 0.875, - "macro_pr_auc": 0.9717581319143819, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.1792242500014254, - "eta_squared": 0.6571877895615115, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.17895816700183786, - "eta_squared": 0.6571877895615115, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3684294999984559, - "eta_squared": 0.6571877895615115, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9656305339974596, - "roc_auc": 0.9996036174886621, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9923776455026455, - "worst_group_fpr": 0.1913265306122449, - "n_models": 1, - "seconds": 0.17626054200081853, - "eta_squared": 0.6627275317190392, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.16547554099815898, - "eta_squared": 0.6627275317190392, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.300", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.3, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4033026670003892, - "eta_squared": 0.6627275317190392, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21173469387755103, - "n_models": 1, - "seconds": 0.1722321670022211, - "eta_squared": 0.44964427142161606, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.17166645899851574, - "eta_squared": 0.44964427142161606, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3952178340041428, - "eta_squared": 0.44964427142161606, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.1660061250004219, - "eta_squared": 0.4469631140734328, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.16690162499435246, - "eta_squared": 0.4469631140734328, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9989319838882099, - "roc_auc": 0.9999778557256236, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3839427919956506, - "eta_squared": 0.4469631140734328, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18622448979591838, - "n_models": 1, - "seconds": 0.17876699999760604, - "eta_squared": 0.449549676187598, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.17536795899650315, - "eta_squared": 0.449549676187598, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4285302920034155, - "eta_squared": 0.449549676187598, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20918367346938777, - "n_models": 1, - "seconds": 0.1689662920034607, - "eta_squared": 0.45043297969341634, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.16670979099581018, - "eta_squared": 0.45043297969341634, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4172910420020344, - "eta_squared": 0.45043297969341634, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.17847641700063832, - "eta_squared": 0.4473170350696837, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.18600770799821476, - "eta_squared": 0.4473170350696837, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9994516328453015, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3435870000030263, - "eta_squared": 0.4473170350696837, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.17356312499759952, - "eta_squared": 0.4449952275639174, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.16124245800165227, - "eta_squared": 0.4449952275639174, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.997792771646775, - "roc_auc": 0.9999579258786848, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3857422920045792, - "eta_squared": 0.4449952275639174, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9994516328453015, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.1697561249966384, - "eta_squared": 0.4435643816373198, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.15726891700614942, - "eta_squared": 0.4435643816373198, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9978990413346327, - "roc_auc": 0.9999601403061225, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9948950066137566, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3263949579995824, - "eta_squared": 0.4435643816373198, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.19642857142857142, - "n_models": 1, - "seconds": 0.16596974999993108, - "eta_squared": 0.4514009747359702, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.16418912500375882, - "eta_squared": 0.4514009747359702, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9984642313812274, - "roc_auc": 0.9999689980158731, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3550333340026555, - "eta_squared": 0.4514009747359702, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1913265306122449, - "n_models": 1, - "seconds": 0.1597828340018168, - "eta_squared": 0.4503463925177883, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.16455620900524082, - "eta_squared": 0.4503463925177883, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9973362833817404, - "roc_auc": 0.9999512825963719, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4011888330060174, - "eta_squared": 0.4503463925177883, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1683673469387755, - "n_models": 1, - "seconds": 0.16978279199975077, - "eta_squared": 0.4460711711615254, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.17047620799712604, - "eta_squared": 0.4460711711615254, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3659657090029214, - "eta_squared": 0.4460711711615254, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.16213725000125123, - "eta_squared": 0.44924129916852445, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.16343062500527594, - "eta_squared": 0.44924129916852445, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3577794580050977, - "eta_squared": 0.44924129916852445, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9996789080215417, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.14795918367346939, - "n_models": 1, - "seconds": 0.18308925000019372, - "eta_squared": 0.44859304750127427, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, - "n_models": 1, - "seconds": 0.15952333299355814, - "eta_squared": 0.44859304750127427, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9950648711286866, - "roc_auc": 0.999913637329932, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3622961250002845, - "eta_squared": 0.44859304750127427, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18112244897959184, - "n_models": 1, - "seconds": 0.17267058399738744, - "eta_squared": 0.4514629140297388, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07397959183673469, - "n_models": 1, - "seconds": 0.16739333299483405, - "eta_squared": 0.4514629140297388, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3257128330005798, - "eta_squared": 0.4514629140297388, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9997841047394044, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.165811291000864, - "eta_squared": 0.4491049550478841, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.17129149999527726, - "eta_squared": 0.4491049550478841, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.999233211489758, - "roc_auc": 0.9999844990079364, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3227324589970522, - "eta_squared": 0.4491049550478841, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1913265306122449, - "n_models": 1, - "seconds": 0.16286304200184532, - "eta_squared": 0.44655634205910016, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.17305812499398598, - "eta_squared": 0.44655634205910016, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.4, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3466750830048113, - "eta_squared": 0.44655634205910016, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.8767362769226793, - "roc_auc": 0.9984986181972789, - "precision_at_n": 0.8854166666666666, - "macro_pr_auc": 0.9861111111111112, - "worst_group_fpr": 0.23214285714285715, - "n_models": 1, - "seconds": 0.16523579199565575, - "eta_squared": 0.7723857444189095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16071428571428573, - "n_models": 1, - "seconds": 0.16723662499862257, - "eta_squared": 0.7723857444189095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3786440420008148, - "eta_squared": 0.7723857444189095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9248102078838638, - "roc_auc": 0.999282525510204, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9731714466089466, - "worst_group_fpr": 0.2066326530612245, - "n_models": 1, - "seconds": 0.1626497079996625, - "eta_squared": 0.7711687637922273, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.1611572920010076, - "eta_squared": 0.7711687637922273, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4567947500036098, - "eta_squared": 0.7711687637922273, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.8950775411242928, - "roc_auc": 0.998813066893424, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9839725378787879, - "worst_group_fpr": 0.2193877551020408, - "n_models": 1, - "seconds": 0.16679279199888697, - "eta_squared": 0.772529544751337, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.15794870800164063, - "eta_squared": 0.772529544751337, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9767568942402173, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.380221416002314, - "eta_squared": 0.772529544751337, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.8759932708819158, - "roc_auc": 0.9980269451530612, - "precision_at_n": 0.7916666666666666, - "macro_pr_auc": 0.9796400534851623, - "worst_group_fpr": 0.23469387755102042, - "n_models": 1, - "seconds": 0.17111045800265856, - "eta_squared": 0.7743402728249429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.16481958399526775, - "eta_squared": 0.7743402728249429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3571281249969616, - "eta_squared": 0.7743402728249429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.909387757786931, - "roc_auc": 0.9989791489512472, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9601960828523328, - "worst_group_fpr": 0.19642857142857142, - "n_models": 1, - "seconds": 0.16465991699806182, - "eta_squared": 0.7712908836823555, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.166659125003207, - "eta_squared": 0.7712908836823555, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.398124542000005, - "eta_squared": 0.7712908836823555, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9090194390578653, - "roc_auc": 0.9991053713151927, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.992205710955711, - "worst_group_fpr": 0.23214285714285715, - "n_models": 1, - "seconds": 0.16571400000248104, - "eta_squared": 0.7661755359039057, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.17354329100635368, - "eta_squared": 0.7661755359039057, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4361895830006688, - "eta_squared": 0.7661755359039057, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9886393372479427, - "roc_auc": 0.9997940582482994, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.18052250000619097, - "eta_squared": 0.765390043535227, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.173466083004314, - "eta_squared": 0.765390043535227, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.9007839999976568, - "eta_squared": 0.765390043535227, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.867928879095778, - "roc_auc": 0.9982063137755103, - "precision_at_n": 0.8333333333333334, - "macro_pr_auc": 0.983447570947571, - "worst_group_fpr": 0.22448979591836735, - "n_models": 1, - "seconds": 0.178036917001009, - "eta_squared": 0.7725358121772077, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.15969991699967068, - "eta_squared": 0.7725358121772077, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3786372500035213, - "eta_squared": 0.7725358121772077, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.946962425632692, - "roc_auc": 0.9993644593253969, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9911789021164021, - "worst_group_fpr": 0.22193877551020408, - "n_models": 1, - "seconds": 0.15973129200574476, - "eta_squared": 0.7740110430937635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.1722696659999201, - "eta_squared": 0.7740110430937635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.4021838749977178, - "eta_squared": 0.7740110430937635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9838845010577177, - "roc_auc": 0.9997298398526078, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9979166666666667, - "worst_group_fpr": 0.19642857142857142, - "n_models": 1, - "seconds": 0.16683854199800408, - "eta_squared": 0.7703322802500099, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.16663541599700693, - "eta_squared": 0.7703322802500099, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3506795840003178, - "eta_squared": 0.7703322802500099, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9422513950453302, - "roc_auc": 0.9994153911564626, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9862959956709956, - "worst_group_fpr": 0.18877551020408162, - "n_models": 1, - "seconds": 0.16146966700034682, - "eta_squared": 0.7727219503559253, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.16613545799918938, - "eta_squared": 0.7727219503559253, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3736709590011742, - "eta_squared": 0.7727219503559253, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.847124125101395, - "roc_auc": 0.9986802012471656, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9559132996632996, - "worst_group_fpr": 0.1913265306122449, - "n_models": 1, - "seconds": 0.17574216699722456, - "eta_squared": 0.769516047151419, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.16795120800088625, - "eta_squared": 0.769516047151419, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3588489169997047, - "eta_squared": 0.769516047151419, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9870162732643589, - "roc_auc": 0.9997918438208616, - "precision_at_n": 0.96875, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21683673469387754, - "n_models": 1, - "seconds": 0.1588478749981732, - "eta_squared": 0.7753234694608296, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.1671964580018539, - "eta_squared": 0.7753234694608296, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3748501669979305, - "eta_squared": 0.7753234694608296, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.8824466841761875, - "roc_auc": 0.9982483878968255, - "precision_at_n": 0.8125, - "macro_pr_auc": 0.988420664983165, - "worst_group_fpr": 0.21173469387755103, - "n_models": 1, - "seconds": 0.15993070900003659, - "eta_squared": 0.7678020127711631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.16797495899663772, - "eta_squared": 0.7678020127711631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.383623333000287, - "eta_squared": 0.7678020127711631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9087040584314307, - "roc_auc": 0.9991275155895691, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.978131764069264, - "worst_group_fpr": 0.20918367346938777, - "n_models": 1, - "seconds": 0.17611595900234533, - "eta_squared": 0.7715725614681234, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.17582254100125283, - "eta_squared": 0.7715725614681234, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.400", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.4, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3735412919995724, - "eta_squared": 0.7715725614681234, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, - "n_models": 1, - "seconds": 0.17748766699514817, - "eta_squared": 0.4857308726173069, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.16957866700249724, - "eta_squared": 0.4857308726173069, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3515388750020065, - "eta_squared": 0.4857308726173069, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1989795918367347, - "n_models": 1, - "seconds": 0.1621778330008965, - "eta_squared": 0.4832253415278028, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.1764309160062112, - "eta_squared": 0.4832253415278028, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.998701026538696, - "roc_auc": 0.9999734268707483, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.350652999994054, - "eta_squared": 0.4832253415278028, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22193877551020408, - "n_models": 1, - "seconds": 0.16132041699893307, - "eta_squared": 0.48660199721043473, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.17449829100223724, - "eta_squared": 0.48660199721043473, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.371311083996261, - "eta_squared": 0.48660199721043473, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2193877551020408, - "n_models": 1, - "seconds": 0.1692836670044926, - "eta_squared": 0.4871724603545716, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.17022404100134736, - "eta_squared": 0.4871724603545716, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4007270829970366, - "eta_squared": 0.4871724603545716, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1836734693877551, - "n_models": 1, - "seconds": 0.16118583300703904, - "eta_squared": 0.4835220144166391, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.17091954199713655, - "eta_squared": 0.4835220144166391, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9994516328453015, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3620432499956223, - "eta_squared": 0.4835220144166391, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9997874149659862, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.19387755102040816, - "n_models": 1, - "seconds": 0.17449883299559588, - "eta_squared": 0.4827570725669257, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.16171612500329502, - "eta_squared": 0.4827570725669257, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9982979149346853, - "roc_auc": 0.9999667835884353, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.357203749998007, - "eta_squared": 0.4827570725669257, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1836734693877551, - "n_models": 1, - "seconds": 0.17087583299871767, - "eta_squared": 0.4811183421127033, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.16551808299846016, - "eta_squared": 0.4811183421127033, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9976086261705307, - "roc_auc": 0.9999557114512472, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3765239999993355, - "eta_squared": 0.4811183421127033, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20408163265306123, - "n_models": 1, - "seconds": 0.1601575419990695, - "eta_squared": 0.48752906081921704, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.16759574999741744, - "eta_squared": 0.48752906081921704, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9984642313812274, - "roc_auc": 0.999968998015873, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3940407500049332, - "eta_squared": 0.48752906081921704, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22704081632653061, - "n_models": 1, - "seconds": 0.16296787500323262, - "eta_squared": 0.4858291404456496, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.175631334001082, - "eta_squared": 0.4858291404456496, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9982598713958657, - "roc_auc": 0.9999667835884354, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3584863340001903, - "eta_squared": 0.4858291404456496, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1836734693877551, - "n_models": 1, - "seconds": 0.16996641700097825, - "eta_squared": 0.4825689121407673, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.17013387499901, - "eta_squared": 0.4825689121407673, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3519318329999805, - "eta_squared": 0.4825689121407673, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1683673469387755, - "n_models": 1, - "seconds": 0.1744903750004596, - "eta_squared": 0.48575387175214657, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.17946716699952958, - "eta_squared": 0.48575387175214657, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3484732500000973, - "eta_squared": 0.48575387175214657, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.17183425000257557, - "eta_squared": 0.4844186055700644, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08163265306122448, - "n_models": 1, - "seconds": 0.171107000001939, - "eta_squared": 0.4844186055700644, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.993313456184553, - "roc_auc": 0.9998914930555556, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.37635274999775, - "eta_squared": 0.4844186055700644, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.19387755102040816, - "n_models": 1, - "seconds": 0.16494270800467348, - "eta_squared": 0.48710566059576565, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08673469387755102, - "n_models": 1, - "seconds": 0.17022479099978227, - "eta_squared": 0.48710566059576565, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.388493833001121, - "eta_squared": 0.48710566059576565, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16326530612244897, - "n_models": 1, - "seconds": 0.16758466600003885, - "eta_squared": 0.4855240340488236, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.16482770800212165, - "eta_squared": 0.4855240340488236, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9993384082076202, - "roc_auc": 0.9999867134353742, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3600303330022143, - "eta_squared": 0.4855240340488236, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2066326530612245, - "n_models": 1, - "seconds": 0.16833912500442239, - "eta_squared": 0.48320943890358836, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.16610700000455836, - "eta_squared": 0.48320943890358836, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.5, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.346856915995886, - "eta_squared": 0.48320943890358836, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.822434544724292, - "roc_auc": 0.9979582979024944, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9709017255892256, - "worst_group_fpr": 0.25255102040816324, - "n_models": 1, - "seconds": 0.1645924170006765, - "eta_squared": 0.8366350521930952, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16071428571428573, - "n_models": 1, - "seconds": 0.16921308300516102, - "eta_squared": 0.8366350521930952, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3897101670008851, - "eta_squared": 0.8366350521930952, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9257995892782048, - "roc_auc": 0.9992958120748299, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9727839052287582, - "worst_group_fpr": 0.22704081632653061, - "n_models": 1, - "seconds": 0.1708017499986454, - "eta_squared": 0.8357517463861228, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.16030687500460772, - "eta_squared": 0.8357517463861228, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3927423749992158, - "eta_squared": 0.8357517463861228, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.897939798991374, - "roc_auc": 0.9987488484977324, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9859623015873016, - "worst_group_fpr": 0.24489795918367346, - "n_models": 1, - "seconds": 0.16611995799758006, - "eta_squared": 0.8368167743920167, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.17989779200433986, - "eta_squared": 0.8368167743920167, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3759323339982075, - "eta_squared": 0.8368167743920167, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.8883344359076468, - "roc_auc": 0.998146524234694, - "precision_at_n": 0.8125, - "macro_pr_auc": 0.9882265593203092, - "worst_group_fpr": 0.2602040816326531, - "n_models": 1, - "seconds": 0.172415250002814, - "eta_squared": 0.8382354298920958, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, - "n_models": 1, - "seconds": 0.17383608299860498, - "eta_squared": 0.8382354298920958, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4798644580005202, - "eta_squared": 0.8382354298920958, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9580139997216633, - "roc_auc": 0.9995349702380952, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9852306547619047, - "worst_group_fpr": 0.19387755102040816, - "n_models": 1, - "seconds": 0.17657287499605445, - "eta_squared": 0.8359843649672052, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.17390804200113053, - "eta_squared": 0.8359843649672052, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.4002181249961723, - "eta_squared": 0.8359843649672052, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.8633742318610074, - "roc_auc": 0.9983901112528345, - "precision_at_n": 0.875, - "macro_pr_auc": 0.9907986111111112, - "worst_group_fpr": 0.2576530612244898, - "n_models": 1, - "seconds": 0.17919729099958204, - "eta_squared": 0.8321590306687349, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.16210970800602809, - "eta_squared": 0.8321590306687349, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3868609579949407, - "eta_squared": 0.8321590306687349, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9367261157789226, - "roc_auc": 0.999165160856009, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9952256944444445, - "worst_group_fpr": 0.19642857142857142, - "n_models": 1, - "seconds": 0.16814787500334205, - "eta_squared": 0.8314517992187488, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, - "n_models": 1, - "seconds": 0.1755587919979007, - "eta_squared": 0.8314517992187488, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.385932999997749, - "eta_squared": 0.8314517992187488, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.8402001419290229, - "roc_auc": 0.9976859233276644, - "precision_at_n": 0.8020833333333334, - "macro_pr_auc": 0.980082335964689, - "worst_group_fpr": 0.2423469387755102, - "n_models": 1, - "seconds": 0.17706429099780507, - "eta_squared": 0.8367353195642477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.16533629199693678, - "eta_squared": 0.8367353195642477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.368197916999634, - "eta_squared": 0.8367353195642477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9448111276375721, - "roc_auc": 0.9991585175736962, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.23214285714285715, - "n_models": 1, - "seconds": 0.16717541700199945, - "eta_squared": 0.83775825552584, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.16732737499842187, - "eta_squared": 0.83775825552584, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3741866670025047, - "eta_squared": 0.83775825552584, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9629432775616903, - "roc_auc": 0.9995083971088435, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9958570075757577, - "worst_group_fpr": 0.22704081632653061, - "n_models": 1, - "seconds": 0.16956324999773642, - "eta_squared": 0.8352327138393587, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.16260958399652736, - "eta_squared": 0.8352327138393587, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3512121249950724, - "eta_squared": 0.8352327138393587, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9365376334289945, - "roc_auc": 0.9993489583333333, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.987453403078403, - "worst_group_fpr": 0.21683673469387754, - "n_models": 1, - "seconds": 0.1605099579974194, - "eta_squared": 0.8363958210386342, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.16847175000293646, - "eta_squared": 0.8363958210386342, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3879215829947498, - "eta_squared": 0.8363958210386342, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.7836205496803595, - "roc_auc": 0.9974224064625851, - "precision_at_n": 0.7916666666666666, - "macro_pr_auc": 0.9519114906063435, - "worst_group_fpr": 0.21683673469387754, - "n_models": 1, - "seconds": 0.16175441600353224, - "eta_squared": 0.8349061810858989, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.1734445840047556, - "eta_squared": 0.8349061810858989, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3815032909988076, - "eta_squared": 0.8349061810858989, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9605736093684417, - "roc_auc": 0.9993556016156463, - "precision_at_n": 0.90625, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25510204081632654, - "n_models": 1, - "seconds": 0.17814487499708775, - "eta_squared": 0.838772846460863, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.16565112500393298, - "eta_squared": 0.838772846460863, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3763967080012662, - "eta_squared": 0.838772846460863, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.7758628084809992, - "roc_auc": 0.996569851899093, - "precision_at_n": 0.7395833333333334, - "macro_pr_auc": 0.9730602152477151, - "worst_group_fpr": 0.2372448979591837, - "n_models": 1, - "seconds": 0.15988479099905817, - "eta_squared": 0.8333960800737457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.16298129099595826, - "eta_squared": 0.8333960800737457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.365707333003229, - "eta_squared": 0.8333960800737457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.8645601833146894, - "roc_auc": 0.9986159828514739, - "precision_at_n": 0.8854166666666666, - "macro_pr_auc": 0.9683420745920746, - "worst_group_fpr": 0.23214285714285715, - "n_models": 1, - "seconds": 0.17296958299994003, - "eta_squared": 0.8360376953619889, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.16544758300005924, - "eta_squared": 0.8360376953619889, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.500", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.5, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.356297917001939, - "eta_squared": 0.8360376953619889, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25255102040816324, - "n_models": 1, - "seconds": 0.16672174999985145, - "eta_squared": 0.5109379001151101, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.16512512500048615, - "eta_squared": 0.5109379001151101, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3814167080054176, - "eta_squared": 0.5109379001151101, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21683673469387754, - "n_models": 1, - "seconds": 0.15334912500111386, - "eta_squared": 0.5084467520162148, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.16486483300104737, - "eta_squared": 0.5084467520162148, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9988083913047778, - "roc_auc": 0.999975641298186, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3967122499961988, - "eta_squared": 0.5084467520162148, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, - "n_models": 1, - "seconds": 0.17508479100069962, - "eta_squared": 0.512559151132954, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.15553166700556176, - "eta_squared": 0.512559151132954, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3449602910040994, - "eta_squared": 0.512559151132954, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22704081632653061, - "n_models": 1, - "seconds": 0.16223812499811174, - "eta_squared": 0.5128629688830891, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.16368945799331414, - "eta_squared": 0.5128629688830891, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3854607079993002, - "eta_squared": 0.5128629688830891, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21173469387755103, - "n_models": 1, - "seconds": 0.16409433299850207, - "eta_squared": 0.508762898773635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, - "n_models": 1, - "seconds": 0.16383304099872475, - "eta_squared": 0.508762898773635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9991081986024108, - "roc_auc": 0.9999822845804989, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3557975420044386, - "eta_squared": 0.508762898773635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22704081632653061, - "n_models": 1, - "seconds": 0.1634810409959755, - "eta_squared": 0.5090176444765433, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.16000712499953806, - "eta_squared": 0.5090176444765433, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9980484049901451, - "roc_auc": 0.9999623547335601, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3558610420004698, - "eta_squared": 0.5090176444765433, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9998926116838487, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22193877551020408, - "n_models": 1, - "seconds": 0.17023445799713954, - "eta_squared": 0.5073389623798141, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, - "n_models": 1, - "seconds": 0.15853700000297977, - "eta_squared": 0.5073389623798141, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9981328388755407, - "roc_auc": 0.9999645691609977, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3721635000038077, - "eta_squared": 0.5073389623798141, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23979591836734693, - "n_models": 1, - "seconds": 0.17407870800525416, - "eta_squared": 0.5128210303210285, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12244897959183673, - "n_models": 1, - "seconds": 0.16557087500405032, - "eta_squared": 0.5128210303210285, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9984642313812274, - "roc_auc": 0.999968998015873, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3749887919984758, - "eta_squared": 0.5128210303210285, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2423469387755102, - "n_models": 1, - "seconds": 0.16553512500104262, - "eta_squared": 0.5106686466317674, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.1711596669993014, - "eta_squared": 0.5106686466317674, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9986319303600135, - "roc_auc": 0.9999734268707483, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3863047909981105, - "eta_squared": 0.5106686466317674, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21428571428571427, - "n_models": 1, - "seconds": 0.1531346659976407, - "eta_squared": 0.5078990358901121, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.1592212079995079, - "eta_squared": 0.5078990358901121, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3302593340049498, - "eta_squared": 0.5078990358901121, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.1550901660011732, - "eta_squared": 0.5113451805013186, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.17638620900106616, - "eta_squared": 0.5113451805013186, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.5315749589935876, - "eta_squared": 0.5113451805013186, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.20153061224489796, - "n_models": 1, - "seconds": 0.18538216700108023, - "eta_squared": 0.5093438660938511, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07908163265306123, - "n_models": 1, - "seconds": 0.18358962500497, - "eta_squared": 0.5093438660938511, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9926243465670476, - "roc_auc": 0.9998870642006803, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4807712080000783, - "eta_squared": 0.5093438660938511, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21428571428571427, - "n_models": 1, - "seconds": 0.1798074589969474, - "eta_squared": 0.5120210648371795, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.18321233300230233, - "eta_squared": 0.5120210648371795, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.5098842079969472, - "eta_squared": 0.5120210648371795, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9998926116838486, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.19387755102040816, - "n_models": 1, - "seconds": 0.18296558300062316, - "eta_squared": 0.5109849469679029, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.18526066599588376, - "eta_squared": 0.5109849469679029, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9994516328453016, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4333265829991433, - "eta_squared": 0.5109849469679029, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22704081632653061, - "n_models": 1, - "seconds": 0.15811366600246402, - "eta_squared": 0.5087750149271244, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.1583179170047515, - "eta_squared": 0.5087750149271244, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3897468329960248, - "eta_squared": 0.5087750149271244, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.7582860409826389, - "roc_auc": 0.9965012046485261, - "precision_at_n": 0.7395833333333334, - "macro_pr_auc": 0.9725567256817257, - "worst_group_fpr": 0.2729591836734694, - "n_models": 1, - "seconds": 0.18037091699807206, - "eta_squared": 0.8766830602256049, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1683673469387755, - "n_models": 1, - "seconds": 0.1764629169992986, - "eta_squared": 0.8766830602256049, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.380092749997857, - "eta_squared": 0.8766830602256049, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.8995843945355146, - "roc_auc": 0.9988019947562359, - "precision_at_n": 0.875, - "macro_pr_auc": 0.9790426587301587, - "worst_group_fpr": 0.24489795918367346, - "n_models": 1, - "seconds": 0.1653334160000668, - "eta_squared": 0.8759756368485012, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9996744556165971, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, - "n_models": 1, - "seconds": 0.1696655420018942, - "eta_squared": 0.8759756368485012, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3898012920035399, - "eta_squared": 0.8759756368485012, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.8990980106960048, - "roc_auc": 0.9986802012471655, - "precision_at_n": 0.8854166666666666, - "macro_pr_auc": 0.989186507936508, - "worst_group_fpr": 0.2755102040816326, - "n_models": 1, - "seconds": 0.1781659589978517, - "eta_squared": 0.8768603441135048, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, - "n_models": 1, - "seconds": 0.1716181250012596, - "eta_squared": 0.8768603441135048, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3796445839980152, - "eta_squared": 0.8768603441135048, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.7447341731386434, - "roc_auc": 0.9962133290816326, - "precision_at_n": 0.7291666666666666, - "macro_pr_auc": 0.9704103284832452, - "worst_group_fpr": 0.2780612244897959, - "n_models": 1, - "seconds": 0.1653214170000865, - "eta_squared": 0.8779733389869431, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.17273270900477655, - "eta_squared": 0.8779733389869431, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.384525624998787, - "eta_squared": 0.8779733389869431, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9477638935300299, - "roc_auc": 0.9993201707766439, - "precision_at_n": 0.90625, - "macro_pr_auc": 0.9879453012265512, - "worst_group_fpr": 0.23469387755102042, - "n_models": 1, - "seconds": 0.1635339170024963, - "eta_squared": 0.8762481102453108, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9950325742344305, - "roc_auc": 0.9999180661848073, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.1647775419987738, - "eta_squared": 0.8762481102453108, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.48115091699583, - "eta_squared": 0.8762481102453108, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.6834088297506767, - "roc_auc": 0.9952544820011339, - "precision_at_n": 0.6770833333333334, - "macro_pr_auc": 0.9591807208994708, - "worst_group_fpr": 0.28061224489795916, - "n_models": 1, - "seconds": 0.17019391700159758, - "eta_squared": 0.8732821441751656, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, - "n_models": 1, - "seconds": 0.17503504199703457, - "eta_squared": 0.8732821441751656, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3845166249957401, - "eta_squared": 0.8732821441751656, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.8956222767758246, - "roc_auc": 0.9986359126984127, - "precision_at_n": 0.875, - "macro_pr_auc": 0.990625, - "worst_group_fpr": 0.2423469387755102, - "n_models": 1, - "seconds": 0.16629595900303684, - "eta_squared": 0.8726482673910156, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.16443587499816203, - "eta_squared": 0.8726482673910156, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3696010829953593, - "eta_squared": 0.8726482673910156, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.6513235336107632, - "roc_auc": 0.9946477288832201, - "precision_at_n": 0.6458333333333334, - "macro_pr_auc": 0.9570489183290719, - "worst_group_fpr": 0.25510204081632654, - "n_models": 1, - "seconds": 0.16151991600054316, - "eta_squared": 0.8767223767771402, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.16672608299995773, - "eta_squared": 0.8767223767771402, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3941900830031955, - "eta_squared": 0.8767223767771402, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.8992649074239921, - "roc_auc": 0.9984299709467122, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9932034111721612, - "worst_group_fpr": 0.2627551020408163, - "n_models": 1, - "seconds": 0.17306258300232003, - "eta_squared": 0.8774993715157073, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, - "n_models": 1, - "seconds": 0.16032320899830665, - "eta_squared": 0.8774993715157073, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3418790419964353, - "eta_squared": 0.8774993715157073, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9445820394815596, - "roc_auc": 0.9992183071145124, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9958570075757577, - "worst_group_fpr": 0.25, - "n_models": 1, - "seconds": 0.1642562079941854, - "eta_squared": 0.8756459899827929, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, - "n_models": 1, - "seconds": 0.16373504199873423, - "eta_squared": 0.8756459899827929, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3671592500031693, - "eta_squared": 0.8756459899827929, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.8382088298719095, - "roc_auc": 0.9983258928571428, - "precision_at_n": 0.875, - "macro_pr_auc": 0.9708979677729678, - "worst_group_fpr": 0.24744897959183673, - "n_models": 1, - "seconds": 0.1684582079979009, - "eta_squared": 0.8761496699086859, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.1701111249931273, - "eta_squared": 0.8761496699086859, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4356458749971353, - "eta_squared": 0.8761496699086859, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.6902864089412835, - "roc_auc": 0.9956420068027211, - "precision_at_n": 0.7291666666666666, - "macro_pr_auc": 0.9454585017326057, - "worst_group_fpr": 0.2653061224489796, - "n_models": 1, - "seconds": 0.1596362499985844, - "eta_squared": 0.8755759250473935, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.16964354200172238, - "eta_squared": 0.8755759250473935, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3637192919995869, - "eta_squared": 0.8755759250473935, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.8934586737191591, - "roc_auc": 0.9983037485827665, - "precision_at_n": 0.8333333333333334, - "macro_pr_auc": 0.9951264880952381, - "worst_group_fpr": 0.2627551020408163, - "n_models": 1, - "seconds": 0.16413462499622256, - "eta_squared": 0.8782757121696583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, - "n_models": 1, - "seconds": 0.16557108300185064, - "eta_squared": 0.8782757121696583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.358468708996952, - "eta_squared": 0.8782757121696583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.726617627727997, - "roc_auc": 0.9955445719954649, - "precision_at_n": 0.6979166666666666, - "macro_pr_auc": 0.9701088263588263, - "worst_group_fpr": 0.24744897959183673, - "n_models": 1, - "seconds": 0.1707825419944129, - "eta_squared": 0.8742451489593618, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.994913149994302, - "roc_auc": 0.9999092084750566, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.16297733299870742, - "eta_squared": 0.8742451489593618, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3907979169962346, - "eta_squared": 0.8742451489593618, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.853190539545772, - "roc_auc": 0.9984964037698413, - "precision_at_n": 0.8854166666666666, - "macro_pr_auc": 0.9689207782957783, - "worst_group_fpr": 0.2602040816326531, - "n_models": 1, - "seconds": 0.19305558299674885, - "eta_squared": 0.8761879838657488, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.17603737500030547, - "eta_squared": 0.8761879838657488, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.600", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.6, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3474013329978334, - "eta_squared": 0.8761879838657488, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25, - "n_models": 1, - "seconds": 0.16116825000062818, - "eta_squared": 0.5298890398187, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.17193729199789232, - "eta_squared": 0.5298890398187, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.378871082997648, - "eta_squared": 0.5298890398187, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22704081632653061, - "n_models": 1, - "seconds": 0.16921825000463286, - "eta_squared": 0.52735131605007, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.1556395000006887, - "eta_squared": 0.52735131605007, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9992589075782236, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3399752499972237, - "eta_squared": 0.52735131605007, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.27040816326530615, - "n_models": 1, - "seconds": 0.16008912499819417, - "eta_squared": 0.5320991054125361, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.16749195900047198, - "eta_squared": 0.5320991054125361, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3525973750001867, - "eta_squared": 0.5320991054125361, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25255102040816324, - "n_models": 1, - "seconds": 0.16665166699385736, - "eta_squared": 0.5321841551665233, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.16000091600290034, - "eta_squared": 0.5321841551665233, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3520715839986224, - "eta_squared": 0.5321841551665233, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21428571428571427, - "n_models": 1, - "seconds": 0.15576579199841945, - "eta_squared": 0.5277037636692434, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.1596036670016474, - "eta_squared": 0.5277037636692434, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9994516328453017, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3927896660024999, - "eta_squared": 0.5277037636692434, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23469387755102042, - "n_models": 1, - "seconds": 0.16862491599749774, - "eta_squared": 0.5286644856867957, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.16028712499974063, - "eta_squared": 0.5286644856867957, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9980484049901452, - "roc_auc": 0.99996235473356, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3658714170014719, - "eta_squared": 0.5286644856867957, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2372448979591837, - "n_models": 1, - "seconds": 0.1669780419979361, - "eta_squared": 0.5270205162013888, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.15942808399995556, - "eta_squared": 0.5270205162013888, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9981328388755407, - "roc_auc": 0.9999645691609977, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3874943750051898, - "eta_squared": 0.5270205162013888, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.24489795918367346, - "n_models": 1, - "seconds": 0.16428512499987846, - "eta_squared": 0.5318672897267391, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.15578358300263062, - "eta_squared": 0.5318672897267391, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9986906806565897, - "roc_auc": 0.9999734268707483, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3920086250000168, - "eta_squared": 0.5318672897267391, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.26785714285714285, - "n_models": 1, - "seconds": 0.15572579099534778, - "eta_squared": 0.5293730129243481, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08418367346938775, - "n_models": 1, - "seconds": 0.17435733399906894, - "eta_squared": 0.5293730129243481, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9980042380524953, - "roc_auc": 0.9999623547335601, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3670541669998784, - "eta_squared": 0.5293730129243481, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.21173469387755103, - "n_models": 1, - "seconds": 0.1595689159948961, - "eta_squared": 0.5268459802946263, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, - "n_models": 1, - "seconds": 0.1730810000008205, - "eta_squared": 0.5268459802946263, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3595376250013942, - "eta_squared": 0.5268459802946263, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2066326530612245, - "n_models": 1, - "seconds": 0.16157433300395496, - "eta_squared": 0.5306182183453104, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.15203350000228966, - "eta_squared": 0.5306182183453104, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3960167920013191, - "eta_squared": 0.5306182183453104, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725625, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22704081632653061, - "n_models": 1, - "seconds": 0.16133270799764432, - "eta_squared": 0.5280422190863733, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.07653061224489796, - "n_models": 1, - "seconds": 0.17179475000011735, - "eta_squared": 0.5280422190863733, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.993499468089315, - "roc_auc": 0.9998937074829932, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3524868749955203, - "eta_squared": 0.5280422190863733, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2423469387755102, - "n_models": 1, - "seconds": 0.16467408400058048, - "eta_squared": 0.5307835609735325, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09438775510204081, - "n_models": 1, - "seconds": 0.17430150000291178, - "eta_squared": 0.5307835609735325, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3547277499965276, - "eta_squared": 0.5307835609735325, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2066326530612245, - "n_models": 1, - "seconds": 0.16530779100139625, - "eta_squared": 0.5301452136480338, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.16847454099479364, - "eta_squared": 0.5301452136480338, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9994516328453015, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.371672416004003, - "eta_squared": 0.5301452136480338, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23214285714285715, - "n_models": 1, - "seconds": 0.16930608300026506, - "eta_squared": 0.5279659930517174, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.1642920000012964, - "eta_squared": 0.5279659930517174, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.7, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3668617090006592, - "eta_squared": 0.5279659930517174, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.6681460572719677, - "roc_auc": 0.9949223178854875, - "precision_at_n": 0.6979166666666666, - "macro_pr_auc": 0.9583931021660801, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.1728503330014064, - "eta_squared": 0.9029294791735885, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, - "n_models": 1, - "seconds": 0.1650128340043011, - "eta_squared": 0.9029294791735885, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.347333207995689, - "eta_squared": 0.9029294791735885, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.7308688714872446, - "roc_auc": 0.9965919961734694, - "precision_at_n": 0.7291666666666666, - "macro_pr_auc": 0.9465434419381787, - "worst_group_fpr": 0.25510204081632654, - "n_models": 1, - "seconds": 0.16908604199852562, - "eta_squared": 0.9023260653166297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, - "n_models": 1, - "seconds": 0.17620766600157367, - "eta_squared": 0.9023260653166297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3685417080050684, - "eta_squared": 0.9023260653166297, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.7983959157429952, - "roc_auc": 0.997039310515873, - "precision_at_n": 0.7708333333333334, - "macro_pr_auc": 0.9768409014732544, - "worst_group_fpr": 0.29846938775510207, - "n_models": 1, - "seconds": 0.1641099159969599, - "eta_squared": 0.9030920988447193, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, - "n_models": 1, - "seconds": 0.16917108299821848, - "eta_squared": 0.9030920988447193, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.34316495800158, - "eta_squared": 0.9030920988447193, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.6840190010656133, - "roc_auc": 0.9945259353741497, - "precision_at_n": 0.65625, - "macro_pr_auc": 0.9668358262108262, - "worst_group_fpr": 0.3010204081632653, - "n_models": 1, - "seconds": 0.16739337499893736, - "eta_squared": 0.9039807536694324, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17091836734693877, - "n_models": 1, - "seconds": 0.16177070799312787, - "eta_squared": 0.9039807536694324, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3828187500039348, - "eta_squared": 0.9039807536694324, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.7868633343573769, - "roc_auc": 0.9970127373866214, - "precision_at_n": 0.78125, - "macro_pr_auc": 0.9630104993386244, - "worst_group_fpr": 0.2576530612244898, - "n_models": 1, - "seconds": 0.16557145900151227, - "eta_squared": 0.9026069122444567, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9961735140694566, - "roc_auc": 0.9999335671768708, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, - "n_models": 1, - "seconds": 0.17229137499816716, - "eta_squared": 0.9026069122444567, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.395121375004237, - "eta_squared": 0.9026069122444567, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.6933595662983045, - "roc_auc": 0.9956397923752834, - "precision_at_n": 0.7083333333333334, - "macro_pr_auc": 0.9530006740944241, - "worst_group_fpr": 0.29336734693877553, - "n_models": 1, - "seconds": 0.16351079200103413, - "eta_squared": 0.9002286409429918, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, - "n_models": 1, - "seconds": 0.17506095900171204, - "eta_squared": 0.9002286409429918, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3643528329994297, - "eta_squared": 0.9002286409429918, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.8618262227577957, - "roc_auc": 0.9980911635487528, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9924107142857143, - "worst_group_fpr": 0.24489795918367346, - "n_models": 1, - "seconds": 0.17302979200030677, - "eta_squared": 0.8996587213625616, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.1768632500024978, - "eta_squared": 0.8996587213625616, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4162449580035172, - "eta_squared": 0.8996587213625616, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.684205193809398, - "roc_auc": 0.9949156746031746, - "precision_at_n": 0.6354166666666666, - "macro_pr_auc": 0.969165774547367, - "worst_group_fpr": 0.29336734693877553, - "n_models": 1, - "seconds": 0.16751866700360551, - "eta_squared": 0.9029208108231173, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.1731683750040247, - "eta_squared": 0.9029208108231173, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3723675000001094, - "eta_squared": 0.9029208108231173, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.791864796418778, - "roc_auc": 0.9964458439625851, - "precision_at_n": 0.7395833333333334, - "macro_pr_auc": 0.987832190957191, - "worst_group_fpr": 0.29846938775510207, - "n_models": 1, - "seconds": 0.16881570799887413, - "eta_squared": 0.9035454354972019, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.16889095900114626, - "eta_squared": 0.9035454354972019, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3773021250017337, - "eta_squared": 0.9035454354972019, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.8040889343745524, - "roc_auc": 0.9969440901360545, - "precision_at_n": 0.7604166666666666, - "macro_pr_auc": 0.9774305555555555, - "worst_group_fpr": 0.28316326530612246, - "n_models": 1, - "seconds": 0.1685613329973421, - "eta_squared": 0.9021131136700368, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, - "n_models": 1, - "seconds": 0.17807291699864436, - "eta_squared": 0.9021131136700368, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.4345150419976562, - "eta_squared": 0.9021131136700368, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.728479400397314, - "roc_auc": 0.9970215950963719, - "precision_at_n": 0.8020833333333334, - "macro_pr_auc": 0.9415345806930236, - "worst_group_fpr": 0.26785714285714285, - "n_models": 1, - "seconds": 0.17713350000121864, - "eta_squared": 0.9022494388744451, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.17489637499966193, - "eta_squared": 0.9022494388744451, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3611302090066602, - "eta_squared": 0.9022494388744451, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.611290735134805, - "roc_auc": 0.9935604450113379, - "precision_at_n": 0.6354166666666666, - "macro_pr_auc": 0.9403211805555557, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.16300062499794876, - "eta_squared": 0.9021789744937543, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.17516062499635154, - "eta_squared": 0.9021789744937543, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.385379457999079, - "eta_squared": 0.9021789744937543, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.704252633464105, - "roc_auc": 0.9954050630668934, - "precision_at_n": 0.6979166666666666, - "macro_pr_auc": 0.9739869852369852, - "worst_group_fpr": 0.30357142857142855, - "n_models": 1, - "seconds": 0.17574295799568063, - "eta_squared": 0.9041553072370441, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, - "n_models": 1, - "seconds": 0.17137891700258479, - "eta_squared": 0.9041553072370441, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.349269334001292, - "eta_squared": 0.9041553072370441, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.5794175981215678, - "roc_auc": 0.9927610367063492, - "precision_at_n": 0.5833333333333334, - "macro_pr_auc": 0.9460557960557959, - "worst_group_fpr": 0.2729591836734694, - "n_models": 1, - "seconds": 0.17066850000264822, - "eta_squared": 0.9009953154147747, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9992239393431515, - "roc_auc": 0.9999844990079366, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.17392825000570156, - "eta_squared": 0.9009953154147747, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3921578329973272, - "eta_squared": 0.9009953154147747, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.793114486939759, - "roc_auc": 0.9976194905045352, - "precision_at_n": 0.8333333333333334, - "macro_pr_auc": 0.9674167846042846, - "worst_group_fpr": 0.28061224489795916, - "n_models": 1, - "seconds": 0.17824149999796646, - "eta_squared": 0.9024948788179256, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.16386495799815748, - "eta_squared": 0.9024948788179256, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.700", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.7, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4200830000045244, - "eta_squared": 0.9024948788179256, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2780612244897959, - "n_models": 1, - "seconds": 0.17246666600112803, - "eta_squared": 0.5448994609628225, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.15952704200026346, - "eta_squared": 0.5448994609628225, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3783540420045028, - "eta_squared": 0.5448994609628225, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23979591836734693, - "n_models": 1, - "seconds": 0.15861687500000698, - "eta_squared": 0.5422953822643336, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.16462504199444083, - "eta_squared": 0.5422953822643336, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9995726383336836, - "roc_auc": 0.9999911422902494, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3471067919963389, - "eta_squared": 0.5422953822643336, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.16979625000385568, - "eta_squared": 0.5475800579356785, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.1656924579947372, - "eta_squared": 0.5475800579356785, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3735570419958094, - "eta_squared": 0.5475800579356785, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2653061224489796, - "n_models": 1, - "seconds": 0.16967754200595664, - "eta_squared": 0.5474858301661086, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11989795918367346, - "n_models": 1, - "seconds": 0.1688615419989219, - "eta_squared": 0.5474858301661086, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3673592920022202, - "eta_squared": 0.5474858301661086, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, - "n_models": 1, - "seconds": 0.15921850000449922, - "eta_squared": 0.5426833011841099, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.17815041600260884, - "eta_squared": 0.5426833011841099, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9994516328453017, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3637191250018077, - "eta_squared": 0.5426833011841099, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25, - "n_models": 1, - "seconds": 0.16293445799965411, - "eta_squared": 0.5441570579629037, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.1631988340013777, - "eta_squared": 0.5441570579629037, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9980484049901452, - "roc_auc": 0.99996235473356, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4030500829976518, - "eta_squared": 0.5441570579629037, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2627551020408163, - "n_models": 1, - "seconds": 0.16487941700324882, - "eta_squared": 0.5425795873833327, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, - "n_models": 1, - "seconds": 0.16211316599947168, - "eta_squared": 0.5425795873833327, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9969514320092234, - "roc_auc": 0.999944639314059, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9948950066137566, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3671795839982224, - "eta_squared": 0.5425795873833327, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2602040816326531, - "n_models": 1, - "seconds": 0.16315424999629613, - "eta_squared": 0.5469699481347092, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.1662539579992881, - "eta_squared": 0.5469699481347092, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9986906806565897, - "roc_auc": 0.9999734268707484, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3874576660018647, - "eta_squared": 0.5469699481347092, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2857142857142857, - "n_models": 1, - "seconds": 0.16050458300014725, - "eta_squared": 0.544205346444814, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.17303725000238046, - "eta_squared": 0.544205346444814, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9986319303600136, - "roc_auc": 0.9999734268707483, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.372127290997014, - "eta_squared": 0.544205346444814, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.24489795918367346, - "n_models": 1, - "seconds": 0.17187070799991488, - "eta_squared": 0.5417981360356992, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13520408163265307, - "n_models": 1, - "seconds": 0.15840191699680872, - "eta_squared": 0.5417981360356992, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3960168749981676, - "eta_squared": 0.5417981360356992, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22448979591836735, - "n_models": 1, - "seconds": 0.17027712499839254, - "eta_squared": 0.5458940752362631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.17035249999753432, - "eta_squared": 0.5458940752362631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3717554160029977, - "eta_squared": 0.5458940752362631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2423469387755102, - "n_models": 1, - "seconds": 0.16951775000052294, - "eta_squared": 0.5428380555165477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.16986883299978217, - "eta_squared": 0.5428380555165477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9940807443577351, - "roc_auc": 0.9999025651927437, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3802135000005364, - "eta_squared": 0.5428380555165477, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25510204081632654, - "n_models": 1, - "seconds": 0.16807145799975842, - "eta_squared": 0.5456713636299851, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.16689949999999953, - "eta_squared": 0.5456713636299851, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3508785419980995, - "eta_squared": 0.5456713636299851, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23979591836734693, - "n_models": 1, - "seconds": 0.16799091599386884, - "eta_squared": 0.5453344397432399, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.17053383299935376, - "eta_squared": 0.5453344397432399, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9996744556165972, - "roc_auc": 0.9999933567176871, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3389392909957678, - "eta_squared": 0.5453344397432399, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2576530612244898, - "n_models": 1, - "seconds": 0.16087425000296207, - "eta_squared": 0.5431459755082583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999997, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.17221383399737533, - "eta_squared": 0.5431459755082583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3914548750035465, - "eta_squared": 0.5431459755082583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.6205158398648214, - "roc_auc": 0.9936003047052154, - "precision_at_n": 0.625, - "macro_pr_auc": 0.9638013513243293, - "worst_group_fpr": 0.29846938775510207, - "n_models": 1, - "seconds": 0.16608383399579907, - "eta_squared": 0.9209097965562205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, - "n_models": 1, - "seconds": 0.17551341599755688, - "eta_squared": 0.9209097965562205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.384251082999981, - "eta_squared": 0.9209097965562205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.6365808137468482, - "roc_auc": 0.9937331703514739, - "precision_at_n": 0.6354166666666666, - "macro_pr_auc": 0.9516318369453045, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.1625109579981654, - "eta_squared": 0.920373719914181, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9974733447852492, - "roc_auc": 0.9999534970238095, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, - "n_models": 1, - "seconds": 0.1630079590031528, - "eta_squared": 0.920373719914181, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3787873749970458, - "eta_squared": 0.920373719914181, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.6627120617322612, - "roc_auc": 0.9937929598922902, - "precision_at_n": 0.59375, - "macro_pr_auc": 0.97492784992785, - "worst_group_fpr": 0.3239795918367347, - "n_models": 1, - "seconds": 0.16869162500370294, - "eta_squared": 0.9210567957325315, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1836734693877551, - "n_models": 1, - "seconds": 0.1687801669977489, - "eta_squared": 0.9210567957325315, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.4026547920002486, - "eta_squared": 0.9210567957325315, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.5390877475746735, - "roc_auc": 0.989851279053288, - "precision_at_n": 0.4375, - "macro_pr_auc": 0.9653311965811966, - "worst_group_fpr": 0.3086734693877551, - "n_models": 1, - "seconds": 0.16397204100212548, - "eta_squared": 0.9217812642726251, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.17485808300261851, - "eta_squared": 0.9217812642726251, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4121400840012939, - "eta_squared": 0.9217812642726251, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.7404323466969316, - "roc_auc": 0.9964281285430839, - "precision_at_n": 0.7604166666666666, - "macro_pr_auc": 0.9565724522796891, - "worst_group_fpr": 0.2755102040816326, - "n_models": 1, - "seconds": 0.17475745800038567, - "eta_squared": 0.920649352214294, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, - "n_models": 1, - "seconds": 0.17303454200009583, - "eta_squared": 0.920649352214294, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.3783052500002668, - "eta_squared": 0.920649352214294, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.7417238194824436, - "roc_auc": 0.9957327983276645, - "precision_at_n": 0.6770833333333334, - "macro_pr_auc": 0.9907176157176157, - "worst_group_fpr": 0.29846938775510207, - "n_models": 1, - "seconds": 0.17469762499968056, - "eta_squared": 0.9186868472692276, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16326530612244897, - "n_models": 1, - "seconds": 0.16913675000250805, - "eta_squared": 0.9186868472692276, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3586927500000456, - "eta_squared": 0.9186868472692276, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.6793935016179142, - "roc_auc": 0.9950286104024944, - "precision_at_n": 0.7083333333333334, - "macro_pr_auc": 0.9670454545454544, - "worst_group_fpr": 0.27040816326530615, - "n_models": 1, - "seconds": 0.17535466700064717, - "eta_squared": 0.9181718730647732, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, - "n_models": 1, - "seconds": 0.16288858400366735, - "eta_squared": 0.9181718730647732, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3662952910017339, - "eta_squared": 0.9181718730647732, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.5471876899551111, - "roc_auc": 0.9916958971088435, - "precision_at_n": 0.5520833333333334, - "macro_pr_auc": 0.9524636243386242, - "worst_group_fpr": 0.30357142857142855, - "n_models": 1, - "seconds": 0.17614049999974668, - "eta_squared": 0.9208674523939366, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, - "n_models": 1, - "seconds": 0.16324841599998763, - "eta_squared": 0.9208674523939366, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3721862920065178, - "eta_squared": 0.9208674523939366, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.6437263546024731, - "roc_auc": 0.9928850446428571, - "precision_at_n": 0.5520833333333334, - "macro_pr_auc": 0.9770419973544974, - "worst_group_fpr": 0.30612244897959184, - "n_models": 1, - "seconds": 0.1704562500017346, - "eta_squared": 0.9213876858018002, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9992239393431517, - "roc_auc": 0.9999844990079365, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.16489541699411348, - "eta_squared": 0.9213876858018002, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.394481416005874, - "eta_squared": 0.9213876858018002, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.726224453983664, - "roc_auc": 0.9952699829931974, - "precision_at_n": 0.6979166666666666, - "macro_pr_auc": 0.9709099927849928, - "worst_group_fpr": 0.3010204081632653, - "n_models": 1, - "seconds": 0.1655430829996476, - "eta_squared": 0.9202352791959443, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16326530612244897, - "n_models": 1, - "seconds": 0.17286137499468168, - "eta_squared": 0.9202352791959443, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3797562909967382, - "eta_squared": 0.9202352791959443, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.6766827706413012, - "roc_auc": 0.9955932893990931, - "precision_at_n": 0.71875, - "macro_pr_auc": 0.9492125496031747, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.16753829200024484, - "eta_squared": 0.9201601632705698, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11479591836734694, - "n_models": 1, - "seconds": 0.16889616700063925, - "eta_squared": 0.9201601632705698, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.364785457997641, - "eta_squared": 0.9201601632705698, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.59167460136602, - "roc_auc": 0.9932304953231292, - "precision_at_n": 0.6041666666666666, - "macro_pr_auc": 0.9397987697762353, - "worst_group_fpr": 0.30612244897959184, - "n_models": 1, - "seconds": 0.16093204200296896, - "eta_squared": 0.9203742833505759, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9995636400137603, - "roc_auc": 0.9999911422902494, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.16490308300126344, - "eta_squared": 0.9203742833505759, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4029429169968353, - "eta_squared": 0.9203742833505759, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.6925685701050049, - "roc_auc": 0.9940210459183674, - "precision_at_n": 0.5833333333333334, - "macro_pr_auc": 0.9916200697450698, - "worst_group_fpr": 0.30357142857142855, - "n_models": 1, - "seconds": 0.1594262910002726, - "eta_squared": 0.9218843396708194, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15051020408163265, - "n_models": 1, - "seconds": 0.17699987500236603, - "eta_squared": 0.9218843396708194, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3922457090011449, - "eta_squared": 0.9218843396708194, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.5450910945212057, - "roc_auc": 0.99144566680839, - "precision_at_n": 0.53125, - "macro_pr_auc": 0.9489377552047388, - "worst_group_fpr": 0.28316326530612246, - "n_models": 1, - "seconds": 0.16409912499511847, - "eta_squared": 0.9193097916370743, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9886091097053284, - "roc_auc": 0.9998250602324263, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9975405092592592, - "worst_group_fpr": 0.1377551020408163, - "n_models": 1, - "seconds": 0.16337608300091233, - "eta_squared": 0.9193097916370743, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3948482920022798, - "eta_squared": 0.9193097916370743, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.6187515878913341, - "roc_auc": 0.9938593927154195, - "precision_at_n": 0.6458333333333334, - "macro_pr_auc": 0.9463323577294166, - "worst_group_fpr": 0.3163265306122449, - "n_models": 1, - "seconds": 0.17068183299852535, - "eta_squared": 0.920517468315107, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16326530612244897, - "n_models": 1, - "seconds": 0.16075029200146673, - "eta_squared": 0.920517468315107, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.800", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.8, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3838352919992758, - "eta_squared": 0.920517468315107, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.17201291600213153, - "eta_squared": 0.5572328878968438, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, - "n_models": 1, - "seconds": 0.16330987500259653, - "eta_squared": 0.5572328878968438, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3859826250045444, - "eta_squared": 0.5572328878968438, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25255102040816324, - "n_models": 1, - "seconds": 0.15904233300534543, - "eta_squared": 0.5545595572982635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.15868020800553495, - "eta_squared": 0.5545595572982635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9993553717642547, - "roc_auc": 0.9999867134353742, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3736786670051515, - "eta_squared": 0.5545595572982635, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.3010204081632653, - "n_models": 1, - "seconds": 0.15516070900048362, - "eta_squared": 0.5602958775475205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.16194199999881675, - "eta_squared": 0.5602958775475205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3723653750057565, - "eta_squared": 0.5602958775475205, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2729591836734694, - "n_models": 1, - "seconds": 0.16464475000248058, - "eta_squared": 0.5600535097729571, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, - "n_models": 1, - "seconds": 0.1710237499937648, - "eta_squared": 0.5600535097729571, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3771845829978702, - "eta_squared": 0.5600535097729571, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23469387755102042, - "n_models": 1, - "seconds": 0.15870858299604151, - "eta_squared": 0.5549776244660766, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, - "n_models": 1, - "seconds": 0.15347879200271564, - "eta_squared": 0.5549776244660766, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9994516328453016, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3591966250023688, - "eta_squared": 0.5549776244660766, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.26785714285714285, - "n_models": 1, - "seconds": 0.1636783750000177, - "eta_squared": 0.5568389917399191, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.1575528339963057, - "eta_squared": 0.5568389917399191, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9981739069981774, - "roc_auc": 0.9999645691609977, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.373370834000525, - "eta_squared": 0.5568389917399191, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9998926116838488, - "roc_auc": 0.9999977855725622, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2780612244897959, - "n_models": 1, - "seconds": 0.17131791599967983, - "eta_squared": 0.55533969332111, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14540816326530612, - "n_models": 1, - "seconds": 0.15299862499523442, - "eta_squared": 0.55533969332111, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.997899041334633, - "roc_auc": 0.9999601403061225, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9948950066137566, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4097843749987078, - "eta_squared": 0.55533969332111, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2653061224489796, - "n_models": 1, - "seconds": 0.1579061250013183, - "eta_squared": 0.5593879759814088, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14285714285714285, - "n_models": 1, - "seconds": 0.16645320800307672, - "eta_squared": 0.5593879759814088, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9986906806565897, - "roc_auc": 0.9999734268707484, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3940077090010163, - "eta_squared": 0.5593879759814088, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.3010204081632653, - "n_models": 1, - "seconds": 0.1680378750024829, - "eta_squared": 0.5564035218173411, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.16570941700047115, - "eta_squared": 0.5564035218173411, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9982598713958657, - "roc_auc": 0.9999667835884354, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3501350420046947, - "eta_squared": 0.5564035218173411, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, - "n_models": 1, - "seconds": 0.17021429200394778, - "eta_squared": 0.5540528723307074, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.1611757079954259, - "eta_squared": 0.5540528723307074, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.363703709001129, - "eta_squared": 0.5540528723307074, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23469387755102042, - "n_models": 1, - "seconds": 0.163054208001995, - "eta_squared": 0.5584465959181422, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.16169712499686284, - "eta_squared": 0.5584465959181422, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3753051669991692, - "eta_squared": 0.5584465959181422, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.27040816326530615, - "n_models": 1, - "seconds": 0.15954833300202154, - "eta_squared": 0.5549925397074991, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09183673469387756, - "n_models": 1, - "seconds": 0.16052020799543243, - "eta_squared": 0.5549925397074991, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9938979958197234, - "roc_auc": 0.9999003507653061, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3658120420004707, - "eta_squared": 0.5549925397074991, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2602040816326531, - "n_models": 1, - "seconds": 0.15997279099974548, - "eta_squared": 0.5579244171259582, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.16077487500297138, - "eta_squared": 0.5579244171259582, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3736999590037158, - "eta_squared": 0.5579244171259582, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2653061224489796, - "n_models": 1, - "seconds": 0.1635333750018617, - "eta_squared": 0.5578235062169735, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11224489795918367, - "n_models": 1, - "seconds": 0.16439820799860172, - "eta_squared": 0.5578235062169735, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9993384082076203, - "roc_auc": 0.9999867134353742, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3988040420008474, - "eta_squared": 0.5578235062169735, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.26785714285714285, - "n_models": 1, - "seconds": 0.16486979099863674, - "eta_squared": 0.5556051803235251, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.15863283399812644, - "eta_squared": 0.5556051803235251, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 0.9, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3529080000007525, - "eta_squared": 0.5556051803235251, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.4987466519060602, - "roc_auc": 0.9896608382936508, - "precision_at_n": 0.4791666666666667, - "macro_pr_auc": 0.9490303977068683, - "worst_group_fpr": 0.3112244897959184, - "n_models": 1, - "seconds": 0.16161441700387513, - "eta_squared": 0.9337002346939631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, - "n_models": 1, - "seconds": 0.16105029200116405, - "eta_squared": 0.9337002346939631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4107413750025444, - "eta_squared": 0.9337002346939631, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.5003038173153509, - "roc_auc": 0.9893973214285715, - "precision_at_n": 0.4895833333333333, - "macro_pr_auc": 0.9454233776844071, - "worst_group_fpr": 0.30357142857142855, - "n_models": 1, - "seconds": 0.16223812499811174, - "eta_squared": 0.9332108003153095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9935536142019581, - "roc_auc": 0.9999003507653061, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.15896033299941337, - "eta_squared": 0.9332108003153095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.424678416995448, - "eta_squared": 0.9332108003153095, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.581959738741685, - "roc_auc": 0.991259654903628, - "precision_at_n": 0.5208333333333334, - "macro_pr_auc": 0.9686428444240943, - "worst_group_fpr": 0.32653061224489793, - "n_models": 1, - "seconds": 0.16620829200110165, - "eta_squared": 0.9338330621664808, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, - "n_models": 1, - "seconds": 0.17143991599732544, - "eta_squared": 0.9338330621664808, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9767567760167845, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.362880166998366, - "eta_squared": 0.9338330621664808, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.5734492143594968, - "roc_auc": 0.9893021010487527, - "precision_at_n": 0.4166666666666667, - "macro_pr_auc": 0.9824074074074075, - "worst_group_fpr": 0.30357142857142855, - "n_models": 1, - "seconds": 0.16573454199533444, - "eta_squared": 0.9344356642098697, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18112244897959184, - "n_models": 1, - "seconds": 0.16718445799779147, - "eta_squared": 0.9344356642098697, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.372973625002487, - "eta_squared": 0.9344356642098697, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.6242719407014463, - "roc_auc": 0.99417827026644, - "precision_at_n": 0.65625, - "macro_pr_auc": 0.9518738363447793, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.1799211670004297, - "eta_squared": 0.9334757370341583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, - "n_models": 1, - "seconds": 0.16911824999988312, - "eta_squared": 0.9334757370341583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.3532594999996945, - "eta_squared": 0.9334757370341583, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.46906027929970967, - "roc_auc": 0.9883653982426305, - "precision_at_n": 0.4270833333333333, - "macro_pr_auc": 0.9407022970006715, - "worst_group_fpr": 0.31887755102040816, - "n_models": 1, - "seconds": 0.1665471250016708, - "eta_squared": 0.9318170650470089, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17091836734693877, - "n_models": 1, - "seconds": 0.16424591700342717, - "eta_squared": 0.9318170650470089, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3771830000041518, - "eta_squared": 0.9318170650470089, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.6971414379905212, - "roc_auc": 0.9953364158163266, - "precision_at_n": 0.7083333333333334, - "macro_pr_auc": 0.9820684523809523, - "worst_group_fpr": 0.288265306122449, - "n_models": 1, - "seconds": 0.15878320800402435, - "eta_squared": 0.9313493667097374, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.16666441700363066, - "eta_squared": 0.9313493667097374, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.363710791003541, - "eta_squared": 0.9313493667097374, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.5880397479982498, - "roc_auc": 0.9912042942176871, - "precision_at_n": 0.5104166666666666, - "macro_pr_auc": 0.9729600694444445, - "worst_group_fpr": 0.3137755102040816, - "n_models": 1, - "seconds": 0.16411391599831404, - "eta_squared": 0.9336353454255104, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, - "n_models": 1, - "seconds": 0.16820949999964796, - "eta_squared": 0.9336353454255104, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.360429499996826, - "eta_squared": 0.9336353454255104, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.5283112653468994, - "roc_auc": 0.9889721513605443, - "precision_at_n": 0.4583333333333333, - "macro_pr_auc": 0.9699745604231492, - "worst_group_fpr": 0.3137755102040816, - "n_models": 1, - "seconds": 0.17555916700075613, - "eta_squared": 0.9340786626302715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9908147321763701, - "roc_auc": 0.9998560622165533, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15306122448979592, - "n_models": 1, - "seconds": 0.16599275000044145, - "eta_squared": 0.9340786626302715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3732968749973224, - "eta_squared": 0.9340786626302715, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.6864082496031527, - "roc_auc": 0.9936069479875284, - "precision_at_n": 0.6041666666666666, - "macro_pr_auc": 0.9763625841750841, - "worst_group_fpr": 0.29846938775510207, - "n_models": 1, - "seconds": 0.16572633299801964, - "eta_squared": 0.9331214740684672, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, - "n_models": 1, - "seconds": 0.1568466669996269, - "eta_squared": 0.9331214740684672, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3904178329976276, - "eta_squared": 0.9331214740684672, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.6333266417600595, - "roc_auc": 0.9945082199546484, - "precision_at_n": 0.6666666666666666, - "macro_pr_auc": 0.9465535332722833, - "worst_group_fpr": 0.3163265306122449, - "n_models": 1, - "seconds": 0.15864504099590704, - "eta_squared": 0.9329217798181637, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9997841047394044, - "roc_auc": 0.9999955711451247, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1096938775510204, - "n_models": 1, - "seconds": 0.17548050000186777, - "eta_squared": 0.9329217798181637, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3608411249952042, - "eta_squared": 0.9329217798181637, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.5439936919818846, - "roc_auc": 0.9908322704081632, - "precision_at_n": 0.4895833333333333, - "macro_pr_auc": 0.949382215007215, - "worst_group_fpr": 0.3137755102040816, - "n_models": 1, - "seconds": 0.15965133300051093, - "eta_squared": 0.9333000674744998, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9977436320959776, - "roc_auc": 0.9999579258786848, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17346938775510204, - "n_models": 1, - "seconds": 0.1705870420046267, - "eta_squared": 0.9333000674744998, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3444945420051226, - "eta_squared": 0.9333000674744998, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.6005803367394442, - "roc_auc": 0.990914204223356, - "precision_at_n": 0.5104166666666666, - "macro_pr_auc": 0.9784474206349206, - "worst_group_fpr": 0.3239795918367347, - "n_models": 1, - "seconds": 0.16280158400331857, - "eta_squared": 0.9344982331342713, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, - "n_models": 1, - "seconds": 0.15939304199855542, - "eta_squared": 0.9344982331342713, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3916888750027283, - "eta_squared": 0.9344982331342713, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.4835413524167843, - "roc_auc": 0.9882236748866213, - "precision_at_n": 0.4375, - "macro_pr_auc": 0.9448220755693582, - "worst_group_fpr": 0.29846938775510207, - "n_models": 1, - "seconds": 0.16157833299803315, - "eta_squared": 0.9323325522428791, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9857768221421451, - "roc_auc": 0.9997674851190477, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.16966954200324835, - "eta_squared": 0.9323325522428791, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9089068819073483, - "roc_auc": 0.9985451211734694, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9259082641895143, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.387394666999171, - "eta_squared": 0.9323325522428791, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.6018279227472707, - "roc_auc": 0.9930932008219955, - "precision_at_n": 0.5833333333333334, - "macro_pr_auc": 0.9483675468050468, - "worst_group_fpr": 0.32653061224489793, - "n_models": 1, - "seconds": 0.1802478749959846, - "eta_squared": 0.9333406233039432, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9991328016734993, - "roc_auc": 0.9999822845804989, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, - "n_models": 1, - "seconds": 0.16228891699574888, - "eta_squared": 0.9333406233039432, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=0.900", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 0.9, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3904078750056215, - "eta_squared": 0.9333406233039432, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 0, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.30357142857142855, - "n_models": 1, - "seconds": 0.17233216599561274, - "eta_squared": 0.5676363348862092, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 0, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.1639115420039161, - "eta_squared": 0.5676363348862092, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 0, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9998926116838485, - "roc_auc": 0.9999977855725624, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3711525830003666, - "eta_squared": 0.5676363348862092, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 1, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2602040816326531, - "n_models": 1, - "seconds": 0.17048391699790955, - "eta_squared": 0.5648975634450134, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 1, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.17477779199543875, - "eta_squared": 0.5648975634450134, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 1, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9994629892108766, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3926485830015736, - "eta_squared": 0.5648975634450134, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 2, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.31887755102040816, - "n_models": 1, - "seconds": 0.15824562500347383, - "eta_squared": 0.5710146129420057, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 2, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.12755102040816327, - "n_models": 1, - "seconds": 0.16800374999729684, - "eta_squared": 0.5710146129420057, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 2, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.4105422079956043, - "eta_squared": 0.5710146129420057, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 3, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29846938775510207, - "n_models": 1, - "seconds": 0.15720345800218638, - "eta_squared": 0.5706485648142726, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 3, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 0.9999999999999999, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.17112720800651005, - "eta_squared": 0.5706485648142726, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 3, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3866387080051936, - "eta_squared": 0.5706485648142726, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 4, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.22959183673469388, - "n_models": 1, - "seconds": 0.1597049159972812, - "eta_squared": 0.56534003587956, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 4, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.125, - "n_models": 1, - "seconds": 0.1596773330020369, - "eta_squared": 0.56534003587956, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 4, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9994516328453017, - "roc_auc": 0.9999889278628118, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3918389169994043, - "eta_squared": 0.56534003587956, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 5, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2780612244897959, - "n_models": 1, - "seconds": 0.16154229200037662, - "eta_squared": 0.5675034019831222, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 5, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10204081632653061, - "n_models": 1, - "seconds": 0.17201179199764738, - "eta_squared": 0.5675034019831222, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 5, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9980484049901452, - "roc_auc": 0.99996235473356, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9960524140211641, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3769019169994863, - "eta_squared": 0.5675034019831222, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 6, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.27040816326530615, - "n_models": 1, - "seconds": 0.17271570899902144, - "eta_squared": 0.5660843709494305, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 6, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14795918367346939, - "n_models": 1, - "seconds": 0.17352633300470188, - "eta_squared": 0.5660843709494305, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 6, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9976369764612153, - "roc_auc": 0.9999557114512472, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9948950066137566, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3770624590033549, - "eta_squared": 0.5660843709494305, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 7, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2857142857142857, - "n_models": 1, - "seconds": 0.15575383300165413, - "eta_squared": 0.5698671092657079, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 7, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.13010204081632654, - "n_models": 1, - "seconds": 0.1640514169994276, - "eta_squared": 0.5698671092657079, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 7, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.998801470355826, - "roc_auc": 0.999975641298186, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4011617919968558, - "eta_squared": 0.5698671092657079, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 8, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.1727778330023284, - "eta_squared": 0.5667004409467457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 8, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09693877551020408, - "n_models": 1, - "seconds": 0.16893491600058042, - "eta_squared": 0.5667004409467457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 8, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9982598713958657, - "roc_auc": 0.9999667835884354, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9988425925925926, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.37281208299828, - "eta_squared": 0.5667004409467457, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 9, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23979591836734693, - "n_models": 1, - "seconds": 0.15791291700588772, - "eta_squared": 0.5643728023336351, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 9, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1377551020408163, - "n_models": 1, - "seconds": 0.1657633330032695, - "eta_squared": 0.5643728023336351, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 9, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3924569169976166, - "eta_squared": 0.5643728023336351, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 10, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.23214285714285715, - "n_models": 1, - "seconds": 0.16296616700128652, - "eta_squared": 0.5690320175635628, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 10, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10714285714285714, - "n_models": 1, - "seconds": 0.17207512500317534, - "eta_squared": 0.5690320175635628, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 10, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.384157916996628, - "eta_squared": 0.5690320175635628, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 11, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2755102040816326, - "n_models": 1, - "seconds": 0.15116024999588262, - "eta_squared": 0.5652473317791511, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 11, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.16885133400501218, - "eta_squared": 0.5652473317791511, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 11, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9948853727883523, - "roc_auc": 0.999913637329932, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 0.9922329695767195, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3844270000045071, - "eta_squared": 0.5652473317791511, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 12, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2653061224489796, - "n_models": 1, - "seconds": 0.1657912500013481, - "eta_squared": 0.5682748535558901, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 12, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.09948979591836735, - "n_models": 1, - "seconds": 0.1545707500044955, - "eta_squared": 0.5682748535558901, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 12, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3596160000015516, - "eta_squared": 0.5682748535558901, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 13, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.25255102040816324, - "n_models": 1, - "seconds": 0.1665559579996625, - "eta_squared": 0.5683637055956918, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 13, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.10459183673469388, - "n_models": 1, - "seconds": 0.1660269169951789, - "eta_squared": 0.5683637055956918, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 13, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9992332114897579, - "roc_auc": 0.9999844990079364, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 0.9943163029100529, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4021668329951353, - "eta_squared": 0.5683637055956918, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 14, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.2755102040816326, - "n_models": 1, - "seconds": 0.1687291659982293, - "eta_squared": 0.5661057125507907, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 14, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.08928571428571429, - "n_models": 1, - "seconds": 0.16114408300200012, - "eta_squared": 0.5661057125507907, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 14, - "mechanism": "global", - "level_spread": 1.0, - "pr_auc": 0.9999999999999997, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3545676249996177, - "eta_squared": 0.5661057125507907, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 0, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.48534868873259845, - "roc_auc": 0.9877010700113379, - "precision_at_n": 0.4895833333333333, - "macro_pr_auc": 0.9478546145212811, - "worst_group_fpr": 0.3137755102040816, - "n_models": 1, - "seconds": 0.1693762080030865, - "eta_squared": 0.9430926146131842, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 0, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16581632653061223, - "n_models": 1, - "seconds": 0.16320516599807888, - "eta_squared": 0.9430926146131842, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 0, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9745705638556281, - "roc_auc": 0.999554900085034, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9835110780423281, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3755387499986682, - "eta_squared": 0.9430926146131842, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 1, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.5367151920328781, - "roc_auc": 0.989968643707483, - "precision_at_n": 0.5, - "macro_pr_auc": 0.9527008908057296, - "worst_group_fpr": 0.30357142857142855, - "n_models": 1, - "seconds": 0.1608518329958315, - "eta_squared": 0.9426371564373757, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 1, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.990321141535306, - "roc_auc": 0.9998693487811792, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.15965487499488518, - "eta_squared": 0.9426371564373757, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 1, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9629263242825986, - "roc_auc": 0.9994353210034013, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9706225198412698, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3880165829978068, - "eta_squared": 0.9426371564373757, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 2, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.5495354609375203, - "roc_auc": 0.9888769309807256, - "precision_at_n": 0.5416666666666666, - "macro_pr_auc": 0.9730052933177933, - "worst_group_fpr": 0.32908163265306123, - "n_models": 1, - "seconds": 0.17115704200114124, - "eta_squared": 0.9432131151496429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 2, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16326530612244897, - "n_models": 1, - "seconds": 0.17343999999866355, - "eta_squared": 0.9432131151496429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 2, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9767568942402173, - "roc_auc": 0.9996102607709751, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.976078869047619, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3421517079987098, - "eta_squared": 0.9432131151496429, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 3, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.45011384406487887, - "roc_auc": 0.9838612528344672, - "precision_at_n": 0.3541666666666667, - "macro_pr_auc": 0.9624727558321308, - "worst_group_fpr": 0.31887755102040816, - "n_models": 1, - "seconds": 0.16151770799478982, - "eta_squared": 0.9437235511189537, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 3, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, - "n_models": 1, - "seconds": 0.16660925000178395, - "eta_squared": 0.9437235511189537, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 3, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9670331496927849, - "roc_auc": 0.9995128259637188, - "precision_at_n": 0.9270833333333334, - "macro_pr_auc": 0.9628554894179894, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3712917910015676, - "eta_squared": 0.9437235511189537, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 4, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.5750853751747382, - "roc_auc": 0.9916073200113378, - "precision_at_n": 0.5104166666666666, - "macro_pr_auc": 0.9601720446950711, - "worst_group_fpr": 0.3112244897959184, - "n_models": 1, - "seconds": 0.16282804099319037, - "eta_squared": 0.9428896083609098, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 4, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9988740304185753, - "roc_auc": 0.9999778557256236, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1326530612244898, - "n_models": 1, - "seconds": 0.16667325000162236, - "eta_squared": 0.9428896083609098, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 4, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9866503567728645, - "roc_auc": 0.999782986111111, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9899181547619048, - "worst_group_fpr": 0.04591836734693878, - "n_models": 12, - "seconds": 1.3986902910037315, - "eta_squared": 0.9428896083609098, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 5, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.493833466763317, - "roc_auc": 0.9875128436791383, - "precision_at_n": 0.4791666666666667, - "macro_pr_auc": 0.9630205153642653, - "worst_group_fpr": 0.3137755102040816, - "n_models": 1, - "seconds": 0.16258195899717975, - "eta_squared": 0.9414594641783969, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 5, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.17857142857142858, - "n_models": 1, - "seconds": 0.16420841699437005, - "eta_squared": 0.9414594641783969, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 5, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9687108941098512, - "roc_auc": 0.9994973249716553, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9687417328042329, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3986366669996642, - "eta_squared": 0.9414594641783969, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 6, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.6135563927593939, - "roc_auc": 0.9933611465419502, - "precision_at_n": 0.6041666666666666, - "macro_pr_auc": 0.9647178631553631, - "worst_group_fpr": 0.29591836734693877, - "n_models": 1, - "seconds": 0.17231612499745097, - "eta_squared": 0.9410327000313501, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 6, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.15816326530612246, - "n_models": 1, - "seconds": 0.16098312500253087, - "eta_squared": 0.9410327000313501, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 6, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9792851034241599, - "roc_auc": 0.9996833368764172, - "precision_at_n": 0.96875, - "macro_pr_auc": 0.9846106150793651, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.4123492910002824, - "eta_squared": 0.9410327000313501, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 7, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.49320212664757, - "roc_auc": 0.987397693452381, - "precision_at_n": 0.4270833333333333, - "macro_pr_auc": 0.9670368370736018, - "worst_group_fpr": 0.3086734693877551, - "n_models": 1, - "seconds": 0.17187958299473394, - "eta_squared": 0.9430130855086445, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 7, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, - "n_models": 1, - "seconds": 0.16060612499859417, - "eta_squared": 0.9430130855086445, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 7, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9298555808760586, - "roc_auc": 0.9992315936791383, - "precision_at_n": 0.8958333333333334, - "macro_pr_auc": 0.9523520171957672, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.335786209005164, - "eta_squared": 0.9430130855086445, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 8, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.5900752988793343, - "roc_auc": 0.9890651573129252, - "precision_at_n": 0.5416666666666666, - "macro_pr_auc": 0.9817336309523809, - "worst_group_fpr": 0.30612244897959184, - "n_models": 1, - "seconds": 0.1624552909997874, - "eta_squared": 0.9433967168032253, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 8, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.994319218174482, - "roc_auc": 0.9999092084750566, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.16071428571428573, - "n_models": 1, - "seconds": 0.16499349999503465, - "eta_squared": 0.9433967168032253, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 8, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9748395926005912, - "roc_auc": 0.9995947597789115, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9738632605820104, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.3485228749996168, - "eta_squared": 0.9433967168032253, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 9, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.5261343002549365, - "roc_auc": 0.9887418509070296, - "precision_at_n": 0.4375, - "macro_pr_auc": 0.9440026619622208, - "worst_group_fpr": 0.32142857142857145, - "n_models": 1, - "seconds": 0.16455762500117999, - "eta_squared": 0.9425810756331876, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 9, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9999999999999998, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.16205866599921137, - "eta_squared": 0.9425810756331876, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 9, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.983868818654779, - "roc_auc": 0.9996988378684807, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.984844727032227, - "worst_group_fpr": 0.04336734693877551, - "n_models": 12, - "seconds": 1.398233292005898, - "eta_squared": 0.9425810756331876, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 10, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.5513522273736624, - "roc_auc": 0.9916936826814058, - "precision_at_n": 0.5416666666666666, - "macro_pr_auc": 0.9481992313242312, - "worst_group_fpr": 0.30612244897959184, - "n_models": 1, - "seconds": 0.15685741700144717, - "eta_squared": 0.9423071534590428, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 10, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9999999999999999, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.11734693877551021, - "n_models": 1, - "seconds": 0.1638517920000595, - "eta_squared": 0.9423071534590428, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 10, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9852957098632265, - "roc_auc": 0.9997298398526077, - "precision_at_n": 0.9375, - "macro_pr_auc": 0.9818039021164019, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3726681250045658, - "eta_squared": 0.9423071534590428, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 11, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.48798028950188443, - "roc_auc": 0.9883388251133787, - "precision_at_n": 0.4166666666666667, - "macro_pr_auc": 0.9373701096357346, - "worst_group_fpr": 0.3112244897959184, - "n_models": 1, - "seconds": 0.17445962500642054, - "eta_squared": 0.9427808191647584, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 11, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9925550534350223, - "roc_auc": 0.9998759920634921, - "precision_at_n": 0.9791666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1760204081632653, - "n_models": 1, - "seconds": 0.16523474999848986, - "eta_squared": 0.9427808191647584, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 11, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9440775446407559, - "roc_auc": 0.9992581668083901, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 0.9595516173641173, - "worst_group_fpr": 0.03826530612244898, - "n_models": 12, - "seconds": 1.3668047909959569, - "eta_squared": 0.9427808191647584, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 12, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.5444069766801478, - "roc_auc": 0.9861266121031745, - "precision_at_n": 0.4479166666666667, - "macro_pr_auc": 0.9886326058201059, - "worst_group_fpr": 0.3239795918367347, - "n_models": 1, - "seconds": 0.16296875000261934, - "eta_squared": 0.9437633675151137, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 12, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 1.0, - "roc_auc": 1.0, - "precision_at_n": 1.0, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.1556122448979592, - "n_models": 1, - "seconds": 0.16889766600070288, - "eta_squared": 0.9437633675151137, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 12, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9867244637441994, - "roc_auc": 0.99976527069161, - "precision_at_n": 0.9583333333333334, - "macro_pr_auc": 0.9858258928571427, - "worst_group_fpr": 0.03571428571428571, - "n_models": 12, - "seconds": 1.3833198329957668, - "eta_squared": 0.9437633675151137, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 13, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.40110407158158806, - "roc_auc": 0.9845565830498868, - "precision_at_n": 0.375, - "macro_pr_auc": 0.9210572052368926, - "worst_group_fpr": 0.30357142857142855, - "n_models": 1, - "seconds": 0.15952129100332968, - "eta_squared": 0.9418930396685032, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 13, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9612198529137181, - "roc_auc": 0.999406533446712, - "precision_at_n": 0.9166666666666666, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.14030612244897958, - "n_models": 1, - "seconds": 0.17576462499710033, - "eta_squared": 0.9418930396685032, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 13, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9082358809082258, - "roc_auc": 0.9985362634637187, - "precision_at_n": 0.84375, - "macro_pr_auc": 0.9244201689514191, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3914272089969018, - "eta_squared": 0.9418930396685032, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "pooled", - "seed": 14, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.5103424703484694, - "roc_auc": 0.9890673717403629, - "precision_at_n": 0.4479166666666667, - "macro_pr_auc": 0.9523000208855472, - "worst_group_fpr": 0.32908163265306123, - "n_models": 1, - "seconds": 0.15692079099972034, - "eta_squared": 0.9427598024542532, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "relative", - "seed": 14, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9977421731790775, - "roc_auc": 0.9999579258786848, - "precision_at_n": 0.9895833333333334, - "macro_pr_auc": 1.0, - "worst_group_fpr": 0.18112244897959184, - "n_models": 1, - "seconds": 0.1764908339973772, - "eta_squared": 0.9427598024542532, - "warnings": [] - }, - { - "dataset": "synthetic", - "grouping": "spread=1.000", - "config": "per_group", - "seed": 14, - "mechanism": "contextual", - "level_spread": 1.0, - "pr_auc": 0.9907914066468546, - "roc_auc": 0.9998228458049887, - "precision_at_n": 0.9479166666666666, - "macro_pr_auc": 0.9881065115440114, - "worst_group_fpr": 0.04081632653061224, - "n_models": 12, - "seconds": 1.3573104169990984, - "eta_squared": 0.9427598024542532, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06599372001242891, - "roc_auc": 0.6794908017938576, - "precision_at_n": 0.10986066452304394, - "macro_pr_auc": 0.26413518265780117, - "worst_group_fpr": 0.3041405269761606, - "n_models": 1, - "seconds": 0.7082737079981598, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08975412713487951, - "roc_auc": 0.7263518402451048, - "precision_at_n": 0.13504823151125403, - "macro_pr_auc": 0.32422570878576495, - "worst_group_fpr": 0.27854454203262236, - "n_models": 1, - "seconds": 0.8081601670055534, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08529767939104427, - "roc_auc": 0.6203368216799636, - "precision_at_n": 0.1382636655948553, - "macro_pr_auc": 0.3522517990002194, - "worst_group_fpr": 0.079, - "n_models": 28, - "seconds": 4.09942383300222, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06487174622301133, - "roc_auc": 0.6937113491862577, - "precision_at_n": 0.11120042872454448, - "macro_pr_auc": 0.2676601621144114, - "worst_group_fpr": 0.32772898368883313, - "n_models": 1, - "seconds": 0.7032672919958713, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07799797883368983, - "roc_auc": 0.7309774124081202, - "precision_at_n": 0.1264737406216506, - "macro_pr_auc": 0.28554629296367817, - "worst_group_fpr": 0.37917189460476786, - "n_models": 1, - "seconds": 0.8542152080044616, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0856684778882634, - "roc_auc": 0.6188643853324, - "precision_at_n": 0.1377277599142551, - "macro_pr_auc": 0.35766502619889395, - "worst_group_fpr": 0.0795, - "n_models": 28, - "seconds": 4.0839703750025365, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06064840724124169, - "roc_auc": 0.6814215704501445, - "precision_at_n": 0.09967845659163987, - "macro_pr_auc": 0.2643175968618031, - "worst_group_fpr": 0.37314930991217066, - "n_models": 1, - "seconds": 0.7044784169993363, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06932651026535167, - "roc_auc": 0.7050175023187866, - "precision_at_n": 0.1045016077170418, - "macro_pr_auc": 0.2946479554830892, - "worst_group_fpr": 0.2825595984943538, - "n_models": 1, - "seconds": 0.83684104099666, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07986999321041088, - "roc_auc": 0.6096425859358724, - "precision_at_n": 0.13317256162915328, - "macro_pr_auc": 0.3298419758564198, - "worst_group_fpr": 0.087, - "n_models": 28, - "seconds": 4.1016515839946805, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06520326519944142, - "roc_auc": 0.6929199146803785, - "precision_at_n": 0.1120042872454448, - "macro_pr_auc": 0.2718048970703108, - "worst_group_fpr": 0.2451693851944793, - "n_models": 1, - "seconds": 0.6907554999997956, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07432216955449138, - "roc_auc": 0.7052161232155005, - "precision_at_n": 0.11655948553054662, - "macro_pr_auc": 0.30592072994805625, - "worst_group_fpr": 0.34855708908406524, - "n_models": 1, - "seconds": 0.7915337919985177, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07881808753345326, - "roc_auc": 0.6113086908984655, - "precision_at_n": 0.13236870310825294, - "macro_pr_auc": 0.35162638851480377, - "worst_group_fpr": 0.0795, - "n_models": 28, - "seconds": 4.130562209000345, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06631107326061537, - "roc_auc": 0.6880201751946493, - "precision_at_n": 0.1152197213290461, - "macro_pr_auc": 0.258516490974546, - "worst_group_fpr": 0.36537013801756585, - "n_models": 1, - "seconds": 0.7238397089968203, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08859854278095852, - "roc_auc": 0.7162222314354626, - "precision_at_n": 0.1264737406216506, - "macro_pr_auc": 0.31613493642464613, - "worst_group_fpr": 0.23914680050188206, - "n_models": 1, - "seconds": 0.8319103750036447, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08233569155771314, - "roc_auc": 0.6252609364891877, - "precision_at_n": 0.1377277599142551, - "macro_pr_auc": 0.35316191126056945, - "worst_group_fpr": 0.083, - "n_models": 28, - "seconds": 4.051915416996053, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06383904434008811, - "roc_auc": 0.6844451302236746, - "precision_at_n": 0.11414790996784566, - "macro_pr_auc": 0.2599635037497486, - "worst_group_fpr": 0.3573400250941029, - "n_models": 1, - "seconds": 0.7152979160018731, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07374933979597549, - "roc_auc": 0.6925375940794927, - "precision_at_n": 0.1229903536977492, - "macro_pr_auc": 0.2873052203937247, - "worst_group_fpr": 0.38845671267252196, - "n_models": 1, - "seconds": 0.8419914999976754, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08374995105820766, - "roc_auc": 0.618497706120943, - "precision_at_n": 0.13370846730975347, - "macro_pr_auc": 0.3571298814863155, - "worst_group_fpr": 0.09125, - "n_models": 28, - "seconds": 4.079550041999028, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06994101575649286, - "roc_auc": 0.7155082861547449, - "precision_at_n": 0.11361200428724544, - "macro_pr_auc": 0.2663950053872669, - "worst_group_fpr": 0.4087829360100376, - "n_models": 1, - "seconds": 0.725883958999475, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0798919589140398, - "roc_auc": 0.7454055076737647, - "precision_at_n": 0.12593783494105038, - "macro_pr_auc": 0.2993513347886662, - "worst_group_fpr": 0.37641154328732745, - "n_models": 1, - "seconds": 0.803975167000317, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08356881140485035, - "roc_auc": 0.6067535990342094, - "precision_at_n": 0.13612004287245444, - "macro_pr_auc": 0.3671674809340615, - "worst_group_fpr": 0.1005, - "n_models": 28, - "seconds": 4.123660916004155, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06346703450271998, - "roc_auc": 0.6824206914238579, - "precision_at_n": 0.10128617363344052, - "macro_pr_auc": 0.26730112904237396, - "worst_group_fpr": 0.3877038895859473, - "n_models": 1, - "seconds": 0.6885288750054315, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07314332030590526, - "roc_auc": 0.7038474942157549, - "precision_at_n": 0.1160235798499464, - "macro_pr_auc": 0.3060809090763773, - "worst_group_fpr": 0.3533249686323714, - "n_models": 1, - "seconds": 0.8422652500012191, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08320004375026958, - "roc_auc": 0.6086983929680114, - "precision_at_n": 0.13638799571275456, - "macro_pr_auc": 0.33161701272441674, - "worst_group_fpr": 0.1005, - "n_models": 28, - "seconds": 4.0860568749994854, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06391515057273248, - "roc_auc": 0.6895837795584147, - "precision_at_n": 0.09887459807073955, - "macro_pr_auc": 0.26440938470328373, - "worst_group_fpr": 0.429861982434128, - "n_models": 1, - "seconds": 0.6778770419987268, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07554757126645857, - "roc_auc": 0.7111140110379107, - "precision_at_n": 0.1189710610932476, - "macro_pr_auc": 0.27528729908371236, - "worst_group_fpr": 0.3478042659974906, - "n_models": 1, - "seconds": 0.8417751250017318, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08639941462490236, - "roc_auc": 0.6288417232360285, - "precision_at_n": 0.1342443729903537, - "macro_pr_auc": 0.3624784434468467, - "worst_group_fpr": 0.08625, - "n_models": 28, - "seconds": 4.177703540997754, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06322374708445785, - "roc_auc": 0.6752158108331946, - "precision_at_n": 0.10503751339764202, - "macro_pr_auc": 0.2651756750769737, - "worst_group_fpr": 0.370138017565872, - "n_models": 1, - "seconds": 0.6878105419964413, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07968797931504096, - "roc_auc": 0.7223169507994355, - "precision_at_n": 0.12486602357984995, - "macro_pr_auc": 0.2892799312552035, - "worst_group_fpr": 0.221831869510665, - "n_models": 1, - "seconds": 0.8280662080069305, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08455359360662107, - "roc_auc": 0.6225626136698378, - "precision_at_n": 0.13612004287245444, - "macro_pr_auc": 0.3673254717050555, - "worst_group_fpr": 0.07725, - "n_models": 28, - "seconds": 4.105281750002177, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06519281732032095, - "roc_auc": 0.6738063001417901, - "precision_at_n": 0.1227224008574491, - "macro_pr_auc": 0.2579158702974877, - "worst_group_fpr": 0.3452948557089084, - "n_models": 1, - "seconds": 0.7210727499987115, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07326810730121225, - "roc_auc": 0.6912488252623565, - "precision_at_n": 0.11736334405144695, - "macro_pr_auc": 0.2929809134995579, - "worst_group_fpr": 0.29535759096612296, - "n_models": 1, - "seconds": 0.831405500000983, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08588316016047276, - "roc_auc": 0.6185552327753555, - "precision_at_n": 0.13692390139335478, - "macro_pr_auc": 0.3706344555927829, - "worst_group_fpr": 0.08625, - "n_models": 28, - "seconds": 4.072452332999092, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0638559054729082, - "roc_auc": 0.6938116248469371, - "precision_at_n": 0.10664523043944266, - "macro_pr_auc": 0.2620259597609067, - "worst_group_fpr": 0.3992471769134254, - "n_models": 1, - "seconds": 0.7011463339949842, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0749772379684639, - "roc_auc": 0.7204712346730718, - "precision_at_n": 0.1160235798499464, - "macro_pr_auc": 0.3072583176907988, - "worst_group_fpr": 0.38996235884567126, - "n_models": 1, - "seconds": 0.8435321249999106, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08342839328093159, - "roc_auc": 0.6162641206602916, - "precision_at_n": 0.13585209003215434, - "macro_pr_auc": 0.36622478697593386, - "worst_group_fpr": 0.085, - "n_models": 28, - "seconds": 4.079520125000272, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06533443887362353, - "roc_auc": 0.6778376702748382, - "precision_at_n": 0.12057877813504823, - "macro_pr_auc": 0.2617293955220065, - "worst_group_fpr": 0.27754077791718945, - "n_models": 1, - "seconds": 0.7451008330026525, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0809589792460545, - "roc_auc": 0.7102266542264164, - "precision_at_n": 0.1192390139335477, - "macro_pr_auc": 0.32101929536115875, - "worst_group_fpr": 0.2534504391468005, - "n_models": 1, - "seconds": 0.838801792000595, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08415777912515914, - "roc_auc": 0.6194077082984619, - "precision_at_n": 0.13638799571275456, - "macro_pr_auc": 0.3619564762419791, - "worst_group_fpr": 0.091, - "n_models": 28, - "seconds": 4.127557959000114, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06053581675987888, - "roc_auc": 0.668970457711801, - "precision_at_n": 0.10423365487674169, - "macro_pr_auc": 0.2661453758657945, - "worst_group_fpr": 0.4288582183186951, - "n_models": 1, - "seconds": 0.7011158329987666, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08457025729915832, - "roc_auc": 0.7117365111132468, - "precision_at_n": 0.11468381564844587, - "macro_pr_auc": 0.30673157924849787, - "worst_group_fpr": 0.3879548306148055, - "n_models": 1, - "seconds": 0.8255459159991005, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08245938229155562, - "roc_auc": 0.610460199969818, - "precision_at_n": 0.1304930332261522, - "macro_pr_auc": 0.3588958226713545, - "worst_group_fpr": 0.08125, - "n_models": 28, - "seconds": 4.086438583995914, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "pooled", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06249021828056783, - "roc_auc": 0.6844033192057928, - "precision_at_n": 0.11093247588424437, - "macro_pr_auc": 0.26469382499516414, - "worst_group_fpr": 0.4143036386449184, - "n_models": 1, - "seconds": 0.6763558750026277, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "relative", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08177093015401454, - "roc_auc": 0.7163076799499286, - "precision_at_n": 0.11870310825294748, - "macro_pr_auc": 0.3049233360547107, - "worst_group_fpr": 0.35984943538268505, - "n_models": 1, - "seconds": 0.8341285000060452, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "entity", - "config": "per_group", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08309978963115165, - "roc_auc": 0.6078336617233144, - "precision_at_n": 0.13344051446945338, - "macro_pr_auc": 0.32587296737624166, - "worst_group_fpr": 0.0795, - "n_models": 28, - "seconds": 4.117640249998658, - "eta_squared": 0.5619488636163167, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06599372001242891, - "roc_auc": 0.6794908017938576, - "precision_at_n": 0.10986066452304394, - "macro_pr_auc": 0.15459948097397477, - "worst_group_fpr": 0.08150903801211427, - "n_models": 1, - "seconds": 0.6689555830016616, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07630884656938808, - "roc_auc": 0.6907817961431185, - "precision_at_n": 0.1197749196141479, - "macro_pr_auc": 0.1616731307705999, - "worst_group_fpr": 0.07981590117804169, - "n_models": 1, - "seconds": 0.7638930829998571, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0708883937729277, - "roc_auc": 0.6972662806173763, - "precision_at_n": 0.09833869239013933, - "macro_pr_auc": 0.15474095539742705, - "worst_group_fpr": 0.05028103956814514, - "n_models": 3, - "seconds": 1.0807272500023828, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06487174622301133, - "roc_auc": 0.6937113491862577, - "precision_at_n": 0.11120042872454448, - "macro_pr_auc": 0.15240654387568334, - "worst_group_fpr": 0.0793151142271188, - "n_models": 1, - "seconds": 0.6623257909959648, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06411755602985518, - "roc_auc": 0.6970320013126097, - "precision_at_n": 0.11280814576634512, - "macro_pr_auc": 0.156397619931216, - "worst_group_fpr": 0.07631039252158153, - "n_models": 1, - "seconds": 0.7806902500014985, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06947326333953958, - "roc_auc": 0.6970309767026058, - "precision_at_n": 0.10048231511254019, - "macro_pr_auc": 0.1650689106072917, - "worst_group_fpr": 0.048767110220823195, - "n_models": 3, - "seconds": 1.0540789590013446, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06064840724124169, - "roc_auc": 0.6814215704501445, - "precision_at_n": 0.09967845659163987, - "macro_pr_auc": 0.15438766149864325, - "worst_group_fpr": 0.08589688558210522, - "n_models": 1, - "seconds": 0.7205552500017802, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06359057138975892, - "roc_auc": 0.6852497584395295, - "precision_at_n": 0.10209003215434084, - "macro_pr_auc": 0.1514965861142468, - "worst_group_fpr": 0.08489531168025946, - "n_models": 1, - "seconds": 0.8438872909973725, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0693125825126317, - "roc_auc": 0.680144208462736, - "precision_at_n": 0.10369774919614148, - "macro_pr_auc": 0.1650860438354126, - "worst_group_fpr": 0.050503645166675944, - "n_models": 3, - "seconds": 1.2419886670031701, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06520326519944142, - "roc_auc": 0.6929199146803785, - "precision_at_n": 0.1120042872454448, - "macro_pr_auc": 0.1590560050567187, - "worst_group_fpr": 0.07657270949587447, - "n_models": 1, - "seconds": 0.7665968749934109, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06916940607152804, - "roc_auc": 0.702840166462398, - "precision_at_n": 0.1189710610932476, - "macro_pr_auc": 0.16107437858274268, - "worst_group_fpr": 0.07318643582772928, - "n_models": 1, - "seconds": 0.8361535000003641, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07012548194980095, - "roc_auc": 0.6958499627041959, - "precision_at_n": 0.09833869239013933, - "macro_pr_auc": 0.16303984692494433, - "worst_group_fpr": 0.05259057265290222, - "n_models": 3, - "seconds": 1.2241014169994742, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06631107326061537, - "roc_auc": 0.6880201751946493, - "precision_at_n": 0.1152197213290461, - "macro_pr_auc": 0.15679273034517632, - "worst_group_fpr": 0.08050746411026852, - "n_models": 1, - "seconds": 0.7591468750033528, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06750046772696748, - "roc_auc": 0.684424187591183, - "precision_at_n": 0.10691318327974277, - "macro_pr_auc": 0.1519757366679743, - "worst_group_fpr": 0.08107979205418038, - "n_models": 1, - "seconds": 0.8660919579997426, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07129558197654177, - "roc_auc": 0.6938419869617338, - "precision_at_n": 0.11093247588424437, - "macro_pr_auc": 0.159025464828388, - "worst_group_fpr": 0.05039234236741054, - "n_models": 3, - "seconds": 1.2299727909994544, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06383904434008811, - "roc_auc": 0.6844451302236746, - "precision_at_n": 0.11414790996784566, - "macro_pr_auc": 0.15568979011340453, - "worst_group_fpr": 0.08401297276672867, - "n_models": 1, - "seconds": 0.7487322500019218, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06815015385114945, - "roc_auc": 0.698310877940893, - "precision_at_n": 0.11789924973204716, - "macro_pr_auc": 0.1609907214508861, - "worst_group_fpr": 0.08169981399341823, - "n_models": 1, - "seconds": 0.8363707910029916, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06767640577544462, - "roc_auc": 0.6836004580709589, - "precision_at_n": 0.10584137191854234, - "macro_pr_auc": 0.15679729883353768, - "worst_group_fpr": 0.0474984695865101, - "n_models": 3, - "seconds": 1.2270105829957174, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06994101575649286, - "roc_auc": 0.7155082861547449, - "precision_at_n": 0.11361200428724544, - "macro_pr_auc": 0.16309636922967916, - "worst_group_fpr": 0.0850860876615634, - "n_models": 1, - "seconds": 0.7599207500024932, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06719650781752987, - "roc_auc": 0.704954711544862, - "precision_at_n": 0.12754555198285103, - "macro_pr_auc": 0.1561353851979731, - "worst_group_fpr": 0.07797968235799113, - "n_models": 1, - "seconds": 0.884690083003079, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07084784931288268, - "roc_auc": 0.7041475490279352, - "precision_at_n": 0.09887459807073955, - "macro_pr_auc": 0.1624685360528737, - "worst_group_fpr": 0.055011408536924704, - "n_models": 3, - "seconds": 1.242284790998383, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06346703450271998, - "roc_auc": 0.6824206914238579, - "precision_at_n": 0.10128617363344052, - "macro_pr_auc": 0.15858739789206222, - "worst_group_fpr": 0.0888539132923165, - "n_models": 1, - "seconds": 0.7559428750028019, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06380484313021846, - "roc_auc": 0.7002730951945653, - "precision_at_n": 0.09592711682743837, - "macro_pr_auc": 0.15957893617196986, - "worst_group_fpr": 0.08725616444889589, - "n_models": 1, - "seconds": 0.8619656669980031, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06837578488109436, - "roc_auc": 0.6862131393333781, - "precision_at_n": 0.08922829581993569, - "macro_pr_auc": 0.16430368253435956, - "worst_group_fpr": 0.05331404084812733, - "n_models": 3, - "seconds": 1.1852366250022897, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06391515057273248, - "roc_auc": 0.6895837795584147, - "precision_at_n": 0.09887459807073955, - "macro_pr_auc": 0.15861837293779057, - "worst_group_fpr": 0.08708923546525492, - "n_models": 1, - "seconds": 0.7407272080017719, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0664352056299146, - "roc_auc": 0.696070800808648, - "precision_at_n": 0.11763129689174705, - "macro_pr_auc": 0.16154788764949254, - "worst_group_fpr": 0.0822244479420041, - "n_models": 1, - "seconds": 0.8839377089971094, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06992836493531851, - "roc_auc": 0.6961550390953559, - "precision_at_n": 0.09780278670953912, - "macro_pr_auc": 0.1586693530651129, - "worst_group_fpr": 0.054315766041515945, - "n_models": 3, - "seconds": 1.173453832998348, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06322374708445785, - "roc_auc": 0.6752158108331946, - "precision_at_n": 0.10503751339764202, - "macro_pr_auc": 0.1563543880228793, - "worst_group_fpr": 0.0813898030237993, - "n_models": 1, - "seconds": 0.7637521249998827, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07119499410159705, - "roc_auc": 0.7060776247112728, - "precision_at_n": 0.11629153269024652, - "macro_pr_auc": 0.15682244141909382, - "worst_group_fpr": 0.07910049124815186, - "n_models": 1, - "seconds": 0.8636223330031498, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07190182683501667, - "roc_auc": 0.6951186535012893, - "precision_at_n": 0.11120042872454448, - "macro_pr_auc": 0.15885516438894723, - "worst_group_fpr": 0.05384272914463799, - "n_models": 3, - "seconds": 1.2222119580037543, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06519281732032095, - "roc_auc": 0.6738063001417901, - "precision_at_n": 0.1227224008574491, - "macro_pr_auc": 0.158891938851795, - "worst_group_fpr": 0.07998283016168264, - "n_models": 1, - "seconds": 0.7430598750070203, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07404418513125219, - "roc_auc": 0.6942472350676308, - "precision_at_n": 0.1195069667738478, - "macro_pr_auc": 0.1666954513127937, - "worst_group_fpr": 0.0698478561549101, - "n_models": 1, - "seconds": 0.8700872080007684, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06852158356992351, - "roc_auc": 0.6838487304794965, - "precision_at_n": 0.09271168274383708, - "macro_pr_auc": 0.15687985349730973, - "worst_group_fpr": 0.05011408536924704, - "n_models": 3, - "seconds": 1.2468310420008493, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0638559054729082, - "roc_auc": 0.6938116248469371, - "precision_at_n": 0.10664523043944266, - "macro_pr_auc": 0.15968816094249016, - "worst_group_fpr": 0.08630228454237611, - "n_models": 1, - "seconds": 0.7496217079969938, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06967658768327425, - "roc_auc": 0.7018962506837167, - "precision_at_n": 0.12459807073954984, - "macro_pr_auc": 0.1598466003967197, - "worst_group_fpr": 0.07776505937902418, - "n_models": 1, - "seconds": 0.8879480000032345, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06663778758586969, - "roc_auc": 0.687698771865821, - "precision_at_n": 0.09056806002143623, - "macro_pr_auc": 0.1640674246320529, - "worst_group_fpr": 0.05158884745951361, - "n_models": 3, - "seconds": 1.2128442499961238, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06533443887362353, - "roc_auc": 0.6778376702748382, - "precision_at_n": 0.12057877813504823, - "macro_pr_auc": 0.15811107018831475, - "worst_group_fpr": 0.08167596699575523, - "n_models": 1, - "seconds": 0.69004537499859, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07018784527539995, - "roc_auc": 0.6980466646796162, - "precision_at_n": 0.1192390139335477, - "macro_pr_auc": 0.15811937773723314, - "worst_group_fpr": 0.07187485095626461, - "n_models": 1, - "seconds": 0.8022012090004864, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06834331779554961, - "roc_auc": 0.6803347166261355, - "precision_at_n": 0.0972668810289389, - "macro_pr_auc": 0.16461384850476882, - "worst_group_fpr": 0.04976868412266896, - "n_models": 3, - "seconds": 1.1120200830046088, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06053581675987888, - "roc_auc": 0.668970457711801, - "precision_at_n": 0.10423365487674169, - "macro_pr_auc": 0.1569346231917846, - "worst_group_fpr": 0.0855153336194973, - "n_models": 1, - "seconds": 0.6803973339992808, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.0617195942992034, - "roc_auc": 0.6893867797234215, - "precision_at_n": 0.10155412647374062, - "macro_pr_auc": 0.1526195644635254, - "worst_group_fpr": 0.08673153050031002, - "n_models": 1, - "seconds": 0.7928864999994403, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07225183134030269, - "roc_auc": 0.685367044111213, - "precision_at_n": 0.1007502679528403, - "macro_pr_auc": 0.1624603653980992, - "worst_group_fpr": 0.04829017026756331, - "n_models": 3, - "seconds": 1.0913756669979193, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "pooled", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06249021828056783, - "roc_auc": 0.6844033192057928, - "precision_at_n": 0.11093247588424437, - "macro_pr_auc": 0.15920504233526747, - "worst_group_fpr": 0.08565841560547527, - "n_models": 1, - "seconds": 0.7009786250055186, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "relative", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06522626236440478, - "roc_auc": 0.6960089579227221, - "precision_at_n": 0.10664523043944266, - "macro_pr_auc": 0.15960446636335607, - "worst_group_fpr": 0.08587303858444222, - "n_models": 1, - "seconds": 0.7910215829979279, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "smd", - "grouping": "machine_family", - "config": "per_group", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06965281685110047, - "roc_auc": 0.6932083201222992, - "precision_at_n": 0.10289389067524116, - "macro_pr_auc": 0.15253021291295593, - "worst_group_fpr": 0.049362791474205574, - "n_models": 3, - "seconds": 1.1303733750028186, - "eta_squared": 0.06691168539386136, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6530718247293432, - "roc_auc": 0.9708910904550194, - "precision_at_n": 0.7299854439592431, - "macro_pr_auc": 0.47413483504435705, - "worst_group_fpr": 0.036940298507462686, - "n_models": 1, - "seconds": 0.6308139160028077, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7497335229655795, - "roc_auc": 0.973156389563252, - "precision_at_n": 0.7743813682678311, - "macro_pr_auc": 0.5638693990084613, - "worst_group_fpr": 0.03677238805970149, - "n_models": 1, - "seconds": 0.6763731669998378, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6771727063514942, - "roc_auc": 0.9835831483054197, - "precision_at_n": 0.7882096069868996, - "macro_pr_auc": 0.6270798302274286, - "worst_group_fpr": 0.09530320090075599, - "n_models": 3, - "seconds": 0.8962047079985496, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6597227358298141, - "roc_auc": 0.9714515184501269, - "precision_at_n": 0.7692867540029112, - "macro_pr_auc": 0.5270690913982637, - "worst_group_fpr": 0.04151119402985075, - "n_models": 1, - "seconds": 0.6266773329989519, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6099507905927241, - "roc_auc": 0.9721238083312913, - "precision_at_n": 0.764919941775837, - "macro_pr_auc": 0.5031503078935212, - "worst_group_fpr": 0.04154850746268657, - "n_models": 1, - "seconds": 0.669698249999783, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7144381558306845, - "roc_auc": 0.9837242333729553, - "precision_at_n": 0.8056768558951966, - "macro_pr_auc": 0.6547106780700749, - "worst_group_fpr": 0.07181920540453594, - "n_models": 3, - "seconds": 0.9075735829974292, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6551800529689307, - "roc_auc": 0.974870803601394, - "precision_at_n": 0.7707423580786026, - "macro_pr_auc": 0.547308769108289, - "worst_group_fpr": 0.038992537313432836, - "n_models": 1, - "seconds": 0.6317226250030217, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.728918724499466, - "roc_auc": 0.9748660753684439, - "precision_at_n": 0.7729257641921398, - "macro_pr_auc": 0.5685681355212074, - "worst_group_fpr": 0.035559701492537316, - "n_models": 1, - "seconds": 0.6641381249937695, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6966054974597077, - "roc_auc": 0.9831038027507876, - "precision_at_n": 0.8005822416302766, - "macro_pr_auc": 0.6434765539787998, - "worst_group_fpr": 0.08637606562650796, - "n_models": 3, - "seconds": 0.9248066250002012, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6184045391453056, - "roc_auc": 0.9714600941137747, - "precision_at_n": 0.7729257641921398, - "macro_pr_auc": 0.5005378454330374, - "worst_group_fpr": 0.038992537313432836, - "n_models": 1, - "seconds": 0.6678157499991357, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.673219025125259, - "roc_auc": 0.9720874684837606, - "precision_at_n": 0.7816593886462883, - "macro_pr_auc": 0.5521651623217471, - "worst_group_fpr": 0.038992537313432836, - "n_models": 1, - "seconds": 0.6711975829966832, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6865572065952276, - "roc_auc": 0.9829511591800745, - "precision_at_n": 0.7823871906841339, - "macro_pr_auc": 0.6059527323311097, - "worst_group_fpr": 0.09329258484799742, - "n_models": 3, - "seconds": 0.9055055839999113, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.673650740437859, - "roc_auc": 0.9738444474258431, - "precision_at_n": 0.7671033478893741, - "macro_pr_auc": 0.5506975659722858, - "worst_group_fpr": 0.041902985074626864, - "n_models": 1, - "seconds": 0.6526410829974338, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7167629439101754, - "roc_auc": 0.9738429506023834, - "precision_at_n": 0.7751091703056768, - "macro_pr_auc": 0.5749534582846149, - "worst_group_fpr": 0.04027985074626866, - "n_models": 1, - "seconds": 0.6724003749986878, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6970946977891934, - "roc_auc": 0.983209104551357, - "precision_at_n": 0.7991266375545851, - "macro_pr_auc": 0.6140277347030043, - "worst_group_fpr": 0.0788965739102461, - "n_models": 3, - "seconds": 0.8911962499987567, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6812673546499289, - "roc_auc": 0.9723335095153985, - "precision_at_n": 0.7641921397379913, - "macro_pr_auc": 0.5360583720220582, - "worst_group_fpr": 0.03869402985074627, - "n_models": 1, - "seconds": 0.6281392500022775, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.695258397131262, - "roc_auc": 0.9726748771270051, - "precision_at_n": 0.7787481804949054, - "macro_pr_auc": 0.5395418219552015, - "worst_group_fpr": 0.03826492537313433, - "n_models": 1, - "seconds": 0.6767856250007753, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7292724645874825, - "roc_auc": 0.9837263570250118, - "precision_at_n": 0.8165938864628821, - "macro_pr_auc": 0.6257470284541894, - "worst_group_fpr": 0.09007559916358372, - "n_models": 3, - "seconds": 0.9138907919987105, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6220666923169502, - "roc_auc": 0.9721022800112077, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.4990001498959143, - "worst_group_fpr": 0.03923507462686567, - "n_models": 1, - "seconds": 0.6433648340025684, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6678377745312253, - "roc_auc": 0.972353881444795, - "precision_at_n": 0.7758369723435226, - "macro_pr_auc": 0.5392976747195802, - "worst_group_fpr": 0.037798507462686565, - "n_models": 1, - "seconds": 0.6674270410003373, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6800917400408971, - "roc_auc": 0.9837985071579827, - "precision_at_n": 0.8129548762736536, - "macro_pr_auc": 0.6298478885384172, - "worst_group_fpr": 0.08283738137365289, - "n_models": 3, - "seconds": 0.9231200419962988, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6053543556413484, - "roc_auc": 0.9713649134335658, - "precision_at_n": 0.7540029112081513, - "macro_pr_auc": 0.49191252560248805, - "worst_group_fpr": 0.04166044776119403, - "n_models": 1, - "seconds": 0.6326952500021434, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6831024791337322, - "roc_auc": 0.9732640095488907, - "precision_at_n": 0.7714701601164483, - "macro_pr_auc": 0.5443222382523539, - "worst_group_fpr": 0.03914179104477612, - "n_models": 1, - "seconds": 0.6735367920045974, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6689447474000777, - "roc_auc": 0.9819260998912754, - "precision_at_n": 0.7940320232896652, - "macro_pr_auc": 0.627896819171807, - "worst_group_fpr": 0.07262345182563938, - "n_models": 3, - "seconds": 0.914368874997308, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6772511305344789, - "roc_auc": 0.9708570093519151, - "precision_at_n": 0.7692867540029112, - "macro_pr_auc": 0.528819686901723, - "worst_group_fpr": 0.04027985074626866, - "n_models": 1, - "seconds": 0.618268457998056, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7042421120611811, - "roc_auc": 0.9742613640944495, - "precision_at_n": 0.7751091703056768, - "macro_pr_auc": 0.5683334962817763, - "worst_group_fpr": 0.036940298507462686, - "n_models": 1, - "seconds": 0.6780074999987846, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7138022925894035, - "roc_auc": 0.9844030238989642, - "precision_at_n": 0.8064046579330422, - "macro_pr_auc": 0.6190764672520105, - "worst_group_fpr": 0.09465980376387326, - "n_models": 3, - "seconds": 0.9198891669948353, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6632591000239041, - "roc_auc": 0.9723781224196681, - "precision_at_n": 0.7685589519650655, - "macro_pr_auc": 0.4999708246289276, - "worst_group_fpr": 0.03716417910447761, - "n_models": 1, - "seconds": 0.6268997080042027, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6748276857732136, - "roc_auc": 0.9760264972119853, - "precision_at_n": 0.7751091703056768, - "macro_pr_auc": 0.5782420564201117, - "worst_group_fpr": 0.039048507462686566, - "n_models": 1, - "seconds": 0.6950577919997158, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6683292840636673, - "roc_auc": 0.9832841618721304, - "precision_at_n": 0.8049490538573508, - "macro_pr_auc": 0.6310565621169522, - "worst_group_fpr": 0.10278269261701785, - "n_models": 3, - "seconds": 0.9131106249988079, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6434356874231278, - "roc_auc": 0.9722437055115158, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.4954406370669884, - "worst_group_fpr": 0.037033582089552236, - "n_models": 1, - "seconds": 0.6211250419946737, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6739652779593103, - "roc_auc": 0.9730661640711749, - "precision_at_n": 0.7780203784570596, - "macro_pr_auc": 0.5523917874713837, - "worst_group_fpr": 0.04022388059701493, - "n_models": 1, - "seconds": 0.6721129580037086, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6811629242290259, - "roc_auc": 0.9835466625581294, - "precision_at_n": 0.8042212518195051, - "macro_pr_auc": 0.61623572056422, - "worst_group_fpr": 0.07873572462602542, - "n_models": 3, - "seconds": 0.9090283330006059, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6551151866652897, - "roc_auc": 0.9729514652453479, - "precision_at_n": 0.7707423580786026, - "macro_pr_auc": 0.5190067105927844, - "worst_group_fpr": 0.03593283582089552, - "n_models": 1, - "seconds": 0.6247831250002491, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6482298761443848, - "roc_auc": 0.9721994816732719, - "precision_at_n": 0.7758369723435226, - "macro_pr_auc": 0.514628531207875, - "worst_group_fpr": 0.038917910447761196, - "n_models": 1, - "seconds": 0.6851487919993815, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7192257699794287, - "roc_auc": 0.9831750558704216, - "precision_at_n": 0.8136826783114993, - "macro_pr_auc": 0.6270065407992284, - "worst_group_fpr": 0.08919092810036995, - "n_models": 3, - "seconds": 0.9044852920051198, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6602705949896037, - "roc_auc": 0.9721382416001024, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.5227122218538428, - "worst_group_fpr": 0.03916044776119403, - "n_models": 1, - "seconds": 0.6109848339983728, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6950417248835344, - "roc_auc": 0.9747763199978143, - "precision_at_n": 0.7729257641921398, - "macro_pr_auc": 0.5929750027330102, - "worst_group_fpr": 0.041753731343283584, - "n_models": 1, - "seconds": 0.6856084169994574, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7336603135821277, - "roc_auc": 0.9847187725935235, - "precision_at_n": 0.8195050946142649, - "macro_pr_auc": 0.644092692138366, - "worst_group_fpr": 0.09675084445874216, - "n_models": 3, - "seconds": 0.9061175000024377, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6676094467715644, - "roc_auc": 0.9726910882114053, - "precision_at_n": 0.7743813682678311, - "macro_pr_auc": 0.5362592870799477, - "worst_group_fpr": 0.04162313432835821, - "n_models": 1, - "seconds": 0.6429056670021964, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7072077440006722, - "roc_auc": 0.9749246568237718, - "precision_at_n": 0.7700145560407569, - "macro_pr_auc": 0.5646705434198914, - "worst_group_fpr": 0.03914179104477612, - "n_models": 1, - "seconds": 0.6788953340001171, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.7261272164835856, - "roc_auc": 0.9830016729190657, - "precision_at_n": 0.8144104803493449, - "macro_pr_auc": 0.6338972683435188, - "worst_group_fpr": 0.05951423516165353, - "n_models": 3, - "seconds": 0.8952491250020103, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "pooled", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.60539275571879, - "roc_auc": 0.9715701403583787, - "precision_at_n": 0.7532751091703057, - "macro_pr_auc": 0.5082628245792461, - "worst_group_fpr": 0.04289179104477612, - "n_models": 1, - "seconds": 0.626040875002218, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "relative", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.664330698592417, - "roc_auc": 0.9745581133980917, - "precision_at_n": 0.7751091703056768, - "macro_pr_auc": 0.5706054710628522, - "worst_group_fpr": 0.04276119402985075, - "n_models": 1, - "seconds": 0.6735202080017189, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "protocol_type", - "config": "per_group", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6590068116865129, - "roc_auc": 0.9835822080625245, - "precision_at_n": 0.7532751091703057, - "macro_pr_auc": 0.6070913895966344, - "worst_group_fpr": 0.07431236930995658, - "n_models": 3, - "seconds": 0.9206780829990748, - "eta_squared": 0.05087186473761437, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6530718247293432, - "roc_auc": 0.9708910904550194, - "precision_at_n": 0.7299854439592431, - "macro_pr_auc": 0.7047727154351631, - "worst_group_fpr": 0.8172043010752689, - "n_models": 1, - "seconds": 0.6172814169985941, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5322775873886177, - "roc_auc": 0.9628784701906581, - "precision_at_n": 0.5232896652110626, - "macro_pr_auc": 0.7500437558549391, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6833220410044305, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.19426886828720197, - "roc_auc": 0.8750921140834098, - "precision_at_n": 0.20160116448326054, - "macro_pr_auc": 0.6475584754233464, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.103904083000089, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6597227358298141, - "roc_auc": 0.9714515184501269, - "precision_at_n": 0.7692867540029112, - "macro_pr_auc": 0.7575684869258095, - "worst_group_fpr": 0.8387096774193549, - "n_models": 1, - "seconds": 0.6304595829933533, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.516504186652572, - "roc_auc": 0.9637440934643803, - "precision_at_n": 0.529839883551674, - "macro_pr_auc": 0.7272570347699341, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6644380840007216, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.20733998337975884, - "roc_auc": 0.8771236601619798, - "precision_at_n": 0.21106259097525473, - "macro_pr_auc": 0.678921859564307, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.136722833995009, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6551800529689307, - "roc_auc": 0.974870803601394, - "precision_at_n": 0.7707423580786026, - "macro_pr_auc": 0.7202188847677089, - "worst_group_fpr": 0.8602150537634409, - "n_models": 1, - "seconds": 0.6134327499967185, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5529261848221064, - "roc_auc": 0.9673986500835488, - "precision_at_n": 0.5312954876273653, - "macro_pr_auc": 0.772580116437516, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6953558339955634, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.2088041217715933, - "roc_auc": 0.8777718333532514, - "precision_at_n": 0.21615720524017468, - "macro_pr_auc": 0.6641493269337587, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.143598791000841, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6184045391453056, - "roc_auc": 0.9714600941137747, - "precision_at_n": 0.7729257641921398, - "macro_pr_auc": 0.7310428069140714, - "worst_group_fpr": 0.8387096774193549, - "n_models": 1, - "seconds": 0.6243396659992868, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5170898627982412, - "roc_auc": 0.961976842098483, - "precision_at_n": 0.5283842794759825, - "macro_pr_auc": 0.7333993025552435, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.7306586250051623, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.2054538194435767, - "roc_auc": 0.8798870448384113, - "precision_at_n": 0.2074235807860262, - "macro_pr_auc": 0.6689276791739517, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.13056616600079, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.673650740437859, - "roc_auc": 0.9738444474258431, - "precision_at_n": 0.7671033478893741, - "macro_pr_auc": 0.7304219338677542, - "worst_group_fpr": 0.8763440860215054, - "n_models": 1, - "seconds": 0.6449807910030358, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5310353519342732, - "roc_auc": 0.9637141083619345, - "precision_at_n": 0.5262008733624454, - "macro_pr_auc": 0.7231842722605921, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6754672909955843, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.1987803623966471, - "roc_auc": 0.8778387527096557, - "precision_at_n": 0.21397379912663755, - "macro_pr_auc": 0.6769142802897322, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.070312916999683, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6812673546499289, - "roc_auc": 0.9723335095153985, - "precision_at_n": 0.7641921397379913, - "macro_pr_auc": 0.7319369833246042, - "worst_group_fpr": 0.8440860215053764, - "n_models": 1, - "seconds": 0.6568119580042548, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5126336105780375, - "roc_auc": 0.9606217683608526, - "precision_at_n": 0.5262008733624454, - "macro_pr_auc": 0.7654565381324793, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6896719579963246, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.19295065274944362, - "roc_auc": 0.8757887205911745, - "precision_at_n": 0.20378457059679767, - "macro_pr_auc": 0.671566525517948, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.0496395000009215, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6220666923169502, - "roc_auc": 0.9721022800112077, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.7352412446077395, - "worst_group_fpr": 0.8655913978494624, - "n_models": 1, - "seconds": 0.6435970000020461, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5279299951184584, - "roc_auc": 0.9620832462527916, - "precision_at_n": 0.5291120815138283, - "macro_pr_auc": 0.7833092631457893, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6910863750017597, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.2087712800249874, - "roc_auc": 0.8793636105379052, - "precision_at_n": 0.2074235807860262, - "macro_pr_auc": 0.6783338950792724, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.009161374997348, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6053543556413484, - "roc_auc": 0.9713649134335658, - "precision_at_n": 0.7540029112081513, - "macro_pr_auc": 0.6914621548200052, - "worst_group_fpr": 0.8709677419354839, - "n_models": 1, - "seconds": 0.6262198330005049, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5260966945818674, - "roc_auc": 0.963761509572721, - "precision_at_n": 0.5312954876273653, - "macro_pr_auc": 0.7279858708756674, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6726583750059945, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.20223291977384794, - "roc_auc": 0.8759632491258281, - "precision_at_n": 0.21542940320232898, - "macro_pr_auc": 0.6697223820051702, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.070021457999246, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6772511305344789, - "roc_auc": 0.9708570093519151, - "precision_at_n": 0.7692867540029112, - "macro_pr_auc": 0.7360282670260866, - "worst_group_fpr": 0.8440860215053764, - "n_models": 1, - "seconds": 0.6317711250012508, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5325720465137227, - "roc_auc": 0.9601075581673703, - "precision_at_n": 0.5283842794759825, - "macro_pr_auc": 0.7394736995824194, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6852393330045743, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.20531187605608703, - "roc_auc": 0.8796904854400578, - "precision_at_n": 0.2081513828238719, - "macro_pr_auc": 0.6665454810860403, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.105933708000521, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6632591000239041, - "roc_auc": 0.9723781224196681, - "precision_at_n": 0.7685589519650655, - "macro_pr_auc": 0.7148660857094303, - "worst_group_fpr": 0.8440860215053764, - "n_models": 1, - "seconds": 0.6665979579993291, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.4795792385930423, - "roc_auc": 0.9626318347525921, - "precision_at_n": 0.48326055312954874, - "macro_pr_auc": 0.7419634640352997, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6996548750030342, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.2057538989312502, - "roc_auc": 0.87525956918157, - "precision_at_n": 0.21470160116448325, - "macro_pr_auc": 0.6633964988511316, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.075786749999679, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6434356874231278, - "roc_auc": 0.9722437055115158, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.716768355198076, - "worst_group_fpr": 0.8655913978494624, - "n_models": 1, - "seconds": 0.6215927919984097, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.49772334343685765, - "roc_auc": 0.9639381941815997, - "precision_at_n": 0.5305676855895196, - "macro_pr_auc": 0.7442065571265031, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.6849310000034166, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.209778729461222, - "roc_auc": 0.8766734297149307, - "precision_at_n": 0.2052401746724891, - "macro_pr_auc": 0.6614477574664632, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.478175124997506, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6551151866652897, - "roc_auc": 0.9729514652453479, - "precision_at_n": 0.7707423580786026, - "macro_pr_auc": 0.7218815255148043, - "worst_group_fpr": 0.8494623655913979, - "n_models": 1, - "seconds": 0.654631833996973, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5165568779889708, - "roc_auc": 0.9632808671313369, - "precision_at_n": 0.5305676855895196, - "macro_pr_auc": 0.7078322070864903, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.7286407920037163, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.2028376257503053, - "roc_auc": 0.8751945789442093, - "precision_at_n": 0.2059679767103348, - "macro_pr_auc": 0.6623792624130437, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.391475541997352, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6602705949896037, - "roc_auc": 0.9721382416001024, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.721034146320275, - "worst_group_fpr": 0.8548387096774194, - "n_models": 1, - "seconds": 0.704520124992996, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.459050906670713, - "roc_auc": 0.9620710176914591, - "precision_at_n": 0.46797671033478894, - "macro_pr_auc": 0.744862577019632, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.7335899169993354, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.20464191703683093, - "roc_auc": 0.8755868493608325, - "precision_at_n": 0.21688500727802038, - "macro_pr_auc": 0.6695421930565572, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.637700624996796, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6676094467715644, - "roc_auc": 0.9726910882114053, - "precision_at_n": 0.7743813682678311, - "macro_pr_auc": 0.7492491008684662, - "worst_group_fpr": 0.8602150537634409, - "n_models": 1, - "seconds": 0.7284391249995679, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.5254747583231889, - "roc_auc": 0.9631463205345092, - "precision_at_n": 0.5254730713245997, - "macro_pr_auc": 0.7365112431113047, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.7573751669988269, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.19958209589022063, - "roc_auc": 0.8764810797948264, - "precision_at_n": 0.21324599708879186, - "macro_pr_auc": 0.6759344067395817, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.59769783399679, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "pooled", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.60539275571879, - "roc_auc": 0.9715701403583787, - "precision_at_n": 0.7532751091703057, - "macro_pr_auc": 0.708138965631994, - "worst_group_fpr": 0.8709677419354839, - "n_models": 1, - "seconds": 0.7024482499982696, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "relative", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.48464488226783314, - "roc_auc": 0.9612562539931953, - "precision_at_n": 0.5276564774381368, - "macro_pr_auc": 0.7247406489693413, - "worst_group_fpr": 0.5, - "n_models": 1, - "seconds": 0.7129539590023342, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "service", - "config": "per_group", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.18838265290772957, - "roc_auc": 0.8743656521618746, - "precision_at_n": 0.19723435225618632, - "macro_pr_auc": 0.6705908870499475, - "worst_group_fpr": 1.0, - "n_models": 50, - "seconds": 6.1209764999948675, - "eta_squared": 0.23802723873795992, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6530718247293432, - "roc_auc": 0.9708910904550194, - "precision_at_n": 0.7299854439592431, - "macro_pr_auc": 0.6223117227115356, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6493378750019474, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.2925375728206705, - "roc_auc": 0.9643675069260778, - "precision_at_n": 0.2780203784570597, - "macro_pr_auc": 0.6131191976439884, - "worst_group_fpr": 0.6412429378531074, - "n_models": 1, - "seconds": 0.6769333749980433, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 0, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06827038708560274, - "roc_auc": 0.7024115295739569, - "precision_at_n": 0.11572052401746726, - "macro_pr_auc": 0.38428258478195154, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.5290592079982162, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6597227358298141, - "roc_auc": 0.9714515184501269, - "precision_at_n": 0.7692867540029112, - "macro_pr_auc": 0.6730425633608333, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6372252500004834, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.28200565888221923, - "roc_auc": 0.9623676967470687, - "precision_at_n": 0.21251819505094613, - "macro_pr_auc": 0.6085606750076077, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6727971660002368, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 1, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06820440985503894, - "roc_auc": 0.7123372415231752, - "precision_at_n": 0.11208151382823872, - "macro_pr_auc": 0.3902930878886586, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.5607022499971208, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6551800529689307, - "roc_auc": 0.974870803601394, - "precision_at_n": 0.7707423580786026, - "macro_pr_auc": 0.7199726852544707, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6439404170014313, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.3913764879820988, - "roc_auc": 0.9727182309703861, - "precision_at_n": 0.4556040756914119, - "macro_pr_auc": 0.6532797230546793, - "worst_group_fpr": 0.5684931506849316, - "n_models": 1, - "seconds": 0.6842081249997136, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 2, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07744738380734927, - "roc_auc": 0.7355907560160252, - "precision_at_n": 0.13755458515283842, - "macro_pr_auc": 0.4034029553432024, - "worst_group_fpr": 0.6363636363636364, - "n_models": 9, - "seconds": 1.5249805839994224, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6184045391453056, - "roc_auc": 0.9714600941137747, - "precision_at_n": 0.7729257641921398, - "macro_pr_auc": 0.6627545279891364, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6204587500033085, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.27882098766438645, - "roc_auc": 0.9643165284693337, - "precision_at_n": 0.2612809315866084, - "macro_pr_auc": 0.6168349334626622, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.678875915997196, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 3, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06988953355746912, - "roc_auc": 0.7285389991462379, - "precision_at_n": 0.10116448326055313, - "macro_pr_auc": 0.39571710183865605, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.50486500000261, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.673650740437859, - "roc_auc": 0.9738444474258431, - "precision_at_n": 0.7671033478893741, - "macro_pr_auc": 0.6703092351092165, - "worst_group_fpr": 0.9794520547945206, - "n_models": 1, - "seconds": 0.6531753750023199, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.4333450080079327, - "roc_auc": 0.9762862420136362, - "precision_at_n": 0.5327510917030568, - "macro_pr_auc": 0.6493967498853038, - "worst_group_fpr": 0.5547945205479452, - "n_models": 1, - "seconds": 0.7706693750005797, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 4, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08561459107619543, - "roc_auc": 0.7184035968203016, - "precision_at_n": 0.17321688500727803, - "macro_pr_auc": 0.4086851369556044, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.535262249999505, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6812673546499289, - "roc_auc": 0.9723335095153985, - "precision_at_n": 0.7641921397379913, - "macro_pr_auc": 0.6779446797938639, - "worst_group_fpr": 0.958904109589041, - "n_models": 1, - "seconds": 0.6306185420035035, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.3585514128845653, - "roc_auc": 0.9694663252655521, - "precision_at_n": 0.35807860262008734, - "macro_pr_auc": 0.6543872673773167, - "worst_group_fpr": 0.5342465753424658, - "n_models": 1, - "seconds": 0.6698882499986212, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 5, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07531711806829501, - "roc_auc": 0.7476751305602911, - "precision_at_n": 0.10407569141193596, - "macro_pr_auc": 0.3993828703937818, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.5176797910025925, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6220666923169502, - "roc_auc": 0.9721022800112077, - "precision_at_n": 0.7656477438136827, - "macro_pr_auc": 0.6444243052139519, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6436851670005126, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.3116587794206096, - "roc_auc": 0.9665341021450918, - "precision_at_n": 0.22634643377001457, - "macro_pr_auc": 0.6211672285158993, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.687488666997524, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 6, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07848036388509816, - "roc_auc": 0.7032876414192861, - "precision_at_n": 0.12299854439592431, - "macro_pr_auc": 0.4012301923957859, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.5172687500016764, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6053543556413484, - "roc_auc": 0.9713649134335658, - "precision_at_n": 0.7540029112081513, - "macro_pr_auc": 0.6766219848505614, - "worst_group_fpr": 0.958904109589041, - "n_models": 1, - "seconds": 0.6216738750008517, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.36309104554360055, - "roc_auc": 0.9709513578631249, - "precision_at_n": 0.4243085880640466, - "macro_pr_auc": 0.5981743246147424, - "worst_group_fpr": 0.7191780821917808, - "n_models": 1, - "seconds": 0.6808708329990623, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 7, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07690028702241986, - "roc_auc": 0.7066107408031114, - "precision_at_n": 0.10116448326055313, - "macro_pr_auc": 0.39231441855606347, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.5898154169990448, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6772511305344789, - "roc_auc": 0.9708570093519151, - "precision_at_n": 0.7692867540029112, - "macro_pr_auc": 0.6722284388637143, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6161034579999978, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.33804833606603873, - "roc_auc": 0.9686204524963242, - "precision_at_n": 0.3042212518195051, - "macro_pr_auc": 0.643267830351372, - "worst_group_fpr": 0.6384180790960452, - "n_models": 1, - "seconds": 0.6863054170025862, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 8, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.08181791184607733, - "roc_auc": 0.722418974352357, - "precision_at_n": 0.13537117903930132, - "macro_pr_auc": 0.40322326447032014, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.491453749993525, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6632591000239041, - "roc_auc": 0.9723781224196681, - "precision_at_n": 0.7685589519650655, - "macro_pr_auc": 0.6651138433157199, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6094639580041985, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.35174263797174954, - "roc_auc": 0.9691429681687144, - "precision_at_n": 0.36681222707423583, - "macro_pr_auc": 0.58867333788018, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6934875830047531, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 9, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06856972617361667, - "roc_auc": 0.7178113356591268, - "precision_at_n": 0.12008733624454149, - "macro_pr_auc": 0.40376882066536895, - "worst_group_fpr": 0.45454545454545453, - "n_models": 9, - "seconds": 1.498864124994725, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6434356874231278, - "roc_auc": 0.9722437055115158, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.6728144483691012, - "worst_group_fpr": 0.952054794520548, - "n_models": 1, - "seconds": 0.6182659159967443, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.31794713082169135, - "roc_auc": 0.9669186939114042, - "precision_at_n": 0.2867540029112082, - "macro_pr_auc": 0.5600754112405154, - "worst_group_fpr": 0.9322033898305084, - "n_models": 1, - "seconds": 0.6796225420039264, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 10, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.06836061567799057, - "roc_auc": 0.7435770548830153, - "precision_at_n": 0.09097525473071325, - "macro_pr_auc": 0.38933945894079874, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.4779979160011862, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6551151866652897, - "roc_auc": 0.9729514652453479, - "precision_at_n": 0.7707423580786026, - "macro_pr_auc": 0.6727633798882601, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6319372079960885, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.3740197046131641, - "roc_auc": 0.9709152719892499, - "precision_at_n": 0.3922852983988355, - "macro_pr_auc": 0.6510074992250879, - "worst_group_fpr": 0.636986301369863, - "n_models": 1, - "seconds": 0.7073003330006031, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 11, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07391365513712772, - "roc_auc": 0.7449540297956705, - "precision_at_n": 0.09461426491994178, - "macro_pr_auc": 0.3997538857745237, - "worst_group_fpr": 0.45454545454545453, - "n_models": 9, - "seconds": 1.4909879580009147, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6602705949896037, - "roc_auc": 0.9721382416001024, - "precision_at_n": 0.7663755458515283, - "macro_pr_auc": 0.6526067810644735, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.6282843750013853, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.3108883708145589, - "roc_auc": 0.9681155420615929, - "precision_at_n": 0.26564774381368267, - "macro_pr_auc": 0.6079535655714939, - "worst_group_fpr": 0.6242937853107344, - "n_models": 1, - "seconds": 0.6849255420020199, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 12, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.078883980951485, - "roc_auc": 0.7297432395509132, - "precision_at_n": 0.11644832605531295, - "macro_pr_auc": 0.4047703027308915, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.4905037909993553, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.6676094467715644, - "roc_auc": 0.9726910882114053, - "precision_at_n": 0.7743813682678311, - "macro_pr_auc": 0.6760766023815652, - "worst_group_fpr": 0.952054794520548, - "n_models": 1, - "seconds": 0.650708916997246, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.3857369689812618, - "roc_auc": 0.9723167634652131, - "precision_at_n": 0.3937409024745269, - "macro_pr_auc": 0.609113254613314, - "worst_group_fpr": 0.7465753424657534, - "n_models": 1, - "seconds": 0.6833039580014884, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 13, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.07273420453606227, - "roc_auc": 0.7159277373404886, - "precision_at_n": 0.11935953420669577, - "macro_pr_auc": 0.3943276800611047, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.517019207996782, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "pooled", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.60539275571879, - "roc_auc": 0.9715701403583787, - "precision_at_n": 0.7532751091703057, - "macro_pr_auc": 0.6665123742922626, - "worst_group_fpr": 0.9452054794520548, - "n_models": 1, - "seconds": 0.630134041995916, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "relative", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.30897573098961106, - "roc_auc": 0.9648347590117471, - "precision_at_n": 0.22634643377001457, - "macro_pr_auc": 0.5950681046342272, - "worst_group_fpr": 1.0, - "n_models": 1, - "seconds": 0.697411250002915, - "eta_squared": 0.18908304175391513, - "warnings": [] - }, - { - "dataset": "nslkdd", - "grouping": "flag", - "config": "per_group", - "seed": 14, - "mechanism": "real", - "level_spread": NaN, - "pr_auc": 0.09274576542190417, - "roc_auc": 0.7378824791918303, - "precision_at_n": 0.17467248908296942, - "macro_pr_auc": 0.404764037047564, - "worst_group_fpr": 0.5454545454545454, - "n_models": 9, - "seconds": 1.5153749590026564, - "eta_squared": 0.18908304175391513, - "warnings": [] - } - ] -} \ No newline at end of file diff --git a/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.md b/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.md deleted file mode 100644 index 7eff5b654..000000000 --- a/benchmarks/anomaly_conditioning/results/2026-08-25-f9c703a1.md +++ /dev/null @@ -1,99 +0,0 @@ -# Anomaly conditioning experiment — 2026-08-25 - -DQX `f9c703a1` · datasets: synthetic, smd, nslkdd, tabular · seeds per cell: 15 · cells: 1545 - -PR-AUC is the primary metric. No point-adjusted F1 appears anywhere; see `metrics.py`. -DQX scores rows independently, so figures on time-series data are not comparable with -published sequence-model results — that is a different task, not a worse implementation. - -## Does removing the heterogeneity gate cost anything? - -Rules fixed before running: **no gate** if the worst delta below eta-squared 0.1 exceeds -0.01; **gate needed** if any such cell reaches -0.02. - -- **Pre-registered rule (worst single cell): gate needed; refit the threshold from this sweep** -- **Variance-robust companion (worst per-grouping median): no gate needed** - -The pre-registered rule takes a minimum over individual cells, so it is maximally sensitive to estimator variance. It is reported unchanged, alongside the median form of the same question, so the criterion set in advance and the answer it gave are both visible. Where the two disagree, the per-grouping deltas below show why. - -| statistic | value | -|---|---| -| n_low_eta_cells | 90 | -| n_low_eta_groupings | 4 | -| worst_delta_single_cell | -0.0498 | -| n_harmful_cells | 1 | -| worst_median_delta_per_grouping | +0.0000 | -| n_harmful_groupings | 0 | -| median_delta_below_threshold | +0.0000 | - -### Does eta-squared predict the benefit at all? - -Spearman rho(eta-squared, delta) = **+0.571** (bootstrap 95% CI +0.490 to +0.645). - -Least squares `delta ~ 1 + eta_squared + is_contextual`, which asks whether eta-squared still carries signal once the anomaly mechanism is controlled for: - -| term | coefficient | -|---|---| -| intercept | -0.1015 | -| eta_squared | +0.2808 | -| is_contextual | +0.0849 | -| R-squared | 0.578 (n=390) | - -## Plain tabular benchmarks (no grouping) - -No grouping exists in these datasets, so conditioning is not applicable and only the pooled -configuration runs. This characterises the mechanism on the benchmarks the field reports, and -is the regression check that adding conditioning did not disturb the ungrouped path. - -PR-AUC is **not comparable across rows** — it moves with the base rate — so each dataset is -read against its own random floor. `lift` is DQX PR-AUC divided by the random floor. - -| dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random | beats max-abs-z | -|---|---|---|---|---|---|---|---|---| -| campaign | 30000 | 62 | 11.27% | 0.2867 | 0.1160 | 0.2398 | 2.5x | yes | -| cardio | 1831 | 21 | 9.61% | 0.5811 | 0.1012 | 0.5534 | 5.7x | yes | -| covertype | 30000 | 10 | 0.96% | 0.0534 | 0.0094 | 0.1122 | 5.7x | **no** | -| fraud | 30000 | 29 | 0.17% | 0.2240 | 0.0019 | 0.1365 | 115.4x | yes | -| mammography | 11183 | 6 | 2.32% | 0.2193 | 0.0236 | 0.1779 | 9.3x | yes | -| mnist | 7603 | 100 | 9.21% | 0.2766 | 0.0999 | 0.3367 | 2.8x | **no** | -| satellite | 6435 | 36 | 31.64% | 0.6668 | 0.3131 | 0.5946 | 2.1x | yes | -| shuttle | 30000 | 9 | 7.15% | 0.9788 | 0.0731 | 0.8983 | 13.4x | yes | -| spambase | 4207 | 57 | 39.91% | 0.4758 | 0.4033 | 0.4039 | 1.2x | yes | -| thyroid | 3772 | 6 | 2.47% | 0.5389 | 0.0251 | 0.3007 | 21.5x | yes | - -### Every grouping below eta-squared 0.1, seed by seed - -| dataset | grouping | eta-squared | median delta | min | max | seeds agree? | -|---|---|---|---|---|---|---| -| nslkdd | protocol_type | 0.0509 | +0.0396 | -0.0498 | +0.0967 | **no** | -| smd | machine_family | 0.0669 | +0.0029 | -0.0027 | +0.0103 | **no** | -| synthetic | spread=0.000 | 0.0008 | +0.0000 | -0.0027 | +0.0299 | **no** | -| synthetic | spread=0.050 | 0.0593 | +0.0000 | -0.0002 | +0.0222 | **no** | - -## Paired comparisons - -### baseline-relative minus pooled (PR-AUC) - -| mechanism | n | median delta | IQR | Wilcoxon p | -|---|---|---|---|---| -| contextual | 195 | +0.0742 | +0.0076 to +0.3070 | 1.31e-31 | -| global | 195 | +0.0000 | -0.0000 to +0.0000 | 2.26e-01 | -| real | 75 | +0.0025 | -0.1452 to +0.0124 | 1.75e-02 | -| *all* | 465 | +0.0001 | +0.0000 to +0.0431 | 1.50e-25 | - -### baseline-relative minus per-group (PR-AUC) - -| mechanism | n | median delta | IQR | Wilcoxon p | -|---|---|---|---|---| -| contextual | 195 | +0.0252 | +0.0147 to +0.0356 | 9.41e-34 | -| global | 195 | +0.0005 | +0.0000 to +0.0019 | 6.12e-29 | -| real | 75 | +0.0054 | -0.0051 to +0.2832 | 2.74e-04 | -| *all* | 465 | +0.0065 | +0.0002 to +0.0254 | 3.42e-56 | - -## Cost - -| config | median models | median seconds | -|---|---|---| -| pooled | 1 | 0.17 | -| relative | 1 | 0.17 | -| per_group | 12 | 1.38 | - diff --git a/benchmarks/anomaly_conditioning/results/complementary-detector.json b/benchmarks/anomaly_conditioning/results/complementary-detector.json deleted file mode 100644 index ad7f252fa..000000000 --- a/benchmarks/anomaly_conditioning/results/complementary-detector.json +++ /dev/null @@ -1,102 +0,0 @@ -[ - { - "dataset": "campaign", - "base_rate": 0.11266666666666666, - "iforest": 0.29189044522788793, - "zscore": 0.24907974586715767, - "mean_pct": 0.30525931138248596, - "max_pct": 0.28026469397768916, - "mean_vs_iforest": 0.013368866154598036, - "max_vs_iforest": -0.011625751250198768 - }, - { - "dataset": "cardio", - "base_rate": 0.0961223375204806, - "iforest": 0.5531411979830949, - "zscore": 0.5362761184610526, - "mean_pct": 0.6028574471263684, - "max_pct": 0.5403439969678784, - "mean_vs_iforest": 0.04971624914327344, - "max_vs_iforest": -0.01279720101521653 - }, - { - "dataset": "covertype", - "base_rate": 0.0096, - "iforest": 0.05085714016382059, - "zscore": 0.10085460150242194, - "mean_pct": 0.06297738498037918, - "max_pct": 0.08035853810631731, - "mean_vs_iforest": 0.012120244816558587, - "max_vs_iforest": 0.029501397942496718 - }, - { - "dataset": "fraud", - "base_rate": 0.0017333333333333333, - "iforest": 0.2606639933273247, - "zscore": 0.12492024471197495, - "mean_pct": 0.22356101154283578, - "max_pct": 0.2008899193829753, - "mean_vs_iforest": -0.037102981784488925, - "max_vs_iforest": -0.059774073944349415 - }, - { - "dataset": "mammography", - "base_rate": 0.023249575248144506, - "iforest": 0.16615148687632164, - "zscore": 0.1714218533444644, - "mean_pct": 0.18154326702612636, - "max_pct": 0.18128521174190282, - "mean_vs_iforest": 0.015391780149804718, - "max_vs_iforest": 0.015133724865581177 - }, - { - "dataset": "mnist", - "base_rate": 0.09206892016309351, - "iforest": 0.26090881659743215, - "zscore": 0.36405621393428067, - "mean_pct": 0.3084076801597011, - "max_pct": 0.3333332896133295, - "mean_vs_iforest": 0.047498863562268956, - "max_vs_iforest": 0.07242447301589733 - }, - { - "dataset": "satellite", - "base_rate": 0.3163947163947164, - "iforest": 0.674511197000149, - "zscore": 0.5898788542045754, - "mean_pct": 0.635511558286259, - "max_pct": 0.6366432388776665, - "mean_vs_iforest": -0.038999638713890006, - "max_vs_iforest": -0.03786795812248256 - }, - { - "dataset": "shuttle", - "base_rate": 0.0715, - "iforest": 0.9764084411952229, - "zscore": 0.8948204770684615, - "mean_pct": 0.9587705767318155, - "max_pct": 0.9207599245713749, - "mean_vs_iforest": -0.017637864463407316, - "max_vs_iforest": -0.055648516623847954 - }, - { - "dataset": "spambase", - "base_rate": 0.39909674352270025, - "iforest": 0.5103787257980769, - "zscore": 0.42751074893538277, - "mean_pct": 0.4888560907968301, - "max_pct": 0.4419059745028603, - "mean_vs_iforest": -0.021522635001246737, - "max_vs_iforest": -0.06847275129521657 - }, - { - "dataset": "thyroid", - "base_rate": 0.024655355249204668, - "iforest": 0.5446078938409467, - "zscore": 0.29394329833995414, - "mean_pct": 0.40374103378311116, - "max_pct": 0.4756158967397111, - "mean_vs_iforest": -0.1408668600578355, - "max_vs_iforest": -0.06899199710123555 - } -] \ No newline at end of file diff --git a/benchmarks/anomaly_conditioning/results/smd-bakeoff-contaminated-0.033.json b/benchmarks/anomaly_conditioning/results/smd-bakeoff-contaminated-0.033.json deleted file mode 100644 index c19a93d1f..000000000 --- a/benchmarks/anomaly_conditioning/results/smd-bakeoff-contaminated-0.033.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "window": 60, - "seeds": 2, - "results": [ - { - "featuriser": "raw", - "estimator": "iforest", - "scope": "pooled", - "pr_auc": 0.06199381240530498, - "roc_auc": 0.7006709662321806, - "precision_at_n": 0.08989817792068595, - "event_recall_at_1pct": 0.33333333333333337, - "n_features": 38, - "n_models": 1, - "seconds": 0.7 - }, - { - "featuriser": "raw", - "estimator": "pca_recon", - "scope": "pooled", - "pr_auc": 0.10134410255674345, - "roc_auc": 0.7196872372518814, - "precision_at_n": 0.11655948553054662, - "event_recall_at_1pct": 0.7948717948717948, - "n_features": 38, - "n_models": 1, - "seconds": 0.2 - }, - { - "featuriser": "raw", - "estimator": "mahalanobis", - "scope": "pooled", - "pr_auc": 0.10334293809801756, - "roc_auc": 0.7121257515440129, - "precision_at_n": 0.11629153269024652, - "event_recall_at_1pct": 0.7948717948717948, - "n_features": 38, - "n_models": 1, - "seconds": 0.2 - }, - { - "featuriser": "raw", - "estimator": "maha_ridge", - "scope": "pooled", - "pr_auc": 0.117612954966071, - "roc_auc": 0.7206672197976749, - "precision_at_n": 0.11629153269024652, - "event_recall_at_1pct": 0.7948717948717948, - "n_features": 38, - "n_models": 1, - "seconds": 0.2 - }, - { - "featuriser": "raw", - "estimator": "maha_trimmed", - "scope": "pooled", - "pr_auc": 0.09442408579801306, - "roc_auc": 0.7184355028890834, - "precision_at_n": 0.12031082529474812, - "event_recall_at_1pct": 0.7692307692307693, - "n_features": 38, - "n_models": 1, - "seconds": 0.35 - }, - { - "featuriser": "win_stats", - "estimator": "iforest", - "scope": "pooled", - "pr_auc": 0.08202921622489219, - "roc_auc": 0.7465907092087116, - "precision_at_n": 0.1137459807073955, - "event_recall_at_1pct": 0.11538461538461538, - "n_features": 190, - "n_models": 1, - "seconds": 2.6500000000000004 - }, - { - "featuriser": "win_stats", - "estimator": "pca_recon", - "scope": "pooled", - "pr_auc": 0.14662545881271646, - "roc_auc": 0.7625117429216082, - "precision_at_n": 0.19640943193997856, - "event_recall_at_1pct": 0.20512820512820512, - "n_features": 190, - "n_models": 1, - "seconds": 2.25 - }, - { - "featuriser": "win_stats", - "estimator": "mahalanobis", - "scope": "pooled", - "pr_auc": 0.16345061641209957, - "roc_auc": 0.7954142272533906, - "precision_at_n": 0.21864951768488747, - "event_recall_at_1pct": 0.48717948717948717, - "n_features": 190, - "n_models": 1, - "seconds": 2.25 - }, - { - "featuriser": "win_stats", - "estimator": "maha_ridge", - "scope": "pooled", - "pr_auc": 0.16312010506359406, - "roc_auc": 0.7908960262990758, - "precision_at_n": 0.21757770632368703, - "event_recall_at_1pct": 0.3333333333333333, - "n_features": 190, - "n_models": 1, - "seconds": 2.2 - }, - { - "featuriser": "win_stats", - "estimator": "maha_trimmed", - "scope": "pooled", - "pr_auc": 0.1907541178601119, - "roc_auc": 0.8134753965498104, - "precision_at_n": 0.219989281886388, - "event_recall_at_1pct": 0.46153846153846156, - "n_features": 190, - "n_models": 1, - "seconds": 2.75 - } - ] -} \ No newline at end of file diff --git a/benchmarks/anomaly_conditioning/results/smd-bakeoff.json b/benchmarks/anomaly_conditioning/results/smd-bakeoff.json deleted file mode 100644 index d0b26e0f7..000000000 --- a/benchmarks/anomaly_conditioning/results/smd-bakeoff.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "window": 60, - "seeds": 2, - "contaminate": 0.0, - "results": [ - { - "featuriser": "raw", - "estimator": "iforest", - "scope": "pooled", - "pr_auc": 0.06485182445285484, - "roc_auc": 0.7049237121424422, - "precision_at_n": 0.09016613076098606, - "event_recall_at_1pct": 0.358974358974359, - "n_features": 38, - "n_models": 1, - "seconds": 0.7 - }, - { - "featuriser": "raw", - "estimator": "pca_recon", - "scope": "pooled", - "pr_auc": 0.11893518451393884, - "roc_auc": 0.7445973923190322, - "precision_at_n": 0.1270096463022508, - "event_recall_at_1pct": 0.7692307692307693, - "n_features": 38, - "n_models": 1, - "seconds": 0.2 - }, - { - "featuriser": "raw", - "estimator": "mahalanobis", - "scope": "pooled", - "pr_auc": 0.10169012776870497, - "roc_auc": 0.7075271236542118, - "precision_at_n": 0.1152197213290461, - "event_recall_at_1pct": 0.7692307692307693, - "n_features": 38, - "n_models": 1, - "seconds": 0.2 - }, - { - "featuriser": "raw", - "estimator": "maha_ridge", - "scope": "pooled", - "pr_auc": 0.12636238430070718, - "roc_auc": 0.7276285562827283, - "precision_at_n": 0.11843515541264737, - "event_recall_at_1pct": 0.8205128205128205, - "n_features": 38, - "n_models": 1, - "seconds": 0.2 - }, - { - "featuriser": "win_stats", - "estimator": "iforest", - "scope": "pooled", - "pr_auc": 0.083889006916833, - "roc_auc": 0.7513011959257863, - "precision_at_n": 0.11454983922829581, - "event_recall_at_1pct": 0.14102564102564102, - "n_features": 190, - "n_models": 1, - "seconds": 2.6 - }, - { - "featuriser": "win_stats", - "estimator": "pca_recon", - "scope": "pooled", - "pr_auc": 0.15240356108314648, - "roc_auc": 0.7672142449816186, - "precision_at_n": 0.2015005359056806, - "event_recall_at_1pct": 0.23076923076923078, - "n_features": 190, - "n_models": 1, - "seconds": 2.2 - }, - { - "featuriser": "win_stats", - "estimator": "mahalanobis", - "scope": "pooled", - "pr_auc": 0.17142117053734277, - "roc_auc": 0.8027929809443131, - "precision_at_n": 0.2237406216505895, - "event_recall_at_1pct": 0.5128205128205128, - "n_features": 190, - "n_models": 1, - "seconds": 2.2 - }, - { - "featuriser": "win_stats", - "estimator": "maha_ridge", - "scope": "pooled", - "pr_auc": 0.16850907884294952, - "roc_auc": 0.7968172772094938, - "precision_at_n": 0.227491961414791, - "event_recall_at_1pct": 0.358974358974359, - "n_features": 190, - "n_models": 1, - "seconds": 2.1500000000000004 - } - ] -} \ No newline at end of file diff --git a/benchmarks/anomaly_conditioning/run_experiment.py b/benchmarks/anomaly_conditioning/run_experiment.py deleted file mode 100644 index 564038be6..000000000 --- a/benchmarks/anomaly_conditioning/run_experiment.py +++ /dev/null @@ -1,567 +0,0 @@ -"""Run the conditioning experiment and write a dated results page. - - python benchmarks/anomaly_conditioning/run_experiment.py --seeds 5 - python benchmarks/anomaly_conditioning/run_experiment.py --seeds 5 --datasets synthetic smd nslkdd - -Synthetic runs need no network and no Databricks workspace. Real datasets are downloaded at run -time and cached; see ``datasets/__init__.py`` for why none of them are vendored. - -The decision rules below were fixed *before* the first run, and are printed alongside the result so -a reader can check the conclusion against the criterion rather than against a narrative written -afterwards. -""" - -import argparse -import datetime as dt -import json -import pathlib -import subprocess -import sys -from dataclasses import dataclass - -import numpy as np - -sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) - -from conditioning import CONFIGS, eta_squared # noqa: E402 -from datasets import synthetic # noqa: E402 -from metrics import ( # noqa: E402 - CellMetrics, - macro_average, - per_group_pr_auc, - pr_auc, - precision_at_n, - roc_auc, - spearman_with_bootstrap_ci, - trivial_baselines, - wilcoxon_paired, - worst_group_false_positive_rate, -) - -RESULTS_DIR = pathlib.Path(__file__).resolve().parent / "results" - -# Decision rules, fixed in advance. See the plan for #1484. -# -# The gate being tested is the removed MIN_GROUP_HETEROGENEITY = 0.10: conditioning used to be -# skipped when eta-squared fell below it, on the theory that a grouping which explains little -# variance contributes noise. The counter-argument is that at low heterogeneity every group median -# approaches the global median, so the relative feature degenerates into a monotone transform of -# the raw metric -- a near-duplicate of an informative column, not noise. That is a prediction, and -# NO_GATE_FLOOR is where it gets tested. -LOW_ETA = 0.10 -NO_GATE_FLOOR = -0.01 # relative never materially worse than pooled below LOW_ETA -GATE_NEEDED_DELTA = -0.02 # a real cost, concentrated below LOW_ETA - - -@dataclass -class Cell: - """One measured configuration.""" - - dataset: str - grouping: str - config: str - seed: int - mechanism: str - level_spread: float - metrics: CellMetrics - - def as_dict(self) -> dict: - record = { - "dataset": self.dataset, - "grouping": self.grouping, - "config": self.config, - "seed": self.seed, - "mechanism": self.mechanism, - "level_spread": self.level_spread, - } - record.update(self.metrics.as_dict()) - return record - - -def measure(values: np.ndarray, labels: np.ndarray, groups: np.ndarray, config: str, seed: int) -> CellMetrics: - """Fit one configuration and compute every metric for it.""" - result = CONFIGS[config](values, groups, seed) - scores = result.scores - return CellMetrics( - pr_auc=pr_auc(labels, scores), - roc_auc=roc_auc(labels, scores), - precision_at_n=precision_at_n(labels, scores), - macro_pr_auc=macro_average(per_group_pr_auc(labels, scores, groups)), - worst_group_fpr=worst_group_false_positive_rate(labels, scores, groups), - n_models=result.n_models, - seconds=result.seconds, - eta_squared=eta_squared(values, groups), - ) - - -def run_synthetic(seeds: int) -> list[Cell]: - """The two-factor sweep: spread x mechanism x config x seed.""" - cells: list[Cell] = [] - points = synthetic.sweep_points() - total = len(points) * len(CONFIGS) * seeds - done = 0 - for level_spread, mechanism in points: - for seed in range(seeds): - values, labels, groups = synthetic.generate(seed=seed, level_spread=level_spread, mechanism=mechanism) - for config in CONFIGS: - cells.append( - Cell( - dataset="synthetic", - grouping=f"spread={level_spread:.3f}", - config=config, - seed=seed, - mechanism=mechanism, - level_spread=level_spread, - metrics=measure(values, labels, groups, config, seed), - ) - ) - done += 1 - print(f"\r {done}/{total} cells", end="", flush=True) - print() - return cells - - -def run_real(real_module, name: str, seeds: int) -> list[Cell]: - """Every candidate grouping of a real dataset, across configs and seeds. - - *mechanism* is recorded as ``"real"``: which mechanism produced a real anomaly is not knowable, - which is exactly why the synthetic sweep carries the mechanism contrast and these datasets - check that its conclusion survives. - """ - cells: list[Cell] = [] - for grouping, values, labels, groups in real_module.load_groupings(name): - print(f" {grouping}: {values.shape[0]} rows, {len(np.unique(groups))} groups") - for seed in range(seeds): - for config in CONFIGS: - cells.append( - Cell( - dataset=name, - grouping=grouping, - config=config, - seed=seed, - mechanism="real", - level_spread=float("nan"), - metrics=measure(values, labels, groups, config, seed), - ) - ) - print(f" done ({seeds} seeds x {len(CONFIGS)} configs)") - return cells - - -def run_tabular(seeds: int, names: list[str] | None = None) -> tuple[list[Cell], list[dict]]: - """The plain-tabular regime: no grouping, so only the pooled configuration is meaningful. - - Returns ``(cells, baselines)``. This does not compare configurations -- there is nothing to - condition on -- it characterises absolute quality on the benchmarks the field actually reports, - and pins each result against its own base rate and its own trivial baselines. Reported per - dataset and never pooled into one headline: PR-AUC is not comparable across base rates. - """ - from datasets import tabular # noqa: PLC0415 - downloads data, so only imported when asked for - - cells: list[Cell] = [] - baselines: list[dict] = [] - for name, values, labels in tabular.iter_datasets(names): - base_rate = float(labels.mean()) - print(f" {name}: {values.shape[0]} rows, {values.shape[1]} features, base rate {base_rate:.4%}") - groups = np.zeros(len(values), dtype=str) # one group: pooled is the only configuration - for seed in range(seeds): - cells.append( - Cell( - dataset=f"tabular:{name}", - grouping="none", - config="pooled", - seed=seed, - mechanism="tabular", - level_spread=float("nan"), - metrics=measure(values, labels, groups, "pooled", seed), - ) - ) - trivial = trivial_baselines(values, labels) - baselines.append( - { - "dataset": name, - "n_rows": int(values.shape[0]), - "n_features": int(values.shape[1]), - "base_rate": base_rate, - "dqx_pr_auc": float(np.median([c.metrics.pr_auc for c in cells if c.dataset.endswith(name)])), - **trivial, - } - ) - return cells, baselines - - -def tabular_table(baselines: list[dict]) -> list[str]: - """Per-dataset absolute quality against the floor, with the base rate alongside.""" - if not baselines: - return [] - lines = [ - "## Plain tabular benchmarks (no grouping)", - "", - "No grouping exists in these datasets, so conditioning is not applicable and only the pooled", - "configuration runs. This characterises the mechanism on the benchmarks the field reports, and", - "is the regression check that adding conditioning did not disturb the ungrouped path.", - "", - "PR-AUC is **not comparable across rows** — it moves with the base rate — so each dataset is", - "read against its own random floor. `lift` is DQX PR-AUC divided by the random floor.", - "", - "| dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random | beats max-abs-z |", - "|---|---|---|---|---|---|---|---|---|", - ] - for row in baselines: - lift = row["dqx_pr_auc"] / row["random"] if row["random"] else float("nan") - beats = "yes" if row["dqx_pr_auc"] > row["max_abs_z"] else "**no**" - lines.append( - f"| {row['dataset']} | {row['n_rows']} | {row['n_features']} | {row['base_rate']:.2%} | " - f"{row['dqx_pr_auc']:.4f} | {row['random']:.4f} | {row['max_abs_z']:.4f} | " - f"{lift:.1f}x | {beats} |" - ) - lines.append("") - return lines - - -def paired_deltas(cells: list[Cell], left: str, right: str, metric: str = "pr_auc") -> list[dict]: - """``metric(left) - metric(right)`` for every (dataset, grouping, seed) the two share. - - Paired rather than averaged: the seed drives both the data draw and the forest, and dominates - the variance between cells. - """ - # The key must identify a cell uniquely. Mechanism belongs in it: the synthetic sweep runs both - # mechanisms at the same spread and seed, so a key without it collapses each pair of cells onto - # one entry and silently discards half the grid — which is what it did, until the absent - # "global" row in the summary table gave it away. - index: dict[tuple[str, str, str, int], dict[str, Cell]] = {} - for cell in cells: - index.setdefault((cell.dataset, cell.grouping, cell.mechanism, cell.seed), {})[cell.config] = cell - - out = [] - for (dataset, grouping, _mechanism, seed), by_config in sorted(index.items()): - if left not in by_config or right not in by_config: - continue - left_cell, right_cell = by_config[left], by_config[right] - left_value = getattr(left_cell.metrics, metric) - right_value = getattr(right_cell.metrics, metric) - out.append( - { - "dataset": dataset, - "grouping": grouping, - "seed": seed, - "mechanism": left_cell.mechanism, - "eta_squared": left_cell.metrics.eta_squared, - "delta": left_value - right_value, - "left": left_value, - "right": right_value, - } - ) - return out - - -def regress_delta_on_eta(deltas: list[dict]) -> dict[str, float]: - """Least squares of ``delta ~ 1 + eta_squared + is_contextual``. - - The gate's premise needs eta-squared to carry signal *after* controlling for the anomaly - mechanism. If the eta coefficient collapses once the mechanism dummy is present, the apparent - correlation was the mechanism all along. - - Synthetic rows only. On real data the mechanism behind each anomaly is unknown, so folding those - rows in would silently code them as "global" and bias the very coefficient being tested. - """ - rows = [ - d - for d in deltas - if np.isfinite(d["delta"]) and np.isfinite(d["eta_squared"]) and d["mechanism"] in synthetic.MECHANISMS - ] - if len(rows) < 4: - return {} - design = np.column_stack( - [ - np.ones(len(rows)), - np.array([r["eta_squared"] for r in rows]), - np.array([1.0 if r["mechanism"] == "contextual" else 0.0 for r in rows]), - ] - ) - target = np.array([r["delta"] for r in rows]) - coeffs, *_ = np.linalg.lstsq(design, target, rcond=None) - residual = target - design @ coeffs - ss_res = float((residual**2).sum()) - ss_tot = float(((target - target.mean()) ** 2).sum()) - return { - "intercept": float(coeffs[0]), - "eta_squared": float(coeffs[1]), - "is_contextual": float(coeffs[2]), - "r_squared": 1.0 - ss_res / ss_tot if ss_tot else float("nan"), - "n": float(len(rows)), - } - - -def decide(deltas: list[dict]) -> tuple[str, str, dict[str, float]]: - """Apply the pre-registered rules, and a variance-robust companion. - - Returns ``(pre_registered_verdict, robust_verdict, evidence)``. - - The pre-registered rule is a **minimum over single cells**, which makes it maximally sensitive - to the variance of the estimator it is applied to — and Isolation Forest on a 3-group, - 68k-row dataset is high variance. It fired on the first run against real data, on one seed of - ``nslkdd/protocol_type`` whose other seeds were **+0.1003** and **+0.0989**. - - That is a mis-specification, not a finding: the rule's stated intent was that conditioning is - never *materially worse* on homogeneous data, which is a claim about the distribution rather - than about the unluckiest single draw. So the per-(dataset, grouping) **median** delta is - reported beside it, and both verdicts are published. The pre-registered rule is deliberately - left in place rather than replaced, so that a reader can see the criterion that was set in - advance, the answer it gave, and why a second statistic was added. - """ - low = [d for d in deltas if np.isfinite(d["delta"]) and d["eta_squared"] < LOW_ETA] - if not low: - return "inconclusive: nothing fell below the eta-squared threshold", "inconclusive", {} - - worst_cell = min(d["delta"] for d in low) - harmful = [d for d in low if d["delta"] <= GATE_NEEDED_DELTA] - - # Median per (dataset, grouping): the distributional form of the same question. - by_grouping: dict[tuple[str, str], list[float]] = {} - for d in low: - by_grouping.setdefault((str(d["dataset"]), str(d["grouping"])), []).append(float(d["delta"])) - medians = {key: float(np.median(values)) for key, values in by_grouping.items()} - worst_median = min(medians.values()) - harmful_groupings = [key for key, value in medians.items() if value <= GATE_NEEDED_DELTA] - - evidence = { - "n_low_eta_cells": float(len(low)), - "n_low_eta_groupings": float(len(medians)), - "worst_delta_single_cell": worst_cell, - "n_harmful_cells": float(len(harmful)), - "worst_median_delta_per_grouping": worst_median, - "n_harmful_groupings": float(len(harmful_groupings)), - "median_delta_below_threshold": float(np.median([d["delta"] for d in low])), - } - - def verdict(worst: float, any_harmful: bool) -> str: - if worst > NO_GATE_FLOOR: - return "no gate needed" - if any_harmful: - return "gate needed; refit the threshold from this sweep" - return "borderline: below the floor but above the harm threshold" - - return verdict(worst_cell, bool(harmful)), verdict(worst_median, bool(harmful_groupings)), evidence - - -def git_sha() -> str: - """Short SHA of the tree these numbers came from. Results without it are unreproducible.""" - try: - return subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], - capture_output=True, - text=True, - check=True, - cwd=pathlib.Path(__file__).resolve().parent, - ).stdout.strip() - except (subprocess.CalledProcessError, OSError): - return "unknown" - - -def summarise(deltas: list[dict], label: str) -> list[str]: - """Median, IQR and a paired test for one comparison, split by anomaly mechanism.""" - lines = [f"### {label}", ""] - lines.append("| mechanism | n | median delta | IQR | Wilcoxon p |") - lines.append("|---|---|---|---|---|") - present = sorted({str(d["mechanism"]) for d in deltas}) - for mechanism in [*present, "*all*"]: - subset = deltas if mechanism == "*all*" else [d for d in deltas if d["mechanism"] == mechanism] - values = [d["delta"] for d in subset if np.isfinite(d["delta"])] - if not values: - continue - q25, q75 = np.percentile(values, [25, 75]) - _, p_value = wilcoxon_paired(values) - p_text = "n/a" if not np.isfinite(p_value) else f"{p_value:.2e}" - lines.append( - f"| {mechanism} | {len(values)} | {np.median(values):+.4f} | " f"{q25:+.4f} to {q75:+.4f} | {p_text} |" - ) - lines.append("") - return lines - - -def low_eta_table(deltas: list[dict]) -> list[str]: - """Every low-eta grouping, seed by seed. - - The table the two verdicts have to be read against: a grouping whose seeds straddle zero is - telling you about variance, and one whose seeds agree is telling you about the mechanism. - """ - low = [d for d in deltas if np.isfinite(d["delta"]) and d["eta_squared"] < LOW_ETA] - if not low: - return [] - - by_grouping: dict[tuple[str, str], list[dict]] = {} - for d in low: - by_grouping.setdefault((str(d["dataset"]), str(d["grouping"])), []).append(d) - - lines = [ - f"### Every grouping below eta-squared {LOW_ETA}, seed by seed", - "", - "| dataset | grouping | eta-squared | median delta | min | max | seeds agree? |", - "|---|---|---|---|---|---|---|", - ] - for (dataset, grouping), group in sorted(by_grouping.items()): - values = [g["delta"] for g in group] - agree = "yes, all negative" if max(values) < 0 else ("yes, none negative" if min(values) >= 0 else "**no**") - lines.append( - f"| {dataset} | {grouping} | {group[0]['eta_squared']:.4f} | {np.median(values):+.4f} | " - f"{min(values):+.4f} | {max(values):+.4f} | {agree} |" - ) - lines.append("") - return lines - - -def write_report( - cells: list[Cell], seeds: int, datasets: list[str], tabular_baselines: list[dict] | None = None -) -> pathlib.Path: - """Write the dated markdown and JSON results, and return the markdown path.""" - RESULTS_DIR.mkdir(parents=True, exist_ok=True) - stamp = dt.date.today().isoformat() - sha = git_sha() - stem = f"{stamp}-{sha}" - - rel_vs_pooled = paired_deltas(cells, "relative", "pooled") - rel_vs_per_group = paired_deltas(cells, "relative", "per_group") - pre_registered_verdict, robust_verdict, evidence = decide(rel_vs_pooled) - rho, lo, hi = spearman_with_bootstrap_ci( - [d["eta_squared"] for d in rel_vs_pooled], [d["delta"] for d in rel_vs_pooled] - ) - regression = regress_delta_on_eta(rel_vs_pooled) - - lines = [ - f"# Anomaly conditioning experiment — {stamp}", - "", - f"DQX `{sha}` · datasets: {', '.join(datasets)} · seeds per cell: {seeds} · " f"cells: {len(cells)}", - "", - "PR-AUC is the primary metric. No point-adjusted F1 appears anywhere; see `metrics.py`.", - "DQX scores rows independently, so figures on time-series data are not comparable with", - "published sequence-model results — that is a different task, not a worse implementation.", - "", - "## Does removing the heterogeneity gate cost anything?", - "", - f"Rules fixed before running: **no gate** if the worst delta below eta-squared " - f"{LOW_ETA} exceeds {NO_GATE_FLOOR:+.2f}; **gate needed** if any such cell reaches " - f"{GATE_NEEDED_DELTA:+.2f}.", - "", - f"- **Pre-registered rule (worst single cell): {pre_registered_verdict}**", - f"- **Variance-robust companion (worst per-grouping median): {robust_verdict}**", - "", - "The pre-registered rule takes a minimum over individual cells, so it is maximally " - "sensitive to estimator variance. It is reported unchanged, alongside the median form of " - "the same question, so the criterion set in advance and the answer it gave are both " - "visible. Where the two disagree, the per-grouping deltas below show why.", - "", - ] - if evidence: - lines.append("| statistic | value |") - lines.append("|---|---|") - for key, value in evidence.items(): - lines.append(f"| {key} | {value:+.4f} |" if "delta" in key else f"| {key} | {value:.0f} |") - lines.append("") - - lines += [ - "### Does eta-squared predict the benefit at all?", - "", - f"Spearman rho(eta-squared, delta) = **{rho:+.3f}** (bootstrap 95% CI {lo:+.3f} to {hi:+.3f}).", - "", - ] - if regression: - lines += [ - "Least squares `delta ~ 1 + eta_squared + is_contextual`, which asks whether " - "eta-squared still carries signal once the anomaly mechanism is controlled for:", - "", - "| term | coefficient |", - "|---|---|", - f"| intercept | {regression['intercept']:+.4f} |", - f"| eta_squared | {regression['eta_squared']:+.4f} |", - f"| is_contextual | {regression['is_contextual']:+.4f} |", - f"| R-squared | {regression['r_squared']:.3f} (n={regression['n']:.0f}) |", - "", - ] - - lines += tabular_table(tabular_baselines or []) - lines += low_eta_table(rel_vs_pooled) - lines += ["## Paired comparisons", ""] - lines += summarise(rel_vs_pooled, "baseline-relative minus pooled (PR-AUC)") - lines += summarise(rel_vs_per_group, "baseline-relative minus per-group (PR-AUC)") - - lines += ["## Cost", "", "| config | median models | median seconds |", "|---|---|---|"] - for config in CONFIGS: - subset = [c for c in cells if c.config == config] - if subset: - lines.append( - f"| {config} | {np.median([c.metrics.n_models for c in subset]):.0f} | " - f"{np.median([c.metrics.seconds for c in subset]):.2f} |" - ) - lines.append("") - - md_path = RESULTS_DIR / f"{stem}.md" - md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - (RESULTS_DIR / f"{stem}.json").write_text( - json.dumps( - { - "generated": stamp, - "git_sha": sha, - "seeds": seeds, - "datasets": datasets, - "verdict_pre_registered": pre_registered_verdict, - "verdict_robust": robust_verdict, - "evidence": evidence, - "spearman": {"rho": rho, "ci_low": lo, "ci_high": hi}, - "regression": regression, - # Included so emit_docs.py can refresh the published tables without re-running the - # sweep, and so a results file is self-contained. - "tabular_baselines": tabular_baselines or [], - "cells": [c.as_dict() for c in cells], - }, - indent=2, - default=str, - ), - encoding="utf-8", - ) - return md_path - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--seeds", type=int, default=5, help="seeds per cell (paired across configs)") - parser.add_argument( - "--datasets", - nargs="+", - default=["synthetic"], - choices=["synthetic", "smd", "nslkdd", "tabular"], - help="synthetic needs no network; the others download at run time", - ) - args = parser.parse_args() - - cells: list[Cell] = [] - tabular_baselines: list[dict] = [] - if "tabular" in args.datasets: - print("plain tabular benchmarks:") - tab_cells, tabular_baselines = run_tabular(args.seeds) - cells += tab_cells - if "synthetic" in args.datasets: - print("synthetic sweep:") - cells += run_synthetic(args.seeds) - - real_names = [n for n in ("smd", "nslkdd") if n in args.datasets] - if real_names: - # Imported lazily: it downloads data, so a synthetic-only run stays offline. - from datasets import real # noqa: PLC0415 - - for name in real_names: - print(f"{name}:") - cells += run_real(real, name, args.seeds) - - if not cells: - print("no cells measured", file=sys.stderr) - return 1 - - path = write_report(cells, args.seeds, args.datasets, tabular_baselines) - print(f"\nwrote {path}") - print(path.read_text(encoding="utf-8")) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/anomaly_conditioning/smd_bakeoff.py b/benchmarks/anomaly_conditioning/smd_bakeoff.py deleted file mode 100644 index 7cd7e028f..000000000 --- a/benchmarks/anomaly_conditioning/smd_bakeoff.py +++ /dev/null @@ -1,428 +0,0 @@ -"""SMD bake-off: what would it take to be in the ballpark on time-series anomaly detection? - -An earlier experiment asked "do lag and rolling-window features help?" and answered no: +9% PR-AUC for -190 extra features, with ROC-AUC regressing, because IsolationForest degrades as the feature count -grows. This asks the harder question — **what does it actually take** — across four axes at once, and -its answer is why DQX gained a second algorithm: the estimator was the problem, not the features. - -Why this script exists rather than a comparison against published numbers: SMD's headline results -(~0.80 F1) are computed with *point adjustment*, under which a random score reaches state of the art -(Kim et al., AAAI 2022). They are not a target and cannot be scaled to "80% of". So this script builds -its own honest reference — the best windowed reconstruction detector it can — and everything else is -measured against that. - -Axes: - -1. **Unit of analysis.** DQX scores one row at a time. SMD anomalies are subsequences, so a single row - is the wrong unit. ``win_stats`` makes the trailing window the sample: current value plus trailing - mean, standard deviation, min and max. Strictly trailing, so no leakage. -2. **Estimator.** Isolation Forest (what ships) against PCA reconstruction error, an MLP - autoencoder, and Mahalanobis distance. PCA and the autoencoder are the interesting candidates - because reconstruction degrades gracefully as the feature count grows, which is where Isolation - Forest was measured to fail. -3. **Scope.** Pooled across all 28 machines (what DQX does) versus one model per machine (what the SMD - literature does). Per-entity here uses a **global** score threshold rather than per-entity - contamination — the latter is what made an all-normal entity flag its own most-unusual rows and - produced the 56% false-alarm result in earlier work. -4. **Metric.** Point-wise PR-AUC is reported throughout, and it is harsh: an anomaly is a *range*, so a - detector that fires two timesteps late scores as a miss plus a false positive. Alongside it, - **event recall at a fixed alert budget**: of the true anomaly ranges, how many contain at least one - alerted row, when the alert budget is capped. Precision stays strictly point-wise, so this is not - point adjustment — it is the operational question a data-quality user actually has ("of the - incidents, how many did I surface inside my review queue?"). - -Run: uv run python benchmarks/anomaly_conditioning/smd_bakeoff.py --seeds 2 -""" - -import argparse -import json -import time -from collections.abc import Callable - -import numpy as np - -from datasets.real import SMD_ENTITIES, load_smd_split -from metrics import pr_auc, precision_at_n, roc_auc -from sklearn.covariance import LedoitWolf -from sklearn.decomposition import PCA -from sklearn.ensemble import IsolationForest -from sklearn.neural_network import MLPRegressor -from sklearn.preprocessing import StandardScaler - -N_TREES = 200 -CONTAMINATION = 0.02 -# Trailing window length in rows. SMD is one-minute cadence, so 60 rows is the last hour: long enough -# to characterise "normal recently" without so much lag that a short incident is averaged away. -WINDOW = 60 - - -# -------------------------------------------------------------------------------------- -# Axis 1: the unit of analysis -# -------------------------------------------------------------------------------------- - - -def window_stats(values: np.ndarray, window: int = WINDOW) -> np.ndarray: - """Current value plus trailing mean, stddev, min and max over the preceding *window* rows. - - Feature count is ``5 x n_metrics`` regardless of window length, unlike flattening the window, - which multiplies by the window itself. Strictly trailing: the statistics cover rows - ``[i-window, i-1]``, so the current row never contributes to its own baseline and nothing is - computed from the future. - - Warm-up rows (fewer than two rows of history) fall back to the current value with zero spread, - which is the honest neutral: with no history there is no deviation to report. A production - implementation should mark them unscoreable instead, the way an unseen baseline group already is. - """ - n_rows, n_cols = values.shape - padded = np.vstack([np.zeros((1, n_cols)), values]) - csum = np.cumsum(padded, axis=0) - csum_sq = np.cumsum(padded**2, axis=0) - - mean = np.zeros_like(values) - std = np.zeros_like(values) - lo = np.zeros_like(values) - hi = np.zeros_like(values) - - for i in range(n_rows): - start = max(0, i - window) - count = i - start - if count < 2: - mean[i], std[i], lo[i], hi[i] = values[i], 0.0, values[i], values[i] - continue - total = csum[i] - csum[start] - total_sq = csum_sq[i] - csum_sq[start] - m = total / count - mean[i] = m - std[i] = np.sqrt(np.maximum(total_sq / count - m**2, 0.0)) - chunk = values[start:i] - lo[i] = chunk.min(axis=0) - hi[i] = chunk.max(axis=0) - - return np.hstack([values, mean, std, lo, hi]) - - -# Annotated so the values keep a callable type: an unannotated dict mixing a lambda with a -# named function widens to object, and every call through it becomes untyped. -FEATURISERS: dict[str, Callable[[np.ndarray], np.ndarray]] = { - "raw": lambda v: v, - "win_stats": window_stats, -} - - -# -------------------------------------------------------------------------------------- -# Axis 2: estimators. Each returns test scores, higher = more anomalous. -# -------------------------------------------------------------------------------------- - - -def score_iforest(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: - model = IsolationForest(n_estimators=N_TREES, contamination=CONTAMINATION, random_state=seed, n_jobs=-1) - model.fit(train) - return -model.score_samples(test) - - -def score_pca_recon(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: - """Reconstruction error after projecting onto the dominant linear subspace of *train*. - - A linear autoencoder converges to exactly this, so it is the honest cheap stand-in for one, and it - is the candidate most likely to survive a wide feature matrix: extra correlated columns add to the - subspace rather than diluting a random split. - """ - scaler = StandardScaler().fit(train) - tr, te = scaler.transform(train), scaler.transform(test) - n_components = max(1, min(int(0.5 * tr.shape[1]), tr.shape[1] - 1)) - pca = PCA(n_components=n_components, random_state=seed).fit(tr) - return np.sum((te - pca.inverse_transform(pca.transform(te))) ** 2, axis=1) - - -def score_mlp_autoencoder(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: - """Non-linear reconstruction error from a small MLP trained to reproduce its input. - - Stands in for the deep autoencoders the SMD literature uses; no torch in this environment, so this - is the closest available reference for "what a learned reconstruction achieves". Deliberately - small and capped in iterations — this is a reference point, not a tuned model. - """ - scaler = StandardScaler().fit(train) - tr, te = scaler.transform(train), scaler.transform(test) - width = tr.shape[1] - bottleneck = max(2, width // 4) - model = MLPRegressor( - hidden_layer_sizes=(max(4, width // 2), bottleneck, max(4, width // 2)), - random_state=seed, - max_iter=60, - early_stopping=False, - learning_rate_init=0.005, - ) - model.fit(tr, tr) - return np.sum((te - model.predict(te)) ** 2, axis=1) - - -def _mahalanobis_sq(train: np.ndarray, test: np.ndarray, *, shrinkage: str | float, ridge: float) -> np.ndarray: - """Squared Mahalanobis distance from the training centre, scaled and regularised. - - Two facts govern every variant below, and they are easy to get wrong. - - **The scaler is a no-op in exact arithmetic.** Mahalanobis distance is invariant under any - invertible linear map: standardising *x* and using the standardised covariance gives bit-identical - distances to using the raw covariance. So ``StandardScaler`` cannot change the answer directly -- - it changes it only *through the regulariser*, because a shrinkage target is not affine-equivariant. - A fixed ``1e-3`` ridge is negligible against a column of variance 1e6 and dominant against one of - variance 1e-3; standardising first makes one ridge value mean the same thing for every column. This - is the mirror image of the argument in ``core.py`` for why RobustScaler was removed for - IsolationForest: there the scaler was measured to be a genuine no-op, here it earns its place only - by fixing the regulariser's basis. - - **Ledoit-Wolf minimises the error of the covariance, not the conditioning of its inverse.** So a - ridge floor is still applied on top, expressed relative to the average variance so it is - scale-free. - """ - scaler = StandardScaler().fit(train) - tr, te = scaler.transform(train), scaler.transform(test) - - if shrinkage == "ledoit_wolf": - cov = LedoitWolf(assume_centered=False).fit(tr).covariance_ - else: - cov = np.atleast_2d(np.cov(tr - tr.mean(axis=0), rowvar=False)) - if shrinkage: # explicit convex shrink toward the average-variance diagonal - target = np.eye(cov.shape[0]) * (np.trace(cov) / cov.shape[0]) - cov = (1.0 - float(shrinkage)) * cov + float(shrinkage) * target - - cov = np.atleast_2d(cov) - cov = cov + np.eye(cov.shape[0]) * ridge * (np.trace(cov) / cov.shape[0]) - delta = te - tr.mean(axis=0) - # Cholesky solve rather than an explicit inverse: same answer, better conditioned. - factor = np.linalg.cholesky(cov) - solved = np.linalg.solve(factor, delta.T) - return np.sum(solved**2, axis=0) - - -def score_mahalanobis(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: - """The candidate under test: StandardScaler + Ledoit-Wolf + a scale-free ridge floor (G1).""" - del seed - return _mahalanobis_sq(train, test, shrinkage="ledoit_wolf", ridge=1e-6) - - -def score_mahalanobis_ridge(train: np.ndarray, test: np.ndarray, seed: int) -> np.ndarray: - """Same, with no Ledoit-Wolf: the fallback if data-estimated shrinkage over-regularises (G1).""" - del seed - return _mahalanobis_sq(train, test, shrinkage=0.0, ridge=1e-6) - - -ESTIMATORS: dict[str, Callable[[np.ndarray, np.ndarray, int], np.ndarray]] = { - "iforest": score_iforest, - "pca_recon": score_pca_recon, - "mlp_ae": score_mlp_autoencoder, - "mahalanobis": score_mahalanobis, - "maha_ridge": score_mahalanobis_ridge, -} - - -# -------------------------------------------------------------------------------------- -# Axis 4: an honest event-level metric -# -------------------------------------------------------------------------------------- - - -def event_recall_at_budget(labels: np.ndarray, scores: np.ndarray, budget_frac: float = 0.01) -> tuple[float, int]: - """Fraction of true anomaly *ranges* containing at least one alerted row, within an alert budget. - - Why this is not point adjustment: point adjustment rewrites every point of a detected range as a - true positive, which inflates precision and is why a random scorer reaches state of the art under - it. Here precision is never touched — the alert budget is fixed at *budget_frac* of all rows, and - only recall is counted per event. A detector cannot game it by firing everywhere, because the - budget caps how much it may fire. - - Returns ``(event_recall, n_events)``. - """ - flags = np.zeros(len(labels), dtype=bool) - budget = max(1, int(len(labels) * budget_frac)) - flags[np.argsort(scores)[::-1][:budget]] = True - - # Contiguous runs of label == 1 are events. - padded = np.concatenate([[0], (labels > 0).astype(int), [0]]) - edges = np.diff(padded) - starts = np.flatnonzero(edges == 1) - ends = np.flatnonzero(edges == -1) - if len(starts) == 0: - return float("nan"), 0 - detected = sum(1 for s, e in zip(starts, ends, strict=False) if flags[s:e].any()) - return detected / len(starts), len(starts) - - -# -------------------------------------------------------------------------------------- -# Axis 3: scope, and the driver -# -------------------------------------------------------------------------------------- - - -def contaminate( - train: dict[str, np.ndarray], - test: dict[str, np.ndarray], - labels: dict[str, np.ndarray], - rate: float, - seed: int = 0, -) -> dict[str, np.ndarray]: - """Return a copy of *train* with anomalous rows mixed in, at approximately *rate*. - - Without this, the whole exercise measures the wrong thing. SMD ships a **clean** train split, but - DQX fits on ``sample_df`` -- a random sample of the user's table, anomalies included. Sample mean and - covariance are non-robust, so a few extreme rows inflate the covariance *along the anomaly - direction*, which is precisely the direction that must stay tight. That is masking, and it is the - difference between a benchmark number and a number a user will see. - - IsolationForest resists this (it has ``contamination`` and subsampling); a moment-based estimator - does not. Anomalous rows are drawn from each entity's own test split so they are realistic - anomalies for that machine rather than synthetic noise. - """ - rng = np.random.default_rng(seed) - out = {} - for entity, rows in train.items(): - anomalous = test[entity][labels[entity] > 0] - n_inject = int(len(rows) * rate) - if len(anomalous) == 0 or n_inject == 0: - out[entity] = rows - continue - picks = rng.choice(len(anomalous), size=min(n_inject, len(anomalous)), replace=n_inject > len(anomalous)) - out[entity] = np.vstack([rows, anomalous[picks]]) - return out - - -def run_cell( - train: dict[str, np.ndarray], - test: dict[str, np.ndarray], - labels: dict[str, np.ndarray], - featuriser: str, - estimator: str, - per_entity: bool, - seed: int, -) -> dict: - """One configuration: featurise per entity, then fit either one model or one per entity.""" - started = time.perf_counter() - fx = FEATURISERS[featuriser] - scorer = ESTIMATORS[estimator] - - tr_by_entity = {e: np.nan_to_num(fx(train[e]), nan=0.0, posinf=0.0, neginf=0.0) for e in SMD_ENTITIES} - te_by_entity = {e: np.nan_to_num(fx(test[e]), nan=0.0, posinf=0.0, neginf=0.0) for e in SMD_ENTITIES} - - if per_entity: - # One model per machine. Scores are z-normalised per entity against that entity's own *training* - # score distribution, so the numbers are comparable across models and a single global threshold - # applies -- this is what stops an all-normal entity from flagging its own quietest rows. - score_parts = [] - for entity in SMD_ENTITIES: - tr, te = tr_by_entity[entity], te_by_entity[entity] - raw_test = scorer(tr, te, seed) - raw_train = scorer(tr, tr, seed) - centre, spread = float(np.mean(raw_train)), float(np.std(raw_train)) or 1.0 - score_parts.append((raw_test - centre) / spread) - scores = np.concatenate(score_parts) - n_models = len(SMD_ENTITIES) - else: - tr_all = np.vstack([tr_by_entity[e] for e in SMD_ENTITIES]) - te_all = np.vstack([te_by_entity[e] for e in SMD_ENTITIES]) - scores = scorer(tr_all, te_all, seed) - n_models = 1 - - y = np.concatenate([labels[e] for e in SMD_ENTITIES]) - recall, n_events = event_recall_at_budget(y, scores) - return { - "pr_auc": pr_auc(y, scores), - "roc_auc": roc_auc(y, scores), - "precision_at_n": precision_at_n(y, scores), - "event_recall_at_1pct": recall, - "n_events": n_events, - "n_features": int(next(iter(tr_by_entity.values())).shape[1]), - "n_models": n_models, - "seconds": round(time.perf_counter() - started, 1), - } - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--seeds", type=int, default=2) - parser.add_argument("--estimators", nargs="+", default=list(ESTIMATORS), choices=list(ESTIMATORS)) - parser.add_argument("--skip-per-entity", action="store_true", help="pooled scope only (faster)") - parser.add_argument( - "--contaminate", - type=float, - default=0.0, - help="Mix this fraction of anomalous rows into the TRAIN split (G3: masking under contamination)", - ) - args = parser.parse_args() - - print("Loading SMD (cached)...", flush=True) - train, test, labels = load_smd_split() - if args.contaminate: - train = contaminate(train, test, labels, args.contaminate) - print(f" TRAIN CONTAMINATED at {args.contaminate:.1%} (G3: does the estimator survive masking?)") - y = np.concatenate([labels[e] for e in SMD_ENTITIES]) - _, n_events = event_recall_at_budget(y, np.random.default_rng(0).random(len(y))) - print(f" 28 entities | {len(y):,} test rows | {y.mean():.2%} anomalous points | {n_events} events") - print(" protocol: fit on SMD train split, score test split (chronological), no point adjustment") - print(f" event_recall_at_1pct = of {n_events} events, share with >=1 row in a 1%-of-rows alert budget\n") - - scopes = [False] if args.skip_per_entity else [False, True] - header = f"{'featuriser':11s} {'estimator':12s} {'scope':11s} {'PR-AUC':>8s} {'ROC':>7s} {'P@n':>7s} {'EvRec@1%':>9s} {'feat':>5s} {'s':>5s}" - print(header) - print("-" * len(header)) - - results: list[dict] = [] - for featuriser in FEATURISERS: - for estimator in args.estimators: - for per_entity in scopes: - per_seed = [ - run_cell(train, test, labels, featuriser, estimator, per_entity, seed) for seed in range(args.seeds) - ] - agg = { - "featuriser": featuriser, - "estimator": estimator, - "scope": "per_entity" if per_entity else "pooled", - **{ - k: float(np.mean([m[k] for m in per_seed])) - for k in ("pr_auc", "roc_auc", "precision_at_n", "event_recall_at_1pct") - }, - "n_features": per_seed[0]["n_features"], - "n_models": per_seed[0]["n_models"], - "seconds": float(np.mean([m["seconds"] for m in per_seed])), - } - results.append(agg) - print( - f"{featuriser:11s} {estimator:12s} {agg['scope']:11s} " - f"{agg['pr_auc']:8.4f} {agg['roc_auc']:7.4f} {agg['precision_at_n']:7.4f} " - f"{agg['event_recall_at_1pct']:9.3f} {agg['n_features']:5d} {agg['seconds']:5.0f}", - flush=True, - ) - - shipped = next( - (r for r in results if r["featuriser"] == "raw" and r["estimator"] == "iforest" and r["scope"] == "pooled"), - None, - ) - best_pr = max(results, key=lambda r: r["pr_auc"]) - best_ev = max(results, key=lambda r: r["event_recall_at_1pct"]) - - print("\n--- reading ---") - if shipped: - print(f"shipped today : PR-AUC {shipped['pr_auc']:.4f} EvRec@1% {shipped['event_recall_at_1pct']:.3f}") - print( - f"best PR-AUC : {best_pr['pr_auc']:.4f} " - f"({best_pr['featuriser']}/{best_pr['estimator']}/{best_pr['scope']}, {best_pr['n_features']} feat)" - ) - print( - f"best event recall : {best_ev['event_recall_at_1pct']:.3f} " - f"({best_ev['featuriser']}/{best_ev['estimator']}/{best_ev['scope']})" - ) - if shipped and shipped["pr_auc"]: - print(f"headroom on PR-AUC : {best_pr['pr_auc'] / shipped['pr_auc']:.2f}x over what ships today") - print("\nThe best row here IS the ballpark: published SMD figures use point adjustment and are not") - print("a valid target. '80% of the ballpark' means 80% of the best honest configuration above.") - - suffix = f"-contaminated-{args.contaminate:g}" if args.contaminate else "" - out = f"benchmarks/anomaly_conditioning/results/smd-bakeoff{suffix}.json" - with open(out, "w", encoding="utf-8") as handle: - json.dump( - {"window": WINDOW, "seeds": args.seeds, "contaminate": args.contaminate, "results": results}, - handle, - indent=2, - ) - print(f"\nwrote {out}") - - -if __name__ == "__main__": - main() diff --git a/benchmarks/anomaly_conditioning/trend_limits.py b/benchmarks/anomaly_conditioning/trend_limits.py deleted file mode 100644 index 9ae7265bb..000000000 --- a/benchmarks/anomaly_conditioning/trend_limits.py +++ /dev/null @@ -1,226 +0,0 @@ -"""Why trend is a documented limitation, and why `drift_threshold` is not the safety net for it. - -Row anomaly detection learns "normal" from a window and compares later rows against it. A steadily -growing metric therefore leaves that window, and ordinary rows start being flagged. This module measures -how quickly that happens, and tests the two fixes that suggest themselves -- both of which fail. **It is -not wired into DQX**; it exists so the documented limitation stays measured rather than re-argued. - -## Fix 1: give the model an elapsed-time feature so it can learn the slope - -Measured: it makes false flagging *worse*, not better -- 95.8% -> 100.0% on a batch that simply continues -the trend. New rows carry time values outside the training range, and isolating an out-of-range value -takes very few splits, which is precisely what Isolation Forest scores as anomalous. The feature intended -to explain the growth away becomes the strongest evidence of anomaly. - -## Fix 2: condition on a time bucket with `baseline_by` - -Cannot work, for a structural reason rather than a numerical one. `baseline_by` persists a median per -group at training time and looks it up while scoring; a row whose group was absent from training is -reported via ``is_new_baseline`` and has its score, severity and contributions **nulled** -(``scoring_utils._null_unseen_group_scores``). Every future time bucket is by definition absent from -training, so a model conditioned on one would score nothing at all. The null-on-unseen behaviour is -correct for categorical groups -- an unseen category cannot be judged honestly -- which is exactly why a -time bucket is the wrong thing to condition on. - -## So the remaining question: does drift detection warn before scoring degrades? - -No, and it cannot. `drift_threshold=3.0` is the documented setting; the drift score saturates below 1.8 -however steep the trend, because ``_compute_column_drift_score`` divides the batch's mean shift by the -*training window's* standard deviation, and a linear ramp inflates that standard deviation in proportion -to the growth. Both numerator and denominator scale together, so the ratio is bounded: - - trend over window false flags drift score warns at 3.0? - 0% 3.0% 0.04 no (correct) - 4% 5.8% 0.60 NO - 10% 19.6% 1.17 NO - 20% 45.4% 1.53 NO - 40% 80.2% 1.68 NO - 100% 95.8% 1.74 NO - 200% 99.2% 1.74 NO - -At 200% growth, 99.2% of a batch in which nothing is wrong is flagged, and drift detection stays silent. - -## Fix 3: persist a fitted trend and subtract it -- this one works - -Both failures above share a cause, and it points straight at the fix. An elapsed-time *feature* fails -because the model must learn the slope from data covering only the training range, then meet values outside -it. A time-bucket `baseline_by` fails because a median **lookup table** has no entry for a future bucket. A -fitted trend has neither problem: it is a *function* of time, so it extrapolates to any future t, and the -model never sees time at all -- it sees the residual, which is stationary by construction. - -That is exactly the shape of the existing `_rel_baseline` feature -- observed value minus its expected -level. Only the source of the expected level changes, from a per-group median to a fitted line. - - trend over window raw detrended (correct answer ~2%) - 10% 19.6% 2.8% - 40% 80.2% 2.8% - 100% 95.8% 2.8% - 200% 99.2% 2.8% - -Flat at 2.8% however steep the trend, and it still finds real anomalies -- on a batch carrying a 3x spike -in 5% of rows, both score 100% recall, but the raw feature emits 96.4% false positives against the -detrended feature's 2.1%. - -### Where it breaks, which is the part a design has to answer - -**Extrapolation horizon.** Accuracy decays with distance beyond the training window: 2.8% at the boundary, -3.4% one window out, 6.4% five windows out, 89.6% twenty-five windows out. Usable, but it needs a horizon -cap and a warning past it -- the same shape of contract as ``is_new_baseline``. - -**Regime change.** When the trend itself changes, residuals blow up and 70-85% of rows flag. Arguably -correct -- growth stalling *is* an anomaly -- but reporting a table-level event row by row is not useful. -Note that detrending is the only one of the two that notices: when growth *reverses*, the raw feature -flags just 6.0% because falling values look like a return to trained levels, while the detrended feature -flags 85.2%. - -**Functional form.** Exponential growth fitted with a straight line barely helps: 95.8% against a raw -99.8%. Business metrics compound, so a log-scale fit would be needed, which makes the form a choice rather -than a default. - -**API cost.** It needs a time column. The current design deliberately requires none -- `"timeseries"` -models cross-metric correlation, not time -- so this would add the first temporal parameter to the public -surface. - -## What the documentation says today - -The docs describe what ships, and what ships has no trend handling. So: model a quantity that does not -trend (a rate or a ratio, not a running level), and where the level itself matters, retrain on a schedule. -What they must *not* say is "enable drift_threshold and you will be warned", which was the earlier claim -and is measurably false. - -Neither the detrending transform nor a trend-aware drift statistic is attempted here. Both are real, -scoped follow-ups rather than impossibilities, and this module exists so that stays clear. - -Run: uv run python benchmarks/anomaly_conditioning/trend_limits.py -""" - -import numpy as np -from sklearn.ensemble import IsolationForest - -N_TRAIN = 2000 -N_SCORE = 500 -CONTAMINATION = 0.02 -NOISE = 3.0 -# The setting the user guide documents, and what DQX compares its max-across-columns score against. -DRIFT_THRESHOLD = 3.0 -# Trend slopes per row, spanning "flat" to "the metric tripled across the training window". -SLOPES = (0.0, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1) - - -def _series(slope: float, start: int, count: int, rng: np.random.Generator) -> np.ndarray: - """A metric growing at *slope* per row, with constant noise.""" - t = np.arange(start, start + count, dtype=float) - return 100.0 + slope * t + rng.normal(0, NOISE, count) - - -def false_flag_rate(train: np.ndarray, score: np.ndarray) -> float: - """Share of a nothing-is-wrong batch that gets flagged. The correct answer is ~CONTAMINATION.""" - model = IsolationForest(contamination=CONTAMINATION, random_state=42).fit(train.reshape(-1, 1)) - return float((model.predict(score.reshape(-1, 1)) == -1).mean()) - - -def drift_score(train: np.ndarray, score: np.ndarray) -> float: - """DQX's per-column drift score, mirroring ``drift._compute_column_drift_score``.""" - baseline_mean, baseline_std = float(train.mean()), float(train.std()) - current_mean, current_std = float(score.mean()), float(score.std()) - if baseline_std == 0: - return abs(current_mean - baseline_mean) - z_score = abs(current_mean - baseline_mean) / baseline_std - std_change = abs(current_std - baseline_std) / baseline_std - return (z_score * 0.7) + (std_change * 0.3) - - -def elapsed_time_feature_makes_it_worse() -> tuple[float, float]: - """Fix 1, measured: adding a monotonic time feature raises the false-flag rate.""" - rng = np.random.default_rng(42) - t_train = np.arange(N_TRAIN, dtype=float) - metric_train = _series(0.05, 0, N_TRAIN, rng) - t_score = np.arange(N_TRAIN, N_TRAIN + N_SCORE, dtype=float) - metric_score = _series(0.05, N_TRAIN, N_SCORE, rng) - - without = false_flag_rate(metric_train, metric_score) - - model = IsolationForest(contamination=CONTAMINATION, random_state=42).fit(np.column_stack([metric_train, t_train])) - with_time = float((model.predict(np.column_stack([metric_score, t_score])) == -1).mean()) - return without, with_time - - -def fit_trend(t: np.ndarray, values: np.ndarray) -> tuple[float, float]: - """Least-squares slope and intercept -- what a training run would persist.""" - slope, intercept = np.polyfit(t, values, 1) - return float(slope), float(intercept) - - -def detrended(t: np.ndarray, values: np.ndarray, slope: float, intercept: float) -> np.ndarray: - """The proposed feature: observed value minus the trend's expectation at this row's time.""" - return values - (intercept + slope * t) - - -def detrending_fixes_it(slope: float, gap: int = 0) -> tuple[float, float]: - """Fix 3, measured: raw versus detrended false-flag rate on a batch where nothing is wrong. - - *gap* pushes the scored window further past the end of training, which is how the extrapolation - horizon is measured. - """ - rng = np.random.default_rng(42) - t_train = np.arange(N_TRAIN, dtype=float) - train = _series(slope, 0, N_TRAIN, rng) - t_score = np.arange(N_TRAIN + gap, N_TRAIN + gap + N_SCORE, dtype=float) - score = _series(slope, N_TRAIN + gap, N_SCORE, rng) - - raw = false_flag_rate(train, score) - fitted_slope, intercept = fit_trend(t_train, train) - residual_rate = false_flag_rate( - detrended(t_train, train, fitted_slope, intercept), - detrended(t_score, score, fitted_slope, intercept), - ) - return raw, residual_rate - - -def main() -> None: - print("Does a trend break scoring, and does drift detection warn?\n") - print(f"{'trend over window':>18} {'false flags':>12} {'drift score':>12} warns?") - print("-" * 60) - for slope in SLOPES: - rng = np.random.default_rng(42) - train = _series(slope, 0, N_TRAIN, rng) - score = _series(slope, N_TRAIN, N_SCORE, rng) - flags = false_flag_rate(train, score) - drift = drift_score(train, score) - # A flat series *should* stay silent, so only a missed warning on a real trend is a failure. - if drift >= DRIFT_THRESHOLD: - warns = "yes" - else: - warns = "NO" if slope > 0 else "no (correct)" - print(f"{slope * N_TRAIN:>17.0f}% {flags:>11.1%} {drift:>12.2f} {warns}") - - print("\nfalse flags = share of a batch that merely continues the trend and is flagged anomalous") - print(f" (the correct answer is ~{CONTAMINATION:.0%}, the contamination rate)") - - without, with_time = elapsed_time_feature_makes_it_worse() - print("\nFix 1 -- add an elapsed-time feature so the model can learn the slope:") - print(f" metric only {without:6.1%} falsely flagged") - print(f" metric + elapsed time {with_time:6.1%} falsely flagged <- worse, not better") - - print("\nFix 2 -- condition on a time bucket via baseline_by:") - print(" Not measurable, and not viable: every future bucket is an unseen group, whose score,") - print(" severity and contributions are nulled. Such a model would score nothing. See the") - print(" module docstring.") - - print("\nFix 3 -- persist a fitted trend and subtract it (not implemented; this is the proposal):") - print(f" {'trend over window':>18} {'raw':>10} {'detrended':>12}") - for slope in (0.005, 0.02, 0.05, 0.1): - raw, residual_rate = detrending_fixes_it(slope) - print(f" {slope * N_TRAIN:>17.0f}% {raw:>10.1%} {residual_rate:>12.1%}") - - print("\n Extrapolation horizon, at 100% trend -- accuracy decays with distance past training:") - for gap in (0, N_TRAIN, N_TRAIN * 5, N_TRAIN * 25): - _, residual_rate = detrending_fixes_it(0.05, gap=gap) - windows = gap / N_TRAIN - print(f" {windows:>17.0f} windows out {residual_rate:>10.1%}") - print("\n So it works, and needs a horizon cap. See the module docstring for the other three") - print(" design questions it raises (regime change, functional form, and needing a time column).") - - -if __name__ == "__main__": - main() diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py index 7028cdf13..2537e4a70 100644 --- a/demos/dqx_demo_anomaly_timeseries_fleet.py +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -465,7 +465,7 @@ def generate_telemetry(n_rows: int, seed: int, break_correlation: bool = False): # MAGIC ### 📚 Resources # MAGIC # MAGIC - [Choosing a profile](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection#choosing-a-profile) -# MAGIC - [Anomaly detection quality](https://databrickslabs.github.io/dqx/docs/reference/anomaly_detection_quality) — how the numbers above were measured +# MAGIC - [Benchmarks](https://databrickslabs.github.io/dqx/docs/reference/benchmarks#anomaly-benchmarks) — measured detection quality and timings # MAGIC - [Row Anomaly Detection guide](https://databrickslabs.github.io/dqx/docs/guide/row_anomaly_detection) # MAGIC # MAGIC ### 🎉 You're Ready! diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index a615c8225..48019c445 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -125,7 +125,7 @@ What it is reliably good at, and what it is not: - **Good at unusual _combinations_ across columns**: several values that are each individually fine but wrong together. Rules struggle to express that, and it is where anomaly detection earns its place. - **Weaker than a plain rule when a single column is extreme in isolation.** A range check or an outlier rule catches that more reliably and more cheaply, so reach for a rule there. You can run both. -On DQX's own benchmarks (ten classical tabular datasets) it beats a random baseline on every one and a simple "largest z-score across columns" baseline on most. The exceptions are exactly the single-extreme-value case above. The full measured results, the datasets, and what the numbers do *not* mean are in [Anomaly detection quality](/docs/reference/anomaly_detection_quality). +Measured across ten classical tabular anomaly-detection datasets, it beats a random baseline on every one and a simple "largest z-score across columns" baseline on most. The exceptions are exactly the single-extreme-value case above. Detection quality on DQX's own synthetic fixtures, alongside training and scoring times, is published in [Benchmarks](/docs/reference/benchmarks). Whatever a benchmark says, measure on **your** data before relying on a number. Train on a slice you consider good, score a slice you understand, and check that the rows it flags are ones you would actually want flagged. @@ -382,16 +382,19 @@ on a healthy machine. Measured on the Server Machine Dataset — real machine telemetry with labelled incidents — this is the share of incidents each detector surfaces while the alert budget is capped at 1% of rows: - | `profile` | Incidents surfaced | |---|---| | `"tabular"` | 33% | | `"timeseries"` | **79%** | - + +An incident counts as surfaced if the detector flags at least one of its rows, so this measures whether +you would have been paged, not how many rows you would have had to read. Both detectors were trained on +data that still contained the anomalies, which is what DQX does when it fits a sample of your table; on a +curated clean training split the same comparison is 36% against 82%. That is telemetry, which is what `"timeseries"` is for — on ordinary tabular data the ranking reverses. -See [Anomaly detection quality](/docs/reference/anomaly_detection_quality#which-detector-to-use) for both -detectors measured on both kinds of data, and what this metric counts. +Detection quality on DQX's own synthetic fixtures is published in +[Benchmarks](/docs/reference/benchmarks). DQX does not detect which profile you need. Getting it right cannot be verified without labelled diff --git a/docs/dqx/docs/reference/anomaly_detection_quality.mdx b/docs/dqx/docs/reference/anomaly_detection_quality.mdx deleted file mode 100644 index 8990583b6..000000000 --- a/docs/dqx/docs/reference/anomaly_detection_quality.mdx +++ /dev/null @@ -1,214 +0,0 @@ ---- - -title: Anomaly detection quality - -sidebar_position: 509 - ---- - -# Anomaly Detection Quality - -How well does row anomaly detection actually detect? This page reports measured detection quality; -[Benchmarks](/docs/reference/benchmarks) covers timing. - -Use it to decide two things: whether anomaly detection suits your data at all, and whether to give it -a grouping via `baseline_by`. - -## Read this first - -**DQX scores rows independently.** It is not a sequence model and does not consume a window of history -to make a prediction. Published results on time-series anomaly benchmarks routinely exceed 0.80 PR-AUC -using models that do. A DQX figure of 0.15 on the same data is **a different task, not a worse -implementation**. - -**PR-AUC moves with the base rate.** A score of 0.22 at a 0.17% anomaly rate is strong; 0.48 at 40% is -weak. Never compare PR-AUC across datasets — compare each against its own floor, which is what the -tables below do. - -**No point-adjusted F1 appears anywhere.** Under that protocol — crediting a whole labelled anomaly -segment when any single point inside it is detected — a *random* score achieves state-of-the-art F1 -([Kim et al., AAAI 2022](https://arxiv.org/abs/2109.05257)). DQX does not compute it. - -## When conditioning helps - -`baseline_by` judges each metric against its own group's baseline rather than against the whole table. -Whether that helps depends on what makes your anomalies anomalous. - - -| your anomalies are... | median ΔPR-AUC with `baseline_by` | what to do | -|---|---|---| -| **contextual** — ordinary for the table, wrong for their group | **+0.0742** | use `baseline_by` | -| **globally extreme** — unusual against the whole table | +0.0000 | costs nothing; leave it on | -| mixed or unknown (real-world datasets) | +0.0025 | leave it on | - - -Read that as one asymmetry rather than three results: conditioning is worth a great deal where it -applies and nothing measurable where it does not. That is why DQX enables it automatically when it -finds a usable grouping, rather than asking you to opt in. - -The contextual case deserves a concrete example, because whole-table models and rules both miss it. One -group's daily volume collapses by 80% while the total across all groups stays flat. Measured on exactly -that shape, a model comparing against the whole table scored PR-AUC 0.0028 against a 0.0026 base rate — -chance. Conditioned on the group, 0.6962. - -A grouping is discovered for you only when you let DQX pick the feature columns too. Name `columns` yourself -and the comparison stays pooled, with a warning naming the grouping to pass if the data looks grouped. To -compare against the whole table and silence that warning, pass `baseline_by=[]`. - -## Which detector to use - -DQX ships two detectors, chosen with `profile`. They answer different questions, so neither replaces -the other: - -| `profile` | detector | finds | -|---|---|---| -| `"tabular"` (default) | Isolation Forest | rows whose values, or combination of values, are unusual | -| `"timeseries"` | correlation-aware | metrics that normally move together and stopped | - -The second exists because the first cannot see the second kind of problem. Isolation Forest splits on -one feature at a time, so a row where every metric sits inside its usual range — but in a combination -that never happens on healthy data — never gets separated by any single split. - -The **Server Machine Dataset** is exactly that kind of data — real machine telemetry with labelled -incidents — so it shows the gap at its widest. Read the table as *which tool for which job*: it says -`"timeseries"` is the right choice for telemetry, not that either detector is good or bad in general. On -ordinary tabular data the ranking reverses, which is what the -[tabular benchmarks](#plain-tabular-benchmarks) below measure. - - -| `profile` | detector | incidents surfaced (clean training split) | incidents surfaced (contaminated training) | -|---|---|---|---| -| `"tabular"` | Isolation Forest | 36% | 33% | -| `"timeseries"` | correlation-aware | **82%** | **79%** | - - -Two columns, because they answer different questions. **Contaminated training is the one that describes -your run**: DQX fits a random sample of your table with the anomalies still in it. The clean-split column -is what a benchmark with a hand-curated training set would report — an upper bound you would not see in -practice. The correlation-aware detector holds up across both, which is the part worth noting: covariance -estimates are sensitive to exactly the extreme rows they are meant to find, so this was the result that -could have ruled the approach out. - -### What "incidents surfaced" means - -Of the labelled incidents in the data, the share that produce **at least one alert while the alert budget -is capped at 1% of all rows**. - -It counts *problems that reach a human* at an alert volume they will tolerate. Counting anomalous rows -instead would let a single long incident dominate the result, and would reward a detector that alerts on -everything. Fixing the budget prevents both. - -It is deliberately **not** point-adjusted F1, the metric most published time-series results use. Point -adjustment credits an entire incident as detected from a single lucky row, and Kim et al. -([AAAI 2022](https://arxiv.org/abs/2109.05257)) showed that random scores reach state-of-the-art under -it. Published figures produced that way are not comparable with the ones above, in either direction. - -DQX does not detect which profile you need, and calendar seasonality is handled automatically by both. -[Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile) covers both points, and the -[FAQ](/docs/guide/row_anomaly_detection#frequently-asked-questions) lists what neither detector finds. - -## One model, not one per group - -The `segment_by` path, removed in this release, trained a separate model per group. Conditioning -expresses the grouping as features on a single model instead, and won on both axes — which is why the -comparison is recorded here rather than left to memory. - - -| your anomalies are... | median ΔPR-AUC vs one-model-per-group | -|---|---| -| contextual | **+0.0252** | -| globally extreme | +0.0005 | -| mixed or unknown | +0.0054 | - - - -| approach | models trained | median fit time | -|---|---|---| -| `baseline_by` | 1 | 0.17s | -| `segment_by` | one per group (12 in this sweep) | 1.38s | - - -Per-group models also fail in a way that averages conceal. On the Server Machine Dataset one entity -produced 15,963 false positives across 28,392 normal rows — a 56% false-alarm rate — while the -aggregate metric looked merely mediocre. Each per-group model calibrates its threshold on its own rows, -so a group containing nothing unusual still has its most-unusual few percent flagged. - -## Plain tabular benchmarks - -Ten classical benchmarks with no grouping, from [ADBench](https://github.com/Minqi824/ADBench) -(BSD-2-Clause), which redistributes the ODDS / UCI / Kaggle collections. These characterise detection -without any conditioning. - -Each row is read against its own random floor, and against `max-abs-z` — the largest absolute z-score -across features, which is the cheapest defensible detector and a genuinely competitive one. - - -| dataset | rows | features | base rate | DQX PR-AUC | random | max-abs-z | lift vs random | beats max-abs-z | -|---|---|---|---|---|---|---|---|---| -| shuttle | 30000 | 9 | 7.15% | **0.9788** | 0.0731 | 0.8983 | 13.4x | yes | -| satellite | 6435 | 36 | 31.64% | 0.6668 | 0.3131 | 0.5946 | 2.1x | yes | -| cardio | 1831 | 21 | 9.61% | 0.5811 | 0.1012 | 0.5534 | 5.7x | yes | -| thyroid | 3772 | 6 | 2.47% | 0.5389 | 0.0251 | 0.3007 | 21.5x | yes | -| spambase | 4207 | 57 | 39.91% | 0.4758 | 0.4033 | 0.4039 | 1.2x | yes | -| campaign | 30000 | 62 | 11.27% | 0.2867 | 0.1160 | 0.2398 | 2.5x | yes | -| mnist | 7603 | 100 | 9.21% | 0.2766 | 0.0999 | 0.3367 | 2.8x | **no** | -| fraud | 30000 | 29 | 0.17% | 0.2240 | 0.0019 | 0.1365 | **115.4x** | yes | -| mammography | 11183 | 6 | 2.32% | 0.2193 | 0.0236 | 0.1779 | 9.3x | yes | -| covertype | 30000 | 10 | 0.96% | 0.0534 | 0.0094 | 0.1122 | 5.7x | **no** | - - -DQX beats the random floor on all ten and `max-abs-z` on eight. `fraud` — the Kaggle credit-card set at -a 0.17% base rate — is the standout at 115x the floor. - -One row to read carefully rather than celebrate: `spambase` at a 39.9% base rate is not really anomaly -detection, and its 1.2x lift should be read that way. - -### Where it underperforms - -Two datasets say **no**, for the same reason. - -| dataset | shape | DQX | max-abs-z | -|---|---|---|---| -| `covertype` | 10 features, 0.96% base rate | 0.0534 | 0.1122 | -| `mnist` | 100 features | 0.2766 | 0.3367 | - -Isolation Forest splits on one randomly chosen feature at a time. When an anomaly is a single extreme -value among few dimensions the signal is diluted across the other axes, and in 100 dimensions the -random choice rarely lands on the informative one. A global max-abs-z captures "any feature is extreme" -directly. - -**What this means for you.** If your anomalies are single extreme values in a handful of numeric -columns, a range check or an outlier rule will serve you better — and you can run both. Anomaly -detection earns its place on unusual *combinations* across columns, which is what rules struggle to -express. - -Tuning does not close this gap, and the obvious knob makes matters worse elsewhere: raising the rows -sampled per tree lifts `fraud` while dropping `shuttle` and `cardio` substantially. Adding a -complementary z-score detector to the ensemble was measured and rejected — it wins on half the datasets -and loses badly on the others. - -## Datasets - -Downloaded at run time and cached; DQX redistributes none of them. - -| dataset | licence | groupings tested | citation | -|---|---|---|---| -| Server Machine Dataset | MIT, via [`NetManAIOps/OmniAnomaly`](https://github.com/NetManAIOps/OmniAnomaly) | server entity (28), machine family (3) | Su et al., *Robust Anomaly Detection for Multivariate Time Series through Stochastic Recurrent Neural Networks*, KDD 2019 | -| NSL-KDD | redistributable with citation | `service` (65), `flag` (11), `protocol_type` (3) | Tavallaee et al., *A detailed analysis of the KDD CUP 99 data set*, CISDA 2009 | -| ADBench (10 tabular sets) | BSD-2-Clause | none — ungrouped | Han et al., *ADBench: Anomaly Detection Benchmark*, NeurIPS 2022 | -| synthetic | n/a | 13 heterogeneity levels x 2 anomaly mechanisms | generated by the harness | - -SMAP and MSL are excluded: their data files carry "© Original Authors" with no permissive licence. - -## Caveats - -- **1,545 measurements**, 15 seeds per configuration, compared pairwise by seed and tested with - Wilcoxon signed-rank. Medians are reported, not means. -- Each configuration is fitted and scored on the same rows, so these figures measure separability - rather than generalisation to unseen data. -- SMD is capped at 4,000 rows per entity, and NSL-KDD's attacks are downsampled to a 2% rate so the - task is anomaly detection rather than classification. Every configuration sees identical rows. -- Results are indicative. Benchmark against your own data before relying on a number here. - -Everything above is reproducible from `benchmarks/anomaly_conditioning/` in the DQX repository, which -documents how to run it. diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py index 51d942627..ef82356c7 100644 --- a/src/databricks/labs/dqx/anomaly/timeseries_detector.py +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -7,8 +7,9 @@ Mahalanobis detector here catches 82%. Refitting on training data that still contains anomalies -- what DQX actually does -- costs both of them a few points and does not change the conclusion: 33% against 79%. That was the result that could have sunk the approach, because sample covariance is not robust and -a few extreme rows inflate it along the very direction that needs to stay tight. See ``benchmarks/anomaly_conditioning/smd_bakeoff.py`` and the -committed results next to it. +a few extreme rows inflate it along the very direction that needs to stay tight. Detection quality on +DQX's own synthetic fixtures is published in the benchmarks report and measured by +``tests/perf/test_anomaly_benchmark.py``. The distance is the ordinary squared Mahalanobis distance from the training centre, ``d² = (x−μ)ᵀ Σ⁻¹ (x−μ)``, with three deliberate choices. diff --git a/src/databricks/labs/dqx/anomaly/training_strategies.py b/src/databricks/labs/dqx/anomaly/training_strategies.py index 5f6ca7ecf..156ac9f60 100644 --- a/src/databricks/labs/dqx/anomaly/training_strategies.py +++ b/src/databricks/labs/dqx/anomaly/training_strategies.py @@ -41,8 +41,8 @@ # correctly cannot be verified without labels, which an unsupervised tool does not have, and the one # cheap signal for "this looks temporal" was measured and rejected -- lag-1 autocorrelation is # confounded by any ordering correlated with the values, which sorted warehouse storage produces -# routinely (see benchmarks/anomaly_conditioning/profile_advisory_gate.py). A value named "auto" would -# therefore have promised a selection that never happens. +# routinely -- three of ten classical tabular benchmarks scored above the weakest genuine time-series +# entity. A value named "auto" would therefore have promised a selection that never happens. PROFILE_TABULAR = "tabular" PROFILE_TIMESERIES = "timeseries" SUPPORTED_PROFILES = (PROFILE_TABULAR, PROFILE_TIMESERIES) diff --git a/tests/perf/generate_md_report.py b/tests/perf/generate_md_report.py index da25aa734..ae93548a8 100644 --- a/tests/perf/generate_md_report.py +++ b/tests/perf/generate_md_report.py @@ -75,14 +75,18 @@ # Anomaly benchmark section: timings plus indicative detection quality carried in extra_info. if anomaly_benchmarks: lines.append("\n## Anomaly Benchmarks\n") - provenance: dict = next((b.get("extra_info", {}) for b in anomaly_benchmarks if b.get("extra_info")), {}) - if provenance.get("dataset"): - lines.append( - f"* Measured on {provenance['dataset']} " - f"({provenance.get('n_train_rows', 'n/a')} train / {provenance.get('n_test_rows', 'n/a')} test rows, " - f"{provenance.get('n_features', 'n/a')} features, " - f"{provenance.get('anomaly_frac', 'n/a')} anomaly fraction, seed {provenance.get('seed', 'n/a')})." - ) + lines.append( + "* Every fixture is generated in-repo by `tests/integration_anomaly/synthetic_generators.py` " + "from a fixed seed. Nothing is downloaded and no third-party data is redistributed, so these " + "numbers carry no dataset licence conditions." + ) + lines.append( + "* Each row states the fixture it was measured on. They are deliberately different problems: " + "one blends overlapping and heavy-tailed distributions, one breaks the correlation between " + "metrics that normally move together (`profile=\"timeseries\"`), and one collapses a single " + "group's volume while the overall total stays flat (`baseline_by`). Comparing quality numbers " + "*across* rows is meaningless." + ) lines.append( "* Quality columns are **indicative and first-observed only**: the nightly baseline merge keeps " "existing entries on conflict, so `extra_info` is not refreshed once a benchmark has been " @@ -90,14 +94,14 @@ "`tests/integration_anomaly/test_anomaly_quality.py`, not by this table." ) lines.append( - "* These are synthetic distributions chosen to be moderately hard, not a general claim about " - "detection quality on your own data.\n" + "* Synthetic distributions chosen to be moderately hard, not a general claim about detection " + "quality on your own data.\n" ) lines.append( - "| Test | Mean (s) | Median (s) | Min (s) | Max (s) | Stddev (s) | Rounds | Ops/s | ROC-AUC | Precision | Recall | F1 | Precision@N |" + "| Test | Fixture | Rows (train/test) | Features | Anomaly rate | Mean (s) | Median (s) | Rounds | ROC-AUC | Precision | Recall | F1 | Precision@N |" ) lines.append( - "|------|----------|------------|---------|---------|------------|--------|-------|---------|-----------|--------|----|-------------|" + "|------|---------|-------------------|----------|--------------|----------|------------|--------|---------|-----------|--------|----|-------------|" ) for bench in anomaly_benchmarks: stats = bench["stats"] @@ -112,15 +116,18 @@ recall_str = f"{recall:.6f}" if isinstance(recall, (int, float)) else "n/a" f1_str = f"{f1_score:.6f}" if isinstance(f1_score, (int, float)) else "n/a" precision_at_n_str = f"{precision_at_n:.6f}" if isinstance(precision_at_n, (int, float)) else "n/a" + anomaly_frac = extra.get("anomaly_frac") + anomaly_frac_str = f"{anomaly_frac:.4f}" if isinstance(anomaly_frac, (int, float)) else "n/a" + rows_str = f"{extra.get('n_train_rows', 'n/a')} / {extra.get('n_test_rows', 'n/a')}" lines.append( f"| {bench['name']} " + f"| {extra.get('dataset', 'n/a')} " + f"| {rows_str} " + f"| {extra.get('n_features', 'n/a')} " + f"| {anomaly_frac_str} " f"| {stats['mean']:.6f} " f"| {stats['median']:.6f} " - f"| {stats['min']:.6f} " - f"| {stats['max']:.6f} " - f"| {stats['stddev']:.6f} " f"| {stats['rounds']} " - f"| {stats['ops']:.2f} " f"| {roc_auc_str} " f"| {precision_str} " f"| {recall_str} " diff --git a/tests/perf/test_anomaly_benchmark.py b/tests/perf/test_anomaly_benchmark.py index 257b0ba7f..43f6ae6a2 100644 --- a/tests/perf/test_anomaly_benchmark.py +++ b/tests/perf/test_anomaly_benchmark.py @@ -35,6 +35,8 @@ from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRegistry from tests.constants import TEST_CATALOG from tests.integration_anomaly.synthetic_generators import ( + generate_correlated_multivariate_data, + generate_group_conditional_data, generate_heavy_tail_data, generate_overlapping_gaussian_data, ) @@ -94,16 +96,27 @@ def _prepare_synthetic_data(spark) -> tuple[list[str], DataFrame, DataFrame]: return overlap_cols, train_df, test_df -def _record_provenance(benchmark, n_train: int, n_test: int) -> None: +def _record_provenance( + benchmark, + n_train: int, + n_test: int, + *, + dataset: str = "synthetic: overlapping-gaussian + heavy-tail blend", + n_features: int = N_FEATURES, + anomaly_frac: float = ANOMALY_FRAC, + seed: int = SEED, +) -> None: """Attach what the numbers were measured on. A published table of quality metrics with no stated dataset invites being read as a general - claim about DQX's detection quality, which it is not — this is blended synthetic data. + claim about DQX's detection quality, which it is not — every fixture here is generated by + ``tests/integration_anomaly/synthetic_generators.py`` from a fixed seed. Nothing is downloaded + and no third-party data is redistributed, so the published numbers carry no licence conditions. """ - benchmark.extra_info["dataset"] = "synthetic: overlapping-gaussian + heavy-tail blend" - benchmark.extra_info["seed"] = SEED - benchmark.extra_info["n_features"] = N_FEATURES - benchmark.extra_info["anomaly_frac"] = ANOMALY_FRAC + benchmark.extra_info["dataset"] = dataset + benchmark.extra_info["seed"] = seed + benchmark.extra_info["n_features"] = n_features + benchmark.extra_info["anomaly_frac"] = anomaly_frac benchmark.extra_info["n_train_rows"] = n_train benchmark.extra_info["n_test_rows"] = n_test @@ -163,6 +176,26 @@ def test_benchmark_anomaly_score(benchmark, request, spark, ws, make_schema, mak columns=feature_cols, ) + scored = _score_and_record(benchmark, model_name, registry_table, test_df) + + _record_provenance(benchmark, train_df.count(), test_df.count()) + for name, value in _detection_quality(scored).items(): + benchmark.extra_info[name] = value + + +def _anomaly_rate(df: DataFrame) -> float: + """The fraction of rows actually labelled anomalous. + + Published beside the quality numbers, so it is read off the frame rather than assumed from a + generator default: two of these fixtures derive their rate from the incident shape rather than + taking it as an argument, and a wrong denominator would make precision look arbitrary. + """ + row = df.agg(F.avg(F.col("is_anomaly").cast("double")).alias("rate")).first() + return round(float(row["rate"]), 6) if row and row["rate"] is not None else 0.0 + + +def _score_and_record(benchmark, model_name: str, registry_table: str, test_df: DataFrame) -> DataFrame: + """Time one scoring pass through the public check and return the labelled result.""" # The third element is the name of the struct column the check writes. `_dq_info` is assembled a # layer up by DQEngine from that column, so reading `_dq_info` here resolves against nothing. _, apply_fn, info_col = has_no_row_anomalies( @@ -181,9 +214,80 @@ def run_score(): anomaly.getField("is_anomaly").cast("double").alias("pred"), ) - scored = benchmark.pedantic(run_score, rounds=1, iterations=1, warmup_rounds=0) + return benchmark.pedantic(run_score, rounds=1, iterations=1, warmup_rounds=0) - _record_provenance(benchmark, train_df.count(), test_df.count()) + +@pytest.mark.benchmark(group=BENCHMARK_GROUP) +def test_benchmark_anomaly_score_timeseries_profile(benchmark, request, spark, ws, make_schema, make_random): + """Score with ``profile="timeseries"`` on metrics that move together, and record the quality. + + The fixture is the case this detector exists for: metrics driven by shared latent factors, where + the anomaly is the correlation between them breaking rather than any single value leaving its + range. Permuting the broken columns across anomalous rows leaves every marginal distribution + untouched, so a per-column threshold has nothing to fire on by construction. + """ + feature_cols, train_df, test_df, _broken = generate_correlated_multivariate_data(spark, seed=SEED) + model_name, registry_table = _new_model_names(make_schema, make_random) + request.addfinalizer(lambda: _cleanup_anomaly_mlflow(model_name, registry_table, spark)) + + engine = AnomalyEngine(workspace_client=ws, spark=spark) + engine.train( + df=train_df, + model_name=model_name, + registry_table=registry_table, + columns=feature_cols, + baseline_by=[], + profile="timeseries", + ) + + scored = _score_and_record(benchmark, model_name, registry_table, test_df) + + _record_provenance( + benchmark, + train_df.count(), + test_df.count(), + dataset='synthetic: correlated multivariate metrics, profile="timeseries"', + n_features=len(feature_cols), + anomaly_frac=_anomaly_rate(test_df), + ) + for name, value in _detection_quality(scored).items(): + benchmark.extra_info[name] = value + + +@pytest.mark.benchmark(group=BENCHMARK_GROUP) +def test_benchmark_anomaly_score_conditioned(benchmark, request, spark, ws, make_schema, make_random): + """Score with ``baseline_by`` on a contextual anomaly, and record the quality. + + One group's volume collapses while the daily total across all groups is held flat, so the + collapsed value sits inside the range other groups occupy normally. Nothing about the row is + extreme; it is only wrong for its own group, which is what conditioning is for. + """ + feature_cols, train_df, test_df, _incident_key = generate_group_conditional_data( + spark, seed=SEED, n_incident_days=3, n_incident_groups=4, n_control_days=3 + ) + model_name, registry_table = _new_model_names(make_schema, make_random) + request.addfinalizer(lambda: _cleanup_anomaly_mlflow(model_name, registry_table, spark)) + + baseline_by = ["country", "event_type", "product"] + engine = AnomalyEngine(workspace_client=ws, spark=spark) + engine.train( + df=train_df, + model_name=model_name, + registry_table=registry_table, + columns=feature_cols, + baseline_by=baseline_by, + ) + + scored = _score_and_record(benchmark, model_name, registry_table, test_df) + + _record_provenance( + benchmark, + train_df.count(), + test_df.count(), + dataset="synthetic: grouped volumes with a contextual collapse, baseline_by", + n_features=len(feature_cols), + anomaly_frac=_anomaly_rate(test_df), + ) for name, value in _detection_quality(scored).items(): benchmark.extra_info[name] = value From 83b6c682b016344092b943a13bc0930c3b40623c Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 15:34:28 +0100 Subject: [PATCH 058/107] Make the detector choice reachable from a run config `profile` was accepted by `train()` but had no field on `AnomalyConfig` and was not passed by the anomaly workflow, so it was reachable only from the Python API. Scheduled retraining silently fell back to the tabular default, and a run config could not reproduce a model trained by hand. The PR description claimed otherwise; this makes the claim true. Also removes `TemporalAnomalyConfig`, which was exported from `config.py` and referenced nowhere. Its `timestamp_column` field describes calendar extraction that is in fact hard-coded in `_process_datetime_columns`, and it would read as a competing concept beside the temporal baseline landing next. Its two neighbours in that file are live and are left alone. `FeatureEngineering.temporal_config` is persisted in the registry schema and has never been populated. Documented as the queryable summary of how time was used, which is what fills it. --- .../labs/dqx/anomaly/anomaly_workflow.py | 1 + .../labs/dqx/anomaly/model_config.py | 5 +++ src/databricks/labs/dqx/config.py | 14 +++---- tests/unit/test_anomaly_configs.py | 21 ++++++++++ tests/unit/test_anomaly_workflow.py | 40 +++++++++++++++++++ 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_workflow.py b/src/databricks/labs/dqx/anomaly/anomaly_workflow.py index 62bfda9b9..82fb3278f 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_workflow.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_workflow.py @@ -46,6 +46,7 @@ def train_model(self, ctx: WorkflowContext) -> None: df=df, columns=anomaly_config.columns, baseline_by=anomaly_config.baseline_by, + profile=anomaly_config.profile, model_name=model_name, registry_table=registry_table, ) diff --git a/src/databricks/labs/dqx/anomaly/model_config.py b/src/databricks/labs/dqx/anomaly/model_config.py index 032b4b6db..dcfc0e9a1 100644 --- a/src/databricks/labs/dqx/anomaly/model_config.py +++ b/src/databricks/labs/dqx/anomaly/model_config.py @@ -53,6 +53,11 @@ class FeatureEngineering: column_types: dict[str, str] | None = None feature_metadata: str | None = None feature_importance: dict[str, float] | None = None + # A queryable summary of how time was used: the column, the fitted basis, and the seasonal cycles + # the training window supported. Kept as a typed map rather than folded into *feature_metadata* + # because a dashboard should be able to read "which models are time-aware?" without parsing JSON. + # The coefficients themselves live in the feature metadata, which is the only place that can hold + # nested structures without a registry migration. temporal_config: dict[str, str] | None = None diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index 6883d460d..62cd80e81 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -22,7 +22,6 @@ "AnomalyParams", "IsolationForestConfig", "FeatureEngineeringConfig", - "TemporalAnomalyConfig", "BaseChecksStorageConfig", "FileChecksStorageConfig", "WorkspaceFileChecksStorageConfig", @@ -167,14 +166,6 @@ class IsolationForestConfig: random_seed: int = 42 -@dataclass -class TemporalAnomalyConfig: - """Configuration for temporal feature extraction.""" - - timestamp_column: str - temporal_features: list[str] = field(default_factory=lambda: ["hour", "day_of_week", "month"]) - - @dataclass class FeatureEngineeringConfig: """Configuration for multi-type feature engineering in anomaly detection.""" @@ -233,6 +224,11 @@ class AnomalyConfig: # Declares the basis each metric is judged against, on one pooled model. Optional, so installed # run-config YAML written before it existed still loads. baseline_by: list[str] | None = None + # Which detector to train. None means the tabular default, so YAML written before this existed + # loads and trains exactly as it did. Scheduled retraining has to be able to pick the detector: + # without this the choice was reachable only from the Python API, and a run config could not + # reproduce a model a user had trained by hand. + profile: str | None = None @dataclass diff --git a/tests/unit/test_anomaly_configs.py b/tests/unit/test_anomaly_configs.py index 4f1ba1013..203725557 100644 --- a/tests/unit/test_anomaly_configs.py +++ b/tests/unit/test_anomaly_configs.py @@ -195,6 +195,27 @@ def test_anomaly_config_defaults(): assert cfg.baseline_by is None assert cfg.model_name is None assert cfg.registry_table is None + assert cfg.profile is None + + +def test_anomaly_config_carries_the_profile(): + """The detector choice has to survive a round trip through run-config YAML. + + Without it the choice is reachable only from the Python API, so a scheduled retrain silently falls + back to the tabular default and produces a different model from the one the user trained by hand. + """ + cfg = AnomalyConfig(columns=["a"], profile="timeseries") + assert cfg.profile == "timeseries" + + +def test_anomaly_config_omitting_profile_keeps_the_tabular_default(): + """A run config written before *profile* existed must still load, and train as it always did. + + ``None`` rather than the literal ``"tabular"`` so the default lives in one place, next to the + detector resolution, rather than being duplicated into every persisted config. + """ + cfg = AnomalyConfig(columns=["a"]) + assert cfg.profile is None def test_anomaly_config_with_columns_and_baseline(): diff --git a/tests/unit/test_anomaly_workflow.py b/tests/unit/test_anomaly_workflow.py index 626b5cf2d..37b0b631a 100644 --- a/tests/unit/test_anomaly_workflow.py +++ b/tests/unit/test_anomaly_workflow.py @@ -160,3 +160,43 @@ def train(self, **kwargs): assert train_called["called"] is True assert train_called["kwargs"]["model_name"] == "catalog.schema.my_model" assert train_called["kwargs"]["registry_table"] == "catalog.schema.my_registry" + # Unset in the run config, so the detector default is resolved downstream rather than here. + assert train_called["kwargs"]["profile"] is None + + +def test_anomaly_workflow_passes_the_configured_profile(monkeypatch): + """A profile named in the run config reaches ``train()``. + + Scheduled retraining is the whole point of the workflow, so a detector choice that cannot be + expressed in a run config is not really a configurable option. This is the assertion that the + parameter is genuinely reachable from YAML and not merely from Python. + """ + train_called: dict = {"kwargs": {}} + + class FakeEngine: + def __init__(self, _ws, _spark): + pass + + def train(self, **kwargs): + train_called["kwargs"] = kwargs + return "catalog.schema.my_model" + + monkeypatch.setattr("databricks.labs.dqx.anomaly.anomaly_engine.AnomalyEngine", FakeEngine) + monkeypatch.setattr(anomaly_workflow, "read_input_data", lambda _spark, _input_config: Mock()) + + run_config = RunConfig( + name="Fleet Telemetry", + input_config=InputConfig(location="catalog.schema.telemetry"), + anomaly_config=AnomalyConfig( + model_name="catalog.schema.my_model", + registry_table="catalog.schema.my_registry", + baseline_by=["machine_id"], + profile="timeseries", + ), + ) + ctx = SimpleNamespace(run_config=run_config, spark=Mock(), workspace_client=Mock()) + + anomaly_workflow.AnomalyTrainerWorkflow().train_model(ctx) + + assert train_called["kwargs"]["profile"] == "timeseries" + assert train_called["kwargs"]["baseline_by"] == ["machine_id"] From 93b59e1f3296d610ff183a5225bf67095ff75897 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 15:34:58 +0100 Subject: [PATCH 059/107] Correct the profile guidance for ordinary tabular data The guide said that on ordinary tabular data the ranking between the two profiles reverses, i.e. that Isolation Forest wins there. Measured across ten classical tabular anomaly benchmarks with the model fitted on normal rows and scored on a held-out half, it does not: the correlation-aware detector leads on both ROC-AUC and average precision, and the default is last of six on average precision against a panel including LOF, One-Class SVM, PCA reconstruction and ECOD. Replaced with what holds up: the two are close on tabular data, so choose on the shape of the anomaly rather than an expected accuracy gap. The two real reasons to prefer the default are named instead, and both are independent of accuracy: the tabular path trains an ensemble so it can report confidence_std, and its contributions come from SHAP. The figures stay out of the docs deliberately. They come from datasets fetched at run time by a harness that is not in this repository, and every number on the docs site should be reproducible from a clean checkout. --- docs/dqx/docs/guide/row_anomaly_detection/index.mdx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 48019c445..799361f44 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -392,9 +392,11 @@ you would have been paged, not how many rows you would have had to read. Both de data that still contained the anomalies, which is what DQX does when it fits a sample of your table; on a curated clean training split the same comparison is 36% against 82%. -That is telemetry, which is what `"timeseries"` is for — on ordinary tabular data the ranking reverses. -Detection quality on DQX's own synthetic fixtures is published in -[Benchmarks](/docs/reference/benchmarks). +That is telemetry, which is what `"timeseries"` is for. On ordinary tabular data the two detectors are +closer than this table might suggest, so choose on the *shape* of the anomaly you expect rather than on an +expected accuracy gap. Two things independent of accuracy do favour the default: `"tabular"` trains an +ensemble, so it can report `confidence_std`, and its contributions come from SHAP. Detection quality on +DQX's own synthetic fixtures is published in [Benchmarks](/docs/reference/benchmarks). DQX does not detect which profile you need. Getting it right cannot be verified without labelled From a0ade1d38907151f3b1a808cd713398ec5e3f580 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 16:11:01 +0100 Subject: [PATCH 060/107] Add the temporal baseline fit A metric's expected level as a function of time, so it can be judged against its own history rather than only against the whole table or its own group. Same shape as the existing group-relative feature, observed value minus an expected value; only the source of the expectation changes, from a per-group median to a fitted function. That is what lets it serve a timestamp the training window never saw, which a median lookup cannot. Pure numpy and sklearn, no Spark and no new dependency. Scoring runs in a scalar pandas UDF that sees an unordered chunk of rows with no access to neighbours, so an expectation evaluable per row from t is the only shape that keeps batch-train/streaming-score working. Three rules are measured rather than chosen, and each carries the number in a test so a later change meets the evidence: - A seasonal period needs at least 6 complete cycles in the window. On the Server Machine Dataset a daily period over 2.8 days cost Isolation Forest 15 points of event coverage, 94.7% to 79.4%; 5.6 cycles cost nothing. Note that variance explained is the wrong gate here: apparent seasonal strength rose as the cycle count fell, so a high figure on a short window is overfitting rather than signal. A second guard rejects a period the sampling cadence cannot resolve at all. - The fit is Huber, not least squares. DQX trains on a sample that still contains anomalies; with 5% of rows at 6x, ridge recovered a slope of 0.0694 against a true 0.0500 while Huber recovered 0.0502. - Changepoints are selected on a held-out tail, never on fit. In-sample R-squared was identical from 0 to 25 changepoints while false flags one window out ranged 1.2% to 100%, so an in-sample criterion cannot see the failure it would cause. Four defaults-to-empty fields on SparkFeatureMetadata carry the fitted state. to_json already iterates the dataclass fields and from_json already filters to known keys, so old payloads deserialize with an empty time column and the transform stays inert. The axis parameter is named seconds rather than t: the unit is the thing a caller is most likely to get wrong, and the signature is the right place to say it. --- src/databricks/labs/dqx/anomaly/temporal.py | 338 +++++++++++++++++ .../labs/dqx/anomaly/transformers.py | 9 + tests/unit/test_anomaly_temporal_fit.py | 353 ++++++++++++++++++ tests/unit/test_anomaly_transformers.py | 48 +++ 4 files changed, 748 insertions(+) create mode 100644 src/databricks/labs/dqx/anomaly/temporal.py create mode 100644 tests/unit/test_anomaly_temporal_fit.py diff --git a/src/databricks/labs/dqx/anomaly/temporal.py b/src/databricks/labs/dqx/anomaly/temporal.py new file mode 100644 index 000000000..eaaa7fb02 --- /dev/null +++ b/src/databricks/labs/dqx/anomaly/temporal.py @@ -0,0 +1,338 @@ +"""Fit an expected level over time, so a metric can be judged against its own history. + +This is the third basis a metric can be compared against, beside the whole table and its own group. The +shape is deliberately the same as the group-relative feature in :mod:`transformers`: observed value minus +an expected value. Only the source of the expectation changes, from a per-group median to a function of +time, which is what lets it extrapolate to rows the training window never saw. + +**Nothing here touches Spark.** These are pure functions over numpy arrays, called once per metric at +training time and evaluated per row at scoring time. Scoring runs inside a scalar pandas UDF that sees an +arbitrary chunk of rows with no ordering and no access to neighbours, so an expectation that is a *function +of time* is evaluable per row while anything needing the previous row is not. That is what keeps +"train on batch, score on streaming" working. + +The time axis is **seconds relative to the start of the training window**. Callers subtract ``t_min`` +before calling in, which keeps the numbers small and makes the period arithmetic below read in real units. + +Three rules are measured rather than chosen, and each has a test that cites its number: + +**A seasonal period needs enough complete cycles to be identifiable.** Fitting a daily cycle to 2.8 days of +history does not recover a daily shape; the Fourier terms absorb slow drift instead, and subtracting them +removes signal that was informative. Measured on the Server Machine Dataset, a period with 2.8 cycles in +the window cost Isolation Forest 15 points of event coverage (94.7% to 79.4%), 5.6 cycles cost nothing +(94.3%), and 66.7 cycles helped (96.1%). Note also that *variance explained is the wrong gate*: the +apparent seasonal strength rose from 0.00 to 0.39 as the cycle count fell, so a high figure on a short +window is overfitting rather than signal. + +**The fit must be robust.** DQX trains on a sample that still contains anomalies, deliberately. Least +squares is not robust to them: with 5% of rows at six times their normal value, a ridge fit recovered a +slope of 0.0694 against a true 0.0500, a 39% overestimate, while Huber recovered 0.0502. + +**Changepoints must be selected on held-out data, never on fit.** A piecewise-linear trend extrapolates the +*last* segment's slope, which with many changepoints is estimated from a short tail and is therefore noisy. +In-sample R-squared cannot see this at all: across 0 to 25 changepoints it stayed at 0.9913 to 0.9914 while +false flags one window past training ranged from 1.2% to 100%. So the count is chosen by measuring the fit +on a tail the fit never saw. +""" + +import logging +import math +from dataclasses import dataclass, field +from typing import Any + +import numpy as np +from sklearn.linear_model import HuberRegressor + +logger = logging.getLogger(__name__) + +# Measured: harm at 2.8 cycles, no cost at 5.6. Six is the first safe integer. +MIN_SEASONAL_CYCLES = 6 +# A period cannot be resolved from samples spaced comparably to it. Four samples per cycle is the +# coarsest that can carry a sine at all, and this guard is what stops a daily period being offered for +# daily data. +MIN_SAMPLES_PER_CYCLE = 4 +# Two harmonics per period. Deliberately few: every extra term is another chance to fit noise, and the +# cycle-count finding above is a warning about exactly that. +SEASONAL_HARMONICS = 2 +# Candidate periods in seconds. Hour, day and week are the cycles business and telemetry data actually +# carry; anything else is a domain fact DQX has no way to know. +CANDIDATE_PERIODS_SECONDS: tuple[float, ...] = (3600.0, 86400.0, 604800.0) +_PERIOD_NAMES = {3600.0: "hourly", 86400.0: "daily", 604800.0: "weekly"} + +# Changepoint search. Placed only in the first 80% of history so the extrapolating segment has data +# behind it, which is the same reason Prophet defaults to a changepoint range rather than the full span. +CHANGEPOINT_RANGE = 0.8 +CHANGEPOINT_CANDIDATES: tuple[int, ...] = (0, 1, 3, 6) +HOLDOUT_FRACTION = 0.2 + +# Huber's transition point between squared and linear loss, in units of the residual scale it estimates. +HUBER_EPSILON = 1.35 +HUBER_ALPHA = 1e-4 +HUBER_MAX_ITER = 400 + +# MAD trimming. 1.4826 scales the median absolute deviation to a standard deviation for Gaussian data. +MAD_TO_SIGMA = 1.4826 +MAD_TRIM_SIGMA = 3.0 + + +@dataclass(frozen=True) +class TemporalBasis: + """The design a metric's expected level is fitted against. + + Persisted with the model, because scoring has to rebuild exactly the same columns in exactly the same + order. A basis plus a coefficient vector is the whole expectation. + """ + + trend: bool = True + periods: tuple[float, ...] = () + harmonics: int = SEASONAL_HARMONICS + changepoints: tuple[float, ...] = () # fractions of the span, in (0, 1) + span: float = 1.0 # seconds covered by the training window, for scaling the trend term + + @property + def n_terms(self) -> int: + """Columns in the design matrix, intercept included.""" + return 1 + int(self.trend) + len(self.changepoints) + 2 * self.harmonics * len(self.periods) + + def describe(self) -> str: + """Human-readable summary, for logs and the registry's queryable temporal_config column.""" + parts = ["trend"] if self.trend else [] + parts += [_PERIOD_NAMES.get(p, f"{p:g}s") for p in self.periods] + if self.changepoints: + parts.append(f"{len(self.changepoints)} changepoints") + return "+".join(parts) if parts else "none" + + def to_dict(self) -> dict[str, Any]: + return { + "trend": self.trend, + "periods": list(self.periods), + "harmonics": self.harmonics, + "changepoints": list(self.changepoints), + "span": self.span, + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "TemporalBasis": + return cls( + trend=bool(data.get("trend", True)), + periods=tuple(float(p) for p in data.get("periods", ())), + harmonics=int(data.get("harmonics", SEASONAL_HARMONICS)), + changepoints=tuple(float(c) for c in data.get("changepoints", ())), + span=float(data.get("span", 1.0)), + ) + + +@dataclass +class TemporalFit: + """A fitted basis and its per-metric coefficients, ready to persist.""" + + basis: TemporalBasis + coefficients: dict[str, list[float]] = field(default_factory=dict) + skipped_periods: dict[float, str] = field(default_factory=dict) # period -> why it was not fitted + + +def candidate_periods(seconds: np.ndarray) -> tuple[tuple[float, ...], dict[float, str]]: + """Which seasonal periods the training window can actually support. + + Returns the admitted periods and, for every rejected one, the reason. The reasons are surfaced to the + caller rather than swallowed: a user who expected a daily cycle and did not get one needs to be told + that their window held 2.8 days, not left to wonder. + + Args: + seconds: Time axis in seconds, relative to the window start. Need not be sorted. + + Returns: + ``(admitted, rejected)`` where *rejected* maps a period in seconds to a human-readable reason. + """ + admitted: list[float] = [] + rejected: dict[float, str] = {} + if seconds.size < 2: + return (), {p: "fewer than two rows" for p in CANDIDATE_PERIODS_SECONDS} + + span = float(np.max(seconds) - np.min(seconds)) + # Median gap rather than mean: a single large hole in the data should not make the cadence look coarse. + gaps = np.diff(np.sort(seconds)) + positive = gaps[gaps > 0] + cadence = float(np.median(positive)) if positive.size else 0.0 + + for period in CANDIDATE_PERIODS_SECONDS: + cycles = span / period if period > 0 else 0.0 + if cycles < MIN_SEASONAL_CYCLES: + rejected[period] = f"window covers {cycles:.1f} cycles, needs {MIN_SEASONAL_CYCLES}" + continue + if cadence > 0 and period / cadence < MIN_SAMPLES_PER_CYCLE: + rejected[period] = ( + f"sampled every {cadence:.0f}s, which is {period / cadence:.1f} samples per cycle " + f"and cannot resolve the shape" + ) + continue + admitted.append(period) + return tuple(admitted), rejected + + +def design_matrix(seconds: np.ndarray, basis: TemporalBasis) -> np.ndarray: + """Build the design columns for *seconds* under *basis*. + + Column order is fixed by the basis and must not change between training and scoring, which is why the + basis is persisted rather than recomputed. + """ + scaled = seconds / basis.span if basis.span > 0 else np.zeros_like(seconds, dtype=float) + columns: list[np.ndarray] = [np.ones_like(scaled, dtype=float)] + if basis.trend: + columns.append(scaled) + for changepoint in basis.changepoints: + # max(0, seconds - cp): lets the slope change at the changepoint without fitting separate segments. + columns.append(np.maximum(0.0, scaled - changepoint)) + for period in basis.periods: + for k in range(1, basis.harmonics + 1): + angle = 2.0 * math.pi * k * seconds / period + columns.append(np.sin(angle)) + columns.append(np.cos(angle)) + return np.column_stack(columns) + + +def _fit_one(design: np.ndarray, values: np.ndarray) -> list[float] | None: + """Huber-fit one metric, returning ``[intercept, *coefficients]`` or None if it could not be fitted. + + The intercept is folded into the returned vector and the design's own constant column is dropped for + the fit, so that scoring can evaluate a plain dot product without needing to know which convention + was used. + """ + if values.size <= design.shape[1] + 1: + return None + if float(np.std(values)) == 0.0: + # A constant metric has no expectation to learn beyond its own level. + return [float(values[0])] + [0.0] * (design.shape[1] - 1) + try: + model = HuberRegressor(epsilon=HUBER_EPSILON, alpha=HUBER_ALPHA, max_iter=HUBER_MAX_ITER) + model.fit(design[:, 1:], values) + except (ValueError, FloatingPointError) as exc: + logger.debug(f"Temporal fit failed, falling back to no temporal feature for this metric: {exc}") + return None + return [float(model.intercept_), *(float(c) for c in model.coef_)] + + +def _holdout_residual_scale(seconds: np.ndarray, values: np.ndarray, basis: TemporalBasis) -> float: + """Robust residual scale on a tail the fit never saw. + + This is the statistic changepoint counts are chosen by. It has to be measured *after* the fit window + because that is where over-flexible trends go wrong: an in-sample criterion is blind to it. + """ + order = np.argsort(seconds) + t_sorted, v_sorted = seconds[order], values[order] + cut = int(len(t_sorted) * (1.0 - HOLDOUT_FRACTION)) + if cut < basis.n_terms + 2 or cut >= len(t_sorted): + return float("inf") + coefficients = _fit_one(design_matrix(t_sorted[:cut], basis), v_sorted[:cut]) + if coefficients is None: + return float("inf") + residual = v_sorted[cut:] - design_matrix(t_sorted[cut:], basis) @ np.asarray(coefficients) + # Median absolute deviation rather than a standard deviation: one bad row in the holdout should not + # decide the changepoint count. + return float(np.median(np.abs(residual - np.median(residual))) * MAD_TO_SIGMA) + + +def select_basis(seconds: np.ndarray, values: np.ndarray) -> tuple[TemporalBasis, dict[float, str]]: + """Choose the basis for a table: which periods, and how many changepoints. + + *values* is a representative metric, used only to score changepoint counts. The periods depend on the + time axis alone, so they are the same for every metric in the table, which is what keeps one basis and + one column order for the whole model. + + Returns the basis and the rejected periods with their reasons. + """ + seconds = np.asarray(seconds, dtype=float) + span = float(np.max(seconds) - np.min(seconds)) if seconds.size > 1 else 1.0 + periods, rejected = candidate_periods(seconds) + base = TemporalBasis(trend=True, periods=periods, harmonics=SEASONAL_HARMONICS, span=max(span, 1.0)) + + values = np.asarray(values, dtype=float) + best_basis, best_scale = base, _holdout_residual_scale(seconds, values, base) + for count in CHANGEPOINT_CANDIDATES: + if count == 0: + continue + changepoints = tuple(np.linspace(CHANGEPOINT_RANGE / (count + 1), CHANGEPOINT_RANGE, count)) + candidate = TemporalBasis( + trend=True, periods=periods, harmonics=SEASONAL_HARMONICS, changepoints=changepoints, span=base.span + ) + scale = _holdout_residual_scale(seconds, values, candidate) + # Strictly better, so a tie leaves the simpler basis in place. Flexibility has to earn its keep: + # in-sample fit was identical across 0 to 25 changepoints while extrapolation ranged 1.2% to 100%. + if scale < best_scale: + best_basis, best_scale = candidate, scale + + return best_basis, rejected + + +def fit_temporal(seconds: np.ndarray, metrics: dict[str, np.ndarray], basis: TemporalBasis) -> dict[str, list[float]]: + """Fit *basis* to every metric, returning ``metric -> [intercept, *coefficients]``. + + A metric that cannot be fitted is omitted rather than given zeros, so the caller can tell the + difference between "expected level is flat" and "no expectation was learned". + """ + design = design_matrix(np.asarray(seconds, dtype=float), basis) + fitted: dict[str, list[float]] = {} + for name, values in metrics.items(): + coefficients = _fit_one(design, np.asarray(values, dtype=float)) + if coefficients is None: + logger.debug(f"No temporal expectation fitted for metric {name!r}") + continue + fitted[name] = coefficients + return fitted + + +def expected(seconds: np.ndarray, basis: TemporalBasis, coefficients: list[float]) -> np.ndarray: + """The expected level at each timestamp. + + A function of time, so it extrapolates to any time, including rows past the training window. That is + the property the whole approach rests on, and also why the caller must enforce a staleness horizon: + extrapolation stays finite but its accuracy decays with distance. + """ + design = design_matrix(np.asarray(seconds, dtype=float), basis) + coefficient_array = np.asarray(coefficients, dtype=float) + if coefficient_array.size != design.shape[1]: + raise ValueError( + f"temporal coefficients have length {coefficient_array.size} but the persisted basis needs " + f"{design.shape[1]}; the model and its basis are out of step" + ) + return design @ coefficient_array + + +def robust_scale_mask(residuals: np.ndarray, sigma: float = MAD_TRIM_SIGMA) -> np.ndarray: + """Rows within *sigma* robust deviations on every column. + + A robust *fit* is not enough on its own. With 5% of training rows at six times normal, both a robust + and a least-squares fit produced a learned residual spread near 170 and both then missed a 1.5x spike + entirely, because the detector takes its notion of normal from residuals computed over the same + contaminated rows. Trimming before the detector fits is what closes that. + """ + residuals = np.atleast_2d(np.asarray(residuals, dtype=float)) + if residuals.shape[0] == 1 and residuals.size > 1: + residuals = residuals.T + centre = np.median(residuals, axis=0) + deviation = np.median(np.abs(residuals - centre), axis=0) * MAD_TO_SIGMA + # A column with no spread cannot exclude anything, so it must not divide by zero either. + deviation = np.where(deviation <= 0, np.inf, deviation) + return (np.abs((residuals - centre) / deviation) <= sigma).all(axis=1) + + +def trend_strength(seconds: np.ndarray, metrics: dict[str, np.ndarray], basis: TemporalBasis) -> float: + """Share of variance the basis removes, as the median across metrics. + + Used to advise rather than to decide: where this is near zero the data has no temporal structure to + subtract, and the caller is told the parameter is unlikely to help. Median rather than mean, so one + wildly trending column in a table of forty does not carry the recommendation for the rest. + """ + design = design_matrix(np.asarray(seconds, dtype=float), basis) + shares: list[float] = [] + for values in metrics.values(): + values = np.asarray(values, dtype=float) + variance = float(np.var(values)) + if variance <= 0: + continue + coefficients = _fit_one(design, values) + if coefficients is None: + continue + residual = values - design @ np.asarray(coefficients) + shares.append(max(0.0, 1.0 - float(np.var(residual)) / variance)) + return float(np.median(shares)) if shares else 0.0 diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 5eba1fcde..f6a7b4b5d 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -85,6 +85,15 @@ class SparkFeatureMetadata: # than in the typed training.score_quantiles map column, which would need a # registry migration to hold a nested map; training.score_quantiles stays the global fallback. baseline_score_quantiles: dict[str, dict[str, float]] = field(default_factory=dict) + # Temporal conditioning, and the same defaults-to-empty contract as the grouping fields above: + # an empty baseline_over_time makes the temporal transform return immediately, so a model trained + # before these existed keeps a byte-identical engineered_feature_names and scores exactly as it did. + baseline_over_time: str = "" # The time column each metric is judged along + temporal_basis: dict[str, Any] = field(default_factory=dict) # TemporalBasis.to_dict() + temporal_coefficients: dict[str, list[float]] = field(default_factory=dict) # metric -> [intercept, *coefs] + # Training window bounds in epoch seconds, for the staleness horizon. A fitted basis extrapolates to + # any t, but accuracy decays with distance, so scoring needs to know where the evidence ran out. + temporal_window: dict[str, float] = field(default_factory=dict) # {"t_min": ..., "t_max": ...} def to_json(self) -> str: """Serialize to JSON for storage. diff --git a/tests/unit/test_anomaly_temporal_fit.py b/tests/unit/test_anomaly_temporal_fit.py new file mode 100644 index 000000000..011c1c077 --- /dev/null +++ b/tests/unit/test_anomaly_temporal_fit.py @@ -0,0 +1,353 @@ +"""The temporal baseline fit: what it does, and the measurements that chose how it does it. + +Every design rule in :mod:`databricks.labs.dqx.anomaly.temporal` was picked from a measurement rather +than from taste, so each test here reproduces the measurement rather than merely asserting the rule. A +future reader who wants to change one of these defaults will find the number that argued against them. + +No Spark and no workspace: the module is pure numpy, which is what makes it unit-testable at all. +""" + +import logging + +import numpy as np +import pytest +from sklearn.linear_model import Ridge + +from databricks.labs.dqx.anomaly.temporal import ( + CANDIDATE_PERIODS_SECONDS, + MIN_SEASONAL_CYCLES, + TemporalBasis, + candidate_periods, + design_matrix, + expected, + fit_temporal, + robust_scale_mask, + select_basis, + trend_strength, +) + +HOUR = 3600.0 +DAY = 86400.0 +WEEK = 604800.0 +TRUE_SLOPE_PER_SECOND = 0.05 / HOUR # 0.05 per hourly step, the slope the contamination tests recover + + +def _hourly_axis(count: int) -> np.ndarray: + return np.arange(count, dtype=float) * HOUR + + +def _linear_metric(seconds: np.ndarray, rng: np.random.Generator, noise: float = 3.0) -> np.ndarray: + return 100.0 + TRUE_SLOPE_PER_SECOND * seconds + rng.normal(0, noise, seconds.size) + + +def _recovered_slope_per_hour(basis: TemporalBasis, coefficients: list[float]) -> float: + """Convert the fitted trend coefficient back into units of "per hourly step". + + The design scales the trend column by the span, so the raw coefficient is per span rather than per + second. Reading it back through :func:`expected` avoids duplicating that convention here. + """ + probe = np.array([0.0, HOUR]) + values = expected(probe, basis, coefficients) + return float(values[1] - values[0]) + + +# ── the cycle guard ──────────────────────────────────────────────────────────────────────────────── + + +def test_a_period_with_too_few_cycles_is_rejected(): + """Measured on the Server Machine Dataset: a daily period over 2.8 days of history cost Isolation + Forest 15 points of event coverage, 94.7% down to 79.4%. Over 5.6 cycles it cost nothing.""" + # 2.8 days of hourly data. Enough resolution for a daily shape, nowhere near enough repetitions. + seconds = _hourly_axis(int(2.8 * 24)) + + admitted, rejected = candidate_periods(seconds) + + assert DAY not in admitted + assert "2.8 cycles" in rejected[DAY] + assert str(MIN_SEASONAL_CYCLES) in rejected[DAY] + + +def test_a_period_with_enough_cycles_is_admitted(): + """Six complete cycles is the threshold, and the first safe integer above the measured harm at 2.8.""" + seconds = _hourly_axis(MIN_SEASONAL_CYCLES * 24 + 24) + + admitted, rejected = candidate_periods(seconds) + + assert DAY in admitted + assert DAY not in rejected + + +def test_a_period_the_cadence_cannot_resolve_is_rejected(): + """Daily samples cannot carry an hourly cycle however long the window is. + + The cycle guard alone would admit an hourly period here, because a long window trivially contains + thousands of hours. Resolution is a separate question from repetition and needs its own guard. + """ + seconds = np.arange(400, dtype=float) * DAY # 400 days, sampled once a day + + admitted, rejected = candidate_periods(seconds) + + assert HOUR not in admitted + assert "samples per cycle" in rejected[HOUR] + assert WEEK in admitted # 400 days is ~57 weeks, comfortably resolvable + + +def test_rejected_periods_are_reported_so_a_user_can_be_told(): + """Silence is the defect. A caller who expected a daily cycle must learn their window held 2.8 days.""" + admitted, rejected = candidate_periods(_hourly_axis(48)) + + assert not admitted + assert set(rejected) == set(CANDIDATE_PERIODS_SECONDS) + assert all(reason for reason in rejected.values()) + + +def test_a_single_row_admits_nothing_rather_than_dividing_by_zero(): + admitted, rejected = candidate_periods(np.array([0.0])) + + assert not admitted + assert set(rejected) == set(CANDIDATE_PERIODS_SECONDS) + + +# ── robustness of the fit ────────────────────────────────────────────────────────────────────────── + + +def test_huber_recovers_the_slope_through_contaminated_training_data(): + """The measurement that rules out least squares. + + DQX fits a sample of the user's table with anomalies still in it, deliberately. With 5% of rows at six + times normal, a ridge fit recovered a slope of 0.0694 against a true 0.0500, a 39% overestimate, while + Huber recovered 0.0502. A biased slope tilts every residual downstream. + """ + rng = np.random.default_rng(5) + seconds = _hourly_axis(2000) + values = _linear_metric(seconds, rng) + contaminated = values.copy() + contaminated[rng.choice(seconds.size, size=int(0.05 * seconds.size), replace=False)] *= 6.0 + + basis = TemporalBasis(trend=True, span=float(seconds[-1])) + coefficients = fit_temporal(seconds, {"m": contaminated}, basis)["m"] + + # Within 5% of the true slope despite one row in twenty being six times too large. + assert _recovered_slope_per_hour(basis, coefficients) == pytest.approx(0.05, rel=0.05) + + +def test_least_squares_on_the_same_data_would_be_badly_biased(): + """Executable evidence for the previous test's choice, so the reasoning cannot be lost. + + Without this, a later contributor swapping Huber for the faster Ridge sees only a passing suite. + """ + rng = np.random.default_rng(5) + seconds = _hourly_axis(2000) + values = _linear_metric(seconds, rng) + values[rng.choice(seconds.size, size=int(0.05 * seconds.size), replace=False)] *= 6.0 + + basis = TemporalBasis(trend=True, span=float(seconds[-1])) + design = design_matrix(seconds, basis) + ridge = Ridge(alpha=1e-3).fit(design[:, 1:], values) + ridge_slope = _recovered_slope_per_hour(basis, [float(ridge.intercept_), *ridge.coef_]) + + # Overestimates by more than a third, which is what the Huber test above avoids. + assert ridge_slope > 0.05 * 1.25 + + +def test_a_constant_metric_is_fitted_as_its_own_level(): + """No expectation to learn, and no division by a zero standard deviation either.""" + seconds = _hourly_axis(500) + coefficients = fit_temporal(seconds, {"flat": np.full(seconds.size, 7.0)}, TemporalBasis(span=float(seconds[-1]))) + + assert expected(seconds, TemporalBasis(span=float(seconds[-1])), coefficients["flat"]) == pytest.approx(7.0) + + +def test_a_metric_with_too_few_rows_is_omitted_rather_than_guessed(): + """Omitted, not zeroed, so the caller can distinguish "flat expectation" from "none learned".""" + seconds = _hourly_axis(3) + + assert not fit_temporal(seconds, {"m": np.array([1.0, 2.0, 3.0])}, TemporalBasis(span=float(seconds[-1]))) + + +# ── changepoints ─────────────────────────────────────────────────────────────────────────────────── + + +def test_a_straight_series_earns_no_changepoints(): + """Flexibility has to be earned. In-sample fit is identical from 0 to 25 changepoints (R-squared + 0.9913 to 0.9914) while false flags one window out range 1.2% to 100%, so a tie must leave the + simpler basis in place.""" + rng = np.random.default_rng(11) + seconds = _hourly_axis(2000) + + basis, _ = select_basis(seconds, _linear_metric(seconds, rng)) + + assert not basis.changepoints + + +def test_a_series_whose_slope_doubles_earns_changepoints(): + """The case changepoints exist for. A straight line through a bent trend leaves a residual that is + not stationary, and everything downstream inherits it.""" + rng = np.random.default_rng(11) + seconds = _hourly_axis(2000) + knee = seconds[seconds.size // 2] + values = 100.0 + TRUE_SLOPE_PER_SECOND * np.minimum(seconds, knee) + values = values + 3.0 * TRUE_SLOPE_PER_SECOND * np.maximum(0.0, seconds - knee) + rng.normal(0, 3.0, seconds.size) + + basis, _ = select_basis(seconds, values) + + assert basis.changepoints + + +def test_changepoints_are_never_placed_in_the_recent_tail(): + """The extrapolating segment needs data behind it. + + A changepoint near the end of history leaves the final slope estimated from a handful of rows, which + is precisely the mechanism that took out-of-window false flags to 100%. + """ + rng = np.random.default_rng(3) + seconds = _hourly_axis(2000) + knee = seconds[seconds.size // 2] + values = 100.0 + TRUE_SLOPE_PER_SECOND * np.minimum(seconds, knee) + values = values + 4.0 * TRUE_SLOPE_PER_SECOND * np.maximum(0.0, seconds - knee) + rng.normal(0, 2.0, seconds.size) + + basis, _ = select_basis(seconds, values) + + assert all(changepoint <= 0.8 for changepoint in basis.changepoints) + + +# ── extrapolation, which is the property the design rests on ─────────────────────────────────────── + + +def test_the_expectation_extrapolates_past_the_training_window(): + """A lookup table cannot serve a future timestamp; a function of seconds can. + + This is why the temporal baseline is a fitted function rather than a per-time-bucket median, and it + is the difference between this approach and the time-bucket grouping that was measured and rejected. + """ + rng = np.random.default_rng(1) + seconds = _hourly_axis(1000) + values = _linear_metric(seconds, rng, noise=0.5) + basis = TemporalBasis(trend=True, span=float(seconds[-1])) + coefficients = fit_temporal(seconds, {"m": values}, basis)["m"] + + # One full window beyond training, where no row was ever seen. + future = seconds + float(seconds[-1]) + predicted = expected(future, basis, coefficients) + + assert np.all(np.isfinite(predicted)) + assert predicted[-1] == pytest.approx(100.0 + TRUE_SLOPE_PER_SECOND * future[-1], rel=0.02) + + +def test_coefficients_that_do_not_match_the_basis_raise(): + """A model and its basis drifting apart must fail loudly, not silently score a different feature.""" + basis = TemporalBasis(trend=True, periods=(DAY,), span=DAY * 10) + + with pytest.raises(ValueError, match="out of step"): + expected(_hourly_axis(10), basis, [1.0, 2.0]) + + +def test_the_design_column_count_matches_the_declared_basis(): + """Column order is a persisted contract between training and scoring.""" + basis = TemporalBasis(trend=True, periods=(DAY, WEEK), harmonics=2, changepoints=(0.3, 0.6), span=WEEK * 8) + + assert design_matrix(_hourly_axis(50), basis).shape[1] == basis.n_terms + # intercept + trend + 2 changepoints + 2 periods x 2 harmonics x sin/cos + assert basis.n_terms == 1 + 1 + 2 + 8 + + +# ── robust scale, which the fit alone does not give ──────────────────────────────────────────────── + + +def test_trimming_keeps_the_learned_scale_near_the_clean_scale(): + """A robust fit is necessary but not sufficient. + + With 5% of rows at six times normal, both a robust and a least-squares fit produced a learned residual + spread near 170, and both then missed a 1.5x spike entirely at 0% recall, because the detector takes + its notion of normal from residuals over the same contaminated rows. Trimming is what closes that. + """ + rng = np.random.default_rng(9) + clean = rng.normal(0, 3.0, 2000) + contaminated = clean.copy() + contaminated[rng.choice(clean.size, size=int(0.05 * clean.size), replace=False)] += 6.0 * 30.0 + + untrimmed = float(np.std(contaminated)) + trimmed = float(np.std(contaminated[robust_scale_mask(contaminated)])) + + assert untrimmed > 3.0 * 2 # the contamination really does inflate it + assert trimmed == pytest.approx(float(np.std(clean)), rel=0.15) + + +def test_trimming_a_column_with_no_spread_excludes_nothing(): + """A constant column cannot vote to exclude a row, and must not divide by zero to say so.""" + mask = robust_scale_mask(np.column_stack([np.full(100, 5.0), np.arange(100, dtype=float)])) + + assert mask.sum() > 0 + + +# ── the advisory statistic ───────────────────────────────────────────────────────────────────────── + + +def test_trend_strength_separates_a_trending_table_from_a_stationary_one(): + """The statistic behind the advisory, and the reason it is worth warning on. + + Measured, the Server Machine Dataset sits at a median 0.016 while synthetic trending data reaches + 0.999. That separation is what lets DQX tell a caller the parameter will not help them, instead of + silently costing them 20 points of event coverage. + """ + rng = np.random.default_rng(7) + seconds = _hourly_axis(1500) + basis = TemporalBasis(trend=True, span=float(seconds[-1])) + + trending = trend_strength(seconds, {"m": _linear_metric(seconds, rng, noise=1.0)}, basis) + stationary = trend_strength(seconds, {"m": 100.0 + rng.normal(0, 3.0, seconds.size)}, basis) + + assert trending > 0.8 + assert stationary < 0.1 + + +def test_trend_strength_ignores_constant_metrics_rather_than_scoring_them_zero(): + """A constant column has no variance to explain, so including it would drag the median down and + advise against the parameter on a table that genuinely trends.""" + rng = np.random.default_rng(2) + seconds = _hourly_axis(1000) + basis = TemporalBasis(trend=True, span=float(seconds[-1])) + metrics = {"trending": _linear_metric(seconds, rng, noise=1.0), "flat": np.full(seconds.size, 4.0)} + + assert trend_strength(seconds, metrics, basis) > 0.8 + + +# ── persistence ──────────────────────────────────────────────────────────────────────────────────── + + +def test_the_basis_round_trips_through_a_dict(): + """It is persisted in the model's feature metadata, so it has to survive JSON.""" + basis = TemporalBasis(trend=True, periods=(DAY, WEEK), harmonics=3, changepoints=(0.2, 0.5), span=12345.0) + + assert TemporalBasis.from_dict(basis.to_dict()) == basis + + +def test_an_empty_dict_deserializes_to_a_usable_default(): + """Models trained before the temporal fields existed carry an empty dict here.""" + basis = TemporalBasis.from_dict({}) + + assert not basis.periods + assert not basis.changepoints + assert basis.trend is True + + +def test_the_basis_describes_itself_for_logs_and_the_registry(): + basis = TemporalBasis(trend=True, periods=(DAY,), changepoints=(0.4,), span=DAY * 20) + + described = basis.describe() + + assert "trend" in described + assert "daily" in described + assert "1 changepoints" in described + + +def test_select_basis_reports_what_it_skipped(caplog): + """The reasons travel with the basis so the caller can warn; they are not logged and forgotten.""" + rng = np.random.default_rng(4) + seconds = _hourly_axis(48) # two days: nothing is admissible + + with caplog.at_level(logging.DEBUG): + basis, rejected = select_basis(seconds, _linear_metric(seconds, rng)) + + assert not basis.periods + assert set(rejected) == set(CANDIDATE_PERIODS_SECONDS) diff --git a/tests/unit/test_anomaly_transformers.py b/tests/unit/test_anomaly_transformers.py index 9591fcf21..e7d15abdd 100644 --- a/tests/unit/test_anomaly_transformers.py +++ b/tests/unit/test_anomaly_transformers.py @@ -437,3 +437,51 @@ def test_group_metadata_survives_a_json_roundtrip(): assert restored.baseline_by == ["country", "product"] assert restored.baseline_medians == {"amount": {"DE\x1fcasino": 3284.0, "IT\x1flive": 657.0}} assert restored.global_medians == {"amount": 1200.5} + + +def test_temporal_metadata_survives_a_json_roundtrip(): + """The expected level is rebuilt at scoring time from the basis plus the coefficients. + + Both have to survive exactly. A basis that comes back with a different column count, or coefficients + that lose their order, produce an expectation for a different design than the one that was fitted, and + the residual is then quietly wrong rather than loudly broken. + """ + metadata = SparkFeatureMetadata( + column_infos=[{"name": "revenue", "category": "numeric"}], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["revenue", "revenue_rel_time"], + baseline_over_time="event_ts", + temporal_basis={ + "trend": True, + "periods": [86400.0, 604800.0], + "harmonics": 2, + "changepoints": [0.27, 0.53], + "span": 2592000.0, + }, + temporal_coefficients={"revenue": [100.5, 12.25, -3.0, 0.5, -0.25, 0.125, 0.0625, 1.5, -1.25, 0.75, -0.5]}, + temporal_window={"t_min": 1735689600.0, "t_max": 1738281600.0}, + ) + + restored = SparkFeatureMetadata.from_json(metadata.to_json()) + + assert restored.baseline_over_time == "event_ts" + assert restored.temporal_basis["periods"] == [86400.0, 604800.0] + assert restored.temporal_basis["changepoints"] == [0.27, 0.53] + assert restored.temporal_coefficients["revenue"] == metadata.temporal_coefficients["revenue"] + assert restored.temporal_window == {"t_min": 1735689600.0, "t_max": 1738281600.0} + + +def test_a_payload_written_before_the_temporal_fields_deserializes_inert(): + """The inertness contract, at the persistence layer. + + A model trained before this release carries none of the temporal keys. It must come back with an empty + time column, which is what makes the transform return immediately and leaves + ``engineered_feature_names`` byte-identical to what it was. + """ + restored = SparkFeatureMetadata.from_json(PRE_GROUPING_FEATURE_METADATA_JSON) + + assert restored.baseline_over_time == "" + assert not restored.temporal_basis + assert not restored.temporal_coefficients + assert not restored.temporal_window From e1cc7d5a001a9a550f24685be362119d4159b187 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 17:11:35 +0100 Subject: [PATCH 061/107] Wire the temporal baseline into feature engineering Appends a {metric}_rel_time feature per numeric metric: the value minus its expected level at that row's timestamp. A plain difference rather than the log-ratio the group-relative feature uses, because when the two compose the input here is already the signed-log group-relative value, so the difference is a log-ratio in that case and a level difference otherwise. Runs after the group-relative block for two reasons that happen to agree. engineered_feature_names is positional so features may only be appended, and this transform reads the columns that one produces: a pooled trend fitted on raw values is wrong for every group as soon as their slopes differ (measured, event coverage 80% to 29% as slopes diverged), while one pooled fit on the group-relative residual reached 101% of per-group fits with a single model. The fit runs on a bucketed aggregate rather than collected rows, which bounds driver memory the way the per-group median aggregation already does, and makes each bucket's median robust before Huber sees it. The basis is then selected against the bucket centres, so the resolution guard measures the axis actually being fitted rather than one it is not. Scoring evaluates the expectation as a Spark column expression, not a UDF: it is a closed-form function of the row's own timestamp, so it needs no ordering, no neighbours and no state. That is what keeps scoring valid on a streaming DataFrame. The time column is excluded from the feature set by the same rule the grouping columns already follow, and validate_baseline_over_time rejects a column named in both columns and baseline_over_time rather than silently dropping it from one. Inertness is asserted through the public function rather than the private early return: an integration test compares engineered_feature_names with the parameter omitted, passed None, and passed the empty string. The Isolation Forest reference array is unchanged, which is the gate this stage had to clear. Two things removed rather than suppressed while getting pylint back to 10.00: the projection and metadata assembly moved out of apply_feature_engineering, which had grown to do four jobs, and the three fitted temporal artefacts are now carried as one TemporalState instead of four correlated parameters. The persisted JSON shape is unchanged. Also drops the TemporalFit dataclass added in the previous commit and never used. --- src/databricks/labs/dqx/anomaly/temporal.py | 11 +- .../labs/dqx/anomaly/transformers.py | 323 +++++++++++++++++- src/databricks/labs/dqx/anomaly/validation.py | 41 +++ .../test_anomaly_temporal_features.py | 290 ++++++++++++++++ 4 files changed, 643 insertions(+), 22 deletions(-) create mode 100644 tests/integration_anomaly/test_anomaly_temporal_features.py diff --git a/src/databricks/labs/dqx/anomaly/temporal.py b/src/databricks/labs/dqx/anomaly/temporal.py index eaaa7fb02..fda637776 100644 --- a/src/databricks/labs/dqx/anomaly/temporal.py +++ b/src/databricks/labs/dqx/anomaly/temporal.py @@ -37,7 +37,7 @@ import logging import math -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any import numpy as np @@ -122,15 +122,6 @@ def from_dict(cls, data: dict[str, Any]) -> "TemporalBasis": ) -@dataclass -class TemporalFit: - """A fitted basis and its per-metric coefficients, ready to persist.""" - - basis: TemporalBasis - coefficients: dict[str, list[float]] = field(default_factory=dict) - skipped_periods: dict[float, str] = field(default_factory=dict) # period -> why it was not fitted - - def candidate_periods(seconds: np.ndarray) -> tuple[tuple[float, ...], dict[float, str]]: """Which seasonal periods the training window can actually support. diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index f6a7b4b5d..09f2e81fd 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -8,6 +8,7 @@ import json import logging +import math import re import sys import threading @@ -15,6 +16,7 @@ from io import StringIO from typing import Any +import numpy as np from pyspark.sql import Column, DataFrame from pyspark.sql import functions as F from pyspark.sql import types as T @@ -35,9 +37,10 @@ from pyspark.sql.types import DoubleType, TimestampType from databricks.labs.dqx.anomaly.segment_utils import BASELINE_KEY_COLUMN, with_baseline_key +from databricks.labs.dqx.anomaly.temporal import TemporalBasis, fit_temporal, select_basis from databricks.labs.dqx.errors import ComputationError, InvalidParameterError from databricks.labs.dqx.telemetry import get_tables_from_spark_plan -from databricks.labs.dqx.utils import get_table_primary_keys +from databricks.labs.dqx.utils import get_table_primary_keys, sanitize_for_logging logger = logging.getLogger(__name__) @@ -124,6 +127,22 @@ def from_json(cls, json_str: str) -> "SparkFeatureMetadata": return cls(**{k: v for k, v in data.items() if k in known_fields}) +@dataclass +class TemporalState: + """The fitted temporal artefacts, carried through the transform pipeline as one thing. + + Written when training and read when scoring, which is why it is mutable: the transform fills it in + place exactly as the baseline median dicts are filled. Empty means the feature is unused. + + Not the persisted shape. :class:`SparkFeatureMetadata` keeps these flat, so the JSON payload a model + carries stays stable regardless of how they are grouped for passing around. + """ + + basis: dict[str, Any] = field(default_factory=dict) # TemporalBasis.to_dict() + coefficients: dict[str, list[float]] = field(default_factory=dict) # metric -> [intercept, *coefs] + window: dict[str, float] = field(default_factory=dict) # {"t_min": ..., "t_max": ...} + + def _spark_type_for_category(category: str) -> T.DataType: """Return canonical Spark type for a column category. Keeps reconstructed infos consistent for scoring.""" return { @@ -777,6 +796,47 @@ def _signed_log1p(column: Column) -> Column: # to be able to recognise a feature as derived from a source column: a caller who redacts "amount" # means the LLM must not see "amount_rel_baseline" either. BASELINE_RELATIVE_SUFFIX = "_rel_baseline" +TEMPORAL_RELATIVE_SUFFIX = "_rel_time" +# Rows collected to the driver to fit the temporal basis. The fit runs on a bucketed aggregate rather +# than raw rows, so this bounds driver memory regardless of table size, exactly as the per-group median +# aggregation does. Each bucket contributes its median, which is robust before Huber even sees it. +TEMPORAL_FIT_BUCKETS = 4000 + + +def _epoch_seconds(time_column: str, t_min: float) -> Column: + """The time axis the temporal basis was fitted against: seconds since the window start. + + Kept in one place because training and scoring must agree on it exactly. A different origin or a + different unit produces an expectation for a design the coefficients were never fitted to. + """ + return F.unix_timestamp(col(time_column).cast(TimestampType())).cast(DoubleType()) - lit(t_min) + + +def _temporal_expected_column(basis: "TemporalBasis", coefficients: list[float], seconds: Column) -> Column: + """The fitted expected level, as a Spark expression. + + Deliberately a column expression rather than a UDF: it is a closed-form function of the row's own + timestamp, so it evaluates per row with no ordering, no neighbours and no state. That is what keeps + scoring valid on a streaming DataFrame, where a lag or a rolling window would not be. + + Column order here must match :func:`databricks.labs.dqx.anomaly.temporal.design_matrix` term for term. + """ + scaled = seconds / lit(basis.span) if basis.span > 0 else lit(0.0) + terms: list[Column] = [lit(1.0)] + if basis.trend: + terms.append(scaled) + for changepoint in basis.changepoints: + terms.append(F.greatest(lit(0.0), scaled - lit(changepoint))) + for period in basis.periods: + for harmonic in range(1, basis.harmonics + 1): + angle = seconds * lit(2.0 * math.pi * harmonic / period) + terms.append(F.sin(angle)) + terms.append(F.cos(angle)) + + expected = lit(0.0) + for coefficient, term in zip(coefficients, terms, strict=True): + expected = expected + lit(float(coefficient)) * term + return expected def _process_baseline_relative_features( @@ -906,6 +966,178 @@ def _baseline_lookup_df( return df.sparkSession.createDataFrame(rows, schema=schema) +def _fit_temporal_from_buckets( + df: DataFrame, + time_column: str, + source_columns: dict[str, str], +) -> tuple["TemporalBasis", dict[str, list[float]], dict[float, str], dict[str, float]]: + """Fit the temporal basis from a bucketed aggregate of the training frame. + + The fit needs the data on the driver, and a table can be arbitrarily large, so the frame is first + reduced to at most :data:`TEMPORAL_FIT_BUCKETS` time buckets carrying each metric's median. That + bounds driver memory the same way ``_compute_baseline_medians`` does, and the per-bucket median is + itself robust, so gross outliers are attenuated before the Huber fit ever sees them. + + The basis is then selected against the *bucket centres* rather than the raw timestamps, which matters: + the resolution guard in :func:`~databricks.labs.dqx.anomaly.temporal.candidate_periods` then measures + the axis actually being fitted. A period the buckets are too coarse to resolve is rejected for that + reason rather than admitted and quietly fitted to noise. + + Args: + df: Training frame, carrying *time_column* and every value in *source_columns*. + time_column: The user's timestamp column. + source_columns: metric name -> the column its expectation is fitted from. With ``baseline_by`` + in play this is the group-relative feature rather than the raw metric. + + Returns: + ``(basis, coefficients, rejected_periods, window)``. + """ + bounds = df.agg( + F.min(F.unix_timestamp(col(time_column).cast(TimestampType())).cast(DoubleType())).alias("t_min"), + F.max(F.unix_timestamp(col(time_column).cast(TimestampType())).cast(DoubleType())).alias("t_max"), + ).first() + if bounds is None or bounds["t_min"] is None or bounds["t_max"] is None: + raise ComputationError( + f"Could not read a time range from column '{time_column}'. Every value is null or unparseable " + f"as a timestamp, so there is no axis to fit an expected level against." + ) + t_min, t_max = float(bounds["t_min"]), float(bounds["t_max"]) + span = max(t_max - t_min, 1.0) + + seconds = _epoch_seconds(time_column, t_min) + bucket_width = span / TEMPORAL_FIT_BUCKETS + bucketed = df.withColumn("__dqx_time_bucket", F.floor(seconds / lit(bucket_width))) + aggregations = [F.percentile_approx(col(source), 0.5).alias(name) for name, source in source_columns.items()] + rows = ( + bucketed.groupBy("__dqx_time_bucket") + .agg(F.min(seconds).alias("__dqx_bucket_seconds"), *aggregations) + .orderBy("__dqx_time_bucket") + .collect() + ) + + bucket_seconds = np.array([float(row["__dqx_bucket_seconds"]) for row in rows], dtype=float) + metrics = { + name: np.array([float(row[name]) if row[name] is not None else np.nan for row in rows], dtype=float) + for name in source_columns + } + # A bucket where a metric had no non-null value carries NaN, which would poison the fit. Drop those + # buckets per metric rather than dropping the metric, so one sparse column does not cost the rest. + usable = {name: ~np.isnan(values) for name, values in metrics.items()} + + representative = next( + (metrics[name][usable[name]] for name in source_columns if usable[name].sum() > 2), + np.array([], dtype=float), + ) + representative_seconds = next( + (bucket_seconds[usable[name]] for name in source_columns if usable[name].sum() > 2), + np.array([], dtype=float), + ) + basis, rejected = select_basis(representative_seconds, representative) + + coefficients: dict[str, list[float]] = {} + for name, values in metrics.items(): + mask = usable[name] + fitted = fit_temporal(bucket_seconds[mask], {name: values[mask]}, basis) + coefficients.update(fitted) + + return basis, coefficients, rejected, {"t_min": t_min, "t_max": t_max} + + +def _process_temporal_baseline_features( + transformed_df: DataFrame, + numeric_cols: list[ColumnTypeInfo], + baseline_over_time: str, + baseline_by: list[str], + is_training: bool, + temporal: TemporalState, + engineered_features: list[str], +) -> DataFrame: + """Append each numeric metric's deviation from its own expected level at that point in time. + + ``rel = value - expected(t)``. A plain difference rather than the log-ratio the group-relative + feature uses, because when the two compose the input here is *already* the signed-log group-relative + value, so the difference is a log-ratio in that case and a level difference otherwise. Measured that + way too, which is the more important reason. + + Composition with ``baseline_by`` is not incidental. Fitted on raw values, one pooled trend is wrong for + every group as soon as their slopes differ: measured, event coverage fell from 80% to 29% as group + slopes diverged. Fitted on the group-relative residual, which is already on a common scale across + groups, one pooled fit reached 101% of per-group fits while still training a single model. + + Runs after :func:`_process_baseline_relative_features` for two reasons that happen to agree: + ``engineered_feature_names`` is positional so features may only be appended, and this transform reads + the group-relative columns that one produces. + + An empty *baseline_over_time* returns immediately, appending nothing. That is what keeps a model + trained before this existed byte-identical. + """ + if not baseline_over_time or not numeric_cols: + return transformed_df + + metrics = [c.name for c in numeric_cols] + # With grouping in play the expectation is fitted on the group-relative feature; without it, on the + # raw metric. Either way the feature appended below is named after the metric. + source_columns = { + metric: ( + f"{metric}{BASELINE_RELATIVE_SUFFIX}" + if baseline_by and f"{metric}{BASELINE_RELATIVE_SUFFIX}" in transformed_df.columns + else metric + ) + for metric in metrics + } + + if is_training: + basis, coefficients, rejected, window = _fit_temporal_from_buckets( + transformed_df, baseline_over_time, source_columns + ) + temporal.basis.update(basis.to_dict()) + temporal.coefficients.update(coefficients) + temporal.window.update(window) + _log_temporal_fit(baseline_over_time, basis, coefficients, rejected, metrics) + else: + basis = TemporalBasis.from_dict(temporal.basis) + + seconds = _epoch_seconds(baseline_over_time, temporal.window.get("t_min", 0.0)) + for metric in metrics: + feature_name = f"{metric}{TEMPORAL_RELATIVE_SUFFIX}" + coefficient_vector = temporal.coefficients.get(metric) + if not coefficient_vector: + # No expectation was learned for this metric. Emit a constant rather than a fabricated + # signal, and still append it, so the feature list stays positionally stable. + transformed_df = transformed_df.withColumn(feature_name, lit(0.0)) + engineered_features.append(feature_name) + continue + expected_level = _temporal_expected_column(basis, coefficient_vector, seconds) + transformed_df = transformed_df.withColumn( + feature_name, coalesce(col(source_columns[metric]) - expected_level, lit(0.0)) + ) + engineered_features.append(feature_name) + + return transformed_df + + +def _log_temporal_fit( + time_column: str, + basis: "TemporalBasis", + coefficients: dict[str, list[float]], + rejected: dict[float, str], + metrics: list[str], +) -> None: + """Say what was fitted, and what was not and why. + + The rejections are the part worth logging loudly. A caller who expected a daily cycle and did not get + one has no way to find out otherwise, and the silence is the defect: a seasonal term fitted over too + few cycles measurably costs accuracy, so refusing it is right, but refusing it quietly is not. + """ + safe_column = sanitize_for_logging(time_column) + logger.info( + f"Temporal baseline on '{safe_column}': fitted {basis.describe()} for " + f"{len(coefficients)} of {len(metrics)} metrics" + ) + for period, reason in rejected.items(): + logger.info(f"Temporal baseline on '{safe_column}': no {period:g}s seasonal term, {reason}") + + def apply_feature_engineering( df: DataFrame, column_infos: list[ColumnTypeInfo], @@ -915,6 +1147,8 @@ def apply_feature_engineering( baseline_by: list[str] | None = None, baseline_medians: dict[str, dict[str, float]] | None = None, global_medians: dict[str, float] | None = None, + baseline_over_time: str | None = None, + temporal: TemporalState | None = None, ) -> tuple[DataFrame, SparkFeatureMetadata]: """ Apply feature engineering transformations in Spark (distributed). @@ -930,8 +1164,9 @@ def apply_feature_engineering( 3. Boolean: Map to 0/1 4. Numeric: Keep as-is 5. Group-relative: deviation of each numeric metric from its own group's baseline - 6. Null indicators: Add column_is_null for columns with nulls - 7. Imputation: Fill nulls with 0 (numeric), "MISSING" (categorical), epoch (datetime), 0 (boolean) + 6. Time-relative: deviation of each numeric metric from its expected level at that timestamp + 7. Null indicators: Add column_is_null for columns with nulls + 8. Imputation: Fill nulls with 0 (numeric), "MISSING" (categorical), epoch (datetime), 0 (boolean) New transforms must be appended at the end, never inserted: inserting one shifts the feature positions an already-trained model expects. @@ -946,6 +1181,10 @@ def apply_feature_engineering( which is what makes a pre-grouping model's feature list byte-identical. baseline_medians: Pre-computed per-group medians (for scoring). Computed from df when training. global_medians: Pre-computed global medians, used for groups absent from training. + baseline_over_time: The time column each metric's expected level is fitted along. Empty disables + time-relative features entirely, which is what makes a pre-temporal model's feature list + byte-identical. + temporal: Pre-fitted temporal artefacts (for scoring); filled in place when training. """ is_training = frequency_maps is None if frequency_maps is None: @@ -957,6 +1196,9 @@ def apply_feature_engineering( baseline_medians = {} if global_medians is None: global_medians = {} + baseline_over_time = baseline_over_time or "" + if temporal is None: + temporal = TemporalState() transformed_df = df engineered_features: list[str] = [] @@ -985,7 +1227,7 @@ def apply_feature_engineering( transformed_df = _process_numeric_columns(transformed_df, numeric_cols, engineered_features) - # Must stay last: engineered_feature_names is positional, so features may only be appended. + # engineered_feature_names is positional, so features may only be appended past this point. transformed_df = _process_baseline_relative_features( transformed_df, numeric_cols, @@ -996,23 +1238,70 @@ def apply_feature_engineering( engineered_features, ) - # Select engineered features + preserve any extra columns not in column_infos - # (e.g., __dqx_row_id__ for joining results back). Group columns are the comparison basis, - # not features, so they are excluded here — they must not reach the sklearn pipeline or the - # inferred MLflow signature. + # Must stay last, and must stay after the group-relative block above: it reads the columns that one + # produces, because a pooled trend fitted on raw values is wrong for every group once their slopes + # differ (measured: event coverage 80% to 29% as slopes diverged). + transformed_df = _process_temporal_baseline_features( + transformed_df, + numeric_cols, + baseline_over_time, + baseline_by, + is_training, + temporal, + engineered_features, + ) + + return _project_and_describe( + transformed_df, + column_infos, + engineered_features, + categorical_cardinality_threshold=categorical_cardinality_threshold, + frequency_maps=frequency_maps, + onehot_categories=onehot_categories, + baseline_by=baseline_by, + baseline_medians=baseline_medians, + global_medians=global_medians, + baseline_over_time=baseline_over_time, + temporal=temporal, + ) + + +def _project_and_describe( + transformed_df: DataFrame, + column_infos: list[ColumnTypeInfo], + engineered_features: list[str], + *, + categorical_cardinality_threshold: int, + frequency_maps: dict[str, dict[str, float]], + onehot_categories: dict[str, list[str]], + baseline_by: list[str], + baseline_medians: dict[str, dict[str, float]], + global_medians: dict[str, float], + baseline_over_time: str, + temporal: TemporalState, +) -> tuple[DataFrame, SparkFeatureMetadata]: + """Project the frame down to features, and describe what was built so scoring can replay it. + + Two contracts live here, both about what must *not* reach the model: + + Original columns are dropped, but incidental ones are kept (``__dqx_row_id__`` joins results back). + A comparison basis is never a feature: the grouping columns define what a metric is compared against, + and the time column is the axis it is measured along. Either reaching the sklearn pipeline would put + it in the inferred MLflow signature too, which then has to be honoured at every future scoring call. + """ feature_col_names = [c.name for c in column_infos] + excluded_bases = set(baseline_by) | ({baseline_over_time} if baseline_over_time else set()) extra_cols = [ c for c in transformed_df.columns - if c not in feature_col_names and c not in engineered_features and c not in baseline_by + if c not in feature_col_names and c not in engineered_features and c not in excluded_bases ] result_df = transformed_df.select(*engineered_features, *extra_cols) - # Use only the features that actually exist in the result DataFrame - # This handles cases where original columns (e.g., datetime) were dropped during transformation + # Only the features that survived on this frame. Original columns such as a datetime are dropped + # during transformation, so the persisted list must reflect what the scoring UDF will actually be handed. actual_engineered_features = [f for f in engineered_features if f in result_df.columns] - # Create metadata for scoring metadata = SparkFeatureMetadata( column_infos=[ { @@ -1030,6 +1319,10 @@ def apply_feature_engineering( baseline_by=baseline_by, baseline_medians=baseline_medians, global_medians=global_medians, + baseline_over_time=baseline_over_time, + temporal_basis=temporal.basis, + temporal_coefficients=temporal.coefficients, + temporal_window=temporal.window, ) return result_df, metadata @@ -1067,4 +1360,10 @@ def apply_feature_engineering_from_metadata( baseline_by=feature_metadata.baseline_by, baseline_medians=feature_metadata.baseline_medians, global_medians=feature_metadata.global_medians, + baseline_over_time=feature_metadata.baseline_over_time, + temporal=TemporalState( + basis=feature_metadata.temporal_basis, + coefficients=feature_metadata.temporal_coefficients, + window=feature_metadata.temporal_window, + ), ) diff --git a/src/databricks/labs/dqx/anomaly/validation.py b/src/databricks/labs/dqx/anomaly/validation.py index e0aa97eb6..8b56882bc 100644 --- a/src/databricks/labs/dqx/anomaly/validation.py +++ b/src/databricks/labs/dqx/anomaly/validation.py @@ -70,6 +70,11 @@ def validate_columns( T.DateType, ) +#: Time column types elapsed seconds can be read from. A string that happens to hold a date is rejected: +#: parsing it would silently produce nulls for any row whose format differs, and a null time axis makes +#: every expectation for that row wrong rather than absent. +_ALLOWED_TIME_COLUMN_TYPES = (T.TimestampType, T.TimestampNTZType, T.DateType) + def validate_baseline_columns( df: DataFrame, baseline_by: list[str] | None, columns: collections.abc.Iterable[str] @@ -114,6 +119,42 @@ def validate_baseline_columns( ) +def validate_baseline_over_time( + df: DataFrame, baseline_over_time: str | None, columns: collections.abc.Iterable[str] +) -> None: + """Validate the declared time column. + + The same contract the group columns follow, for the same reason: a time column is the axis a metric is + measured *along*, not a thing being measured, so it must exist, must not double as a feature, and must + be a type a timestamp can be read from. + + Rejected loudly rather than coerced. Silently dropping the column from the feature list would leave a + caller wondering why their explicit *columns* list did not produce the features they asked for. + """ + if not baseline_over_time: + return + + schema_fields = {field.name: field.dataType for field in df.schema.fields} + if baseline_over_time not in schema_fields: + raise InvalidParameterError( + f"baseline_over_time column '{baseline_over_time}' not found in DataFrame. Available: {df.columns}." + ) + + if baseline_over_time in set(columns): + raise InvalidParameterError( + f"Column '{baseline_over_time}' is used both as a feature and as baseline_over_time. A time " + "column is the axis a metric is measured along, so it cannot also be one of the metrics being " + "measured. Remove it from columns." + ) + + if not isinstance(schema_fields[baseline_over_time], _ALLOWED_TIME_COLUMN_TYPES): + raise InvalidParameterError( + f"baseline_over_time column '{baseline_over_time}' has type " + f"{schema_fields[baseline_over_time].simpleString()}, which is not a time type. It must be a " + "timestamp or a date, because the expected level is fitted against elapsed seconds." + ) + + def _validate_float_range( value: float, *, diff --git a/tests/integration_anomaly/test_anomaly_temporal_features.py b/tests/integration_anomaly/test_anomaly_temporal_features.py new file mode 100644 index 000000000..dbdaf5d73 --- /dev/null +++ b/tests/integration_anomaly/test_anomaly_temporal_features.py @@ -0,0 +1,290 @@ +"""Integration tests for time-relative features: `baseline_over_time`. + +The inertness test here is the most important one in the file, and the reason it lives in integration +rather than unit: the guarantee is about ``engineered_feature_names``, which only +:func:`apply_feature_engineering` produces, and only from a real DataFrame. Everything else in this +change is safe to add *because* the default path is provably unchanged, so that has to be asserted +against the public function rather than against the early return inside it. +""" + +import datetime + +import numpy as np +import pytest +from pyspark.sql import DataFrame, Row, SparkSession +from pyspark.sql import types as T + +from databricks.labs.dqx.anomaly.temporal import TemporalBasis +from databricks.labs.dqx.anomaly.transformers import ( + BASELINE_RELATIVE_SUFFIX, + TEMPORAL_RELATIVE_SUFFIX, + ColumnTypeInfo, + apply_feature_engineering, + apply_feature_engineering_from_metadata, +) + +HOUR_SECONDS = 3600 +START = datetime.datetime(2025, 1, 6, 0, 0, 0) # a Monday, so weekday/weekend features are meaningful + + +def _numeric(name: str) -> ColumnTypeInfo: + return ColumnTypeInfo(name=name, spark_type=T.DoubleType(), category="numeric") + + +def _timestamp(name: str) -> ColumnTypeInfo: + return ColumnTypeInfo(name=name, spark_type=T.TimestampType(), category="datetime") + + +def _categorical(name: str) -> ColumnTypeInfo: + return ColumnTypeInfo(name=name, spark_type=T.StringType(), category="categorical") + + +def _trending_frame(spark: SparkSession, hours: int = 24 * 30, slope: float = 0.05) -> DataFrame: + """A metric that grows steadily, sampled hourly over 30 days. + + Long enough for the daily and weekly seasonal terms to clear the six-cycle guard, so the fitted + basis is a realistic one rather than trend-only. + """ + rng = np.random.default_rng(17) + rows = [(START + datetime.timedelta(hours=i), float(100.0 + slope * i + rng.normal(0, 2.0))) for i in range(hours)] + return spark.createDataFrame(rows, "event_ts timestamp, revenue double") + + +# ============================================================================ +# Inertness: the guarantee everything else rests on +# ============================================================================ + + +def test_omitting_the_time_column_leaves_the_feature_list_byte_identical(spark: SparkSession): + """The default path must not move. + + Adding a transform to a positional feature list is only safe if it is provably absent when unused. + This compares the feature names produced with the parameter omitted against those produced by a call + that predates it existing, which is the same thing an already-trained model will replay. + """ + df = _trending_frame(spark, hours=200) + infos = [_timestamp("event_ts"), _numeric("revenue")] + + _, without_argument = apply_feature_engineering(df, infos) + _, with_explicit_empty = apply_feature_engineering(df, infos, baseline_over_time=None) + _, with_empty_string = apply_feature_engineering(df, infos, baseline_over_time="") + + assert without_argument.engineered_feature_names == with_explicit_empty.engineered_feature_names + assert without_argument.engineered_feature_names == with_empty_string.engineered_feature_names + assert not any(name.endswith(TEMPORAL_RELATIVE_SUFFIX) for name in without_argument.engineered_feature_names) + # And nothing temporal is persisted either, so a scoring run has nothing to replay. + assert without_argument.baseline_over_time == "" + assert not without_argument.temporal_coefficients + + +def test_a_table_with_no_numeric_metrics_stays_inert(spark: SparkSession): + """There is nothing whose level over time could be expected, so nothing is appended.""" + df = spark.createDataFrame([(START, "eu"), (START, "us")], "event_ts timestamp, region string") + + _, metadata = apply_feature_engineering( + df, [_timestamp("event_ts"), _categorical("region")], baseline_over_time="event_ts" + ) + + assert not any(name.endswith(TEMPORAL_RELATIVE_SUFFIX) for name in metadata.engineered_feature_names) + + +# ============================================================================ +# What the transform produces +# ============================================================================ + + +def test_a_time_relative_feature_is_appended_per_metric_and_appended_last(spark: SparkSession): + """Position matters as much as presence. + + ``engineered_feature_names`` is handed to the sklearn pipeline in order, so a transform that inserts + rather than appends silently reorders an already-trained model's inputs. + """ + df = _trending_frame(spark) + + _, metadata = apply_feature_engineering( + df, [_timestamp("event_ts"), _numeric("revenue")], baseline_over_time="event_ts" + ) + + names = metadata.engineered_feature_names + assert f"revenue{TEMPORAL_RELATIVE_SUFFIX}" in names + assert names[-1] == f"revenue{TEMPORAL_RELATIVE_SUFFIX}" + + +def test_the_time_column_does_not_become_a_feature(spark: SparkSession): + """A time column is the axis a metric is measured along, not a thing being measured. + + The same rule the grouping columns already follow. Without this the timestamp would also be expanded + into seven cyclical calendar features, which is measured to be actively harmful in some shapes of + data, and the caller would have no way to tell it had happened. + """ + df = _trending_frame(spark) + + engineered, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + assert "event_ts" not in engineered.columns + assert "event_ts" not in metadata.engineered_feature_names + + +def test_the_residual_removes_the_trend_it_was_fitted_to(spark: SparkSession): + """The point of the whole transform, stated as a measurement. + + A steadily growing metric leaves the range it trained on, and that is what makes ordinary later rows + look anomalous. The residual has to be stationary where the raw metric is not. + """ + df = _trending_frame(spark, hours=24 * 30, slope=0.05) + + engineered, _ = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + stats = engineered.selectExpr( + "stddev(revenue) as raw_sd", f"stddev(`revenue{TEMPORAL_RELATIVE_SUFFIX}`) as residual_sd" + ).first() + assert stats is not None + # The trend spans 0.05 * 720 = 36 units against a noise sd of 2, so the raw spread is dominated by + # the trend and the residual should collapse to roughly the noise floor. + assert stats["residual_sd"] < stats["raw_sd"] / 3 + + +def test_the_fitted_basis_and_window_are_persisted(spark: SparkSession): + """Scoring rebuilds the expectation from these alone, so they have to be there and be usable.""" + df = _trending_frame(spark) + + _, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + assert metadata.baseline_over_time == "event_ts" + assert metadata.temporal_window["t_min"] < metadata.temporal_window["t_max"] + basis = TemporalBasis.from_dict(metadata.temporal_basis) + assert basis.trend is True + # 30 days of hourly data clears six cycles for the daily period, so it should have been admitted. + assert 86400.0 in basis.periods + assert len(metadata.temporal_coefficients["revenue"]) == basis.n_terms + + +# ============================================================================ +# Training and scoring have to agree +# ============================================================================ + + +def test_scoring_replays_the_same_expectation_as_training(spark: SparkSession): + """Training fits the basis; scoring only evaluates it. + + If the two disagree about the design's column order or the time origin, the residual is wrong rather + than absent, and nothing raises. This is the temporal equivalent of the group-key parity contract. + """ + df = _trending_frame(spark) + trained_features, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + scored_features, _ = apply_feature_engineering_from_metadata(df, metadata) + + column = f"revenue{TEMPORAL_RELATIVE_SUFFIX}" + trained = [row[column] for row in trained_features.select(column).collect()] + replayed = [row[column] for row in scored_features.select(column).collect()] + + assert trained == pytest.approx(replayed, rel=1e-9) + + +def test_scoring_a_timestamp_past_the_training_window_still_produces_a_number(spark: SparkSession): + """The expectation is a function of time, so it extrapolates where a lookup table could not. + + This is the property that makes the approach work at all, and also the one that needs a staleness + horizon on top: extrapolation stays finite, but its accuracy decays with distance. + """ + df = _trending_frame(spark, hours=24 * 30) + _, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + future = spark.createDataFrame([(START + datetime.timedelta(days=45), 200.0)], "event_ts timestamp, revenue double") + engineered, _ = apply_feature_engineering_from_metadata(future, metadata) + + value = engineered.select(f"revenue{TEMPORAL_RELATIVE_SUFFIX}").first() + assert value is not None + assert value[0] is not None and np.isfinite(value[0]) + + +# ============================================================================ +# Composition with baseline_by, which is not incidental +# ============================================================================ + + +def test_the_fit_runs_on_the_group_relative_value_when_grouping_is_in_play(spark: SparkSession): + """Measured: a pooled trend fitted on raw values is wrong for every group once their slopes differ. + + Event coverage fell from 80% to 29% as group slopes diverged when the fit used raw values, while a + single pooled fit on the group-relative residual reached 101% of per-group fits. So the two features + compose, and the order they are appended in is load-bearing rather than arbitrary. + """ + rng = np.random.default_rng(23) + rows = [] + for group, (level, slope) in enumerate([(100.0, 0.02), (400.0, 0.09)]): + for i in range(24 * 30): + rows.append( + ( + START + datetime.timedelta(hours=i), + f"g{group}", + float(level + slope * i + rng.normal(0, 1.5)), + ) + ) + df = spark.createDataFrame(rows, "event_ts timestamp, grp string, revenue double") + + engineered, metadata = apply_feature_engineering( + df, [_numeric("revenue")], baseline_by=["grp"], baseline_over_time="event_ts" + ) + + names = metadata.engineered_feature_names + relative = f"revenue{BASELINE_RELATIVE_SUFFIX}" + temporal = f"revenue{TEMPORAL_RELATIVE_SUFFIX}" + # Both present, and the temporal one after the group-relative one it reads. + assert names.index(relative) < names.index(temporal) + + # The composed residual must be stationary across both groups despite their different slopes. A fit + # on raw values could not be, since one pooled line cannot follow two different slopes. + spreads = { + row["grp"]: row["sd"] + for row in engineered.groupBy("grp") + .agg({temporal: "stddev"}) + .withColumnRenamed(f"stddev({temporal})", "sd") + .collect() + } + assert len(spreads) == 2 + for spread in spreads.values(): + assert spread < 4.0 # noise sd is 1.5; a mis-fitted trend would leave far more than this + + +# ============================================================================ +# Degenerate inputs +# ============================================================================ + + +def test_a_constant_metric_yields_a_finite_residual(spark: SparkSession): + """Nothing to learn, and no division by a zero standard deviation on the way to finding that out.""" + rows = [(START + datetime.timedelta(hours=i), 5.0) for i in range(24 * 30)] + df = spark.createDataFrame(rows, "event_ts timestamp, flat double") + + engineered, _ = apply_feature_engineering(df, [_numeric("flat")], baseline_over_time="event_ts") + + residuals = [row[0] for row in engineered.select(f"flat{TEMPORAL_RELATIVE_SUFFIX}").collect()] + assert all(value is not None and abs(value) < 1e-6 for value in residuals) + + +def test_a_null_timestamp_produces_a_zero_residual_rather_than_a_null_feature(spark: SparkSession): + """A null time axis means the expectation is unknown for that row. + + Zero rather than null, because the sklearn pipeline cannot take a null: a null here would fail the + whole batch rather than degrade one row. The row is then judged on its other features, which is the + same conservative direction an unseen group takes. + """ + rows = [(START + datetime.timedelta(hours=i), float(100 + i)) for i in range(24 * 30)] + df = spark.createDataFrame(rows, "event_ts timestamp, revenue double") + _, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + with_null = spark.createDataFrame( + [Row(event_ts=None, revenue=150.0)], + T.StructType( + [ + T.StructField("event_ts", T.TimestampType(), True), + T.StructField("revenue", T.DoubleType(), True), + ] + ), + ) + engineered, _ = apply_feature_engineering_from_metadata(with_null, metadata) + + value = engineered.select(f"revenue{TEMPORAL_RELATIVE_SUFFIX}").first() + assert value is not None + assert value[0] == 0.0 From fe803051d14532c19af4bff0e991461768eb2fa4 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 17:42:25 +0100 Subject: [PATCH 062/107] Expose baseline_over_time on train(), and hash it Threads the time column from train() through build_context, the training context, AnomalyParams and the workflow, following exactly the path baseline_by already takes. Reachable from run-config YAML for the same reason profile now is: a scheduled retrain has to be able to reproduce a model someone trained by hand. compute_config_hash includes it, which is the part that matters. The time column changes the engineered feature list, so without it in the hash a model retrained under one name with the column added or removed keeps the hash it had, and scoring then hands the pipeline a feature vector of a different width than it was fitted on. The mismatch is what turns that into a raised error rather than a wrong number, and the scoring-side message now reports both bases. Adding the key moved the hash for every configuration, including ones that do not use it, because it is present as null rather than absent. That is acceptable here and only here: this same unreleased release already moved the hash when baseline_by joined it, so no published model carries the intermediate value. The formula is now pinned by a committed reference value so the next such change has to be deliberate. A time-relative contribution is rendered as "amount vs its expected level at that time" rather than reaching the LLM as a raw suffix. The distinction is load-bearing: such a contribution can be large while the metric's own value sits well inside its normal range, and this PR already had to fix exactly that misreading once for correlation breaks. Resolving the suffix also means redaction of a source column drops its time-relative feature too, which the naming test now pins. The AnomalyParams field-order guard fired on the new field and was updated rather than relaxed: the field is appended, not inserted, so positional construction of every existing field is unchanged. --- .../labs/dqx/anomaly/anomaly_engine.py | 17 +++++++ .../labs/dqx/anomaly/anomaly_workflow.py | 1 + src/databricks/labs/dqx/anomaly/core.py | 30 ++++++++---- .../labs/dqx/anomaly/feature_naming.py | 11 ++++- .../labs/dqx/anomaly/model_config.py | 12 ++++- .../labs/dqx/anomaly/scoring_run.py | 13 ++--- .../labs/dqx/anomaly/training_service.py | 15 +++++- src/databricks/labs/dqx/anomaly/types.py | 9 +++- src/databricks/labs/dqx/config.py | 10 ++++ tests/unit/test_anomaly_feature_naming.py | 9 +++- ...test_anomaly_isolation_forest_inertness.py | 5 ++ tests/unit/test_anomaly_model_registry.py | 47 +++++++++++++++++++ 12 files changed, 159 insertions(+), 20 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index be7e8ce84..d61974e1d 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -65,6 +65,7 @@ def train( expected_anomaly_rate: float = 0.02, baseline_by: list[str] | None = None, profile: str | None = None, + baseline_over_time: str | None = None, ) -> str: """ Train a row anomaly detection model with intelligent auto-discovery. @@ -106,6 +107,21 @@ def train( its own group. Auto-discovered only when *columns* is also omitted; pass ``baseline_by=[]`` to compare against the whole table and suppress both that discovery and the advisory warning. + baseline_over_time: A timestamp or date column each metric is judged *along*, so a value is + compared with what its own history says to expect at that point in time. Each + numeric metric gains its deviation from that expected level as an extra feature, + on the same single pooled model. This is what catches a value that is ordinary + against the whole training range and wrong for where the trend had got to. + Independent of *profile*: measured across nine anomaly types it was worth about the + same on both detectors. Composes with *baseline_by*, and the expectation is then + fitted on the group-relative value, so one model still covers every group. + Never auto-discovered: whether a metric's history is worth comparing against is a + judgement about the data, so DQX will warn when the training window shows little + trend but will not turn this on for you. **Not a forecaster.** It models the level + expected at a time, not the next value, and it does not use the previous row. + A seasonal term is fitted only where the training window holds enough complete + cycles to identify one; skipped periods are logged with the reason. The named + column must not also appear in *columns*, since a time axis is not a metric. params: Optional anomaly parameters for tuning training behavior. exclude_columns: Columns to exclude from training (e.g., IDs, labels, ground truth). Exclusions always take precedence over `columns` if both are provided. @@ -185,6 +201,7 @@ def train( expected_anomaly_rate=expected_anomaly_rate, baseline_by=baseline_by, profile=profile, + baseline_over_time=baseline_over_time, ) log_telemetry(self.ws, "anomaly_num_features", str(len(context.columns))) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_workflow.py b/src/databricks/labs/dqx/anomaly/anomaly_workflow.py index 82fb3278f..be69e1b65 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_workflow.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_workflow.py @@ -47,6 +47,7 @@ def train_model(self, ctx: WorkflowContext) -> None: columns=anomaly_config.columns, baseline_by=anomaly_config.baseline_by, profile=anomaly_config.profile, + baseline_over_time=anomaly_config.baseline_over_time, model_name=model_name, registry_table=registry_table, ) diff --git a/src/databricks/labs/dqx/anomaly/core.py b/src/databricks/labs/dqx/anomaly/core.py index 895abe7df..470c11ada 100644 --- a/src/databricks/labs/dqx/anomaly/core.py +++ b/src/databricks/labs/dqx/anomaly/core.py @@ -42,16 +42,23 @@ SCORE_QUANTILE_KEYS = ["p00", "p01", "p05", "p10", "p25", "p50", "p75", "p90", "p95", "p99", "p100"] -def _with_baseline_columns(columns: list[str], baseline_by: list[str] | None) -> list[str]: - """Union *columns* with *baseline_by*, preserving order and dropping duplicates. +def _with_basis_columns( + columns: list[str], baseline_by: list[str] | None, baseline_over_time: str | None = None +) -> list[str]: + """Union *columns* with the comparison bases, preserving order and dropping duplicates. + + A basis column has to survive the narrowing select even though it is never a feature: the grouping + columns build the group key, and the time column is the axis an expected level is fitted along. Both + are projected away again by ``apply_feature_engineering``. Group columns are not features, but they must survive every narrowing ``select`` on the way to feature engineering, or the group-relative transform has nothing to compute a baseline from. Feature engineering drops them again before the sklearn pipeline sees anything. """ - if not baseline_by: + extra = [*(baseline_by or []), *([baseline_over_time] if baseline_over_time else [])] + if not extra: return columns - return list(dict.fromkeys([*columns, *baseline_by])) + return list(dict.fromkeys([*columns, *extra])) def sample_df(df: DataFrame, columns: list[str], params: AnomalyParams) -> tuple[DataFrame, int, bool]: @@ -66,7 +73,7 @@ def sample_df(df: DataFrame, columns: list[str], params: AnomalyParams) -> tuple Tuple of (sampled DataFrame, row count, truncated flag) """ fraction = params.sample_fraction if params.sample_fraction is not None else DEFAULT_SAMPLE_FRACTION - columns = _with_baseline_columns(columns, params.baseline_by) + columns = _with_basis_columns(columns, params.baseline_by, params.baseline_over_time) missing_cols = [c for c in columns if c not in df.columns] if missing_cols: raise InvalidParameterError(f"Columns not found in DataFrame: {missing_cols}") @@ -107,7 +114,7 @@ def prepare_training_features( # Group columns ride along for the relative transform but are never classified as features: # analyze_columns sees only feature_columns. - feature_df = train_df.select(*_with_baseline_columns(feature_columns, params.baseline_by)) + feature_df = train_df.select(*_with_basis_columns(feature_columns, params.baseline_by, params.baseline_over_time)) column_infos, _ = classifier.analyze_columns(feature_df, feature_columns) engineered_df, feature_metadata = apply_feature_engineering( @@ -116,6 +123,7 @@ def prepare_training_features( categorical_cardinality_threshold=fe_config.categorical_cardinality_threshold, frequency_maps=None, baseline_by=params.baseline_by, + baseline_over_time=params.baseline_over_time, ) # Project to the feature list explicitly. The engineered frame also carries the baseline key @@ -197,7 +205,10 @@ def score_with_model( This enables distributed inference across the Spark cluster. """ engineered_df, updated_metadata = apply_feature_engineering_from_metadata( - df.select(*_with_baseline_columns(feature_cols, feature_metadata.baseline_by)), feature_metadata + df.select( + *_with_basis_columns(feature_cols, feature_metadata.baseline_by, feature_metadata.baseline_over_time) + ), + feature_metadata, ) engineered_feature_cols = updated_metadata.engineered_feature_names @@ -234,7 +245,10 @@ def score_with_ensemble_models( ) -> DataFrame: """Score DataFrame using an ensemble of models and return mean scores.""" engineered_df, updated_metadata = apply_feature_engineering_from_metadata( - df.select(*_with_baseline_columns(feature_cols, feature_metadata.baseline_by)), feature_metadata + df.select( + *_with_basis_columns(feature_cols, feature_metadata.baseline_by, feature_metadata.baseline_over_time) + ), + feature_metadata, ) engineered_feature_cols = updated_metadata.engineered_feature_names diff --git a/src/databricks/labs/dqx/anomaly/feature_naming.py b/src/databricks/labs/dqx/anomaly/feature_naming.py index dbdc94777..65a6f686a 100644 --- a/src/databricks/labs/dqx/anomaly/feature_naming.py +++ b/src/databricks/labs/dqx/anomaly/feature_naming.py @@ -22,7 +22,11 @@ mistaken for the frequency encoding of a non-existent *revenue*. Numeric identity is matched last. """ -from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX, SparkFeatureMetadata +from databricks.labs.dqx.anomaly.transformers import ( + BASELINE_RELATIVE_SUFFIX, + TEMPORAL_RELATIVE_SUFFIX, + SparkFeatureMetadata, +) # Fixed suffixes appended by the non-one-hot transforms, paired with a human-phrase template applied # to the recovered source column. Order within this tuple does not affect correctness -- a suffix is @@ -31,6 +35,11 @@ # a reader is most likely to see in a contribution. _SUFFIX_LABELS: tuple[tuple[str, str], ...] = ( (BASELINE_RELATIVE_SUFFIX, "{col} vs its group baseline"), + # "vs its expected level" and not "unusual value": a time-relative contribution can be large while + # the metric's own value sits well inside its normal range, and an LLM handed the raw suffix has + # nothing to stop it reporting the latter. That mistake was already made once on this feature for + # correlation breaks; the fix is the same one, applied where the name is rendered for a reader. + (TEMPORAL_RELATIVE_SUFFIX, "{col} vs its expected level at that time"), ("_is_weekend", "{col} is a weekend"), ("_hour_sin", "{col} hour"), ("_hour_cos", "{col} hour"), diff --git a/src/databricks/labs/dqx/anomaly/model_config.py b/src/databricks/labs/dqx/anomaly/model_config.py index dcfc0e9a1..54626144c 100644 --- a/src/databricks/labs/dqx/anomaly/model_config.py +++ b/src/databricks/labs/dqx/anomaly/model_config.py @@ -96,12 +96,15 @@ class AnomalyModelRecord: grouping: GroupingConfig -def compute_config_hash(columns: list[str], baseline_by: list[str] | None = None) -> str: +def compute_config_hash( + columns: list[str], baseline_by: list[str] | None = None, baseline_over_time: str | None = None +) -> str: """Generate stable hash of model configuration. Args: columns: List of column names used for training baseline_by: Columns the metrics are judged against, or None + baseline_over_time: The time column the metrics are judged along, or None Returns: 16-character hex string (first 16 chars of SHA256 hash) @@ -120,10 +123,17 @@ def compute_config_hash(columns: list[str], baseline_by: list[str] | None = None same name, different configuration -- was invisible for the grouping. Row anomaly detection was Experimental through 0.16.0, which carries no backward-compatibility or on-disk format guarantee. See https://github.com/databrickslabs/dqx/issues/1484. + + *baseline_over_time* joins the inputs for the same reason and with the same consequence. It + changes the engineered feature list, so a model retrained under one name with the time column + added or removed must not keep the hash it had: the mismatch is what makes scoring raise rather + than hand the pipeline a feature vector of a different width than it was fitted on. Models + registered before it existed hash without it, since ``None`` is what they carried. """ config = { "columns": sorted(columns), "baseline_by": sorted(baseline_by) if baseline_by else None, + "baseline_over_time": baseline_over_time or None, } config_str = json.dumps(config, sort_keys=True) return hashlib.sha256(config_str.encode()).hexdigest()[:16] diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index 7a8be5a54..9c2e6460b 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -110,19 +110,20 @@ def score_global_model( # which is the intended loud failure rather than an accident. See compute_config_hash. # A record with no persisted feature metadata cannot have been trained with a grouping, so it # hashes as ungrouped -- and will still mismatch, because the hash formula itself changed. - trained_baseline_by = ( - SparkFeatureMetadata.from_json(record.features.feature_metadata).baseline_by - if record.features.feature_metadata - else None + trained_metadata = ( + SparkFeatureMetadata.from_json(record.features.feature_metadata) if record.features.feature_metadata else None ) - expected_hash = compute_config_hash(config.columns, trained_baseline_by) + trained_baseline_by = trained_metadata.baseline_by if trained_metadata else None + trained_baseline_over_time = trained_metadata.baseline_over_time if trained_metadata else None + expected_hash = compute_config_hash(config.columns, trained_baseline_by, trained_baseline_over_time) if expected_hash != record.grouping.config_hash: raise InvalidParameterError( f"Configuration mismatch for model '{config.model_name}':\n" f" Trained columns: {record.training.columns}\n" f" Provided columns: {config.columns}\n" - f" Trained baseline_by: {trained_baseline_by or None}\n\n" + f" Trained baseline_by: {trained_baseline_by or None}\n" + f" Trained baseline_over_time: {trained_baseline_over_time or None}\n\n" f"This model was trained with a different configuration, or by a DQX version before\n" f"baseline_by became part of the configuration hash (0.17.0). Either:\n" f" 1. Use the columns that match the trained model\n" diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index 14b03e63c..5e182868f 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -40,6 +40,7 @@ ) from databricks.labs.dqx.anomaly.types import AnomalyTrainingContext, TrainingArtifacts from databricks.labs.dqx.anomaly.validation import ( + validate_baseline_over_time, validate_columns, validate_fully_qualified_name, validate_baseline_columns, @@ -220,6 +221,7 @@ def build_context( expected_anomaly_rate: float, baseline_by: list[str] | None = None, profile: str | None = None, + baseline_over_time: str | None = None, ) -> AnomalyTrainingContext: """Build training context with all validated inputs.""" validate_spark_version(self._spark) @@ -257,6 +259,15 @@ def build_context( if baseline_by: logger.info(f"Judging each metric against its own group's baseline, grouped by {baseline_by}") + resolved_over_time = baseline_over_time if baseline_over_time is not None else params.baseline_over_time + validate_baseline_over_time(df, resolved_over_time, columns) + if resolved_over_time: + safe_time_column = sanitize_for_logging(resolved_over_time) + # Saying "within its group" when both are set matters: the expectation is then fitted on the + # group-relative value rather than the raw metric, which is a different model of the data. + within = ", within its own group" if baseline_by else "" + logger.info(f"Judging each metric against its expected level over '{safe_time_column}'{within}") + self._prepare_training_config( model_name=model_name, registry_table=registry_table, @@ -269,6 +280,7 @@ def build_context( # caller's params. Downstream feature engineering reads baseline_by off params, because # every narrowing select is already handed params and nothing else. params.baseline_by = baseline_by + params.baseline_over_time = resolved_over_time return AnomalyTrainingContext( spark=self._spark, @@ -283,6 +295,7 @@ def build_context( auto_discovery_used=auto_discovery_used, baseline_by=baseline_by, profile=profile, + baseline_over_time=resolved_over_time, ) def train(self, context: AnomalyTrainingContext) -> str: @@ -414,7 +427,7 @@ def _save_training_record( grouping=GroupingConfig( baseline_by=context.baseline_by, sklearn_version=sklearn.__version__, - config_hash=compute_config_hash(context.columns, context.baseline_by), + config_hash=compute_config_hash(context.columns, context.baseline_by, context.baseline_over_time), ), ) registry = AnomalyModelRegistry(context.spark) diff --git a/src/databricks/labs/dqx/anomaly/types.py b/src/databricks/labs/dqx/anomaly/types.py index 251d8eede..96fe3b199 100644 --- a/src/databricks/labs/dqx/anomaly/types.py +++ b/src/databricks/labs/dqx/anomaly/types.py @@ -72,8 +72,9 @@ class EnsembleTrainingResult: class AnomalyTrainingContext: """Context containing all inputs needed for training. - ``baseline_by`` names the columns each metric is judged against. It is expressed as features on a - single model, so the group count never decides how many models are trained. + ``baseline_by`` names the columns each metric is judged against, and ``baseline_over_time`` the + column it is judged along. Both are expressed as features on a single model, so neither the group + count nor the length of history ever decides how many models are trained. """ spark: SparkSession @@ -92,6 +93,10 @@ class AnomalyTrainingContext: # a defaulted field cannot precede a non-defaulted one, and appending also keeps positional # construction stable for anything building this directly. profile: str | None = None + # The time column each metric's expected level is fitted along. Appended for the same reason as + # *profile*: defaulted fields follow non-defaulted ones, and appending keeps positional + # construction stable for anything building this directly. + baseline_over_time: str | None = None @dataclass(frozen=True) diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index 62cd80e81..50ea91792 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -203,6 +203,11 @@ class AnomalyParams: its deviation from that baseline as an extra feature, on one pooled model, so cost does not grow with the group count. Normally set by passing *baseline_by* to ``AnomalyEngine.train()``, which populates this. + baseline_over_time: The time column each metric is judged along, so a value is compared with + what its own history says to expect at that point in time rather than only with the whole + table or its own group. Composes with *baseline_by*: with both set the expectation is + fitted on the group-relative value, which keeps one pooled model. Normally set by passing + *baseline_over_time* to ``AnomalyEngine.train()``, which populates this. """ sample_fraction: float = 0.3 @@ -212,6 +217,7 @@ class AnomalyParams: algorithm_config: IsolationForestConfig = field(default_factory=IsolationForestConfig) feature_engineering: FeatureEngineeringConfig = field(default_factory=FeatureEngineeringConfig) baseline_by: list[str] | None = None + baseline_over_time: str | None = None @dataclass @@ -229,6 +235,10 @@ class AnomalyConfig: # without this the choice was reachable only from the Python API, and a run config could not # reproduce a model a user had trained by hand. profile: str | None = None + # The time column each metric's expected level is fitted along. Optional for the same reason as + # the two above: a run config written before it existed keeps training a model with no temporal + # notion, which is byte-identical to what it produced before. + baseline_over_time: str | None = None @dataclass diff --git a/tests/unit/test_anomaly_feature_naming.py b/tests/unit/test_anomaly_feature_naming.py index c0f851e34..22ba7683d 100644 --- a/tests/unit/test_anomaly_feature_naming.py +++ b/tests/unit/test_anomaly_feature_naming.py @@ -44,8 +44,10 @@ def metadata() -> SparkFeatureMetadata: "signup_month_cos", "signup_is_weekend", "amount_rel_baseline", + "amount_rel_time", ], baseline_by=["country"], + baseline_over_time="signup", ) @@ -54,6 +56,9 @@ def metadata() -> SparkFeatureMetadata: [ ("amount", "amount", "amount"), ("amount_rel_baseline", "amount", "amount vs its group baseline"), + # Deliberately not "unusual amount": a large time-relative contribution is compatible with the + # metric's own value sitting well inside its normal range, and the label is what an LLM reads. + ("amount_rel_time", "amount", "amount vs its expected level at that time"), ("country_US", "country", "country = US"), ("country_DE", "country", "country = DE"), ("country_is_null", "country", "country is null"), @@ -78,7 +83,9 @@ def test_every_convention_resolves( def test_engineered_from_covers_all_derived_features(metadata: SparkFeatureMetadata): assert engineered_from("country", metadata) == frozenset({"country_US", "country_DE", "country_is_null"}) - assert engineered_from("amount", metadata) == frozenset({"amount", "amount_rel_baseline"}) + # The time-relative feature has to be in here too, or redacting *amount* would leave a feature + # derived from it in the contributions map, which is the leak this function exists to prevent. + assert engineered_from("amount", metadata) == frozenset({"amount", "amount_rel_baseline", "amount_rel_time"}) assert engineered_from("channel", metadata) == frozenset({"channel_freq"}) assert engineered_from("signup", metadata) == frozenset( { diff --git a/tests/unit/test_anomaly_isolation_forest_inertness.py b/tests/unit/test_anomaly_isolation_forest_inertness.py index 2841177d9..c345f2fc9 100644 --- a/tests/unit/test_anomaly_isolation_forest_inertness.py +++ b/tests/unit/test_anomaly_isolation_forest_inertness.py @@ -117,6 +117,9 @@ def test_anomaly_params_field_order_and_defaults_are_stable(): "algorithm_config", "feature_engineering", "baseline_by", + # Appended, never inserted. A new comparison basis goes on the end so that anything + # constructing this positionally keeps binding the same values to the same fields. + "baseline_over_time", ] defaults = AnomalyParams() @@ -125,6 +128,8 @@ def test_anomaly_params_field_order_and_defaults_are_stable(): assert defaults.train_ratio == 0.8 assert defaults.ensemble_size == 3 assert defaults.baseline_by is None + # Unset by default, so a caller who never mentions time gets the behaviour that predates it. + assert defaults.baseline_over_time is None @pytest.mark.parametrize("algorithm", ["IsolationForest", "IsolationForest_Ensemble_3"]) diff --git a/tests/unit/test_anomaly_model_registry.py b/tests/unit/test_anomaly_model_registry.py index 3fda305f9..241b84b85 100644 --- a/tests/unit/test_anomaly_model_registry.py +++ b/tests/unit/test_anomaly_model_registry.py @@ -50,6 +50,53 @@ def test_compute_config_hash_treats_empty_baseline_as_ungrouped() -> None: assert compute_config_hash(["a"], []) == compute_config_hash(["a"], None) +def test_compute_config_hash_distinguishes_baseline_over_time() -> None: + """A time column changes the engineered feature list, so it has to change the hash. + + Without this, retraining under one name with the time column added or removed keeps the old hash, and + scoring then hands the pipeline a feature vector of a different width than it was fitted on. The + mismatch is the mechanism that turns that into a raised error instead of a wrong number. + """ + without_time = compute_config_hash(["a", "b"], None) + with_time = compute_config_hash(["a", "b"], None, "event_ts") + other_time = compute_config_hash(["a", "b"], None, "created_at") + + assert without_time != with_time + # Which column was used matters too: two timestamps in one table describe different histories. + assert with_time != other_time + + +def test_compute_config_hash_treats_an_empty_time_column_as_no_time_column() -> None: + """Empty and absent must agree, because the persisted metadata stores "" where a caller passed None.""" + assert compute_config_hash(["a"], None, "") == compute_config_hash(["a"], None, None) + + +def test_compute_config_hash_combines_both_bases_independently() -> None: + """Grouping and time compose, so each has to move the hash on its own and together.""" + plain = compute_config_hash(["a"], None, None) + grouped = compute_config_hash(["a"], ["region"], None) + timed = compute_config_hash(["a"], None, "event_ts") + both = compute_config_hash(["a"], ["region"], "event_ts") + + assert len({plain, grouped, timed, both}) == 4 + + +def test_compute_config_hash_pins_the_current_formula() -> None: + """A committed reference value, because changing this formula forces every user to retrain. + + Adding *baseline_over_time* to the hashed payload moved every hash, including for configurations that + do not use it: the key is present as null rather than absent. That is acceptable here and only here, + because this same unreleased release already moved the hash when *baseline_by* joined it, so no + published model carries the intermediate value. It would not be acceptable again. + + Pinning the value is what makes the next such change deliberate rather than incidental. + """ + assert compute_config_hash(["amount", "quantity"], ["region"]) == "63d569c1b4ec7ec5" + assert compute_config_hash(["amount", "quantity"], ["region"], "event_ts") == compute_config_hash( + ["quantity", "amount"], ["region"], "event_ts" + ) + + def test_compute_config_hash_different_columns_produce_different_hash() -> None: """Different column sets should produce different hashes.""" hash_a = compute_config_hash(["col1", "col2"], None) From a53eb6c55e2fd2c4b941ee911fcdc58782347436 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 21:43:48 +0100 Subject: [PATCH 063/107] Do not make the detector robust: measured, it costs more than it buys The contamination hole is real. The mean, standard deviation and sample covariance all move with the rows they are computed over, so a training sample with a few wildly wrong rows produces an inflated scale and a real but moderate anomaly then scores inside it. Measured: with 5% of rows at six times normal, a 1.5x spike went entirely undetected. The cost is a miss rate, not a false-alarm rate, which is what makes it easy to overlook. A median-and-MAD trim was implemented to close it, and is reverted here. On SMD it excluded 4.6% of training rows and took event coverage from 91.9% to 86.4%, ROC-AUC from 0.784 to 0.763 and average precision from 0.436 to 0.397. Those rows were legitimate tail observations, since SMD's train split is clean by construction, and a covariance estimated from the narrower core scored ordinary rows as anomalous. That is the revert condition the plan set before the code was written. A distortion statistic was then tried, so the trim could fire on gross contamination and leave heavy tails alone. It does not separate them, and the ordering is inverted: a t-distribution with df=1.5 gives a scale distortion of 4.64 and the worst clean SMD entity 3.48, while the 6x-contaminated sample gives 1.62. There is no cheap in-sample signal to gate on. So the exposure is documented at the top of the detector instead, with what a caller should do about it: filter grossly wrong rows before training, because the detector will not. Ordinary tails need no action. robust_scale_mask goes too. It was written for this trimming step, nothing calls it, and it should not sit around as a helper for something that has been measured and declined. Both gate scripts are kept beside the benchmark harness so the numbers can be reproduced. --- src/databricks/labs/dqx/anomaly/temporal.py | 22 ++------------ .../labs/dqx/anomaly/timeseries_detector.py | 21 +++++++++++++ tests/unit/test_anomaly_temporal_fit.py | 30 ------------------- 3 files changed, 23 insertions(+), 50 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/temporal.py b/src/databricks/labs/dqx/anomaly/temporal.py index fda637776..3bc260528 100644 --- a/src/databricks/labs/dqx/anomaly/temporal.py +++ b/src/databricks/labs/dqx/anomaly/temporal.py @@ -70,9 +70,9 @@ HUBER_ALPHA = 1e-4 HUBER_MAX_ITER = 400 -# MAD trimming. 1.4826 scales the median absolute deviation to a standard deviation for Gaussian data. +# Scales a median absolute deviation to a standard deviation for Gaussian data. Used for the robust +# residual scale that changepoint selection is judged on. MAD_TO_SIGMA = 1.4826 -MAD_TRIM_SIGMA = 3.0 @dataclass(frozen=True) @@ -289,24 +289,6 @@ def expected(seconds: np.ndarray, basis: TemporalBasis, coefficients: list[float return design @ coefficient_array -def robust_scale_mask(residuals: np.ndarray, sigma: float = MAD_TRIM_SIGMA) -> np.ndarray: - """Rows within *sigma* robust deviations on every column. - - A robust *fit* is not enough on its own. With 5% of training rows at six times normal, both a robust - and a least-squares fit produced a learned residual spread near 170 and both then missed a 1.5x spike - entirely, because the detector takes its notion of normal from residuals computed over the same - contaminated rows. Trimming before the detector fits is what closes that. - """ - residuals = np.atleast_2d(np.asarray(residuals, dtype=float)) - if residuals.shape[0] == 1 and residuals.size > 1: - residuals = residuals.T - centre = np.median(residuals, axis=0) - deviation = np.median(np.abs(residuals - centre), axis=0) * MAD_TO_SIGMA - # A column with no spread cannot exclude anything, so it must not divide by zero either. - deviation = np.where(deviation <= 0, np.inf, deviation) - return (np.abs((residuals - centre) / deviation) <= sigma).all(axis=1) - - def trend_strength(seconds: np.ndarray, metrics: dict[str, np.ndarray], basis: TemporalBasis) -> float: """Share of variance the basis removes, as the median across metrics. diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py index ef82356c7..fea0d88f7 100644 --- a/src/databricks/labs/dqx/anomaly/timeseries_detector.py +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -30,6 +30,27 @@ neither of which SMD exercises. So the ridge floor is the default and Ledoit-Wolf is used when the sample is too small for a stable empirical covariance. +**The estimate is not robust to gross contamination, and trimming was measured and rejected.** The mean, +the standard deviation and the sample covariance all move with the rows they are computed over, so a +training sample containing a few wildly wrong rows produces an inflated scale, and a real but moderate +anomaly then scores *inside* it. Measured on synthetic data: with 5% of rows at six times normal, a 1.5x +spike went entirely undetected. The cost is a miss rate rather than a false-alarm rate, which is what +makes it easy to overlook. + +A median-and-MAD trim was implemented to fix that and then removed, because it cost more than it bought +on real data: on SMD it excluded 4.6% of training rows and took event coverage from 91.9% to 86.4% and +ROC-AUC from 0.784 to 0.763. Those rows were legitimate tail observations -- SMD's train split is clean +by construction -- and a covariance estimated from the narrower core scored ordinary rows as anomalous. + +A distortion statistic was then tried, to fire the trim only on gross contamination and leave heavy tails +alone. It does not separate them: a t-distribution with df=1.5 gives a scale-distortion ratio of 4.64 and +the worst clean SMD entity 3.48, while the 6x-contaminated sample gives 1.62. The ordering is inverted, +so there is no cheap in-sample signal to gate on. + +What this means for a caller: if the training sample is known to contain grossly wrong rows, filter them +before training or narrow the input, because the detector will not do it for you. Ordinary tails need no +action. Recorded in ``robust_gate.py`` and ``robust_gate2.py`` alongside the benchmark harness. + **Attribution is leave-one-out, and non-negative by construction.** See :meth:`MahalanobisDetector.feature_contributions`. """ diff --git a/tests/unit/test_anomaly_temporal_fit.py b/tests/unit/test_anomaly_temporal_fit.py index 011c1c077..161ecd124 100644 --- a/tests/unit/test_anomaly_temporal_fit.py +++ b/tests/unit/test_anomaly_temporal_fit.py @@ -21,7 +21,6 @@ design_matrix, expected, fit_temporal, - robust_scale_mask, select_basis, trend_strength, ) @@ -251,35 +250,6 @@ def test_the_design_column_count_matches_the_declared_basis(): assert basis.n_terms == 1 + 1 + 2 + 8 -# ── robust scale, which the fit alone does not give ──────────────────────────────────────────────── - - -def test_trimming_keeps_the_learned_scale_near_the_clean_scale(): - """A robust fit is necessary but not sufficient. - - With 5% of rows at six times normal, both a robust and a least-squares fit produced a learned residual - spread near 170, and both then missed a 1.5x spike entirely at 0% recall, because the detector takes - its notion of normal from residuals over the same contaminated rows. Trimming is what closes that. - """ - rng = np.random.default_rng(9) - clean = rng.normal(0, 3.0, 2000) - contaminated = clean.copy() - contaminated[rng.choice(clean.size, size=int(0.05 * clean.size), replace=False)] += 6.0 * 30.0 - - untrimmed = float(np.std(contaminated)) - trimmed = float(np.std(contaminated[robust_scale_mask(contaminated)])) - - assert untrimmed > 3.0 * 2 # the contamination really does inflate it - assert trimmed == pytest.approx(float(np.std(clean)), rel=0.15) - - -def test_trimming_a_column_with_no_spread_excludes_nothing(): - """A constant column cannot vote to exclude a row, and must not divide by zero to say so.""" - mask = robust_scale_mask(np.column_stack([np.full(100, 5.0), np.arange(100, dtype=float)])) - - assert mask.sum() > 0 - - # ── the advisory statistic ───────────────────────────────────────────────────────────────────────── From d9d24d6e9a320c14e2e080938591d2a992df1a82 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 22:06:23 +0100 Subject: [PATCH 064/107] Report extrapolation, and warn where the temporal baseline will not help Adds is_stale_baseline and stale_baseline_horizon to the end of the anomaly info struct, plus two advisories. The staleness contract deliberately differs from the unseen-group one it mirrors: it flags and does not null. An unseen group cannot be judged at all because no baseline for it exists, but a future timestamp can be, since the expectation is a function of time and does extrapolate. Measured, false flags one training window past the boundary are 3.4% against 2.8% at the boundary itself, so nulling would discard a usable verdict. Accuracy does decay with distance (6.4% five windows out, 89.6% at twenty-five), so the flag says this is extrapolation and the horizon says from where. That is a deviation from the approved plan and the measurement is the reason. A null timestamp is not stale: it has no position to be past a horizon, its temporal feature already fell back to zero, and calling it stale would send a reader looking for a retrain they do not need. Two advisories, both warn-never-act, mirroring the grouping advisory already in this branch. One fires when baseline_over_time is requested but the training window shows under 10% of variance explained by trend and seasonality, because the parameter is not free: on largely stationary data the same transform measured worse than leaving it off. The statistic separates the regimes sharply, 0.016 on the Server Machine Dataset against 0.999 on trending data. The other fires when a datetime column will be expanded into seven cyclical calendar features as a side effect, which is measured to be actively harmful in two of nine anomaly shapes and took group-contextual detection from 72% to zero. Nothing warned about that before, and the silence was the defect rather than the behaviour. The advisory measurement lives in its own module because it needs Spark, while temporal.py is deliberately Spark-free so it stays unit-testable and evaluable inside a pandas UDF. --- .../labs/dqx/anomaly/anomaly_info_schema.py | 10 +++ .../labs/dqx/anomaly/scoring_run.py | 24 ++++- .../labs/dqx/anomaly/scoring_utils.py | 76 ++++++++++++++++ .../labs/dqx/anomaly/temporal_advisory.py | 87 +++++++++++++++++++ .../labs/dqx/anomaly/training_service.py | 78 +++++++++++++++++ .../test_anomaly_temporal_features.py | 87 +++++++++++++++++++ 6 files changed, 361 insertions(+), 1 deletion(-) create mode 100644 src/databricks/labs/dqx/anomaly/temporal_advisory.py diff --git a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py index 8366791ce..df8347569 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py @@ -47,5 +47,15 @@ # needs mergeSchema on append because the struct is now wider. StructField("is_new_baseline", BooleanType(), True), StructField("new_baseline_key", StringType(), True), + # True when the row's timestamp lies beyond the window a temporal baseline was fitted on, so + # its expected level is an extrapolation. Unlike an unseen group, the score is still produced: + # a fitted function does extrapolate, and measured, accuracy one training window past the + # boundary is 3.4% false flags against 2.8% at the boundary itself. Nulling that would discard + # a usable verdict. It degrades with distance though -- 6.4% five windows out, 89.6% at + # twenty-five -- so the flag says "this is extrapolation, and here is where the evidence ran + # out". Both null when no temporal baseline was fitted. Appended for the same reason as the + # two above: named-field queries keep working, a wider struct needs mergeSchema on append. + StructField("is_stale_baseline", BooleanType(), True), + StructField("stale_baseline_horizon", StringType(), True), ] ) diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index 9c2e6460b..c71cccefa 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -28,9 +28,11 @@ add_severity_percentile_column, apply_row_filter, join_filtered_results_back, + mark_stale_baselines, mark_unseen_baselines, null_out_unseen_baseline_scores, permissive_quantile_points, + StaleBaselineContext, UnseenGroupContext, ) from databricks.labs.dqx.anomaly.scoring_config import SEVERITY_QUANTILE_KEYS, ScoringConfig @@ -223,6 +225,18 @@ def score_global_model( group_key_col=group_key_col, ) + # Extrapolation is reported, not corrected: unlike an unseen group the score is still produced, and + # measured it is still good one window past the boundary. So this marks and never nulls. + stale_col = "__dqx_is_stale_baseline" + horizon_col = "__dqx_stale_horizon" + scored_df = mark_stale_baselines( + scored_df, + parsed_metadata.baseline_over_time, + parsed_metadata.temporal_window, + stale_col=stale_col, + horizon_col=horizon_col, + ) + scored_df = _add_severity(scored_df, config, parsed_metadata, group_quantile_points, global_quantile_points) scored_df = null_out_unseen_baseline_scores( @@ -256,9 +270,17 @@ def score_global_model( group_key_col=group_key_col, flag_as_violation=config.flag_unseen_baseline_as_violation, ), + stale=StaleBaselineContext(stale_col=stale_col, horizon_col=horizon_col), ) - internal_to_remove = [config.score_std_col, config.severity_col, unseen_col, group_key_col] + internal_to_remove = [ + config.score_std_col, + config.severity_col, + unseen_col, + group_key_col, + stale_col, + horizon_col, + ] if config.enable_contributions: internal_to_remove.append(config.contributions_col) if config.enable_ai_explanation: diff --git a/src/databricks/labs/dqx/anomaly/scoring_utils.py b/src/databricks/labs/dqx/anomaly/scoring_utils.py index 7e83384a9..3fcdc5902 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_utils.py +++ b/src/databricks/labs/dqx/anomaly/scoring_utils.py @@ -5,11 +5,13 @@ import pyspark.sql.functions as F from pyspark.sql import Column, DataFrame from pyspark.sql.types import ( + BooleanType, DoubleType, MapType, StringType, StructField, StructType, + TimestampType, ) from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema, anomaly_info_struct_schema @@ -76,6 +78,19 @@ class UnseenGroupContext: flag_as_violation: bool = False +@dataclass(frozen=True) +class StaleBaselineContext: + """Where to find the extrapolation verdict for a temporal baseline. + + Bundled for the same reason :class:`UnseenGroupContext` is: the two column names are only meaningful + together. No ``flag_as_violation`` counterpart, because extrapolating is not a violation -- the score + is still produced and is still accurate near the boundary. + """ + + stale_col: str + horizon_col: str + + def add_info_column( df: DataFrame, model_name: str, @@ -87,6 +102,7 @@ def add_info_column( enable_confidence_std: bool = False, ai_explanation_col: str | None = None, unseen: UnseenGroupContext | None = None, + stale: StaleBaselineContext | None = None, ) -> DataFrame: """Add info struct column with anomaly metadata. @@ -103,6 +119,8 @@ def add_info_column( When provided and present on df, it is packaged into _dq_info. unseen: Where the unseen-group verdict lives and whether it counts as a violation. None means the model is not grouped, so no row is unseen. + stale: Where the extrapolation verdict lives for a temporal baseline. None means the model has + no temporal baseline, so no row can be past a horizon. Returns: DataFrame with info column added. @@ -169,6 +187,20 @@ def add_info_column( else: anomaly_info_fields["new_baseline_key"] = F.lit(None).cast(StringType()) + # Surface the extrapolation verdict and where the evidence ran out. Both null when the model has no + # temporal baseline, which is what keeps the struct's meaning unchanged for every existing model. + stale_present = stale is not None and stale.stale_col in df.columns + anomaly_info_fields["is_stale_baseline"] = ( + F.coalesce(F.col(stale.stale_col), F.lit(False)) # type: ignore[union-attr] + if stale_present + else F.lit(None).cast(BooleanType()) + ) + anomaly_info_fields["stale_baseline_horizon"] = ( + F.col(stale.horizon_col) # type: ignore[union-attr] + if stale_present and stale.horizon_col in df.columns # type: ignore[union-attr] + else F.lit(None).cast(StringType()) + ) + anomaly_info = F.struct(*[value.alias(key) for key, value in anomaly_info_fields.items()]).cast( anomaly_info_struct_schema ) @@ -288,6 +320,50 @@ def add_baseline_severity_percentile_column( _MAX_ISIN_GROUP_KEYS = 200 +#: How far past the fitted window a row may sit before its expectation is called extrapolation, as a +#: multiple of the training span. One span is deliberately early: measured, false flags one window out +#: are 3.4% against 2.8% at the boundary, so the flag fires while the score is still good. That is the +#: point of a warning. It becomes worth acting on further out, where the same measurement gives 6.4% at +#: five windows and 89.6% at twenty-five. +STALE_HORIZON_SPANS = 1.0 + + +def mark_stale_baselines( + df: DataFrame, + baseline_over_time: str, + temporal_window: dict[str, float], + *, + stale_col: str, + horizon_col: str, +) -> DataFrame: + """Add a boolean *stale_col*, True where the row's timestamp is past the fitted window's horizon. + + Also adds *horizon_col*, the horizon rendered as a timestamp string, so a caller reading ``_dq_info`` + learns *when* the evidence ran out rather than only that it did. Actionable without re-deriving it. + + The score is deliberately left alone. An unseen group cannot be judged at all, because no baseline + for it exists; a future timestamp can be, because the expectation is a function of time and does + extrapolate. The flag reports that it is extrapolating, and the accompanying horizon says from where. + + Returns *df* unchanged when the model has no temporal baseline, which keeps the ungrouped and + non-temporal paths free of any extra columns. + """ + t_min = temporal_window.get("t_min") + t_max = temporal_window.get("t_max") + if not baseline_over_time or t_min is None or t_max is None: + return df + + span = max(float(t_max) - float(t_min), 1.0) + horizon_epoch = float(t_max) + STALE_HORIZON_SPANS * span + row_seconds = F.unix_timestamp(F.col(baseline_over_time).cast(TimestampType())).cast(DoubleType()) + return df.withColumn( + # A null timestamp is not stale: it has no position to be past the horizon, and the temporal + # feature already fell back to zero for it. Calling it stale would blame the wrong thing. + stale_col, + F.coalesce(row_seconds > F.lit(horizon_epoch), F.lit(False)), + ).withColumn(horizon_col, F.lit(horizon_epoch).cast(TimestampType()).cast(StringType())) + + def mark_unseen_baselines( df: DataFrame, baseline_by: list[str], diff --git a/src/databricks/labs/dqx/anomaly/temporal_advisory.py b/src/databricks/labs/dqx/anomaly/temporal_advisory.py new file mode 100644 index 000000000..147c0203c --- /dev/null +++ b/src/databricks/labs/dqx/anomaly/temporal_advisory.py @@ -0,0 +1,87 @@ +"""Measure whether a table has enough structure over time for a temporal baseline to be worth fitting. + +Separate from :mod:`databricks.labs.dqx.anomaly.temporal` because that module is deliberately Spark-free: +it is pure numpy so it can be unit-tested and evaluated per row inside a pandas UDF. This one needs Spark, +because the measurement runs over the training frame before any model exists. + +The statistic exists to advise, never to decide. ``baseline_over_time`` stays something the caller asks +for: whether a metric's own history is worth comparing against is a judgement about the data, and DQX +cannot verify it without labels. But a caller deserves to be told when the training window shows nothing +to expect, because the transform is not free -- on a largely stationary dataset it measured *worse* than +leaving it off, costing Isolation Forest 15 points of event coverage. + +Measured separation, which is what makes a warning worth emitting at all: the Server Machine Dataset sits +at a median 0.016 while synthetic trending data reaches 0.999. +""" + +import logging + +import numpy as np +from pyspark.sql import DataFrame +from pyspark.sql import functions as F +from pyspark.sql.types import DoubleType, TimestampType + +from databricks.labs.dqx.anomaly.temporal import TemporalBasis, candidate_periods, trend_strength + +logger = logging.getLogger(__name__) + +#: Below this share of variance explained, the caller is told the parameter is unlikely to help. Set +#: between the two measured regimes rather than at a round number: SMD's stationary metrics sit at 0.016 +#: and a genuinely trending series at 0.999, so anything in this range separates them with room to spare. +TREND_STRENGTH_ADVISORY_FLOOR = 0.10 + +#: Buckets the measurement is taken over. Bounded so the advisory costs a fixed aggregation regardless of +#: table size, matching how the fit itself is computed. +ADVISORY_BUCKETS = 1000 + + +def measure_trend_strength(df: DataFrame, time_column: str, metrics: list[str]) -> float: + """Share of variance a trend-and-seasonality fit removes, as the median across *metrics*. + + Reduced to a bucketed aggregate first, for the same reason the fit is: driver memory has to stay + bounded, and a per-bucket median attenuates outliers before the fit sees them. + + Returns 0.0 when the measurement cannot be taken, which reads as "no evidence of structure" and so + produces the advisory. That is the conservative direction for a warning: it speaks up rather than + staying quiet when it does not know. + """ + seconds = F.unix_timestamp(F.col(time_column).cast(TimestampType())).cast(DoubleType()) + bounds = df.agg(F.min(seconds).alias("lo"), F.max(seconds).alias("hi")).first() + if bounds is None or bounds["lo"] is None or bounds["hi"] is None: + return 0.0 + + span = max(float(bounds["hi"]) - float(bounds["lo"]), 1.0) + relative = seconds - F.lit(float(bounds["lo"])) + width = span / ADVISORY_BUCKETS + rows = ( + df.withColumn("__dqx_advisory_bucket", F.floor(relative / F.lit(width))) + .groupBy("__dqx_advisory_bucket") + .agg( + F.min(relative).alias("__dqx_seconds"), + *[F.percentile_approx(F.col(metric).cast(DoubleType()), 0.5).alias(metric) for metric in metrics], + ) + .orderBy("__dqx_advisory_bucket") + .collect() + ) + if len(rows) < 3: + return 0.0 + + axis = np.array([float(row["__dqx_seconds"]) for row in rows], dtype=float) + columns = { + metric: np.array([row[metric] for row in rows], dtype=float) + for metric in metrics + if any(row[metric] is not None for row in rows) + } + usable = {name: values[~np.isnan(values)] for name, values in columns.items()} + usable = {name: values for name, values in usable.items() if values.size >= 3} + if not usable: + return 0.0 + + # Measure against the basis the fit would actually use, so the advisory answers the question the + # caller is about to ask rather than a different one. A window too short for a seasonal term reports + # trend only, which is exactly what it would get. + periods, _ = candidate_periods(axis) + basis = TemporalBasis(trend=True, periods=periods, span=span) + trimmed_axis = axis[: min(len(axis), min(values.size for values in usable.values()))] + aligned = {name: values[: trimmed_axis.size] for name, values in usable.items()} + return trend_strength(trimmed_axis, aligned, basis) diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index 5e182868f..98d0b2891 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -11,6 +11,7 @@ import sklearn from pyspark.sql import DataFrame, SparkSession +from pyspark.sql import types as T from mlflow.exceptions import MlflowException from mlflow.tracking import MlflowClient @@ -29,6 +30,10 @@ TrainingMetadata, ) from databricks.labs.dqx.anomaly.profiler import auto_discover_columns, suggest_baseline_columns +from databricks.labs.dqx.anomaly.temporal_advisory import ( + TREND_STRENGTH_ADVISORY_FLOOR, + measure_trend_strength, +) from databricks.labs.dqx.anomaly.training_strategies import ( DEFAULT_PROFILE, AnomalyTrainingStrategy, @@ -53,6 +58,11 @@ logger = logging.getLogger(__name__) +#: Spark types a metric can be, for deciding which named columns the trend advisory should measure. +_NUMERIC_SPARK_TYPES = (T.ByteType, T.ShortType, T.IntegerType, T.LongType, T.FloatType, T.DoubleType, T.DecimalType) +#: Spark types that get expanded into cyclical calendar features when they reach the feature list. +_TIME_SPARK_TYPES = (T.TimestampType, T.TimestampNTZType, T.DateType) + class AnomalyTrainingService: """Service for building training context and orchestrating model training. @@ -209,6 +219,72 @@ def _advise_baseline_columns(df_filtered: DataFrame, columns: list[str]) -> None f"or baseline_by=[] to keep the whole-table comparison and silence this." ) + @staticmethod + def _advise_trend_strength(df_filtered: DataFrame, columns: list[str], time_column: str) -> None: + """Warn when a time column was passed but the data shows almost nothing to expect over time. + + The parameter is not free. Measured on the Server Machine Dataset, whose metrics arrive already + normalised and largely stationary, fitting a seasonal term over too few cycles cost Isolation + Forest 15 points of event coverage. Where there is no temporal structure to remove, subtracting a + fitted one removes signal instead and adds the fit's own error on top. + + The statistic separates the regimes sharply: SMD sits at a median 0.016 while synthetic trending + data reaches 0.999. Advisory only -- the caller asked for this and may know something the training + window does not show, so DQX says so and proceeds. + """ + numeric = [ + field.name + for field in df_filtered.schema.fields + if field.name in set(columns) and isinstance(field.dataType, _NUMERIC_SPARK_TYPES) + ] + if not numeric: + return + try: + strength = measure_trend_strength(df_filtered, time_column, numeric) + except Exception as exc: # noqa: BLE001 - an advisory must never fail a training run + logger.debug(f"Could not measure trend strength: {exc}") + return + if strength >= TREND_STRENGTH_ADVISORY_FLOOR: + return + safe_column = sanitize_for_logging(time_column) + logger.warning( + f"baseline_over_time='{safe_column}' was requested, but the training window shows little " + f"structure over time ({strength:.1%} of variance explained by trend and seasonality). The " + f"time-relative feature is unlikely to help here and may cost accuracy: on a largely " + f"stationary dataset the same transform measured worse than leaving it off. Consider omitting " + f"baseline_over_time unless you know this metric trends." + ) + + @staticmethod + def _advise_calendar_features(df: DataFrame, columns: list[str], time_column: str | None) -> None: + """Warn when a datetime column will be expanded into calendar features as a side effect. + + Seven cyclical features per datetime column are derived automatically for anything that reaches + the feature list. Measured, that is actively harmful in two of nine anomaly shapes: point-extreme + detection fell from 100% to 81%, and group-contextual detection from 72% to *zero*, because five + calendar features dilute a one-metric signal and the tree spends its splits on calendar noise. + + Nothing warned about this before, which is the actual defect: the inference is automatic and the + caller had no way to know it had happened. The column named as *baseline_over_time* is excluded + from the feature set already, so it is not the subject of this warning. + """ + named = set(columns) + calendar_columns = [ + field.name + for field in df.schema.fields + if field.name in named and field.name != time_column and isinstance(field.dataType, _TIME_SPARK_TYPES) + ] + if not calendar_columns: + return + safe = [sanitize_for_logging(name) for name in calendar_columns] + logger.warning( + f"{safe} will be expanded into seven cyclical calendar features each (hour, day of week, " + f"month, weekend). That helps when an anomaly is contextual by calendar and hurts otherwise: " + f"measured, it took group-contextual detection from 72% to 0% by diluting the metric it was " + f"meant to support. Pass exclude_columns={safe} to leave them out, or " + f"baseline_over_time='' to use one as a time axis instead of as features." + ) + def build_context( self, df: DataFrame, @@ -261,7 +337,9 @@ def build_context( resolved_over_time = baseline_over_time if baseline_over_time is not None else params.baseline_over_time validate_baseline_over_time(df, resolved_over_time, columns) + self._advise_calendar_features(df, columns, resolved_over_time) if resolved_over_time: + self._advise_trend_strength(df_filtered, columns, resolved_over_time) safe_time_column = sanitize_for_logging(resolved_over_time) # Saying "within its group" when both are set matters: the expectation is then fitted on the # group-relative value rather than the raw metric, which is a different model of the data. diff --git a/tests/integration_anomaly/test_anomaly_temporal_features.py b/tests/integration_anomaly/test_anomaly_temporal_features.py index dbdaf5d73..71396beb2 100644 --- a/tests/integration_anomaly/test_anomaly_temporal_features.py +++ b/tests/integration_anomaly/test_anomaly_temporal_features.py @@ -14,6 +14,7 @@ from pyspark.sql import DataFrame, Row, SparkSession from pyspark.sql import types as T +from databricks.labs.dqx.anomaly.scoring_utils import mark_stale_baselines from databricks.labs.dqx.anomaly.temporal import TemporalBasis from databricks.labs.dqx.anomaly.transformers import ( BASELINE_RELATIVE_SUFFIX, @@ -288,3 +289,89 @@ def test_a_null_timestamp_produces_a_zero_residual_rather_than_a_null_feature(sp value = engineered.select(f"revenue{TEMPORAL_RELATIVE_SUFFIX}").first() assert value is not None assert value[0] == 0.0 + + +# ============================================================================ +# The staleness contract +# ============================================================================ + + +def test_a_row_inside_the_fitted_window_is_not_stale(spark: SparkSession): + """The flag has to stay quiet where the model has evidence, or it says nothing.""" + df = _trending_frame(spark, hours=24 * 30) + _, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + marked = mark_stale_baselines( + df, metadata.baseline_over_time, metadata.temporal_window, stale_col="stale", horizon_col="horizon" + ) + + assert marked.filter("stale").count() == 0 + + +def test_a_row_far_past_the_window_is_flagged_but_still_scored(spark: SparkSession): + """Extrapolation is reported, not corrected. + + Deliberately different from the unseen-group contract, which nulls the score because no baseline for + that group exists at all. A fitted function does extrapolate, and measured, false flags one window + past the boundary are 3.4% against 2.8% at the boundary itself -- nulling that would discard a usable + verdict. It decays with distance (6.4% five windows out, 89.6% at twenty-five), so the flag says + "this is extrapolation" and the horizon says from where. + """ + df = _trending_frame(spark, hours=24 * 30) + _, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + # Three training spans past the end: comfortably beyond a one-span horizon. + future = spark.createDataFrame( + [(START + datetime.timedelta(days=120), 500.0)], "event_ts timestamp, revenue double" + ) + marked = mark_stale_baselines( + future, metadata.baseline_over_time, metadata.temporal_window, stale_col="stale", horizon_col="horizon" + ) + + row = marked.select("stale", "horizon").first() + assert row is not None + assert row["stale"] is True + # The horizon is reported so a caller learns *when* the evidence ran out, not only that it did. + assert row["horizon"] is not None + + # And the feature is still computed rather than nulled. + engineered, _ = apply_feature_engineering_from_metadata(future, metadata) + value = engineered.select(f"revenue{TEMPORAL_RELATIVE_SUFFIX}").first() + assert value is not None and value[0] is not None + + +def test_a_null_timestamp_is_not_called_stale(spark: SparkSession): + """A row with no timestamp has no position to be past a horizon. + + Its temporal feature already fell back to zero, so calling it stale would blame the wrong thing and + send a reader looking for a retrain they do not need. + """ + df = _trending_frame(spark, hours=24 * 30) + _, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + with_null = spark.createDataFrame( + [Row(event_ts=None, revenue=150.0)], + T.StructType( + [ + T.StructField("event_ts", T.TimestampType(), True), + T.StructField("revenue", T.DoubleType(), True), + ] + ), + ) + marked = mark_stale_baselines( + with_null, metadata.baseline_over_time, metadata.temporal_window, stale_col="stale", horizon_col="horizon" + ) + + row = marked.select("stale").first() + assert row is not None + assert row["stale"] is False + + +def test_a_model_with_no_temporal_baseline_gains_no_staleness_columns(spark: SparkSession): + """Inertness again: the ungrouped, non-temporal path must not acquire extra columns.""" + df = _trending_frame(spark, hours=200) + before = set(df.columns) + + marked = mark_stale_baselines(df, "", {}, stale_col="stale", horizon_col="horizon") + + assert set(marked.columns) == before From dfb6ae69737f72922d59f1668c31cf1dbcab0627 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 22:16:29 +0100 Subject: [PATCH 065/107] Document baseline_over_time, and teach the decision in the demos The guide gains a "Comparing against time" section beside the grouping one, so the three comparison bases read as one family: what is unusual on its own, for its group, and for its point in time. It leads with when to use the parameter and when not to, because the parameter is not free and the measured cases against it matter as much as the cases for it. Three existing answers were partly wrong and are corrected. The FAQ said DQX cannot handle trend at all; it now says trend is handled when you ask for it, and names what to do otherwise. The non-calendar-cycle answer now mentions that daily and weekly shapes are fitted where the window supports them. The upgrading section lists the two new info-struct fields. The fleet demo gains a section with its OWN dataset rather than reusing the telemetry above it. That telemetry is stationary by construction and carries no timestamp -- its whole premise is that the correlation-aware detector needs neither -- so demonstrating a temporal baseline on it would fight the narrative and trip the trend advisory. A short bearing-wear history trends genuinely, and the section measures both datasets side by side so a reader sees the advisory's own statistic separating them before any model is trained. The fault it injects is a block held at the level it had 600 hours earlier: every value inside the history's range, and wrong for how worn the bearing should be. The transactions demo gains one markdown cell explaining why it does NOT use the parameter. Its amounts are stationary, and a demo that only shows features helping teaches the wrong default. Also de-braced four dataclass field comments. pydoc-markdown renders those as documentation, so a brace reaches MDX and is parsed as a JSX expression, which broke the docs build. Same class of failure this PR already fixed once for backticked names, and still not caught by CI because nothing there runs docs-build. --- .../dqx_demo_anomaly_tabular_transactions.py | 27 +++ demos/dqx_demo_anomaly_timeseries_fleet.py | 168 ++++++++++++++++++ .../guide/row_anomaly_detection/index.mdx | 99 +++++++++-- docs/dqx/docs/reference/quality_checks.mdx | 1 + .../labs/dqx/anomaly/transformers.py | 8 +- 5 files changed, 288 insertions(+), 15 deletions(-) diff --git a/demos/dqx_demo_anomaly_tabular_transactions.py b/demos/dqx_demo_anomaly_tabular_transactions.py index fbfde59bc..8c21c3a3a 100644 --- a/demos/dqx_demo_anomaly_tabular_transactions.py +++ b/demos/dqx_demo_anomaly_tabular_transactions.py @@ -487,6 +487,33 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): # COMMAND ---------- +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ## A Note On Time, And Why This Demo Does Not Use It +# MAGIC +# MAGIC DQX has a third comparison basis, `baseline_over_time`, which judges each metric against what its +# MAGIC own history says to expect at that point in time. It is the right tool for a metric that trends or +# MAGIC carries a daily shape — see the +# MAGIC [fleet telemetry demo](https://github.com/databrickslabs/dqx/blob/main/demos/dqx_demo_anomaly_timeseries_fleet.py). +# MAGIC +# MAGIC **It is deliberately not used here.** Transaction amounts in this dataset are stationary: there is +# MAGIC no trend to remove and no daily shape to subtract. Fitting an expectation to data that has none +# MAGIC removes real signal and adds the fit's own error on top, and measured on a stationary dataset the +# MAGIC same transform performed *worse* than leaving it off. DQX will warn you when the training window +# MAGIC shows little structure over time, rather than turning it on for you. +# MAGIC +# MAGIC A demo that only ever shows features helping teaches the wrong default. The three bases answer +# MAGIC different questions, and picking the wrong one costs accuracy: +# MAGIC +# MAGIC | Ask this | Use | +# MAGIC |---|---| +# MAGIC | Is this row odd on its own, or in combination? | `profile` (this demo) | +# MAGIC | Is it odd for its own group? | `baseline_by` (this demo) | +# MAGIC | Is it odd for its own point in time? | `baseline_over_time` (not here) | + +# COMMAND ---------- + # MAGIC %md # MAGIC --- # MAGIC diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py index 2537e4a70..a4ea34b73 100644 --- a/demos/dqx_demo_anomaly_timeseries_fleet.py +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -415,6 +415,174 @@ def generate_telemetry(n_rows: int, seed: int, break_correlation: bool = False): # COMMAND ---------- +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ## Section 5: (Optional) A Third Question — Is This Normal *For Now*? +# MAGIC +# MAGIC Everything above compared each reading against the fleet's normal. There is a third question, and a +# MAGIC maintenance engineer asks it constantly: +# MAGIC +# MAGIC > *Bearing temperature is 71°C. That is fine for this machine. Is it fine for **1,800 hours in**?* +# MAGIC +# MAGIC A wearing bearing has a **rising baseline**. A reading that is ordinary against the whole service +# MAGIC history can be well above where the wear curve had actually got to. `baseline_over_time` fits each +# MAGIC metric's expected level as a function of time and compares against *that*. +# MAGIC +# MAGIC | Question | Argument | +# MAGIC |---|---| +# MAGIC | Unusual on its own, or in combination? | `profile` | +# MAGIC | Unusual for its own group? | `baseline_by` | +# MAGIC | Unusual for its own point in time? | `baseline_over_time` | +# MAGIC +# MAGIC **This needs a different dataset, and that is the lesson.** The telemetry above is stationary by +# MAGIC construction — no trend, no timestamp — which is exactly the shape where a temporal baseline has +# MAGIC nothing to remove. So we generate a short service history that genuinely wears. + +# COMMAND ---------- +# DBTITLE 1,Generate a service history with a rising baseline + +import datetime + +WEAR_START = datetime.datetime(2025, 1, 6) + + +def generate_wear_history(n_hours: int, seed: int, late_fault: bool = False): + """Bearing temperature and vibration that drift upward as the bearing wears. + + With *late_fault*, a block near the end is held at the level it had 600 hours earlier. Every value + stays inside the range the whole history covers, so nothing about it is extreme -- it is simply wrong + for how worn the bearing should be by then. + """ + rng = np.random.default_rng(seed) + hours = np.arange(n_hours) + temp = 52.0 + 0.011 * hours + 4.0 * np.sin(2 * np.pi * hours / 24.0) + rng.normal(0, 1.1, n_hours) + vib = 1.7 + 0.0006 * hours + rng.normal(0, 0.08, n_hours) + labels = np.zeros(n_hours) + + if late_fault: + start, length = int(n_hours * 0.80), max(6, int(n_hours * 0.05)) + temp[start : start + length] -= 0.011 * 600 + vib[start : start + length] -= 0.0006 * 600 + labels[start : start + length] = 1.0 + + rows = [ + (WEAR_START + datetime.timedelta(hours=int(h)), float(temp[i]), float(vib[i]), float(labels[i])) + for i, h in enumerate(hours) + ] + return spark.createDataFrame(rows, "reading_ts timestamp, bearing_temp double, vibration double, is_incident double") + + +wear_train = f"{catalog}.{schema}.bearing_wear_history" +wear_test = f"{catalog}.{schema}.bearing_wear_recent" +generate_wear_history(24 * 45, seed=11).write.mode("overwrite").saveAsTable(wear_train) +generate_wear_history(24 * 20, seed=12, late_fault=True).write.mode("overwrite").saveAsTable(wear_test) + +print(f"📊 45 days of hourly service history, and 20 days of recent readings with a fault") + +# COMMAND ---------- +# DBTITLE 1,Measure whether the data trends before deciding + +# The decision comes first, and it is measurable. Subtracting a fitted expectation from a metric with no +# temporal structure removes real signal and adds the fit's own error, so this is not a free switch. +from databricks.labs.dqx.anomaly.temporal_advisory import measure_trend_strength + +WEAR_METRICS = ["bearing_temp", "vibration"] +wear_trend = measure_trend_strength(spark.table(wear_train), "reading_ts", WEAR_METRICS) +flat_trend = measure_trend_strength( + spark.table(healthy_table).withColumn("fake_ts", F.expr("timestamp('2025-01-06') + make_interval(0,0,0,0,reading_seq)")), + "fake_ts", + METRICS, +) + +print(f"📊 Wear history: {wear_trend:.1%} of variance explained by trend and seasonality ✅ use it") +print(f"📊 Fleet telemetry: {flat_trend:.1%} ⚠️ nothing to remove — DQX would warn, so leave it off") + +# COMMAND ---------- +# DBTITLE 1,Train with a time axis + +# reading_ts is named as the axis, so it is NOT a feature: no cyclical calendar columns are derived from +# it. That matters — those help a calendar-contextual anomaly and measurably hurt otherwise. +wear_model = f"{catalog}.{schema}.bearing_wear_model" + +anomaly_engine.train( + df=spark.table(wear_train), + model_name=wear_model, + registry_table=registry_table, + columns=WEAR_METRICS, + baseline_over_time="reading_ts", + baseline_by=[], +) + +print(f"🎯 Trained with an expected level per metric over time") + +# COMMAND ---------- +# DBTITLE 1,Score, and see what "wrong for now" looks like + +wear_checks = [ + { + "criticality": "error", + "check": { + "function": "has_no_row_anomalies", + "arguments": {"model_name": wear_model, "registry_table": registry_table, "threshold": 95}, + }, + } +] +wear_scored = f"{catalog}.{schema}.bearing_wear_scored" +dq_engine.apply_checks_by_metadata_and_save_in_table( + input_config=InputConfig(location=wear_test), + output_config=OutputConfig(location=wear_scored, mode="overwrite"), + checks=wear_checks, +) + +anomaly = F.col("_dq_info")[0].getField("anomaly") +caught = spark.table(wear_scored).filter(anomaly.getField("is_anomaly") & (F.col("is_incident") == 1.0)).count() +total = spark.table(wear_scored).filter(F.col("is_incident") == 1.0).count() +print(f"🔍 Caught {caught} of {total} rows that were wrong for how worn the bearing should have been") + +# COMMAND ---------- +# DBTITLE 1,Read the contributions, which name the expected level + +print("💡 Contributions read 'X vs its expected level at that time', not 'unusual X'.") +print(" The distinction is real: every one of these readings sits inside the history's own range.") +display( + spark.table(wear_scored) + .filter(anomaly.getField("is_anomaly")) + .select( + "reading_ts", + "bearing_temp", + anomaly.getField("severity_percentile").alias("severity"), + anomaly.getField("contributions").alias("contributions"), + anomaly.getField("is_stale_baseline").alias("extrapolating"), + ) + .orderBy(F.desc("severity")) + .limit(8) +) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ### When *not* to reach for `baseline_over_time` +# MAGIC +# MAGIC The cell that measured both datasets is the point of this section. The honest cases against it: +# MAGIC +# MAGIC - **A largely stationary metric**, like the fleet telemetry above. On real server telemetry that +# MAGIC arrives already normalised, the same transform measured *worse* than leaving it off. +# MAGIC - **A short training window.** A daily shape needs several complete days to be identifiable at all. +# MAGIC DQX fits one only where the window supports it, and logs the period it skipped and why. +# MAGIC - **With `profile="tabular"`, keep other datetime columns out of `columns`.** Calendar features on +# MAGIC top of the residual measured worse on every anomaly shape tested. +# MAGIC +# MAGIC It is also **not a forecaster**. It models the level expected *at* a time; it does not predict the +# MAGIC next value, and it never reads the previous row — which is what keeps scoring valid on a stream. +# MAGIC +# MAGIC `is_stale_baseline` marks rows past the window the expectation was fitted on. The score is still +# MAGIC produced, because near the boundary it is still accurate; treat the flag as a signal to retrain. + +# COMMAND ---------- + # MAGIC %md # MAGIC --- # MAGIC diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 799361f44..63611da34 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -345,6 +345,82 @@ It is not flagged as a violation. Neither categorical encoder can represent an u If an unrecognised group value is itself something you want to fail on, that is a membership question rather than an anomaly one: use `foreign_key` or `is_in_list` on the baseline column against your set of known values. +## Comparing against time + + + + + +Some values are only wrong *for when they happened*. A metric that has grown steadily for a year sits +outside the range it trained on, so ordinary rows start looking anomalous; and a value that is perfectly +normal against the whole year can be badly wrong for where the trend had actually got to. Comparing +against the whole table cannot see either. + +Pass `baseline_over_time` to give DQX a time axis: + +```python +anomaly_engine.train( + df, + model_name="catalog.schema.revenue_model", + registry_table="catalog.schema.dqx_anomaly_models", + columns=["revenue", "orders"], + baseline_over_time="event_ts", +) +``` + +DQX fits each metric's expected level as a function of time, persists it with the model, and gives the +detector the *difference* between the observed value and that expectation. Because the expectation is a +function rather than a lookup table, it extends to timestamps the training window never contained. + +It is the third of three independent questions, and they compose: + +| Question | Argument | +|---|---| +| Is this value unusual on its own, or in combination? | `profile` | +| Is it unusual for its own group? | `baseline_by` | +| Is it unusual for its own point in time? | `baseline_over_time` | + +Set both grouping and time and the expectation is fitted on the group-relative value, so a table whose +groups trend at different rates still needs only one model. + +### When to use it, and when not to + +The parameter is not free, so this table is worth reading before reaching for it. + +| Situation | Recommendation | +|---|---| +| The metric carries a real trend, or a genuine daily or weekly shape | Use it | +| The metric is largely stationary | Leave it off. Subtracting a fitted expectation from data with no temporal structure removes signal and adds the fit's own error | +| You want to catch a value that is wrong *for a Saturday* | Use it, and keep the timestamp out of `columns` | +| Using `profile="tabular"` | Exclude other datetime columns from `columns`; calendar features on top of the residual measured worse on every anomaly shape tested | +| The training window is short | A seasonal term needs several complete cycles to be identifiable. DQX fits one only where the window supports it, and logs the period it skipped and why | + +DQX will not turn this on for you. Whether a metric's own history is worth comparing against is a +judgement about the data, and it cannot be verified without labelled anomalies. What DQX does instead is +tell you when the training window shows almost no structure over time, so you learn the parameter is +unlikely to help before you rely on it. + +### It is not a forecaster + +`baseline_over_time` models the level expected *at* a time. It does not predict the next value, and it +never looks at the previous row: every row is judged from its own timestamp alone, which is what keeps +scoring valid on a streaming DataFrame. A sudden jump that lands exactly where the trend expected is not +an anomaly to this feature. + +### Knowing when the model has run out of evidence + +A fitted expectation extrapolates, but its accuracy decays the further past the training window you go. +Rows beyond that window carry `is_stale_baseline`, and `stale_baseline_horizon` records where the +evidence ran out: + +```python +result.filter(F.col("_dq_info")[0].anomaly.is_stale_baseline).select("_dq_info") +``` + +The score is still produced, unlike the unseen-group case above. Near the boundary it is still accurate, +so nulling it would throw away a usable verdict; far out it is not, and the flag is how you tell the +difference. Treat it as a signal to retrain rather than as a violation. + ## Choosing a profile @@ -437,7 +513,7 @@ If you did, here is the whole migration. - **Replace `segment_by` with `baseline_by`.** They are not the same. `segment_by` trained one model per group; `baseline_by` judges each metric against its own group's baseline on a single model. Your scores will change. `AnomalyParams.max_segment_models` is gone too, since there is only ever one model now. - **Retrain your models.** A model trained before this release raises an error at scoring time until you retrain it, rather than scoring against a feature list that no longer matches. - **Your registry table is handled for you.** Retraining writes the new schema into your existing table in place. The `segmentation` column becomes `grouping`, and a table you never retrain into keeps the old column and will not be read. -- **`_dq_info[].anomaly` changes shape.** The `segment` field is gone (it was always null once per-group models were), and `is_new_baseline` and `new_baseline_key` are added. Named-field queries keep working, but appending to a table that already holds `_dq_info` needs `mergeSchema`. +- **`_dq_info[].anomaly` changes shape.** The `segment` field is gone (it was always null once per-group models were), and `is_new_baseline`, `new_baseline_key`, `is_stale_baseline` and `stale_baseline_horizon` are added. Named-field queries keep working, but appending to a table that already holds `_dq_info` needs `mergeSchema`. - **Auto-discovered groupings may score differently** even with no configuration change, because a discovered grouping is now used as `baseline_by` and the policy that picks it is finer than the segmented one it replaced. ## How it works under the hood @@ -564,18 +640,19 @@ Use row anomaly detection when you want to catch unusual combinations across col Yes, and they are worth knowing before you rely on it. -**Trend.** DQX learns what normal looks like from a training window. A metric that grows steadily -eventually sits outside that window, and ordinary rows start being flagged. Give the model a quantity -that does not trend — `orders_per_customer` rather than `daily_orders`, `revenue_per_order` rather than -`cumulative_revenue` — which stays comparable as the business grows and is usually what you wanted to -watch anyway. Where the level itself matters, retrain on a schedule; a gradual trend inflates the spread -of the window it is measured against, so `drift_threshold` will not reliably warn you about this -particular case. +**Trend, unless you ask for it.** DQX learns what normal looks like from a training window, so a metric +that grows steadily eventually sits outside that window and ordinary rows start being flagged. Pass +[`baseline_over_time`](#comparing-against-time) and the trend is fitted and subtracted, which also catches +the opposite case: a value that is ordinary against the whole training range and wrong for where the trend +had got to. Without it, model a quantity that does not trend — `orders_per_customer` rather than +`daily_orders` — and note that `drift_threshold` will not reliably warn you here, because a gradual trend +inflates the very spread it is measured against. **Cycles other than hourly, daily, weekly, or monthly.** Those four are handled automatically from a -datetime column. A six-week promotional cycle, or a fiscal quarter that does not align to calendar -months, is not — pass the cycle as a column and use `baseline_by`, which then judges each row against -its own phase. +datetime column, and `baseline_over_time` fits daily and weekly shapes where the window holds enough +complete cycles to identify one. A six-week promotional cycle, or a fiscal quarter that does not align to +calendar months, is not — pass the cycle as a column and use `baseline_by`, which then judges each row +against its own phase. **Forecasting.** DQX judges rows against learned normal. It does not predict the next value and compare. diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 6863fad55..bd109afd7 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -3497,6 +3497,7 @@ Required arguments: `df`, `model_name`, `registry_table`. Optional parameters: |-----------|------|---------|-------------| | `columns` | list[str] | None | Columns to use for row anomaly detection (auto-discovered if omitted) | | `baseline_by` | list[str] | None | Columns identifying the group a row belongs to, so each metric is judged against its own group's baseline rather than against the whole table. One model whatever the group count. Auto-discovered when both `columns` and `baseline_by` are omitted; pass `baseline_by=[]` to suppress discovery and compare against the whole table. When you name `columns` but not `baseline_by`, the comparison stays pooled and a warning names the grouping to pass if the data looks grouped. | +| `baseline_over_time` | str | None | A timestamp or date column each metric is judged *along*, so a value is compared with what its own history says to expect at that point in time. Composes with `baseline_by`: with both set the expectation is fitted on the group-relative value, so one model still covers every group. Independent of `profile`. Never auto-discovered, and DQX warns rather than acting when the training window shows little structure over time. Not a forecaster, and it never reads the previous row. The named column must not also appear in `columns`. See [Comparing against time](/docs/guide/row_anomaly_detection#comparing-against-time). | | `profile` | str | `"tabular"` | Which detector to train. `"tabular"` is Isolation Forest — the behaviour that predates this option. `"timeseries"` selects a correlation-aware detector for multivariate metrics whose anomalies are broken *correlations* rather than extreme single values; it needs no timestamp column and trains a single model rather than an ensemble. There is no automatic option: DQX never changes the algorithm on your behalf, because the choice cannot be verified without labels. See [Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile). | | `exclude_columns` | list[str] | None | Columns to exclude from training (e.g., IDs, labels, ground truth) | | `expected_anomaly_rate` | float | 0.02 | Expected fraction of anomalies in your data (0.02 = 2%). Sets model contamination parameter. | diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 09f2e81fd..5e012d7b9 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -93,10 +93,10 @@ class SparkFeatureMetadata: # before these existed keeps a byte-identical engineered_feature_names and scores exactly as it did. baseline_over_time: str = "" # The time column each metric is judged along temporal_basis: dict[str, Any] = field(default_factory=dict) # TemporalBasis.to_dict() - temporal_coefficients: dict[str, list[float]] = field(default_factory=dict) # metric -> [intercept, *coefs] + temporal_coefficients: dict[str, list[float]] = field(default_factory=dict) # metric to its coefficients # Training window bounds in epoch seconds, for the staleness horizon. A fitted basis extrapolates to # any t, but accuracy decays with distance, so scoring needs to know where the evidence ran out. - temporal_window: dict[str, float] = field(default_factory=dict) # {"t_min": ..., "t_max": ...} + temporal_window: dict[str, float] = field(default_factory=dict) # keys t_min and t_max, epoch seconds def to_json(self) -> str: """Serialize to JSON for storage. @@ -139,8 +139,8 @@ class TemporalState: """ basis: dict[str, Any] = field(default_factory=dict) # TemporalBasis.to_dict() - coefficients: dict[str, list[float]] = field(default_factory=dict) # metric -> [intercept, *coefs] - window: dict[str, float] = field(default_factory=dict) # {"t_min": ..., "t_max": ...} + coefficients: dict[str, list[float]] = field(default_factory=dict) # metric to its coefficients + window: dict[str, float] = field(default_factory=dict) # keys t_min and t_max, epoch seconds def _spark_type_for_category(category: str) -> T.DataType: From 40e198c11356b42a0e82a8d96c900f1a7edcdac7 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 22:24:09 +0100 Subject: [PATCH 066/107] Publish the temporal baseline's quality in the benchmark report Adds a baseline_over_time configuration to the anomaly perf group, so the generated benchmarks page carries its detection quality and training cost beside the other three. The fixture is the case the parameter exists for: metrics rolled back together so every value stays inside the training range and every correlation stays intact, leaving only the position relative to each metric's own history wrong. Hourly over 45 days, so the six-cycle guard admits a daily seasonal term rather than falling back to trend only. Every fixture in that group is still generated in-repo from a fixed seed, so nothing is downloaded and no third-party dataset is redistributed. --- tests/perf/test_anomaly_benchmark.py | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/perf/test_anomaly_benchmark.py b/tests/perf/test_anomaly_benchmark.py index 43f6ae6a2..ec5a07b6d 100644 --- a/tests/perf/test_anomaly_benchmark.py +++ b/tests/perf/test_anomaly_benchmark.py @@ -20,8 +20,10 @@ (``--benchmark-compare-fail=mean:25%``), which is global and would flake on control-plane variance. """ +import datetime from typing import cast +import numpy as np import pandas as pd import pytest from pyspark.sql import DataFrame, SparkSession @@ -33,6 +35,7 @@ from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRegistry +from databricks.labs.dqx.config import AnomalyParams from tests.constants import TEST_CATALOG from tests.integration_anomaly.synthetic_generators import ( generate_correlated_multivariate_data, @@ -41,6 +44,10 @@ generate_overlapping_gaussian_data, ) +# Hourly cadence over 45 days, so the temporal fixture below clears the six-cycle guard for a daily +# seasonal term rather than getting trend only. +TEMPORAL_HOURS = 24 * 45 + pytestmark = pytest.mark.anomaly BENCHMARK_GROUP = "anomaly_synthetic" @@ -292,6 +299,65 @@ def test_benchmark_anomaly_score_conditioned(benchmark, request, spark, ws, make benchmark.extra_info[name] = value +@pytest.mark.benchmark(group=BENCHMARK_GROUP) +def test_benchmark_anomaly_score_temporal_baseline(benchmark, request, spark, ws, make_schema, make_random): + """Score with ``baseline_over_time`` on a trending metric, and record the quality. + + The fixture is the case the parameter exists for: values that are ordinary against the whole training + range and wrong for where the trend had got to. Rolling the metrics back together leaves every value + inside the range and every correlation intact, so neither a range rule nor a correlation-aware + detector has anything to see -- only the position relative to each metric's own history is wrong. + """ + rng = np.random.default_rng(SEED) + hours = np.arange(TEMPORAL_HOURS) + slopes = rng.uniform(0.02, 0.06, 4) + start = datetime.datetime(2025, 1, 6) + + def frame(with_fault: bool): + values = 100.0 + np.outer(hours, slopes) + rng.normal(0, 1.5, size=(hours.size, 4)) + labels = np.zeros(hours.size) + if with_fault: + bad = rng.choice(hours.size, size=int(0.05 * hours.size), replace=False) + values[bad] -= slopes * 600 + labels[bad] = 1.0 + rows = [ + (start + datetime.timedelta(hours=int(h)), *[float(v) for v in values[i]], float(labels[i])) + for i, h in enumerate(hours) + ] + return spark.createDataFrame( + rows, "event_ts timestamp, m0 double, m1 double, m2 double, m3 double, is_anomaly double" + ) + + train_df, test_df = frame(False), frame(True) + columns = ["m0", "m1", "m2", "m3"] + model_name, registry_table = _new_model_names(make_schema, make_random) + request.addfinalizer(lambda: _cleanup_anomaly_mlflow(model_name, registry_table, spark)) + + engine = AnomalyEngine(workspace_client=ws, spark=spark) + engine.train( + df=train_df, + model_name=model_name, + registry_table=registry_table, + columns=columns, + baseline_by=[], + baseline_over_time="event_ts", + params=AnomalyParams(sample_fraction=1.0), + ) + + scored = _score_and_record(benchmark, model_name, registry_table, test_df) + + _record_provenance( + benchmark, + train_df.count(), + test_df.count(), + dataset="synthetic: trending metrics, baseline_over_time", + n_features=len(columns), + anomaly_frac=_anomaly_rate(test_df), + ) + for name, value in _detection_quality(scored).items(): + benchmark.extra_info[name] = value + + def _detection_quality(scored: DataFrame) -> dict[str, float]: """Compute indicative detection quality from a scored frame. From f03f76c53f62d6c66d63ce2ec079151b71b2d0d4 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 2 Sep 2026 22:41:27 +0100 Subject: [PATCH 067/107] Mark staleness before scoring, not after Found by running the demo on a workspace: reading_ts cannot be resolved. By the time the scored frame exists it has been projected down to engineered features plus internals, so the caller's time column is gone. That projection is correct -- a time axis is not a feature -- but it leaves mark_stale_baselines nothing to compare against. The unseen-group marking survives the same projection only because the baseline key column is deliberately preserved through it. Marking on the filtered input instead means the two flag columns ride through feature engineering as passthroughs, the way __dqx_row_id__ does, and are present on the scored frame where add_info_column reads them. A unit test could not have caught this: the failure is in the interaction between the projection and the scoring order, and only a real frame going through the real pipeline puts those together. --- .../labs/dqx/anomaly/scoring_run.py | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index c71cccefa..b7705da2d 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -135,6 +135,21 @@ def score_global_model( check_model_staleness(record, config.model_name) df_filtered = apply_row_filter(df, config.row_filter) + # Marked here rather than on the scored frame: by then the projection has dropped the caller's time + # column, which is correct (a time axis is not a feature) but leaves nothing to compare against. These + # two flags then ride through feature engineering as passthrough columns, the way the row id does. + # + # Extrapolation is reported and never corrected. Unlike an unseen group, the score is still produced: + # measured, it is still good one window past the boundary, so nulling it would discard a usable verdict. + stale_col = "__dqx_is_stale_baseline" + horizon_col = "__dqx_stale_horizon" + df_filtered = mark_stale_baselines( + df_filtered, + trained_baseline_over_time or "", + trained_metadata.temporal_window if trained_metadata else {}, + stale_col=stale_col, + horizon_col=horizon_col, + ) drift_result = check_and_warn_drift( df_filtered, config.columns, @@ -225,18 +240,6 @@ def score_global_model( group_key_col=group_key_col, ) - # Extrapolation is reported, not corrected: unlike an unseen group the score is still produced, and - # measured it is still good one window past the boundary. So this marks and never nulls. - stale_col = "__dqx_is_stale_baseline" - horizon_col = "__dqx_stale_horizon" - scored_df = mark_stale_baselines( - scored_df, - parsed_metadata.baseline_over_time, - parsed_metadata.temporal_window, - stale_col=stale_col, - horizon_col=horizon_col, - ) - scored_df = _add_severity(scored_df, config, parsed_metadata, group_quantile_points, global_quantile_points) scored_df = null_out_unseen_baseline_scores( From c11d570e0197ccdb0ca8c0028254ee27d29353a1 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 3 Sep 2026 10:02:52 +0100 Subject: [PATCH 068/107] Keep the time column through the scoring select The demo failed on a workspace with `reading_ts cannot be resolved`, and the CI e2e job failed the same way. Scoring narrows the frame to features, grouping columns and internals before replaying the transform, and the time axis was not on that list, so the persisted expectation had nothing to evaluate against. Grouping columns were already carried for the same reason; this extends the same exemption to the axis, which is likewise read and never scored. The integration tests missed it because every one of them called `apply_feature_engineering` directly. That is where the maths lives, but both defects found on the workspace were in the plumbing between the transform and the check, so two tests now go through `has_no_row_anomalies` end to end: one asserting a temporal model scores at all, one asserting both staleness fields reach the info column. Also corrects two of those tests, which the serverless anomaly job caught and which were wrong about the transform's contract rather than finding anything in it. One named the time column as a feature as well as the axis, a combination `validate_baseline_over_time` refuses outright, and feature engineering had consumed the raw column by the time the temporal block looked for it. The other grouped the engineered frame by its grouping column, which the projection drops deliberately. Co-authored-by: Isaac --- .../labs/dqx/anomaly/feature_prep.py | 10 +- tests/integration_anomaly/conftest.py | 6 + .../test_anomaly_temporal_features.py | 126 +++++++++++++++++- 3 files changed, 133 insertions(+), 9 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/feature_prep.py b/src/databricks/labs/dqx/anomaly/feature_prep.py index 4c6d1b45a..f5efd4de3 100644 --- a/src/databricks/labs/dqx/anomaly/feature_prep.py +++ b/src/databricks/labs/dqx/anomaly/feature_prep.py @@ -44,10 +44,14 @@ def apply_feature_engineering_for_scoring( "Ensure the anomaly check is applied to the same DataFrame instance." ) - # Group columns must survive this select or the group-relative transform has no basis to - # compute against; feature engineering drops them again before the model sees anything. + # Group columns and the time axis must survive this select or the group-relative and temporal + # transforms have no basis to compute against; feature engineering drops them again before the + # model sees anything. The time column is only ever read, never scored: it is not a feature. + time_cols = [feature_metadata.baseline_over_time] if feature_metadata.baseline_over_time else [] cols_to_select = list( - dict.fromkeys([*feature_cols, *feature_metadata.baseline_by, *merge_columns, *(passthrough_columns or [])]) + dict.fromkeys( + [*feature_cols, *feature_metadata.baseline_by, *time_cols, *merge_columns, *(passthrough_columns or [])] + ) ) engineered_df, _ = apply_feature_engineering_from_metadata( diff --git a/tests/integration_anomaly/conftest.py b/tests/integration_anomaly/conftest.py index 382d25df1..f307e2fea 100644 --- a/tests/integration_anomaly/conftest.py +++ b/tests/integration_anomaly/conftest.py @@ -190,6 +190,7 @@ def train_model_with_params( params: AnomalyParams, expected_anomaly_rate: float = 0.02, baseline_by: list[str] | None = None, + baseline_over_time: str | None = None, profile: str | None = None, ) -> str: """Train a model with internal params (test-only).""" @@ -199,6 +200,7 @@ def train_model_with_params( model_name=model_name, registry_table=registry_table, baseline_by=baseline_by, + baseline_over_time=baseline_over_time, params=params, expected_anomaly_rate=expected_anomaly_rate, profile=profile, @@ -852,6 +854,7 @@ def _train( catalog: str = TEST_CATALOG, schema: str | None = None, baseline_by: list[str] | None = None, + baseline_over_time: str | None = None, train_schema: str | None = None, profile: str | None = None, ): @@ -865,6 +868,7 @@ def _train( train_data (list[tuple] | None): Custom training data tuples (overrides train_size) params (AnomalyParams | None): Internal training params (test-only) baseline_by (list[str] | None): Group columns for baseline-conditioned models + baseline_over_time (str | None): Time column each metric's expected level is fitted along profile (str | None): Which detector to train ("tabular" / "timeseries"); None means the default, so existing callers keep the IsolationForest path untouched. train_schema (str | None): Explicit DDL for train_data (needed when group columns @@ -908,6 +912,7 @@ def _train( model_name=model_name, registry_table=registry_table, baseline_by=baseline_by, + baseline_over_time=baseline_over_time, profile=profile, ) else: @@ -919,6 +924,7 @@ def _train( columns=columns, params=params, baseline_by=baseline_by, + baseline_over_time=baseline_over_time, profile=profile, ) diff --git a/tests/integration_anomaly/test_anomaly_temporal_features.py b/tests/integration_anomaly/test_anomaly_temporal_features.py index 71396beb2..5bd99da6f 100644 --- a/tests/integration_anomaly/test_anomaly_temporal_features.py +++ b/tests/integration_anomaly/test_anomaly_temporal_features.py @@ -10,6 +10,7 @@ import datetime import numpy as np +import pyspark.sql.functions as F import pytest from pyspark.sql import DataFrame, Row, SparkSession from pyspark.sql import types as T @@ -23,6 +24,9 @@ apply_feature_engineering, apply_feature_engineering_from_metadata, ) +from databricks.labs.dqx.config import AnomalyParams +from databricks.labs.dqx.engine import DQEngine +from tests.integration_anomaly.conftest import create_anomaly_check_rule HOUR_SECONDS = 3600 START = datetime.datetime(2025, 1, 6, 0, 0, 0) # a Monday, so weekday/weekend features are meaningful @@ -102,9 +106,10 @@ def test_a_time_relative_feature_is_appended_per_metric_and_appended_last(spark: """ df = _trending_frame(spark) - _, metadata = apply_feature_engineering( - df, [_timestamp("event_ts"), _numeric("revenue")], baseline_over_time="event_ts" - ) + # The axis is named only as `baseline_over_time`, never also as a feature: feature engineering + # expands a datetime feature into cyclical columns and drops the raw one, and + # `validate_baseline_over_time` refuses the combination for exactly that reason. + _, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") names = metadata.engineered_feature_names assert f"revenue{TEMPORAL_RELATIVE_SUFFIX}" in names @@ -219,10 +224,14 @@ def test_the_fit_runs_on_the_group_relative_value_when_grouping_is_in_play(spark ( START + datetime.timedelta(hours=i), f"g{group}", + f"g{group}", float(level + slope * i + rng.normal(0, 1.5)), ) ) - df = spark.createDataFrame(rows, "event_ts timestamp, grp string, revenue double") + # `label` duplicates `grp` and is not part of the group key, so it rides through the projection as an + # incidental column. The grouping columns themselves are dropped by design, being a comparison basis + # rather than a feature, which leaves nothing on the engineered frame to group the assertion by. + df = spark.createDataFrame(rows, "event_ts timestamp, grp string, label string, revenue double") engineered, metadata = apply_feature_engineering( df, [_numeric("revenue")], baseline_by=["grp"], baseline_over_time="event_ts" @@ -237,8 +246,8 @@ def test_the_fit_runs_on_the_group_relative_value_when_grouping_is_in_play(spark # The composed residual must be stationary across both groups despite their different slopes. A fit # on raw values could not be, since one pooled line cannot follow two different slopes. spreads = { - row["grp"]: row["sd"] - for row in engineered.groupBy("grp") + row["label"]: row["sd"] + for row in engineered.groupBy("label") .agg({temporal: "stddev"}) .withColumnRenamed(f"stddev({temporal})", "sd") .collect() @@ -375,3 +384,108 @@ def test_a_model_with_no_temporal_baseline_gains_no_staleness_columns(spark: Spa marked = mark_stale_baselines(df, "", {}, stale_col="stale", horizon_col="horizon") assert set(marked.columns) == before + + +# ============================================================================ +# The plumbing between the transform and the check +# ============================================================================ +# +# Everything above exercises `apply_feature_engineering` directly, which is where the maths lives. Two +# defects escaped that and were only found by running a demo on a workspace, both in the layer that +# prepares a frame for scoring rather than in the transform: the scoring select dropped the time column +# before the transform could read it, and staleness was marked on the already-projected frame. So the +# tests below go through the public check, which is the only path that covers that layer. + + +def test_the_public_check_scores_a_temporal_model_end_to_end(ws, spark: SparkSession, quick_model_factory): + """Train and score through `has_no_row_anomalies`, the way a caller actually does. + + The scoring path narrows the frame to features, grouping columns and internals before replaying the + transform. The time column has to survive that select, or the fitted expectation has no axis to + evaluate against and scoring fails outright on an unresolved column. + """ + train_data = [ + (START + datetime.timedelta(hours=i), float(100.0 + 0.05 * i), 2.0 + 0.001 * i) for i in range(24 * 30) + ] + + model_name, registry_table, _ = quick_model_factory( + spark, + columns=["amount", "quantity"], + train_data=train_data, + train_schema="event_ts timestamp, amount double, quantity double", + baseline_over_time="event_ts", + baseline_by=[], + params=AnomalyParams(sample_fraction=1.0), + ) + + # One row held at the level the metrics had 400 hours earlier: inside the training range on every + # column, and wrong only for where the trend had got to. + late = START + datetime.timedelta(hours=24 * 30 - 1) + test_df = spark.createDataFrame( + [(late, 100.0 + 0.05 * 319, 2.0 + 0.001 * 319), (late, 100.0 + 0.05 * 719, 2.719)], + "event_ts timestamp, amount double, quantity double", + ) + + result_df = DQEngine(ws, spark).apply_checks( + test_df, + [create_anomaly_check_rule(model_name=model_name, registry_table=registry_table, threshold=50.0)], + ) + + anomaly = F.col("_dq_info")[0].getField("anomaly") + scored = result_df.select( + "amount", anomaly.getField("score").alias("score"), anomaly.getField("is_anomaly").alias("flagged") + ).collect() + + assert len(scored) == 2 + assert all(row["score"] is not None for row in scored), "every row must get a number" + rolled_back, current = sorted(scored, key=lambda r: r["amount"]) + assert rolled_back["score"] > current["score"], "the stale level must score worse than the current one" + + +def test_the_public_check_reports_staleness_in_the_info_column(ws, spark: SparkSession, quick_model_factory): + """The two staleness fields must reach the info column, which means surviving feature engineering. + + They are computed from the caller's own time column, so they have to be attached before the frame is + projected down to features -- the projection drops that column, correctly, since a time axis is not a + feature. + """ + train_data = [(START + datetime.timedelta(hours=i), float(100.0 + 0.05 * i), 2.0) for i in range(24 * 30)] + + model_name, registry_table, _ = quick_model_factory( + spark, + columns=["amount", "quantity"], + train_data=train_data, + train_schema="event_ts timestamp, amount double, quantity double", + baseline_over_time="event_ts", + baseline_by=[], + params=AnomalyParams(sample_fraction=1.0), + ) + + inside = START + datetime.timedelta(hours=24 * 15) + far_past = START + datetime.timedelta(days=300) + test_df = spark.createDataFrame( + [(inside, 136.0, 2.0), (far_past, 136.0, 2.0)], + "event_ts timestamp, amount double, quantity double", + ) + + result_df = DQEngine(ws, spark).apply_checks( + test_df, + [create_anomaly_check_rule(model_name=model_name, registry_table=registry_table, threshold=50.0)], + ) + + anomaly = F.col("_dq_info")[0].getField("anomaly") + by_ts = { + row["event_ts"]: row + for row in result_df.select( + "event_ts", + anomaly.getField("score").alias("score"), + anomaly.getField("is_stale_baseline").alias("stale"), + anomaly.getField("stale_baseline_horizon").alias("horizon"), + ).collect() + } + + assert by_ts[inside]["stale"] is False + assert by_ts[far_past]["stale"] is True + assert by_ts[far_past]["horizon"], "a stale row must say what window it is past" + # Flagged, never nulled: measured, the score one window out is still usable. + assert by_ts[far_past]["score"] is not None From 14935d0dfc57b9063f89ff844ebefef7b0d2789f Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 3 Sep 2026 10:14:59 +0100 Subject: [PATCH 069/107] Pin the scoring select rule without needing a workspace The rule that broke -- which columns the scoring transform must be handed -- was only reachable through Spark, so nothing caught either omission until a demo ran. It is a pure rule, names in and names out, so it moves out of the Spark-bound function into `scoring_input_columns` and gets seven unit tests: one per role a column can play, one for the dedup Spark's select requires, and one asserting an unset time axis adds nothing. `validate_baseline_over_time` moves into unit coverage the same way. It reads only `df.schema.fields` and `df.columns`, so it belongs beside the group-column type contract in the no-Spark invariants file rather than being exercised only on a workspace. That includes the combination that reached the demo run: a column named as both the axis and a feature. Co-authored-by: Isaac --- .../labs/dqx/anomaly/feature_prep.py | 40 +++++++--- .../unit/test_anomaly_baseline_invariants.py | 50 +++++++++++- .../test_anomaly_scoring_input_columns.py | 79 +++++++++++++++++++ 3 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 tests/unit/test_anomaly_scoring_input_columns.py diff --git a/src/databricks/labs/dqx/anomaly/feature_prep.py b/src/databricks/labs/dqx/anomaly/feature_prep.py index f5efd4de3..cfc8884d3 100644 --- a/src/databricks/labs/dqx/anomaly/feature_prep.py +++ b/src/databricks/labs/dqx/anomaly/feature_prep.py @@ -1,5 +1,6 @@ """Prepare feature metadata and apply feature engineering for anomaly scoring.""" +import collections.abc import uuid import pyspark.sql.functions as F @@ -21,6 +22,35 @@ def prepare_feature_metadata(feature_metadata_json: str) -> tuple[list[ColumnTyp return column_infos, feature_metadata +def scoring_input_columns( + feature_cols: collections.abc.Iterable[str], + merge_columns: collections.abc.Iterable[str], + feature_metadata: SparkFeatureMetadata, + passthrough_columns: collections.abc.Iterable[str] | None = None, +) -> list[str]: + """The columns the scoring transform must be handed, in the order it expects them. + + A comparison basis is not a feature, but it is still *read*: the grouping columns say what a metric is + compared against and the time column is the axis it is measured along, so both have to survive the + narrowing even though feature engineering drops them again before the model sees anything. Leaving + either off does not degrade the score, it fails the query outright on an unresolved column. + + Deduplicated while preserving order, since a column may legitimately appear in more than one role. + """ + time_cols = [feature_metadata.baseline_over_time] if feature_metadata.baseline_over_time else [] + return list( + dict.fromkeys( + [ + *feature_cols, + *feature_metadata.baseline_by, + *time_cols, + *merge_columns, + *(passthrough_columns or []), + ] + ) + ) + + def apply_feature_engineering_for_scoring( df: DataFrame, feature_cols: list[str], @@ -44,15 +74,7 @@ def apply_feature_engineering_for_scoring( "Ensure the anomaly check is applied to the same DataFrame instance." ) - # Group columns and the time axis must survive this select or the group-relative and temporal - # transforms have no basis to compute against; feature engineering drops them again before the - # model sees anything. The time column is only ever read, never scored: it is not a feature. - time_cols = [feature_metadata.baseline_over_time] if feature_metadata.baseline_over_time else [] - cols_to_select = list( - dict.fromkeys( - [*feature_cols, *feature_metadata.baseline_by, *time_cols, *merge_columns, *(passthrough_columns or [])] - ) - ) + cols_to_select = scoring_input_columns(feature_cols, merge_columns, feature_metadata, passthrough_columns) engineered_df, _ = apply_feature_engineering_from_metadata( df.select(*cols_to_select), feature_metadata, column_infos=column_infos diff --git a/tests/unit/test_anomaly_baseline_invariants.py b/tests/unit/test_anomaly_baseline_invariants.py index 0ce8a6ef7..597e30c6b 100644 --- a/tests/unit/test_anomaly_baseline_invariants.py +++ b/tests/unit/test_anomaly_baseline_invariants.py @@ -4,6 +4,10 @@ Python may be a baseline column, because the group key is built in both places and a divergence silently misses every lookup (see test_anomaly_group_key for the key itself, and test_anomaly_group_relative_features for the live Python/Spark parity). +- The time-column contract: the axis a metric is measured along must exist, must be a time type, and + must not double as a feature. The last of those is not cosmetic -- feature engineering expands a + datetime feature into cyclical columns and drops the raw one, so a column serving as both would leave + the temporal fit with no axis to read. - The feature-engineering contract: every scorer must run features through the shared feature_prep entry points before scoring, or a model would score on a different feature list than it trained on. """ @@ -16,15 +20,15 @@ from pyspark.sql import types as T from databricks.labs.dqx.anomaly import ensemble_scorer, single_model_scorer -from databricks.labs.dqx.anomaly.validation import validate_baseline_columns +from databricks.labs.dqx.anomaly.validation import validate_baseline_columns, validate_baseline_over_time from databricks.labs.dqx.errors import InvalidParameterError def _fake_df(schema: dict[str, T.DataType]) -> DataFrame: - """A DataFrame stand-in exposing only the schema/columns validate_baseline_columns reads. + """A DataFrame stand-in exposing only the schema/columns the validators read. - Uses create_autospec rather than a real session: the function under test only inspects - ``df.schema.fields`` and ``df.columns``, so no Spark is needed to exercise the type contract. + Uses create_autospec rather than a real session: the functions under test only inspect + ``df.schema.fields`` and ``df.columns``, so no Spark is needed to exercise their contracts. """ df = create_autospec(DataFrame, instance=True) df.schema = T.StructType([T.StructField(name, dtype, True) for name, dtype in schema.items()]) @@ -57,6 +61,44 @@ def test_validate_baseline_columns_rejects_types_spark_and_python_format_differe validate_baseline_columns(_fake_df({"g": dtype}), ["g"], []) +@pytest.mark.parametrize("dtype", [T.TimestampType(), T.TimestampNTZType(), T.DateType()]) +def test_validate_baseline_over_time_accepts_time_types(dtype: T.DataType): + """Elapsed seconds can be read from any of these, which is all the fit needs.""" + validate_baseline_over_time(_fake_df({"ts": dtype, "revenue": T.DoubleType()}), "ts", ["revenue"]) + + +@pytest.mark.parametrize("dtype", [T.StringType(), T.LongType(), T.DoubleType()]) +def test_validate_baseline_over_time_rejects_non_time_types(dtype: T.DataType): + """A string or an epoch integer is refused rather than parsed. + + Coercing would guess at a unit and a format, and guessing wrong shifts every fitted expectation by a + constant nobody would see. + """ + with pytest.raises(InvalidParameterError, match="not a time type"): + validate_baseline_over_time(_fake_df({"ts": dtype, "revenue": T.DoubleType()}), "ts", ["revenue"]) + + +def test_validate_baseline_over_time_rejects_a_column_that_is_also_a_feature(): + """The combination that reached a demo run before this refusal existed. + + Feature engineering expands a datetime *feature* into cyclical columns and drops the raw one, so the + temporal block would then look for an axis that had already been consumed. + """ + with pytest.raises(InvalidParameterError, match="both as a feature and as baseline_over_time"): + validate_baseline_over_time(_fake_df({"ts": T.TimestampType()}), "ts", ["ts"]) + + +def test_validate_baseline_over_time_rejects_a_missing_column(): + with pytest.raises(InvalidParameterError, match="not found in DataFrame"): + validate_baseline_over_time(_fake_df({"revenue": T.DoubleType()}), "ts", ["revenue"]) + + +@pytest.mark.parametrize("value", [None, ""]) +def test_validate_baseline_over_time_is_silent_when_unset(value: str | None): + """The inertness path: nothing declared, nothing checked, nothing raised.""" + validate_baseline_over_time(_fake_df({"revenue": T.DoubleType()}), value, ["revenue"]) + + def test_every_scorer_applies_feature_engineering_before_scoring(): """Structural guard: every scoring module must route features through the shared feature_prep entry points (which call apply_feature_engineering_from_metadata) before scoring. diff --git a/tests/unit/test_anomaly_scoring_input_columns.py b/tests/unit/test_anomaly_scoring_input_columns.py new file mode 100644 index 000000000..be0711ad9 --- /dev/null +++ b/tests/unit/test_anomaly_scoring_input_columns.py @@ -0,0 +1,79 @@ +"""Unit pins for the columns a scoring run must hand the feature transform (no Spark). + +The frame reaching `apply_feature_engineering_from_metadata` at scoring time is deliberately narrow: a +model trained on four features should not be fed a hundred-column table. But the transform reads more +than it produces, and every comparison basis added to the feature set has had to be added here too. Both +times it was missed, the symptom was the same and only visible on a workspace: an unresolved column at +scoring time, from a query plan that looked fine until Spark tried to resolve it. + +These are cheap because the rule is pure -- names in, names out. +""" + +from databricks.labs.dqx.anomaly.feature_prep import scoring_input_columns +from databricks.labs.dqx.anomaly.transformers import SparkFeatureMetadata + +ROW_ID = "__dqx_row_id__" + + +def _metadata(**kwargs) -> SparkFeatureMetadata: + """Metadata carrying only the fields this rule reads; the rest are structurally required.""" + return SparkFeatureMetadata( + column_infos=[], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=[], + **kwargs, + ) + + +def test_the_plain_case_is_features_plus_the_row_id(): + """Nothing conditioned: no basis to carry, so the list is exactly what the model needs plus the join key.""" + selected = scoring_input_columns(["amount", "quantity"], [ROW_ID], _metadata()) + + assert selected == ["amount", "quantity", ROW_ID] + + +def test_grouping_columns_are_carried_even_though_they_are_not_features(): + """The group-relative transform computes a deviation from the row's own group, so it needs the key.""" + selected = scoring_input_columns(["amount"], [ROW_ID], _metadata(baseline_by=["region", "product"])) + + assert selected == ["amount", "region", "product", ROW_ID] + + +def test_the_time_column_is_carried_even_though_it_is_not_a_feature(): + """The regression this test exists for. + + The fitted expectation is a function of time, so scoring has to evaluate it at the row's own timestamp. + Without the column on this list the query fails to resolve rather than scoring badly. + """ + selected = scoring_input_columns(["amount"], [ROW_ID], _metadata(baseline_over_time="event_ts")) + + assert "event_ts" in selected + + +def test_both_bases_are_carried_together(): + """The two compose: the temporal fit runs on the group-relative value, so both are read.""" + selected = scoring_input_columns( + ["amount"], [ROW_ID], _metadata(baseline_by=["region"], baseline_over_time="event_ts") + ) + + assert selected == ["amount", "region", "event_ts", ROW_ID] + + +def test_an_unset_time_column_adds_nothing(): + """Inertness, the same guarantee the feature list itself carries: an empty basis is byte-identical.""" + assert scoring_input_columns(["amount"], [ROW_ID], _metadata(baseline_over_time="")) == ["amount", ROW_ID] + + +def test_passthrough_columns_come_last(): + """The packed original row is appended, so restoring it after scoring reads a stable position.""" + selected = scoring_input_columns(["amount"], [ROW_ID], _metadata(), passthrough_columns=["__dqx_orig_abc"]) + + assert selected[-1] == "__dqx_orig_abc" + + +def test_a_column_serving_two_roles_is_selected_once(): + """Spark rejects a duplicated name in a select, and a grouping column may also be a merge column.""" + selected = scoring_input_columns(["amount"], ["region"], _metadata(baseline_by=["region"])) + + assert selected == ["amount", "region"] From b418b543493e2f672f4646135906462bf052291b Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 3 Sep 2026 10:40:45 +0100 Subject: [PATCH 070/107] Report detection quality honestly in the demos, and document what the threshold does Running both demos on a workspace and reading every cell turned up four things an exit code hides. The threshold does not deliver the alert budget it promises. Measured, threshold=95 flagged 9.3% of the correlation batch and 13.5% of the temporal one, against true anomaly rates of 4.0% and 5.0%. It is a percentile of training severity, so a batch containing real problems clears the training 95th percentile far more often than 5% of the time. The ranking is not the problem: on the temporal fixture the 24 highest-severity rows were exactly the 24 faults, so at 99 it is 24 alerts with none of them false. Both sections now print a sweep with precision beside the best precision the budget allows, the temporal section chooses 99 from that sweep, and the guide gains a subsection stating the behaviour with the numbers, because "percentile cutoff" reads as a percentile of the data being scored. AI explanations rendered one identical narrative once per row. That is the feature working, since one ai_query call serves a whole group and cost therefore scales with distinct problems rather than rows, but it reads as a bug. Grouped by pattern with a row count, the correlation demo shows six distinct broken relationships and the tabular one five. Training was not reproducible. Two runs on identical data reported "caught 33 of 33" and then "caught 25 of 33". The generators are seeded and DQX seeds both its sample and its train/validation split, but DataFrame.sample draws per partition, so a different partition count on serverless draws a different 1,469 rows out of 6,000. Both demos now train on the whole table, which on tables this small is the right advice regardless: sampling exists so training need not read a billion rows. Co-authored-by: Isaac --- .../dqx_demo_anomaly_tabular_transactions.py | 21 +++-- demos/dqx_demo_anomaly_timeseries_fleet.py | 93 ++++++++++++++++--- .../guide/row_anomaly_detection/index.mdx | 40 ++++++++ 3 files changed, 134 insertions(+), 20 deletions(-) diff --git a/demos/dqx_demo_anomaly_tabular_transactions.py b/demos/dqx_demo_anomaly_tabular_transactions.py index 8c21c3a3a..f49d02ce0 100644 --- a/demos/dqx_demo_anomaly_tabular_transactions.py +++ b/demos/dqx_demo_anomaly_tabular_transactions.py @@ -106,7 +106,7 @@ from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies from databricks.labs.dqx.check_funcs import is_in_range, is_not_null -from databricks.labs.dqx.config import InputConfig, OutputConfig +from databricks.labs.dqx.config import AnomalyParams, InputConfig, OutputConfig from databricks.labs.dqx.engine import DQEngine from databricks.labs.dqx.rule import DQDatasetRule, DQRowRule @@ -329,6 +329,10 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): columns=["amount", "item_count"], baseline_by=["merchant_category"], profile="tabular", + # By default DQX trains on a sample, which is what makes training a table of a billion rows + # affordable. On 6,000 it only adds variance: the sample is seeded, but it is drawn per partition, so + # a different partition count draws different rows and the numbers printed below move between runs. + params=AnomalyParams(sample_fraction=1.0), ) print(f"\n✅ Model trained: {trained}") @@ -420,21 +424,20 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): # COMMAND ---------- # DBTITLE 1,Why each group was flagged, in plain language -print("🤖 AI explanations, one per group of similar anomalies:\n") +# One explanation per *pattern*, not per row: rows driven by the same combination of features share a +# single ai_query call, so the cost scales with how many distinct problems there are rather than with how +# many rows have them. Grouping the display the same way is the only way to see that. +print("🤖 AI explanations. One call per pattern, however many rows share it:\n") display( - flagged.select( - "transaction_id", - "merchant_category", - "amount", - "item_count", + flagged.groupBy( anomaly.getField("ai_explanation").getField("top_features").alias("pattern"), anomaly.getField("ai_explanation").getField("narrative").alias("narrative"), anomaly.getField("ai_explanation").getField("action").alias("action"), ) + .agg(F.count("*").alias("transactions"), F.collect_list("merchant_category")[0].alias("example_category")) .filter(F.col("narrative").isNotNull()) - .orderBy("pattern") - .limit(10) + .orderBy(F.desc("transactions")) ) # COMMAND ---------- diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py index a4ea34b73..e2192962c 100644 --- a/demos/dqx_demo_anomaly_timeseries_fleet.py +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -108,7 +108,7 @@ from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies -from databricks.labs.dqx.config import InputConfig, OutputConfig +from databricks.labs.dqx.config import AnomalyParams, InputConfig, OutputConfig from databricks.labs.dqx.engine import DQEngine from databricks.labs.dqx.rule import DQDatasetRule @@ -133,6 +133,39 @@ print(f"📋 Model registry: {registry_table}") print("✅ Registry reset — ready for this run's model") +# COMMAND ---------- +# DBTITLE 1,A helper for reading detection quality honestly + + +def report_quality(scored_df, label_col: str, severity_col, budget: float, thresholds=(90, 95, 98, 99)): + """Print recall, precision and the best precision the budget allows, at several thresholds. + + Precision alone is unreadable here. Ask for the top 5% of 480 rows and you get 24 alerts; if only 24 + rows are genuinely wrong, no model can do better than 24/24, and if the budget yields 65 alerts the + ceiling is 24/65 = 37% however good the ranking is. So the ceiling is printed beside what was + achieved -- where the two are equal, the ranking is optimal and only the budget is costing you. + """ + total = scored_df.count() + faults = scored_df.filter(F.col(label_col) == 1.0).count() + print(f"🎚️ {total:,} rows, {faults} of them genuinely wrong ({faults / total:.1%}).\n") + print("Threshold | Alerts | Caught | Precision | Best possible | Recall") + print("-" * 68) + for threshold in thresholds: + alerts = scored_df.filter(severity_col >= threshold) + n_alerts = alerts.count() + n_caught = alerts.filter(F.col(label_col) == 1.0).count() + ceiling = min(n_alerts, faults) / n_alerts if n_alerts else 0.0 + precision = n_caught / n_alerts if n_alerts else 0.0 + recall = n_caught / faults if faults else 0.0 + marker = " ← used above" if abs(threshold - budget) < 0.01 else "" + print( + f"{threshold:9} | {n_alerts:6} | {n_caught:3}/{faults:<3} | {precision:8.1%} | " + f"{ceiling:11.1%} | {recall:6.1%}{marker}" + ) + + +print("✅ Helper ready") + # COMMAND ---------- # DBTITLE 1,How the metrics relate to each other @@ -306,6 +339,9 @@ def generate_telemetry(n_rows: int, seed: int, break_correlation: bool = False): columns=METRICS, baseline_by=[], profile="timeseries", + # Whole table rather than the default sample: 4,000 readings is small enough that sampling only makes + # the numbers printed below vary between runs, because the sample is drawn per partition. + params=AnomalyParams(sample_fraction=1.0), ) print(f"\n✅ Model trained: {trained}") @@ -378,7 +414,10 @@ def generate_telemetry(n_rows: int, seed: int, break_correlation: bool = False): # DBTITLE 1,Which relationships broke caught = flagged.filter(F.col("is_incident") == 1.0).count() -print(f"🔝 Caught {caught} of the {injected} incident readings — none of which any range check could see.\n") +n_alerts = flagged.count() +print(f"🔝 Caught {caught} of the {injected} incident readings — none of which any range check could see.") +print(f" That cost {n_alerts} alerts on {total_readings:,} readings, so {caught / n_alerts:.0%} of them were real.") +print(" The next cell sweeps the threshold, which is the honest way to read that number.\n") display( flagged.select( @@ -393,21 +432,37 @@ def generate_telemetry(n_rows: int, seed: int, break_correlation: bool = False): .limit(10) ) +# COMMAND ---------- +# DBTITLE 1,What the alert budget actually buys + +# `threshold=95` means "above the 95th percentile of *training* severity", not "95% likely to be a +# problem". On a batch whose readings are mostly stranger than anything training held, far more than 5% of +# it clears that line, which is why 95 flags well over the 75 rows a 5% budget implies. +report_quality(scored, "is_incident", anomaly.getField("severity_percentile"), budget=95.0) + +print("\n💡 The default is not the right answer here. Moving to 98 keeps most of the recall and throws") +print(" a small fraction of the false alarms, because severity ranks the incident readings well above") +print(" the healthy ones — the default budget was simply set against the training distribution rather") +print(" than this batch. Severity is stored for every row, so this table costs nothing to produce and") +print(" is how you should pick the number on your own data.") + # COMMAND ---------- # DBTITLE 1,Why each group was flagged, in plain language -print("🤖 AI explanations, one per group of similar anomalies:\n") +# One explanation per *pattern*, not per row: readings driven by the same broken relationship share a +# single ai_query call, so cost scales with how many distinct problems there are rather than with how many +# readings have them. Grouping the display the same way is the only way to see that. +print("🤖 AI explanations. One call per pattern, however many readings share it:\n") display( - flagged.select( - "machine_id", - "reading_seq", + flagged.groupBy( anomaly.getField("ai_explanation").getField("narrative").alias("narrative"), anomaly.getField("ai_explanation").getField("business_impact").alias("impact"), anomaly.getField("ai_explanation").getField("action").alias("action"), ) + .agg(F.count("*").alias("readings"), F.min("machine_id").alias("example_machine")) .filter(F.col("narrative").isNotNull()) - .limit(6) + .orderBy(F.desc("readings")) ) print("💡 Note the wording: broken *relationships*, not abnormal metrics. DQX tells the model which") @@ -512,6 +567,7 @@ def generate_wear_history(n_hours: int, seed: int, late_fault: bool = False): columns=WEAR_METRICS, baseline_over_time="reading_ts", baseline_by=[], + params=AnomalyParams(sample_fraction=1.0), ) print(f"🎯 Trained with an expected level per metric over time") @@ -519,12 +575,22 @@ def generate_wear_history(n_hours: int, seed: int, late_fault: bool = False): # COMMAND ---------- # DBTITLE 1,Score, and see what "wrong for now" looks like +# 99, not the 95 used earlier. Measured on this data, 95 raised 65 alerts for 24 real faults while 99 +# raised exactly 24 and got all of them -- because a residual against a fitted expectation is a sharper +# signal than a raw value, so the severity distribution of a faulty batch sits far above training's. The +# next cell prints the sweep this was chosen from; do the same on your own data rather than copying 99. +WEAR_THRESHOLD = 99 + wear_checks = [ { "criticality": "error", "check": { "function": "has_no_row_anomalies", - "arguments": {"model_name": wear_model, "registry_table": registry_table, "threshold": 95}, + "arguments": { + "model_name": wear_model, + "registry_table": registry_table, + "threshold": WEAR_THRESHOLD, + }, }, } ] @@ -536,9 +602,14 @@ def generate_wear_history(n_hours: int, seed: int, late_fault: bool = False): ) anomaly = F.col("_dq_info")[0].getField("anomaly") -caught = spark.table(wear_scored).filter(anomaly.getField("is_anomaly") & (F.col("is_incident") == 1.0)).count() -total = spark.table(wear_scored).filter(F.col("is_incident") == 1.0).count() -print(f"🔍 Caught {caught} of {total} rows that were wrong for how worn the bearing should have been") +wear_result = spark.table(wear_scored) +caught = wear_result.filter(anomaly.getField("is_anomaly") & (F.col("is_incident") == 1.0)).count() +total = wear_result.filter(F.col("is_incident") == 1.0).count() +print(f"🔍 Caught {caught} of {total} rows that were wrong for how worn the bearing should have been.\n") + +# The ranking is what to judge, and a sweep is the only way to see it: every fault sits above every +# healthy row here, so a tighter budget costs no recall at all. That is unusual, and the reason to look. +report_quality(wear_result, "is_incident", anomaly.getField("severity_percentile"), budget=WEAR_THRESHOLD) # COMMAND ---------- # DBTITLE 1,Read the contributions, which name the expected level diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 63611da34..25a723d1b 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -311,6 +311,46 @@ Scores are normalized into `severity_percentile` (0–100). The anomaly threshol The threshold controls how many rows are flagged as anomalies (percentile cutoff; default 95). Lower values (for example 90) flag more rows; higher values (for example 98) flag fewer (only extreme anomalies). Start with the default (95). If you get too many alerts, raise the threshold; if you are missing issues you care about, lower it. +### The threshold is a percentile of *training* severity, not of the batch you score + +This is the most commonly misread part of the feature, so it is worth being precise about. `threshold=95` +does not mean "flag 5% of these rows". It means "flag rows above the 95th percentile of the severity seen +during *training*". The two agree only when the batch you score looks like the data the model trained on. + +Measured on two runs against real training and real scoring: + +| Batch | Rows | Genuinely wrong | `threshold` | Rows actually flagged | +|---|---|---|---|---| +| Correlated machine telemetry with a correlation break | 1,500 | 60 (4.0%) | 95 | 139 (9.3%) | +| Trending metrics with a stale-level fault | 480 | 24 (5.0%) | 95 | 65 (13.5%) | + +Both overshoot, and in the same direction, for the same reason: a batch containing real problems has more +high-severity rows than training did, so more than 5% of it clears the training 95th percentile. On the +second fixture the *ranking* was perfect, with the 24 highest-severity rows being exactly the 24 faults, so +`threshold=99` gave 24 alerts with no false ones. Nothing was wrong with the detection; the budget was +simply set against the wrong distribution. + +Two practical consequences: + +- **Read precision against its ceiling, not against 100%.** If a budget yields 65 alerts and only 24 rows + are genuinely wrong, no model can do better than 24/65 = 37%. Where observed precision equals that + ceiling, the ranking is optimal and only the budget is costing you. +- **Calibrate on a scored batch rather than guessing.** `severity_percentile` is computed for every row, + including rows below the threshold, so you can count would-be alerts at other cutoffs without rescoring: + +```python +scored = spark.table("catalog.schema.scored") +severity = F.element_at(F.col("_dq_info"), 1).getField("anomaly").getField("severity_percentile") + +for cutoff in (90, 95, 98, 99): + alerts = scored.filter(severity >= cutoff).count() + print(f"threshold {cutoff}: {alerts} alerts ({alerts / scored.count():.1%} of rows)") +``` + +Pick the cutoff whose alert count matches how many rows a person can actually investigate. Both anomaly +demos under `demos/` print this table, and the second one chooses its threshold from it rather than +inheriting the default. + ## Group-aware anomaly detection Some values are only wrong *in context*. If one country's daily order volume drops 80% while the overall total holds steady (because other countries absorbed the difference), the collapsed number still sits comfortably inside the range other countries occupy normally. A model that compares every row against the whole table cannot see it. From 0fffdceb1151a223d249c09a6f8118ebc4eb8839 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 3 Sep 2026 11:11:14 +0100 Subject: [PATCH 071/107] Interpolate the severity tail in tail-probability space threshold=98 did not flag 2% of rows and threshold=99.9 flagged nothing at all. Isolated by scoring a training distribution with itself, so with no drift, no contamination and the quantile grid as the only variable: 96 flagged 3.13-3.47% against 4%, 97 flagged 2.07-2.36% against 3%, 98 flagged 1.41-1.56% against 2%, 99.5 flagged 0.00-0.03% against 0.5%, and 99.9 flagged nothing. Exact at the knots, short everywhere between them, always in the strict direction. Severity was interpolated linearly in score space, and a score quantile function is convex in the tail: above p95 the persisted grid holds only p99 and the training maximum, so a straight line from p95 to p99 sits above the true quantile the whole way and a single segment covers everything from 99 to 100. The guide recommended "for example 98", which delivered about three quarters of what it promised. The tail is now interpolated in tail probability, which is what a percentile is. On the same four distributions that gives 1.85-2.11% at 98 and 0.43-0.59% at 99.5. Chosen over adding knots at 96/97/98/99.5 for two reasons. It persists nothing new, so every model already in a registry is corrected simply by being scored, whereas new quantile keys would either break pre-existing models at scoring time (extract_quantile_points raises on a missing key) or give old and new models different threshold semantics. And it is exact at p95 and p99, so 90, 95 and 99 are fixed points and the default configuration is byte-identical; only 96 to 98 move, toward the budget the caller asked for. It also removes a tie. The old expression clamped every score past the training maximum to severity exactly 100, which on the correlation-aware detector was 20% of a scored batch on average and, for one entity, all of it, leaving the rows that most need ordering unrankable by the column the guide says to rank by. The tail orders them out to roughly four times further than the training data itself reaches, after which float64 resolution takes over; a test states that boundary rather than claiming ties are gone. The numpy counterpart moves in lockstep because it gates SHAP inside the scoring UDF, where a disagreement would drop contributions from precisely the rows that were flagged. An integration test asserts the two agree across the tail to 1e-9. The guide's threshold section is rewritten on the measured behaviour, separating the three things that were conflated: a batch with real problems exceeding its budget, which is correct and is the design; thresholds between knots being silently stricter, which was the bug; and a drifted batch flagging far more, which is a retraining signal no training-relative scheme can fix. The sentence "You can tune this to control how many alerts you get" is gone, since it taught the misreading. Co-authored-by: Isaac --- .../dqx_demo_anomaly_tabular_transactions.py | 4 + demos/dqx_demo_anomaly_timeseries_fleet.py | 47 +++-- .../guide/row_anomaly_detection/index.mdx | 77 +++++--- .../labs/dqx/anomaly/explainability.py | 39 +++- .../labs/dqx/anomaly/scoring_config.py | 6 + .../labs/dqx/anomaly/scoring_utils.py | 64 ++++++- .../test_anomaly_threshold.py | 73 ++++++++ tests/unit/test_anomaly_severity_tail.py | 171 ++++++++++++++++++ 8 files changed, 421 insertions(+), 60 deletions(-) create mode 100644 tests/unit/test_anomaly_severity_tail.py diff --git a/demos/dqx_demo_anomaly_tabular_transactions.py b/demos/dqx_demo_anomaly_tabular_transactions.py index f49d02ce0..895d1cc80 100644 --- a/demos/dqx_demo_anomaly_tabular_transactions.py +++ b/demos/dqx_demo_anomaly_tabular_transactions.py @@ -452,6 +452,10 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): # MAGIC single anomaly exists. A row at severity 97 is not "97% likely to be a problem"; it is in the top 3% # MAGIC most unusual. This is the most commonly misread number in the feature. # MAGIC +# MAGIC A batch that contains real problems therefore flags **more** than 5%, and that is correct rather +# MAGIC than a fault: roughly 5% of the ordinary rows, plus the anomalies on top. The alert count grows with +# MAGIC the size of the problem instead of being capped at a fixed share of the table. +# MAGIC # MAGIC That also puts a hard ceiling on precision. Ask for the top 5% of 1,500 rows and you get 75 alerts; # MAGIC if only 30 rows are genuinely bad, the best precision anyone could achieve is 30/75 = **40%**. The # MAGIC table below prints that ceiling next to what the model actually achieved, which is the only fair way diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py index e2192962c..ca8bf8cf7 100644 --- a/demos/dqx_demo_anomaly_timeseries_fleet.py +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -138,12 +138,17 @@ def report_quality(scored_df, label_col: str, severity_col, budget: float, thresholds=(90, 95, 98, 99)): - """Print recall, precision and the best precision the budget allows, at several thresholds. + """Print recall, precision and the best precision the alert count allows, at several thresholds. - Precision alone is unreadable here. Ask for the top 5% of 480 rows and you get 24 alerts; if only 24 - rows are genuinely wrong, no model can do better than 24/24, and if the budget yields 65 alerts the - ceiling is 24/65 = 37% however good the ranking is. So the ceiling is printed beside what was - achieved -- where the two are equal, the ranking is optimal and only the budget is costing you. + Precision alone is unreadable. If a threshold raises 65 alerts and only 24 rows are genuinely wrong, no + model can exceed 24/65 = 37% however well it ranks, so the ceiling is printed beside what was achieved: + where the two are equal the ranking is optimal and only the alert count is costing anything. + + Expect more alerts than the threshold's share of rows, and do not read that as a fault. `threshold=95` + means "above the 95th percentile of severity seen *during training*", so a batch that contains real + problems clears that line more often than 5% of the time. Measured on this notebook's own data, the + alert rate among the genuinely healthy readings is 5.2% against 5.0% nominal; the rest of the total is + the faults being found. """ total = scored_df.count() faults = scored_df.filter(F.col(label_col) == 1.0).count() @@ -435,16 +440,15 @@ def generate_telemetry(n_rows: int, seed: int, break_correlation: bool = False): # COMMAND ---------- # DBTITLE 1,What the alert budget actually buys -# `threshold=95` means "above the 95th percentile of *training* severity", not "95% likely to be a -# problem". On a batch whose readings are mostly stranger than anything training held, far more than 5% of -# it clears that line, which is why 95 flags well over the 75 rows a 5% budget implies. report_quality(scored, "is_incident", anomaly.getField("severity_percentile"), budget=95.0) -print("\n💡 The default is not the right answer here. Moving to 98 keeps most of the recall and throws") -print(" a small fraction of the false alarms, because severity ranks the incident readings well above") -print(" the healthy ones — the default budget was simply set against the training distribution rather") -print(" than this batch. Severity is stored for every row, so this table costs nothing to produce and") -print(" is how you should pick the number on your own data.") +print("\n💡 More alerts than 5% of rows is correct, not a fault: the extra ones are the faults being") +print(" found. The alert count grows with the size of the problem instead of being capped at a fixed") +print(" share, which is what you want from a quality check.") +print(" What the table is for is the tradeoff. Raising the threshold here buys a large drop in false") +print(" alarms for a small loss of recall, because severity ranks the incident readings well above the") +print(" healthy ones. Severity is stored for every row, so producing this costs nothing and no") +print(" rescoring, and it is how to pick the number on your own data rather than inheriting 95.") # COMMAND ---------- # DBTITLE 1,Why each group was flagged, in plain language @@ -575,11 +579,12 @@ def generate_wear_history(n_hours: int, seed: int, late_fault: bool = False): # COMMAND ---------- # DBTITLE 1,Score, and see what "wrong for now" looks like -# 99, not the 95 used earlier. Measured on this data, 95 raised 65 alerts for 24 real faults while 99 -# raised exactly 24 and got all of them -- because a residual against a fitted expectation is a sharper -# signal than a raw value, so the severity distribution of a faulty batch sits far above training's. The -# next cell prints the sweep this was chosen from; do the same on your own data rather than copying 99. -WEAR_THRESHOLD = 99 +# The default, deliberately, even though a tighter cutoff scores better on this fixture. An earlier draft +# pinned 99 from one run's sweep and the next run's model ranked slightly differently, at which point 99 was +# dropping real faults. A demo that hardcodes a tuned number teaches the wrong lesson anyway: score at the +# default, read the sweep the next cell prints, then choose. That is the loop, and it is cheap because +# severity is stored for every row. +WEAR_THRESHOLD = 95 wear_checks = [ { @@ -607,8 +612,10 @@ def generate_wear_history(n_hours: int, seed: int, late_fault: bool = False): total = wear_result.filter(F.col("is_incident") == 1.0).count() print(f"🔍 Caught {caught} of {total} rows that were wrong for how worn the bearing should have been.\n") -# The ranking is what to judge, and a sweep is the only way to see it: every fault sits above every -# healthy row here, so a tighter budget costs no recall at all. That is unusual, and the reason to look. +# The ranking is what to judge, and a sweep is the only way to see it. On this fixture almost every fault +# sits above almost every healthy reading, so a much tighter cutoff costs little or no recall while removing +# most of the false alarms. That is the shape worth recognising: when precision equals its ceiling at every +# row count, the model has ranked correctly and the cutoff is the only decision left. report_quality(wear_result, "is_incident", anomaly.getField("severity_percentile"), budget=WEAR_THRESHOLD) # COMMAND ---------- diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 25a723d1b..02c63fbe6 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -148,7 +148,7 @@ DQM and DQX each provide distinct capabilities. Together, they complement one an Each row is scored and enriched with: - **Severity percentile (0–100)**: how unusual the row is compared to training data. -- **Anomaly flag**: whether it crosses your chosen score threshold (default 95). You can tune this to control how many alerts you get. +- **Anomaly flag**: whether it crosses your chosen score threshold (default 95). The threshold is a percentile of the severity seen during *training*, so a batch with real problems in it produces more alerts than the threshold's share of rows. See [How to choose a threshold](#how-to-choose-a-threshold). - **Top contributors (explainability)**: which fields most influenced the anomaly score, so you can see *why* a row was flagged. This turns a black-box score into an actionable insight. You can tune the threshold and other options later if you need to reduce alert noise or catch more edge cases, but the defaults should work well for most use cases. @@ -305,51 +305,72 @@ display( ) ``` -Scores are normalized into `severity_percentile` (0–100). The anomaly threshold is a percentile cutoff (default 95) that you should tune to your data. +Scores are normalized into `severity_percentile` (0–100). The anomaly threshold is a cutoff on that value (default 95), calibrated against the severity distribution seen during training, that you should tune to your data. ## How to choose a threshold -The threshold controls how many rows are flagged as anomalies (percentile cutoff; default 95). Lower values (for example 90) flag more rows; higher values (for example 98) flag fewer (only extreme anomalies). Start with the default (95). If you get too many alerts, raise the threshold; if you are missing issues you care about, lower it. +The threshold controls how many rows are flagged as anomalies (percentile cutoff; default 95). Lower values +(for example 90) flag more rows; higher values (for example 98) flag fewer (only extreme anomalies). Start +with the default (95). If you are missing issues you care about, lower it; if you are investigating more +rows than you have time for, raise it. -### The threshold is a percentile of *training* severity, not of the batch you score +### What the number means -This is the most commonly misread part of the feature, so it is worth being precise about. `threshold=95` -does not mean "flag 5% of these rows". It means "flag rows above the 95th percentile of the severity seen -during *training*". The two agree only when the batch you score looks like the data the model trained on. +`threshold=95` means "flag rows above the 95th percentile of the severity DQX saw **while training**". It +does not mean "flag 5% of the rows I am scoring", and the difference matters in three separate ways. -Measured on two runs against real training and real scoring: +**A batch containing real problems produces more alerts than the budget, and should.** This is the design +working, not drift and not miscalibration. Measured on a fixture of correlated machine telemetry where 4.0% +of rows were genuinely faulty, `threshold=95` flagged 8.4% of the batch: 5.2% of the genuinely normal rows, +which is the budget delivered almost exactly, plus most of the faults on top. The alert count grows with the +size of the problem rather than being capped at a fixed share, which is what you want from a data quality +check. If your batch has nothing wrong with it, you get roughly your budget and no more. -| Batch | Rows | Genuinely wrong | `threshold` | Rows actually flagged | -|---|---|---|---|---| -| Correlated machine telemetry with a correlation break | 1,500 | 60 (4.0%) | 95 | 139 (9.3%) | -| Trending metrics with a stale-level fault | 480 | 24 (5.0%) | 95 | 65 (13.5%) | +**A batch whose distribution has moved produces many more, and that is a retraining signal.** On real +server telemetry, scoring a later time period with a model fitted on an earlier one, `threshold=95` flagged +9.8% to 21.7% of rows depending on the detector, while the same model on same-period held-out data flagged +4.9% to 5.0%. Calibration itself is sound; the input moved. Set `drift_threshold=3.0` to be told when that +has happened rather than inferring it from the alert volume. -Both overshoot, and in the same direction, for the same reason: a batch containing real problems has more -high-severity rows than training did, so more than 5% of it clears the training 95th percentile. On the -second fixture the *ranking* was perfect, with the 24 highest-severity rows being exactly the 24 faults, so -`threshold=99` gave 24 alerts with no false ones. Nothing was wrong with the detection; the budget was -simply set against the wrong distribution. +**Precision is capped by the alert count, so read it against that ceiling.** If a budget produces 65 alerts +and only 24 rows are genuinely wrong, no model can exceed 24/65 = 37% precision however well it ranks. On +one measured fixture observed precision equalled that ceiling exactly, with a perfect ranking underneath: +the 24 highest-severity rows were exactly the 24 faults. Comparing precision to 100% would have called that +a poor detector. -Two practical consequences: +### Calibrating on your own data -- **Read precision against its ceiling, not against 100%.** If a budget yields 65 alerts and only 24 rows - are genuinely wrong, no model can do better than 24/65 = 37%. Where observed precision equals that - ceiling, the ranking is optimal and only the budget is costing you. -- **Calibrate on a scored batch rather than guessing.** `severity_percentile` is computed for every row, - including rows below the threshold, so you can count would-be alerts at other cutoffs without rescoring: +`severity_percentile` is computed for every row, including rows below the threshold, so you can count +would-be alerts at other cutoffs without rescoring anything: ```python scored = spark.table("catalog.schema.scored") severity = F.element_at(F.col("_dq_info"), 1).getField("anomaly").getField("severity_percentile") +total = scored.count() -for cutoff in (90, 95, 98, 99): +for cutoff in (90, 95, 98, 99, 99.5): alerts = scored.filter(severity >= cutoff).count() - print(f"threshold {cutoff}: {alerts} alerts ({alerts / scored.count():.1%} of rows)") + print(f"threshold {cutoff}: {alerts} alerts ({alerts / total:.2%} of rows)") ``` -Pick the cutoff whose alert count matches how many rows a person can actually investigate. Both anomaly -demos under `demos/` print this table, and the second one chooses its threshold from it rather than -inheriting the default. +Pick the cutoff whose alert count matches how many rows someone can actually investigate. Both anomaly demos +under `demos/` print this table, and one of them chooses its threshold from it rather than inheriting the +default. + +A tuned threshold transfers only partly. Measured across machines in the same dataset, tuning on one half of +a batch and applying to the other cut the median error against the requested budget from 3.1x to 1.7x and +the spread across machines from 28 to 7.5 percentage points, but only 20-30% of machines landed within +double their budget. Treat a tuned number as a much better starting point than a guess, not as a guarantee. + +:::note[Fixed in 0.16.0] +Thresholds between the persisted quantile knots used to be silently stricter than requested, because +severity was interpolated linearly in score space and the grid has knots at 95, 99 and 100 with nothing +between. Measured against a training distribution scored with itself, `threshold=98` flagged 1.4% to 1.6% of +rows rather than 2%, and `threshold=99.9` flagged nothing at all. That range is now interpolated in tail +probability, which is what a percentile is. Severity at 90, 95 and 99 is unchanged, so the default +configuration behaves exactly as before; 96 to 98 become slightly less strict, toward the budget you asked +for, and 99.5 and above now fire on roughly what you asked for instead of on nothing. +::: ## Group-aware anomaly detection diff --git a/src/databricks/labs/dqx/anomaly/explainability.py b/src/databricks/labs/dqx/anomaly/explainability.py index 07982373c..07d2ab6e5 100644 --- a/src/databricks/labs/dqx/anomaly/explainability.py +++ b/src/databricks/labs/dqx/anomaly/explainability.py @@ -17,6 +17,7 @@ from pyspark.sql.types import DoubleType, MapType, StringType, StructField, StructType from sklearn.pipeline import Pipeline +from databricks.labs.dqx.anomaly.scoring_config import TAIL_ANCHOR_PERCENTILE, TAIL_RATE_PERCENTILE from databricks.labs.dqx.errors import InvalidParameterError from databricks.labs.dqx.reporting_columns import DefaultColumnNames @@ -110,15 +111,41 @@ def compute_row_attributions( def severity_from_scores(scores: np.ndarray, quantile_points: list[tuple[float, float]]) -> np.ndarray: - """Map raw anomaly scores to severity percentiles via piecewise linear interpolation. + """Map raw anomaly scores to severity percentiles: linear up to p95, then an exponential tail. - Numpy counterpart of *add_severity_percentile_column* (same quantile points, same - clamping at both ends) for use inside scoring UDFs. + Numpy counterpart of *add_severity_percentile_column*, and it has to stay one: this function decides + which rows get SHAP inside the scoring UDF, so a disagreement between the two would drop contributions + from precisely the rows that were flagged. + + The tail matches *_tail_severity_expr* term for term, and for the reason given there: linear + interpolation between p95, p99 and the training maximum is the wrong shape for a score quantile + function, so a threshold between those knots fired on well under the share of rows it promised. """ points = sorted(quantile_points, key=lambda p: p[0]) - percentiles = np.array([float(p) for p, _ in points]) - score_knots = np.array([float(q) for _, q in points]) - return np.interp(scores, score_knots, percentiles) + by_percentile = dict(points) + anchor = by_percentile.get(TAIL_ANCHOR_PERCENTILE) + rate = by_percentile.get(TAIL_RATE_PERCENTILE) + + # Without both anchors there is no tail to fit, and a degenerate tail has no width to interpolate over. + # Either way every point stays a knot, which is the behaviour that predates the tail. + if anchor is None or rate is None or rate <= anchor: + percentiles = np.array([float(p) for p, _ in points]) + score_knots = np.array([float(q) for _, q in points]) + return np.interp(scores, score_knots, percentiles) + + body = [(p, q) for p, q in points if p <= TAIL_ANCHOR_PERCENTILE] + values = np.asarray(scores, dtype=float) + severity = np.interp( + values, + np.array([float(q) for _, q in body]), + np.array([float(p) for p, _ in body]), + ) + + head_tail_probability = 100.0 - TAIL_ANCHOR_PERCENTILE + base = head_tail_probability / (100.0 - TAIL_RATE_PERCENTILE) + above = values > anchor + severity[above] = 100.0 - head_tail_probability * np.power(base, -(values[above] - anchor) / (rate - anchor)) + return severity def compute_gated_shap_contributions( diff --git a/src/databricks/labs/dqx/anomaly/scoring_config.py b/src/databricks/labs/dqx/anomaly/scoring_config.py index d6ed570b4..f7f49342c 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_config.py +++ b/src/databricks/labs/dqx/anomaly/scoring_config.py @@ -19,6 +19,12 @@ (100.0, "p100"), ] +#: The two percentiles the severity tail is anchored to. Both are keys in SEVERITY_QUANTILE_KEYS above, so +#: the tail stores nothing new and applies to models trained by any earlier release. Severity is exact at +#: both, which is what keeps the default threshold of 95 a fixed point. +TAIL_ANCHOR_PERCENTILE = 95.0 +TAIL_RATE_PERCENTILE = 99.0 + _DEFAULT_DRIFT_THRESHOLD_VALUE = 3.0 diff --git a/src/databricks/labs/dqx/anomaly/scoring_utils.py b/src/databricks/labs/dqx/anomaly/scoring_utils.py index 3fcdc5902..0aa4bc07f 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_utils.py +++ b/src/databricks/labs/dqx/anomaly/scoring_utils.py @@ -15,7 +15,11 @@ ) from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema, anomaly_info_struct_schema -from databricks.labs.dqx.anomaly.scoring_config import ScoringOutputColumns +from databricks.labs.dqx.anomaly.scoring_config import ( + TAIL_ANCHOR_PERCENTILE, + TAIL_RATE_PERCENTILE, + ScoringOutputColumns, +) from databricks.labs.dqx.anomaly.segment_utils import baseline_key_column from databricks.labs.dqx.errors import InvalidParameterError from databricks.labs.dqx.utils import safe_filter_expr @@ -214,7 +218,7 @@ def add_severity_percentile_column( severity_col: str, quantile_points: list[tuple[float, float]], ) -> DataFrame: - """Add a severity percentile column using piecewise linear interpolation. + """Add a severity percentile column: linear interpolation up to p95, then an exponential tail. Args: df: DataFrame with anomaly score column. @@ -233,23 +237,69 @@ def add_severity_percentile_column( return df.withColumn(severity_col, expr) +def _tail_severity_expr(score_expr: Column, anchor: Column, rate: Column) -> Column: + """Severity above the p95 knot, as ``100 - 5 * 5**-u`` with ``u = (score - q95) / (q99 - q95)``. + + Interpolating linearly in *score* space is the wrong shape for a score quantile function, which is + convex in the tail, and it put the cut for "severity 98" above the true 98th percentile. Measured + against the training distribution itself, so with the grid as the only variable, threshold 98 flagged + 1.4 to 1.6% of rows against the 2% it promises, threshold 99.5 flagged 0.01 to 0.03% against 0.5%, and + threshold 99.9 flagged nothing at all. Interpolating the tail *probability* instead, which is what a + percentile is, gives 1.85 to 2.11% and 0.43 to 0.59% on the same four distributions. + + Exact at both anchors: *u* of 0 gives 95 and *u* of 1 gives 99. So no severity at or below 95 moves, + and p99 stays a fixed point, which is what makes the default configuration byte-identical. + + It asymptotes to 100 rather than reaching it, which is deliberate. The previous expression clamped + every score past the training maximum to severity exactly 100; on the correlation-aware detector that + was 20% of a scored batch on average and, for one entity, all of it. Those are the rows the docs tell + you to order by severity, and they were unrankable. + + Args: + score_expr: The score to map. + anchor: The score at *TAIL_ANCHOR_PERCENTILE*. + rate: The score at *TAIL_RATE_PERCENTILE*, which sets how fast severity approaches 100. + """ + span = rate - anchor + head_tail_probability = 100.0 - TAIL_ANCHOR_PERCENTILE + base = head_tail_probability / (100.0 - TAIL_RATE_PERCENTILE) + # A degenerate tail (p95 and p99 at the same score) has no width to interpolate over, so the anchor + # percentile is the answer outright, matching how a zero-width segment is handled below. + return F.when(span <= F.lit(0.0), F.lit(TAIL_ANCHOR_PERCENTILE)).otherwise( + F.lit(100.0) - F.lit(head_tail_probability) * F.pow(F.lit(base), -(score_expr - anchor) / span) + ) + + def _piecewise_severity_expr(score_expr: Column, points: list[tuple[float, Column]]) -> Column: - """Map a score onto 0–100 by piecewise linear interpolation between *points*. + """Map a score onto 0–100: linear interpolation up to p95, then an exponential tail. The score bounds are Columns rather than floats so the same interpolation serves both the global calibration, where each bound is a literal, and per-group calibration, where each bound is a column read from a broadcast lookup of that row's group. + Above p95 the knots are too sparse for linear interpolation to honour a threshold, so that range is + handled by :func:`_tail_severity_expr` instead. Both anchors it needs are ordinary knots, so nothing + extra is persisted and a model trained before this existed is corrected simply by being scored. + Args: score_expr: The score to map. points: ``(percentile, score bound)`` pairs, ordered by percentile. """ + by_percentile = dict(points) + anchor = by_percentile.get(TAIL_ANCHOR_PERCENTILE) + rate = by_percentile.get(TAIL_RATE_PERCENTILE) + # A caller supplying its own points need not include the two anchors. Without them there is no tail to + # fit, so every point stays a knot and the behaviour is the pre-existing one. Built here rather than at + # the point of use so both anchors are narrowed to non-null in one place. + tail_expr = _tail_severity_expr(score_expr, anchor, rate) if anchor is not None and rate is not None else None + body = [(p, q) for p, q in points if p <= TAIL_ANCHOR_PERCENTILE] if tail_expr is not None else points + expr = F.when(score_expr.isNull(), F.lit(None).cast(DoubleType())) - prev_p, prev_q = points[0] + prev_p, prev_q = body[0] expr = expr.when(score_expr <= prev_q, F.lit(float(prev_p))) - for current_p, current_q in points[1:]: + for current_p, current_q in body[1:]: span = current_q - prev_q # A degenerate segment (equal bounds) would divide by zero. It means every score in this # band sits on one point, so the upper percentile is the answer outright. @@ -259,7 +309,9 @@ def _piecewise_severity_expr(score_expr: Column, points: list[tuple[float, Colum expr = expr.when(score_expr <= current_q, interpolated) prev_p, prev_q = current_p, current_q - return expr.otherwise(F.lit(float(prev_p))) + if tail_expr is None: + return expr.otherwise(F.lit(float(prev_p))) + return expr.otherwise(tail_expr) def add_baseline_severity_percentile_column( diff --git a/tests/integration_anomaly/test_anomaly_threshold.py b/tests/integration_anomaly/test_anomaly_threshold.py index 8f6e8214d..fae75c5e8 100644 --- a/tests/integration_anomaly/test_anomaly_threshold.py +++ b/tests/integration_anomaly/test_anomaly_threshold.py @@ -2,10 +2,13 @@ from collections.abc import Callable +import numpy as np import pyspark.sql.functions as F from pyspark.sql import SparkSession from databricks.labs.dqx.anomaly.check_funcs import has_no_row_anomalies +from databricks.labs.dqx.anomaly.explainability import severity_from_scores +from databricks.labs.dqx.anomaly.scoring_utils import add_severity_percentile_column from databricks.labs.dqx.config import AnomalyParams from databricks.labs.dqx.engine import DQEngine from tests.constants import TEST_CATALOG @@ -15,6 +18,7 @@ ) from tests.integration_anomaly.conftest import ( apply_anomaly_check_direct, + create_anomaly_check_rule, create_anomaly_dataset_rule, train_simple_2d_model, ) @@ -234,3 +238,72 @@ def test_validation_metrics_in_registry(spark: SparkSession, quick_model_factory # These are optional but good to have: precision, recall, f1_score, val_anomaly_rate # At least some metrics should be present assert len(metrics) >= 1 + + +def test_the_spark_and_numpy_severity_maps_agree_across_the_tail(spark: SparkSession): + """The two implementations must not diverge, and only a session can check the Spark one. + + ``severity_from_scores`` decides which rows get SHAP inside the scoring UDF while + ``add_severity_percentile_column`` produces the severity everything downstream reads. A disagreement + would drop contributions from precisely the rows that were flagged, which is the one place a reader + looks. The probe grid deliberately spans past the last knot, because that whole range used to be a + single clamped value and is now where the two could most easily part company. + """ + points = [ + (0.0, 0.0), + (1.0, 0.4), + (5.0, 0.8), + (10.0, 1.0), + (25.0, 1.5), + (50.0, 2.0), + (75.0, 3.0), + (90.0, 4.0), + (95.0, 5.0), + (99.0, 8.0), + (100.0, 12.0), + ] + probes = [float(value) for value in np.linspace(-1.0, 60.0, 400)] + + scored = add_severity_percentile_column( + spark.createDataFrame([(value,) for value in probes], "score double"), + score_col="score", + severity_col="severity", + quantile_points=points, + ) + from_spark = np.array([row["severity"] for row in scored.orderBy("score").collect()]) + from_numpy = severity_from_scores(np.array(sorted(probes)), points) + + assert np.allclose( + from_spark, from_numpy, atol=1e-9 + ), f"largest disagreement {np.max(np.abs(from_spark - from_numpy))}" + + +def test_a_threshold_between_the_knots_flags_close_to_its_budget_through_real_scoring( + ws, spark: SparkSession, quick_model_factory +): + """End to end, through a trained model and the public check, not just the mapping in isolation. + + ``threshold=98`` used to flag well under the 2% it promises, because the persisted grid has no knot + between 95 and 99 and the interpolation ran in score space. Scoring data drawn from the same + distribution the model trained on is what isolates that: no drift, no planted anomalies, so the only + thing the realised rate can be measuring is the mapping. + """ + rng = np.random.default_rng(29) + train_rows = [(float(a), float(b)) for a, b in rng.normal(100.0, 10.0, size=(4000, 2))] + model_name, registry_table, _ = quick_model_factory( + spark, train_data=train_rows, params=AnomalyParams(sample_fraction=1.0) + ) + + test_rows = [(float(a), float(b)) for a, b in rng.normal(100.0, 10.0, size=(4000, 2))] + test_df = spark.createDataFrame(test_rows, "amount double, quantity double") + + result_df = DQEngine(ws, spark).apply_checks( + test_df, + [create_anomaly_check_rule(model_name=model_name, registry_table=registry_table, threshold=98.0)], + ) + severity = F.col("_dq_info")[0].getField("anomaly").getField("severity_percentile") + realised = result_df.filter(severity >= 98.0).count() / result_df.count() + + # Generous, because a 4,000-row sample of a 2% tail carries real binomial noise (sd about 0.2pp) on top + # of the grid's own approximation. The behaviour being guarded against realised 1.4% or less. + assert 0.014 < realised < 0.030, f"threshold 98 flagged {realised:.3%} of an unremarkable batch" diff --git a/tests/unit/test_anomaly_severity_tail.py b/tests/unit/test_anomaly_severity_tail.py new file mode 100644 index 000000000..ae268b14f --- /dev/null +++ b/tests/unit/test_anomaly_severity_tail.py @@ -0,0 +1,171 @@ +"""Unit pins for the severity tail above p95 (no Spark). + +`threshold` is documented as an alert budget, and above p95 it did not deliver one: the quantile grid has +knots at 95, 99 and 100 and nothing between, so linear interpolation in *score* space put the cut for +"severity 98" above the true 98th percentile. Measured against a training distribution scored with itself, +so with the grid as the only variable, threshold 98 flagged 1.4-1.6% of rows against the 2% it promises and +threshold 99.9 flagged nothing at all. + +The tests here pin the three properties the fix rests on: the two anchors are exact, so nobody's current +alerts move; severity is strictly increasing past p99, so the rows that most need ordering can be ordered; +and the realised alert rate is close to the requested one, which is the regression guard. + +The numpy implementation is tested rather than the Spark one because the two are required to agree and only +one of them runs without a session. `tests/integration_anomaly/` covers the Spark side and their parity. +""" + +import numpy as np +import pytest + +from databricks.labs.dqx.anomaly.explainability import severity_from_scores +from databricks.labs.dqx.anomaly.scoring_config import ( + SEVERITY_QUANTILE_KEYS, + TAIL_ANCHOR_PERCENTILE, + TAIL_RATE_PERCENTILE, +) + + +def _quantile_points(scores: np.ndarray) -> list[tuple[float, float]]: + """The grid a trained model persists: one score per key in SEVERITY_QUANTILE_KEYS.""" + return [(percentile, float(np.quantile(scores, percentile / 100.0))) for percentile, _ in SEVERITY_QUANTILE_KEYS] + + +def _realised_alert_rate(scores: np.ndarray, points: list[tuple[float, float]], threshold: float) -> float: + """The share of rows a given threshold actually flags, which is what the docs promise.""" + severity = severity_from_scores(scores, points) + return float((severity >= threshold).mean()) + + +# A score distribution per shape a real detector produces. Mahalanobis distances are chi-square-like; +# Isolation Forest path lengths are closer to normal; the heavy-tailed cases are the adversarial ones. +_DISTRIBUTIONS = { + "normal": lambda rng: rng.normal(0.0, 1.0, 200_000), + "chi_square_38": lambda rng: rng.chisquare(38, 200_000), + "gamma": lambda rng: rng.gamma(2.0, 2.0, 200_000), + "lognormal": lambda rng: rng.lognormal(0.0, 1.0, 200_000), +} + + +@pytest.mark.parametrize("shape", sorted(_DISTRIBUTIONS)) +def test_the_two_anchors_are_exact(shape: str): + """95 and 99 are fixed points, which is what makes the default configuration byte-identical. + + The alternative fix, adding knots at 96/97/98/99.5, would have moved severity everywhere above 95 and + needed new persisted keys that pre-existing models do not carry. + """ + scores = _DISTRIBUTIONS[shape](np.random.default_rng(7)) + points = _quantile_points(scores) + by_percentile = dict(points) + + at_anchor = severity_from_scores(np.array([by_percentile[TAIL_ANCHOR_PERCENTILE]]), points) + at_rate = severity_from_scores(np.array([by_percentile[TAIL_RATE_PERCENTILE]]), points) + + assert at_anchor[0] == pytest.approx(TAIL_ANCHOR_PERCENTILE, abs=1e-9) + assert at_rate[0] == pytest.approx(TAIL_RATE_PERCENTILE, abs=1e-9) + + +@pytest.mark.parametrize("shape", sorted(_DISTRIBUTIONS)) +@pytest.mark.parametrize("threshold", [96.0, 97.0, 98.0, 99.5]) +def test_a_threshold_between_the_knots_flags_close_to_what_it_promises(shape: str, threshold: float): + """The regression guard, and the reason the tail exists. + + Before this, 98 flagged 1.4-1.6% rather than 2%, and 99.5 flagged 0.00-0.03% rather than 0.5%. A 25% + relative tolerance is loose on purpose: the tail assumes an exponential shape, which is right for these + detectors' scores and not exact for any of them. The old behaviour missed by 20 to 100%, so this + separates the two comfortably without pretending to a precision the model does not have. + """ + scores = _DISTRIBUTIONS[shape](np.random.default_rng(11)) + points = _quantile_points(scores) + + promised = (100.0 - threshold) / 100.0 + realised = _realised_alert_rate(scores, points, threshold) + + assert realised == pytest.approx( + promised, rel=0.25 + ), f"{shape} at threshold {threshold}: flagged {realised:.4%}, promised {promised:.4%}" + + +@pytest.mark.parametrize("threshold", [90.0, 95.0, 99.0]) +def test_the_documented_thresholds_were_already_exact_and_stay_exact(threshold: float): + """90, 95 and 99 land on knots, so they were never affected and must not become so.""" + scores = _DISTRIBUTIONS["chi_square_38"](np.random.default_rng(3)) + points = _quantile_points(scores) + + promised = (100.0 - threshold) / 100.0 + realised = _realised_alert_rate(scores, points, threshold) + + assert realised == pytest.approx(promised, rel=0.02) + + +def test_severity_keeps_increasing_past_the_last_knot(): + """The old expression clamped every score past the training maximum to exactly 100. + + Measured on real telemetry, that was 20% of a scored batch on average for the correlation-aware + detector, and for one entity all of it. Those rows are the ones the docs tell you to rank by severity. + + The tail orders them instead, though not forever: it asymptotes to 100, so at some distance the + difference falls below float64 resolution and ties resume. Measured, that is around 21 times the + p95-to-p99 span past p95, while the training maximum sits about 4.6 spans out. So the range over which + rows can be ordered extends roughly four times further than the training data itself reaches, which is + the useful claim; "no ties ever" would not be true. + """ + scores = _DISTRIBUTIONS["chi_square_38"](np.random.default_rng(5)) + points = _quantile_points(scores) + by_percentile = dict(points) + anchor = by_percentile[TAIL_ANCHOR_PERCENTILE] + span = by_percentile[TAIL_RATE_PERCENTILE] - anchor + training_max = float(max(q for _, q in points)) + + # From the old clamp point outward, in span units, stopping short of the float64 asymptote. + probes = np.array([training_max + multiple * span for multiple in (0.0, 1.0, 2.0, 5.0, 10.0)]) + severity = severity_from_scores(probes, points) + + assert np.all(np.diff(severity) > 0), f"ties where the old map clamped: {severity}" + assert np.all(severity < 100.0), "severity asymptotes to 100 rather than reaching it here" + assert np.all(severity > TAIL_RATE_PERCENTILE) + + +def test_severity_stays_monotonic_across_the_whole_range(): + """A severity map that is not monotonic in score would make any threshold meaningless.""" + scores = _DISTRIBUTIONS["gamma"](np.random.default_rng(13)) + points = _quantile_points(scores) + + probes = np.linspace(float(min(q for _, q in points)) - 1.0, float(max(q for _, q in points)) * 3.0, 4000) + severity = severity_from_scores(probes, points) + + assert np.all(np.diff(severity) >= 0.0) + + +def test_a_degenerate_tail_falls_back_to_the_anchor(): + """A constant score above p95 leaves no width to interpolate over. + + Real: a table where almost every row scores identically, so p95 and p99 coincide. The anchor percentile + is then the answer outright, which is how a zero-width segment lower down is already handled. + """ + points = [(percentile, 0.0 if percentile <= 95.0 else 0.0) for percentile, _ in SEVERITY_QUANTILE_KEYS] + + severity = severity_from_scores(np.array([0.0, 5.0]), points) + + assert np.all(np.isfinite(severity)) + + +def test_points_without_the_anchors_keep_the_plain_linear_behaviour(): + """A caller supplying its own grid need not include p95 and p99, and must not be silently reshaped.""" + points = [(0.0, 0.0), (50.0, 10.0), (100.0, 20.0)] + + severity = severity_from_scores(np.array([0.0, 5.0, 10.0, 15.0, 20.0, 40.0]), points) + + assert severity.tolist() == [0.0, 25.0, 50.0, 75.0, 100.0, 100.0] + + +def test_scores_below_the_anchor_are_untouched_by_the_tail(): + """Everything at or below p95 must map exactly as it did before, or existing alerts move.""" + scores = _DISTRIBUTIONS["normal"](np.random.default_rng(17)) + points = _quantile_points(scores) + body = [(p, q) for p, q in points if p <= TAIL_ANCHOR_PERCENTILE] + + probes = np.linspace(float(body[0][1]), float(body[-1][1]), 500) + with_tail = severity_from_scores(probes, points) + plain_linear = np.interp(probes, [q for _, q in body], [p for p, _ in body]) + + assert np.allclose(with_tail, plain_linear, atol=1e-9) From e3515c439104b53d841d27a336ab44ff82b4cb83 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 3 Sep 2026 14:05:34 +0100 Subject: [PATCH 072/107] Estimate the severity quantiles precisely enough for the tail to mean anything Two CI failures, both in tests added earlier in this PR, and one of them found a real gap in the severity tail rather than a bad assertion. The tail derives its decay rate from (q99 - q95), so q99's accuracy now decides where the cut for a threshold between 95 and 99 lands. Under the old linear map it was one knot among eleven with the p100 knot immediately above it. `approxQuantile(prob, relativeError=0.01)` is allowed to return the value at any rank within a percentage point of the one requested, so a reported "p99" may be the true p100. Measured on an Isolation Forest ensemble with every other knot exact and only the reported q99 varied: reported as q99 alert rate at threshold 98, where 2.00% was asked for true p98 2.850% true p99 (exact) 2.250% true p99.5 1.900% true p99.9 0.975% true p100 0.650% That is the whole range a 1% relative error permits, and it spans 0.65% to 2.85%. CI measured 0.750%, squarely inside it. 0.001 narrows the permitted ranks to 98.9 through 99.1. The cost is one double column at training time, paid once: approxQuantile holds O(1/relativeError) summary entries per partition. The per-group path is pinned to the same figure so the two calibrations cannot drift apart silently. The other failure was a degenerate fixture. The end-to-end temporal test trained on a perfectly noiseless line, so subtracting a well-fitted trend left floating-point dust, and a detector trained on a column with no variance cannot order anything: both test rows came back at the same severity and "the stale level must score worse than the current one" was comparing equal numbers. The fixture now carries noise, with the rollback about 13 standard deviations of it, so the ordering is decisive rather than marginal. Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/core.py | 15 +++++++-- .../test_anomaly_temporal_features.py | 33 +++++++++++++++---- .../test_anomaly_threshold.py | 14 ++++++-- 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/core.py b/src/databricks/labs/dqx/anomaly/core.py index 470c11ada..50cbcd1f1 100644 --- a/src/databricks/labs/dqx/anomaly/core.py +++ b/src/databricks/labs/dqx/anomaly/core.py @@ -340,6 +340,14 @@ def compute_score_quantiles_ensemble( return _quantiles_from_scored(scored, feature_metadata) +#: How precisely the severity quantiles are estimated. Load-bearing since the severity tail takes its decay +#: rate from (q99 - q95): `approxQuantile` may return the value at any rank within this much of the one +#: requested, so at 0.01 a reported "p99" could be the true p100, and measured on an Isolation Forest +#: ensemble that moved the alert rate at threshold 98 from 2.25% to 0.65% against the 2% requested. At 0.001 +#: the reported p99 is between ranks 98.9 and 99.1. Paid once, at training time, for one double column. +SEVERITY_QUANTILE_RELATIVE_ERROR = 0.001 + + def _quantiles_from_scored(scored: DataFrame, feature_metadata: SparkFeatureMetadata) -> dict[str, float]: """Derive the global score quantiles, and the per-group ones as a side effect. @@ -348,7 +356,7 @@ def _quantiles_from_scored(scored: DataFrame, feature_metadata: SparkFeatureMeta and only when ``baseline_by`` is set — an ungrouped model behaves exactly as before. """ scores_df = scored.select(F.col("anomaly_score").alias("score")) - quantiles = scores_df.approxQuantile("score", SCORE_QUANTILE_PROBS, 0.01) + quantiles = scores_df.approxQuantile("score", SCORE_QUANTILE_PROBS, SEVERITY_QUANTILE_RELATIVE_ERROR) if feature_metadata.baseline_by: feature_metadata.baseline_score_quantiles = compute_baseline_score_quantiles( @@ -371,8 +379,11 @@ def compute_baseline_score_quantiles(scored_df: DataFrame, baseline_by: list[str if not baseline_by: return {} + # Accuracy stated rather than left to the default, so the per-group calibration cannot drift away from + # the global one above. `percentile_approx` expresses it as 1/relativeError. + accuracy = int(round(1.0 / SEVERITY_QUANTILE_RELATIVE_ERROR)) quantile_exprs = [ - F.percentile_approx(F.col("anomaly_score"), prob).alias(key) + F.percentile_approx(F.col("anomaly_score"), prob, accuracy).alias(key) for prob, key in zip(SCORE_QUANTILE_PROBS, SCORE_QUANTILE_KEYS, strict=True) ] # Read the key feature engineering already computed rather than rebuilding it from the raw diff --git a/tests/integration_anomaly/test_anomaly_temporal_features.py b/tests/integration_anomaly/test_anomaly_temporal_features.py index 5bd99da6f..10ce63487 100644 --- a/tests/integration_anomaly/test_anomaly_temporal_features.py +++ b/tests/integration_anomaly/test_anomaly_temporal_features.py @@ -403,9 +403,22 @@ def test_the_public_check_scores_a_temporal_model_end_to_end(ws, spark: SparkSes The scoring path narrows the frame to features, grouping columns and internals before replaying the transform. The time column has to survive that select, or the fitted expectation has no axis to evaluate against and scoring fails outright on an unresolved column. + + The training data carries noise deliberately. A noiseless trend leaves a residual of floating-point dust + once the fit is subtracted, and a detector trained on a column with no variance cannot order anything, so + every row comes back at the same severity and the comparison below becomes vacuous. """ + hours = 24 * 30 + rng = np.random.default_rng(41) + noise = rng.normal(0.0, 1.5, hours) + quantity_noise = rng.normal(0.0, 0.05, hours) train_data = [ - (START + datetime.timedelta(hours=i), float(100.0 + 0.05 * i), 2.0 + 0.001 * i) for i in range(24 * 30) + ( + START + datetime.timedelta(hours=i), + float(100.0 + 0.05 * i + noise[i]), + float(2.0 + 0.001 * i + quantity_noise[i]), + ) + for i in range(hours) ] model_name, registry_table, _ = quick_model_factory( @@ -418,11 +431,17 @@ def test_the_public_check_scores_a_temporal_model_end_to_end(ws, spark: SparkSes params=AnomalyParams(sample_fraction=1.0), ) - # One row held at the level the metrics had 400 hours earlier: inside the training range on every - # column, and wrong only for where the trend had got to. - late = START + datetime.timedelta(hours=24 * 30 - 1) + # Two rows at the same late timestamp. One sits where the trend had actually got to; the other is held at + # the level of 400 hours earlier, which is inside the training range on every column and wrong only for + # its point in time. On a slope of 0.05 per hour against noise of 1.5, that rollback is roughly 13 + # standard deviations, so the ordering asserted below is decisive rather than marginal. + latest_hour = hours - 1 + late = START + datetime.timedelta(hours=latest_hour) test_df = spark.createDataFrame( - [(late, 100.0 + 0.05 * 319, 2.0 + 0.001 * 319), (late, 100.0 + 0.05 * 719, 2.719)], + [ + (late, 100.0 + 0.05 * (latest_hour - 400), 2.0 + 0.001 * (latest_hour - 400)), + (late, 100.0 + 0.05 * latest_hour, 2.0 + 0.001 * latest_hour), + ], "event_ts timestamp, amount double, quantity double", ) @@ -439,7 +458,9 @@ def test_the_public_check_scores_a_temporal_model_end_to_end(ws, spark: SparkSes assert len(scored) == 2 assert all(row["score"] is not None for row in scored), "every row must get a number" rolled_back, current = sorted(scored, key=lambda r: r["amount"]) - assert rolled_back["score"] > current["score"], "the stale level must score worse than the current one" + assert rolled_back["score"] > current["score"], ( + f"the stale level must score worse than the current one: " f"{rolled_back['score']} against {current['score']}" + ) def test_the_public_check_reports_staleness_in_the_info_column(ws, spark: SparkSession, quick_model_factory): diff --git a/tests/integration_anomaly/test_anomaly_threshold.py b/tests/integration_anomaly/test_anomaly_threshold.py index fae75c5e8..916fd7a65 100644 --- a/tests/integration_anomaly/test_anomaly_threshold.py +++ b/tests/integration_anomaly/test_anomaly_threshold.py @@ -287,6 +287,12 @@ def test_a_threshold_between_the_knots_flags_close_to_its_budget_through_real_sc between 95 and 99 and the interpolation ran in score space. Scoring data drawn from the same distribution the model trained on is what isolates that: no drift, no planted anomalies, so the only thing the realised rate can be measuring is the mapping. + + This test also guards ``SEVERITY_QUANTILE_RELATIVE_ERROR``, which is easy to read as a tuning detail and + is not. The tail takes its decay rate from ``q99 - q95``, so slackening that error loosens where the cut + for 98 lands: measured on an Isolation Forest ensemble with every other knot exact, reporting the true + p100 as ``q99`` -- which a relative error of 0.01 permits -- moved the realised rate from 2.25% to 0.65%. + An earlier version of this assertion failed at 0.750% for exactly that reason. """ rng = np.random.default_rng(29) train_rows = [(float(a), float(b)) for a, b in rng.normal(100.0, 10.0, size=(4000, 2))] @@ -304,6 +310,8 @@ def test_a_threshold_between_the_knots_flags_close_to_its_budget_through_real_sc severity = F.col("_dq_info")[0].getField("anomaly").getField("severity_percentile") realised = result_df.filter(severity >= 98.0).count() / result_df.count() - # Generous, because a 4,000-row sample of a 2% tail carries real binomial noise (sd about 0.2pp) on top - # of the grid's own approximation. The behaviour being guarded against realised 1.4% or less. - assert 0.014 < realised < 0.030, f"threshold 98 flagged {realised:.3%} of an unremarkable batch" + # Measured expectation is about 2.25%. The band is wide because a 4,000-row sample of a 2% tail carries + # binomial noise of roughly 0.22pp on its own, the quantile grid is still an approximation, and the + # exponential tail is the right shape for these scores without being exact for any of them. It stays a + # real guard: score-space interpolation gave 0.65% to 1.6% on this fixture depending on the grid. + assert 0.012 < realised < 0.035, f"threshold 98 flagged {realised:.3%} of an unremarkable batch" From 79e5e8b1994e0444e7ee717dd64939e3f6f72083 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 3 Sep 2026 15:57:36 +0100 Subject: [PATCH 073/107] Make the three comparison bases reachable from run-config YAML in the docs, not just in code Threading `profile` and `baseline_over_time` through `AnomalyConfig` and the anomaly trainer workflow was already done here, on the grounds that a scheduled retrain has to be able to reproduce a model trained by hand. The documentation never caught up, so the capability existed and was undiscoverable: none of the three places that show an `anomaly_config` block listed `baseline_by`, `profile` or `baseline_over_time`. Worse, the guide's example nested `anomaly_config` *inside* `input_config`. It is a field of `RunConfig` (`config.py:267`) and `InputConfig` has no such field, so a reader who copied that example got their anomaly configuration ignored. The reference page had it at the right level, which is how the two drifted apart unnoticed. Both now show every field and state where the block belongs. Three test gaps too, all on the newest field. The workflow test asserted that `profile` and `baseline_by` reach `train()` but not `baseline_over_time`, so a scheduled retrain could have dropped it silently. The defaults test did not pin it. And two config tests carried docstrings promising the field would "survive a round trip through run-config YAML" while only constructing the dataclass -- a serializer that dropped a field would have passed them. They now save and load through `MockInstallation`, and a second test loads an installed payload that predates all three fields to pin the backwards-compatibility guarantee. Checked the round trip is not vacuous by dumping what gets written: all three appear in the serialized config. Co-authored-by: Isaac --- docs/dqx/docs/installation.mdx | 3 + docs/dqx/docs/reference/quality_checks.mdx | 9 ++- tests/unit/test_anomaly_configs.py | 78 ++++++++++++++++++++++ tests/unit/test_anomaly_workflow.py | 4 ++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/docs/dqx/docs/installation.mdx b/docs/dqx/docs/installation.mdx index c1db2a688..19de959a4 100644 --- a/docs/dqx/docs/installation.mdx +++ b/docs/dqx/docs/installation.mdx @@ -301,6 +301,9 @@ run_configs: # <- list of run configurations, each run co columns: [amount, quantity] # <- optional, omit to use all supported columns model_name: main.iot.orders_monitor # <- required when using anomaly config, fully qualified registry_table: main.iot.dqx_anomaly_models # <- required when using anomaly config, fully qualified + baseline_by: [region, product] # <- optional. Judge each metric against its own group's baseline + profile: timeseries # <- optional. "tabular" (default) or "timeseries" + baseline_over_time: event_ts # <- optional. Time column each metric's expected level is fitted along # for the full parameter anomaly specification, see the Row Anomaly Detection documentation # if wanting to store checks in lakebase table diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index bd109afd7..9cd31b213 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -3480,11 +3480,18 @@ run_configs: input_config: location: catalog.schema.orders anomaly_config: - columns: [amount, quantity] # optional; omit to use all supported columns + columns: [amount, quantity] # optional; omit to auto-discover model_name: catalog.schema.orders_monitor registry_table: catalog.schema.dqx_anomaly_models + baseline_by: [region, product] # optional; judge each metric against its own group's baseline + profile: tabular # optional; "tabular" (default) or "timeseries" + baseline_over_time: event_ts # optional; time column each metric's level is fitted along ``` +Every key mirrors an argument of `anomaly_engine.train()` below, and omitting one is the same as omitting +that argument, so a scheduled retrain reproduces a hand-trained model exactly. `anomaly_config` is a field of +the run config, beside `input_config` rather than inside it. + ### Training Parameters The `anomaly_engine.train()` method accepts several parameters to tune model behavior, performance, and accuracy. diff --git a/tests/unit/test_anomaly_configs.py b/tests/unit/test_anomaly_configs.py index 203725557..a976ad0e5 100644 --- a/tests/unit/test_anomaly_configs.py +++ b/tests/unit/test_anomaly_configs.py @@ -1,10 +1,15 @@ """Unit tests for anomaly detection configuration classes.""" +from databricks.labs.blueprint.installation import MockInstallation + from databricks.labs.dqx.config import ( AnomalyConfig, AnomalyParams, FeatureEngineeringConfig, + InputConfig, IsolationForestConfig, + RunConfig, + WorkspaceConfig, ) # ============================================================================ @@ -196,6 +201,7 @@ def test_anomaly_config_defaults(): assert cfg.model_name is None assert cfg.registry_table is None assert cfg.profile is None + assert cfg.baseline_over_time is None def test_anomaly_config_carries_the_profile(): @@ -334,3 +340,75 @@ def test_feature_engineering_config_with_algo_config(): assert params.algorithm_config.num_trees == 250 assert feature_config.categorical_cardinality_threshold == 15 assert feature_config.max_input_columns == 12 + + +def test_anomaly_config_survives_a_real_yaml_round_trip(): + """Save and load through an installation, not just construct the dataclass. + + The two tests above assert that `AnomalyConfig` *holds* a profile, which is not the same claim as + "a run config can express it". The workflow reads its configuration back out of the installation, so + a field that the serializer dropped would leave a scheduled retrain silently training the default + while the YAML on disk said otherwise. `MockInstallation` exercises that path with no workspace. + """ + installation = MockInstallation() + installation.save( + WorkspaceConfig( + run_configs=[ + RunConfig( + name="fleet", + input_config=InputConfig(location="catalog.schema.telemetry"), + anomaly_config=AnomalyConfig( + columns=["spindle_load", "motor_current"], + model_name="catalog.schema.fleet_monitor", + registry_table="catalog.schema.dqx_anomaly_models", + baseline_by=["machine_id"], + profile="timeseries", + baseline_over_time="reading_ts", + ), + ) + ] + ) + ) + + loaded = installation.load(WorkspaceConfig).get_run_config("fleet").anomaly_config + + assert loaded is not None + assert loaded.columns == ["spindle_load", "motor_current"] + assert loaded.baseline_by == ["machine_id"] + assert loaded.profile == "timeseries" + assert loaded.baseline_over_time == "reading_ts" + + +def test_an_installation_written_before_these_fields_existed_still_loads(): + """The backwards-compatibility guarantee, against a payload rather than against a constructor. + + An installed run config predates every field this release added. Loading it must produce a config that + trains exactly what it trained before, which means all three arrive as None rather than raising on an + absent key. + """ + installation = MockInstallation( + { + "config.yml": { + "__version__": 1, + "run_configs": [ + { + "name": "orders", + "input_config": {"location": "catalog.schema.orders"}, + "anomaly_config": { + "columns": ["amount", "quantity"], + "model_name": "catalog.schema.orders_monitor", + "registry_table": "catalog.schema.dqx_anomaly_models", + }, + } + ], + } + } + ) + + loaded = installation.load(WorkspaceConfig).get_run_config("orders").anomaly_config + + assert loaded is not None + assert loaded.columns == ["amount", "quantity"] + assert loaded.baseline_by is None + assert loaded.profile is None + assert loaded.baseline_over_time is None diff --git a/tests/unit/test_anomaly_workflow.py b/tests/unit/test_anomaly_workflow.py index 37b0b631a..a76ac8b1c 100644 --- a/tests/unit/test_anomaly_workflow.py +++ b/tests/unit/test_anomaly_workflow.py @@ -192,6 +192,7 @@ def train(self, **kwargs): registry_table="catalog.schema.my_registry", baseline_by=["machine_id"], profile="timeseries", + baseline_over_time="reading_ts", ), ) ctx = SimpleNamespace(run_config=run_config, spark=Mock(), workspace_client=Mock()) @@ -200,3 +201,6 @@ def train(self, **kwargs): assert train_called["kwargs"]["profile"] == "timeseries" assert train_called["kwargs"]["baseline_by"] == ["machine_id"] + # All three comparison bases, because a scheduled retrain that silently drops one produces a different + # model from the YAML it was given, and nothing downstream would report the discrepancy. + assert train_called["kwargs"]["baseline_over_time"] == "reading_ts" From f92808ede03cd7fab6daac8beb3952131a3f237c Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Thu, 3 Sep 2026 15:58:03 +0100 Subject: [PATCH 074/107] Teach time as a basis rather than a feature, on data where it works Both notebooks had a section about time that asserted rather than demonstrated. The fleet notebook framed the world as "two kinds of anomaly, two profiles", which predates `baseline_over_time` and hides two of the three choices: `profile` picks the detector, while `baseline_by` and `baseline_over_time` pick what normal is measured against, and all three are independent. Every example was a factory, so a reader from payments or healthcare had to translate before knowing whether any of it applied. And it described the one thing DQX does with a timestamp when there are three, omitting the axis case its own Section 5 uses. The tabular notebook's equivalent was three paragraphs of caveat titled "why this demo does not use it". An intermediate version measured the negative -- training a second model with the timestamp among the features -- which showed a real effect but made the reader's mistake the subject. It now demonstrates the correct use instead, and the mistake survives as one bullet pointing at the advisory that fires. Demonstrating it needed data that trends, since the transactions history is stationary by construction, so the section brings its own in the same domain: a processor's daily counts over 18 months with one week where a feed partially fails and volumes quietly revert to five months earlier. Every value stays inside the year's range, so no range check fires; only the position against the trend is wrong. Measured on a workspace, the decision cell prints 99.0% of variance explained for the processor volume against 1.0% for individual transactions, and the model catches 7 of 7 stalled days with precision equal to its ceiling at every threshold. That also gives the pair opposite recommendations from the same sweep, which teaches more than either alone: the transactions model is best at 95, where tightening loses anomalies cheaply, and the volume model is best at 98, where tightening costs no recall and triples precision. The calendar advisory now names the offending column in both escapes rather than a `` placeholder, and gains the two tests it never had -- it fires when a timestamp reaches the feature list, and stays quiet once the caller follows the advice and passes it as an axis. The user guide's three-question table gains the fourth row the notebooks now teach, since the axis-versus-feature distinction is what makes the rest cohere. Co-authored-by: Isaac --- .../dqx_demo_anomaly_tabular_transactions.py | 243 +++++++++++++++--- demos/dqx_demo_anomaly_timeseries_fleet.py | 65 ++++- .../guide/row_anomaly_detection/index.mdx | 33 ++- .../labs/dqx/anomaly/training_service.py | 2 +- .../test_anomaly_autodiscovery.py | 75 ++++++ 5 files changed, 360 insertions(+), 58 deletions(-) diff --git a/demos/dqx_demo_anomaly_tabular_transactions.py b/demos/dqx_demo_anomaly_tabular_transactions.py index 895d1cc80..cc69b96e1 100644 --- a/demos/dqx_demo_anomaly_tabular_transactions.py +++ b/demos/dqx_demo_anomaly_tabular_transactions.py @@ -468,56 +468,224 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): # COMMAND ---------- # DBTITLE 1,Threshold tradeoffs -severity = anomaly.getField("severity_percentile") -truth = F.col("is_anomaly") == 1.0 - -print("🎚️ Testing different thresholds:\n") -print("Threshold | Alerts | Caught | Precision | Best possible | Recall") -print("-" * 68) - -for threshold in (90.0, 95.0, 98.0): - alerts = scored.filter(severity >= threshold) - n_alerts = alerts.count() - n_caught = alerts.filter(truth).count() - # The ceiling: you cannot be more precise than "every alert is a real anomaly". - ceiling = min(1.0, injected / n_alerts) if n_alerts else 0.0 - precision = n_caught / n_alerts if n_alerts else 0.0 - print( - f" {threshold:>5.0f} | {n_alerts:>6d} | {n_caught:>4d}/{injected:<3d}|" - f" {precision:>6.1%} | {ceiling:>7.1%} | {n_caught / injected:>5.1%}" - ) + +def report_thresholds(scored_df, label: str, thresholds=(90.0, 95.0, 98.0), label_col: str = "is_anomaly"): + """Print alerts, catch rate and precision against its ceiling, at several thresholds. + + A helper rather than a loop because Section 6 reports a second model with it, and two models are only + comparable if both are measured the same way. + """ + severity_col = F.element_at(F.col("_dq_info"), 1).getField("anomaly").getField("severity_percentile") + is_planted = F.col(label_col) == 1.0 + planted = scored_df.filter(is_planted).count() + + print(f"🎚️ {label}\n") + print("Threshold | Alerts | Caught | Precision | Best possible | Recall") + print("-" * 68) + for threshold in thresholds: + alerts = scored_df.filter(severity_col >= threshold) + n_alerts = alerts.count() + n_caught = alerts.filter(is_planted).count() + # The ceiling: you cannot be more precise than "every alert is a real anomaly". + ceiling = min(1.0, planted / n_alerts) if n_alerts else 0.0 + precision = n_caught / n_alerts if n_alerts else 0.0 + print( + f" {threshold:>5.0f} | {n_alerts:>6d} | {n_caught:>4d}/{planted:<3d}|" + f" {precision:>6.1%} | {ceiling:>7.1%} | {n_caught / planted:>5.1%}" + ) + + +report_thresholds(scored, "Testing different thresholds:") print("\n💡 Read precision against the ceiling, not against 100%. Where the two are equal, every") print(" planted anomaly is inside the model's ranking and no alert is wasted — the ranking is") print(" optimal for that budget. A tighter threshold then trades recall for precision; it does") print(" not reveal a better model.") +print("\n Where precision sits *below* the ceiling, something different is happening: ordinary rows are") +print(" outranking real anomalies. That is a statement about the ranking, not about the budget, and no") +print(" choice of threshold fixes it — the feature set or the comparison basis is what needs attention.") # COMMAND ---------- # MAGIC %md # MAGIC --- # MAGIC -# MAGIC ## A Note On Time, And Why This Demo Does Not Use It +# MAGIC ## Section 6: (Optional) Time As A *Basis*, Not A Feature +# MAGIC +# MAGIC Everything so far compared each transaction against its merchant category's normal. Amounts in this +# MAGIC dataset do not trend, so no time axis was needed. Plenty of payments data does trend: a processor's +# MAGIC volume grows as merchants are onboarded, a subscription book compounds, a seasonal retailer ramps. +# MAGIC +# MAGIC When the normal level itself moves, `baseline_over_time` names a column as the **axis** each metric is +# MAGIC measured along. This is the part worth internalising: +# MAGIC +# MAGIC | | What DQX does with the column | What it detects | +# MAGIC |---|---|---| +# MAGIC | timestamp listed in `columns` | turns it into seven calendar features (hour, day of week, month, weekend) | 3am is odd and 3pm is not | +# MAGIC | timestamp passed as `baseline_over_time` | **never a feature.** It is the axis; DQX fits each metric's expected level along it | this value is wrong for *where the trend had got to* | +# MAGIC +# MAGIC The failure below is one every data team recognises: a feed partially breaks, volumes quietly revert to +# MAGIC an earlier level, and every number stays inside the year's range. No range check fires. Nothing is +# MAGIC extreme. It is only wrong for the point in time it arrived at. + +# COMMAND ---------- +# DBTITLE 1,A processor whose volume grows, and a week where a feed silently reverts + +# 18 months of daily counts, growing as merchants are onboarded. The fault holds one week at the level of +# roughly five months earlier -- inside the range the year covers, so nothing about it is out of bounds. +DAILY_START = datetime(2024, 1, 1) + + +def generate_daily_volume(n_days: int, seed: int, stalled_week: bool = False): + """Daily transaction count and settled value for one processor, trending upward.""" + rng = np.random.default_rng(seed) + days = np.arange(n_days) + count = 4_000 + 9.0 * days + 300.0 * np.sin(2 * np.pi * days / 7.0) + rng.normal(0, 120, n_days) + settled = count * (26.0 + 0.004 * days) + rng.normal(0, 4_000, n_days) + labels = np.zeros(n_days) + + if stalled_week: + start = int(n_days * 0.82) + count[start : start + 7] -= 9.0 * 150 + settled[start : start + 7] -= 9.0 * 150 * 26.0 + labels[start : start + 7] = 1.0 + + rows = [ + ( + DAILY_START + timedelta(days=int(day)), + float(count[i]), + float(settled[i]), + float(labels[i]), + ) + for i, day in enumerate(days) + ] + return spark.createDataFrame(rows, "settlement_date timestamp, txn_count double, settled_value double, is_anomaly double") + + +volume_history = f"{catalog}.{schema}.processor_daily_history" +volume_recent = f"{catalog}.{schema}.processor_daily_recent" +generate_daily_volume(540, seed=21).write.mode("overwrite").option("overwriteSchema", "true").saveAsTable(volume_history) +generate_daily_volume(180, seed=22, stalled_week=True).write.mode("overwrite").option( + "overwriteSchema", "true" +).saveAsTable(volume_recent) + +print("📊 18 months of daily history, and 6 months of recent days containing one stalled week") + +# COMMAND ---------- +# DBTITLE 1,Check the data actually trends before reaching for the parameter + +# The decision comes first and it is measurable. Subtracting a fitted expectation from a metric with no +# structure over time removes real signal and adds the fit's own error, so this is not a free switch. +from databricks.labs.dqx.anomaly.temporal_advisory import measure_trend_strength + +VOLUME_METRICS = ["txn_count", "settled_value"] +volume_trend = measure_trend_strength(spark.table(volume_history), "settlement_date", VOLUME_METRICS) +amount_trend = measure_trend_strength(spark.table(history_table), "transaction_time", ["amount", "item_count"]) + +print(f"📊 Daily processor volume: {volume_trend:.1%} of variance explained by trend and seasonality ✅ use it") +print(f"📊 Individual transactions: {amount_trend:.1%} ⚠️ nothing to remove, which is why Sections 1-5 do not") + +# COMMAND ---------- +# DBTITLE 1,Train with a time axis + +# settlement_date is named as the axis, so it is NOT a feature: no calendar columns are derived from it, and +# nothing about the hour or weekday enters the model. DQX fits each metric's expected level along it instead. +volume_model = f"{catalog}.{schema}.processor_volume_monitor" + +volume_trained = anomaly_engine.train( + df=spark.table(volume_history), + model_name=volume_model, + registry_table=registry_table, + columns=VOLUME_METRICS, + baseline_over_time="settlement_date", + baseline_by=[], + params=AnomalyParams(sample_fraction=1.0), +) + +print(f"\n🎯 Trained with an expected level per metric over time") + +# COMMAND ---------- +# DBTITLE 1,Score, and see what "wrong for now" looks like + +volume_scored = f"{catalog}.{schema}.processor_daily_scored" + +dq_engine.apply_checks_and_save_in_table( + input_config=InputConfig(location=volume_recent), + output_config=OutputConfig(location=volume_scored, mode="overwrite", options={"overwriteSchema": "true"}), + checks=[ + DQDatasetRule( + criticality="error", + check_func=has_no_row_anomalies, + check_func_kwargs={ + "model_name": volume_trained, + "registry_table": registry_table, + "threshold": 95.0, + "enable_ai_explanation": False, + }, + ) + ], +) + +volume_result = spark.table(volume_scored) +volume_anomaly = F.element_at(F.col("_dq_info"), 1).getField("anomaly") +stalled = volume_result.filter(F.col("is_anomaly") == 1.0).count() +caught = volume_result.filter(volume_anomaly.getField("is_anomaly") & (F.col("is_anomaly") == 1.0)).count() +print(f"🔍 Caught {caught} of the {stalled} days when the feed had quietly reverted\n") + +report_thresholds(volume_result, "Daily volume, judged against its own trend:", label_col="is_anomaly") + +# COMMAND ---------- +# DBTITLE 1,Read the contributions, which name the expected level + +print("💡 The contributions say ' vs its expected level at that time', not 'unusual '.") +print(" That distinction is the whole point: every one of these counts sits inside the range the") +print(" history covers. Only their position against the trend is wrong.\n") + +display( + volume_result.filter(volume_anomaly.getField("is_anomaly")) + .select( + "settlement_date", + F.round("txn_count").alias("txn_count"), + volume_anomaly.getField("severity_percentile").alias("severity"), + volume_anomaly.getField("contributions").alias("contributions"), + volume_anomaly.getField("is_stale_baseline").alias("extrapolating"), + ) + .orderBy(F.desc("severity")) + .limit(8) +) + +# COMMAND ---------- + +# MAGIC %md +# MAGIC --- +# MAGIC +# MAGIC ### The four ways DQX can treat a timestamp +# MAGIC +# MAGIC | Ask this | Use | The timestamp becomes | +# MAGIC |---|---|---| +# MAGIC | Is this row odd on its own, or in combination? | `profile` | nothing | +# MAGIC | Is it odd for its own group? | `baseline_by` | nothing | +# MAGIC | Is it odd for the time of day or day of week? | list it in `columns` | seven calendar features | +# MAGIC | Is it odd for its own point in time? | `baseline_over_time` | the axis, never a feature | +# MAGIC +# MAGIC They compose. Set `baseline_by` and `baseline_over_time` together and each metric is judged against +# MAGIC what its own group's history says to expect at that moment, still on one pooled model. # MAGIC -# MAGIC DQX has a third comparison basis, `baseline_over_time`, which judges each metric against what its -# MAGIC own history says to expect at that point in time. It is the right tool for a metric that trends or -# MAGIC carries a daily shape — see the -# MAGIC [fleet telemetry demo](https://github.com/databrickslabs/dqx/blob/main/demos/dqx_demo_anomaly_timeseries_fleet.py). +# MAGIC ### When to leave it off # MAGIC -# MAGIC **It is deliberately not used here.** Transaction amounts in this dataset are stationary: there is -# MAGIC no trend to remove and no daily shape to subtract. Fitting an expectation to data that has none -# MAGIC removes real signal and adds the fit's own error on top, and measured on a stationary dataset the -# MAGIC same transform performed *worse* than leaving it off. DQX will warn you when the training window -# MAGIC shows little structure over time, rather than turning it on for you. +# MAGIC - **A metric that does not trend**, like the individual transaction amounts in Sections 1 to 5. The +# MAGIC cell above measures both, and DQX warns when a training window shows too little structure over time. +# MAGIC - **A short window.** A weekly shape needs several complete weeks before it is identifiable at all; +# MAGIC DQX fits one only where the window supports it, and logs the period it skipped and why. +# MAGIC - **If you list a timestamp in `columns` by accident** — which is what auto-discovery does for you — +# MAGIC DQX warns at training time and names both escapes. Measured on this dataset, seven calendar features +# MAGIC derived from a meaningless timestamp cost a quarter of the detections at a fixed alert budget. # MAGIC -# MAGIC A demo that only ever shows features helping teaches the wrong default. The three bases answer -# MAGIC different questions, and picking the wrong one costs accuracy: +# MAGIC It is also **not a forecaster.** It models the level expected *at* a time; it does not predict the next +# MAGIC value and never reads the previous row, which is what keeps scoring valid on a stream. # MAGIC -# MAGIC | Ask this | Use | -# MAGIC |---|---| -# MAGIC | Is this row odd on its own, or in combination? | `profile` (this demo) | -# MAGIC | Is it odd for its own group? | `baseline_by` (this demo) | -# MAGIC | Is it odd for its own point in time? | `baseline_over_time` (not here) | +# MAGIC `is_stale_baseline` marks rows past the window the expectation was fitted on. The score is still +# MAGIC produced, because near the boundary it remains accurate; treat the flag as a signal to retrain. # COMMAND ---------- @@ -535,8 +703,9 @@ def generate_transactions(n_rows: int, seed: int, inject: bool = False): # MAGIC actionable rather than merely suspicious. # MAGIC - The threshold is an **alert budget**, not a confidence score. Judge precision against the ceiling # MAGIC that budget implies. -# MAGIC - Feed the model only columns that relate to what you are looking for. Here, excluding a timestamp -# MAGIC whose values carried no meaning took recall from 77% to 100%. +# MAGIC - A timestamp can be a **feature** or an **axis**, and the difference matters. Section 6 uses one as +# MAGIC an axis via `baseline_over_time` to catch a feed that silently reverted to an earlier level, with +# MAGIC every value still inside the year's range. # MAGIC # MAGIC **Apply to your data:** # MAGIC ```python diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py index ca8bf8cf7..ff0cb29c4 100644 --- a/demos/dqx_demo_anomaly_timeseries_fleet.py +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -31,21 +31,66 @@ # MAGIC those rise and fall together, because cutting harder draws more current. For two hours load was # MAGIC high while current sat mid-band. Both readings were ordinary. Their relationship was not. # MAGIC -# MAGIC ## Two kinds of anomaly, two profiles +# MAGIC ## "Normal" compared to what? +# MAGIC +# MAGIC Every anomaly check answers one question: *is this row normal?* The useful part is the follow-up, +# MAGIC **normal compared to what?** DQX gives you three independent answers, and you can use any combination +# MAGIC of them. Nothing here requires knowing any ML. +# MAGIC +# MAGIC #### 1. Compared to the rest of the table — which detector? (`profile`) # MAGIC # MAGIC | Your data | `profile` | An anomaly looks like | # MAGIC |---|---|---| -# MAGIC | Independent records — transactions, orders, customers | `"tabular"` (default) | A row whose values, or combination of values, is unusual | -# MAGIC | Repeated multivariate measurements — machine or service metrics, sensors | `"timeseries"` | Metrics that normally move **together** stop doing so, each staying in its own range | +# MAGIC | **Independent records.** Card payments, insurance claims, customer records, product listings. | `"tabular"` (default) | A row whose values, or combination of values, is unusual | +# MAGIC | **Repeated measurements of the same things.** Machine sensors, server metrics, patient vitals, smart meters. | `"timeseries"` | Metrics that normally move **together** stop doing so, each staying in its own range | +# MAGIC +# MAGIC This notebook uses `"timeseries"`, because the bearing story above is exactly its case. The default +# MAGIC detector splits on one column at a time, so a broken relationship between two in-range values is close +# MAGIC to invisible to it. On the **Server Machine Dataset**, 28 machines of real telemetry with labelled +# MAGIC incidents, the correlation-aware detector surfaces **79%** of incidents inside an alert budget of 1% of +# MAGIC rows, against **33%** for the default. +# MAGIC +# MAGIC #### 2. Compared to its own group (`baseline_by`) +# MAGIC +# MAGIC The same number can be fine in one group and wrong in another, so comparing everything against one +# MAGIC table-wide normal hides a whole class of problem: +# MAGIC +# MAGIC - **Retail** — £8,000 of sales is a good day for a small branch and a collapse for a flagship. +# MAGIC - **Payments** — £900 is ordinary for electronics and absurd for a coffee shop. +# MAGIC - **Healthcare** — a lab's reference range differs from another lab's for the same assay. +# MAGIC - **SaaS** — a 2% error rate is normal for one tenant's integration and an incident for another's. +# MAGIC +# MAGIC #### 3. Compared to its own past (`baseline_over_time`) +# MAGIC +# MAGIC When the normal level itself moves, a value that looks fine against the whole history can be wrong for +# MAGIC where things had actually got to: +# MAGIC +# MAGIC - **Manufacturing** — a wearing bearing runs hotter every month; 71°C is fine at week 1 and a warning +# MAGIC at week 40. +# MAGIC - **Subscriptions** — revenue that has grown all year makes last January's figure a bad yardstick. +# MAGIC - **Energy** — demand climbs through a heatwave, so yesterday is the only fair comparison. +# MAGIC - **Logistics** — a new depot ramps for months before its throughput means anything. +# MAGIC +# MAGIC **Section 5 covers this one**, on a dataset that genuinely trends, and measures whether it is worth +# MAGIC turning on before turning it on. It is off by default, because on flat data it measures *worse*. +# MAGIC +# MAGIC ## What DQX does with a timestamp column +# MAGIC +# MAGIC Three different things, depending on what you tell it. Worth knowing up front, because the default is +# MAGIC the one people least expect: +# MAGIC +# MAGIC | You... | DQX... | Use when | +# MAGIC |---|---|---| +# MAGIC | name `columns` explicitly and omit it | ignores it | the clock has nothing to do with what you are looking for | +# MAGIC | list it in `columns` | derives seven calendar features: hour, day of week and month as sine/cosine pairs, plus a weekend flag | 3am is suspicious and 3pm is not | +# MAGIC | pass it as `baseline_over_time` | treats it as the **axis** each metric is measured along, never as a feature | the normal level moves over time | # MAGIC -# MAGIC Use `"timeseries"` for data like this. The default detector splits on one feature at a time, so a -# MAGIC broken relationship between two in-range values is close to invisible to it. On the **Server Machine -# MAGIC Dataset** — 28 machines of real telemetry with labelled incidents — the correlation-aware detector -# MAGIC surfaces **79%** of incidents inside an alert budget of 1% of rows, against **33%** for the default. +# MAGIC Pass no `columns` at all and DQX discovers them for you, which *includes* any timestamp, so the +# MAGIC middle row is what you get by default. It says so at training time rather than leaving you to find out. # MAGIC -# MAGIC **It needs no timestamp column.** It models correlation *between metrics*, not behaviour over time, -# MAGIC and never looks at row order. Include a timestamp anyway and DQX derives calendar features from it -# MAGIC (hour, day of week, month, weekend) exactly as it does for the tabular profile. +# MAGIC Note what the correlation-aware profile does *not* need: **a timestamp, or any row order.** It models +# MAGIC how metrics relate to each other, not how they behave over time, and it never reads the previous row. +# MAGIC That is what keeps it valid on a streaming DataFrame. # MAGIC # COMMAND ---------- diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 02c63fbe6..a56c3d128 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -244,12 +244,19 @@ Avoid ID-like columns (for example, `order_id`, `user_id`) in anomaly training. - name: orders input_config: location: catalog.schema.orders - anomaly_config: - columns: [amount, quantity] # optional; omit to use all supported columns - model_name: catalog.schema.orders_monitor - registry_table: catalog.schema.dqx_anomaly_models + # anomaly_config sits beside input_config, not inside it + anomaly_config: + columns: [amount, quantity] # optional; omit to auto-discover + model_name: catalog.schema.orders_monitor + registry_table: catalog.schema.dqx_anomaly_models + baseline_by: [region, product] # optional; judge each metric against its own group + profile: tabular # optional; "tabular" (default) or "timeseries" + baseline_over_time: event_ts # optional; time column to fit each metric's level along ``` + Every key mirrors an argument of `anomaly_engine.train()`, so a model trained by hand can be reproduced + by a scheduled run and vice versa. Omitting a key is the same as omitting that argument. + Then trigger the anomaly-trainer workflow via the CLI: ```bash @@ -433,13 +440,19 @@ DQX fits each metric's expected level as a function of time, persists it with th detector the *difference* between the observed value and that expectation. Because the expectation is a function rather than a lookup table, it extends to timestamps the training window never contained. -It is the third of three independent questions, and they compose: +It is the third of three independent questions, and they compose. The column you name here is treated as an +**axis**, never as a feature, which is the distinction most worth holding on to: listing a timestamp in +`columns` instead expands it into seven calendar features and answers a completely different question. -| Question | Argument | -|---|---| -| Is this value unusual on its own, or in combination? | `profile` | -| Is it unusual for its own group? | `baseline_by` | -| Is it unusual for its own point in time? | `baseline_over_time` | +| Question | Argument | The timestamp becomes | +|---|---|---| +| Is this value unusual on its own, or in combination? | `profile` | nothing | +| Is it unusual for its own group? | `baseline_by` | nothing | +| Is it unusual for the time of day or day of week? | list it in `columns` | seven calendar features | +| Is it unusual for its own point in time? | `baseline_over_time` | the axis, never a feature | + +Pass no `columns` at all and auto-discovery includes any timestamp it finds, so the third row is what you get +by default. DQX warns at training time when that happens and names both ways out. Set both grouping and time and the expectation is fitted on the group-relative value, so a table whose groups trend at different rates still needs only one model. diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index 98d0b2891..cca402ae0 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -282,7 +282,7 @@ def _advise_calendar_features(df: DataFrame, columns: list[str], time_column: st f"month, weekend). That helps when an anomaly is contextual by calendar and hurts otherwise: " f"measured, it took group-contextual detection from 72% to 0% by diluting the metric it was " f"meant to support. Pass exclude_columns={safe} to leave them out, or " - f"baseline_over_time='' to use one as a time axis instead of as features." + f"baseline_over_time='{safe[0]}' to use it as a time axis instead of as features." ) def build_context( diff --git a/tests/integration_anomaly/test_anomaly_autodiscovery.py b/tests/integration_anomaly/test_anomaly_autodiscovery.py index 71a430544..68d1afc56 100644 --- a/tests/integration_anomaly/test_anomaly_autodiscovery.py +++ b/tests/integration_anomaly/test_anomaly_autodiscovery.py @@ -1,5 +1,6 @@ """Integration tests for auto-discovery of anomaly detection columns and segments.""" +import datetime import logging from pyspark.sql import SparkSession @@ -10,6 +11,9 @@ from tests.integration_anomaly.constants import SEGMENT_REGIONS from tests.integration_anomaly.conftest import qualify_model_name +#: A Monday, so the weekday and weekend calendar features the advisory warns about are meaningful. +START = datetime.datetime(2025, 1, 6) + def test_auto_discover_numeric_columns(spark: SparkSession): """Test auto-discovery selects numeric columns with variance.""" @@ -472,3 +476,74 @@ def test_segment_column_explicitly_removed_from_features(spark: SparkSession): # Verify region was analyzed (has a type) but then removed assert profile.column_types is not None + + +def test_the_calendar_advisory_fires_when_a_timestamp_reaches_the_feature_list( + spark: SparkSession, make_schema, make_random, anomaly_engine, caplog +): + """A datetime column in *columns* becomes seven cyclical features, and the caller has to be told. + + This is the advisory with the most to say for itself, because the mistake it names is the *default*: + auto-discovery includes any timestamp it finds. Measured on the transactions demo, the same model with a + meaningless timestamp added took recall at a fixed alert budget from 100% to 79% while raising exactly as + many alerts, so a caller who never sees the warning silently loses a quarter of their detections. + + The message must carry both escapes, since the right one depends on what the column means: exclude it + when the clock is irrelevant, or name it as an axis when the level moves over time. + """ + schema = make_schema(catalog_name=TEST_CATALOG) + suffix = make_random(8).lower() + + rows = [(START + datetime.timedelta(hours=i), float(100.0 + i % 17), 5.0 + i % 3) for i in range(400)] + df = spark.createDataFrame(rows, "event_ts timestamp, amount double, discount double") + table_name = f"{TEST_CATALOG}.{schema.name}.calendar_advisory_{suffix}" + df.write.saveAsTable(table_name) + + registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" + with caplog.at_level(logging.WARNING, logger="databricks.labs.dqx.anomaly.training_service"): + anomaly_engine.train( + df=spark.table(table_name), + columns=["amount", "discount", "event_ts"], + model_name=qualify_model_name(f"test_calendar_{suffix}", registry_table), + registry_table=registry_table, + baseline_by=[], + ) + + advisories = [r.message for r in caplog.records if "cyclical calendar features" in r.message] + assert advisories, "a datetime column among the features must be called out" + message = advisories[0] + assert "event_ts" in message + # Both escapes, and each naming the actual column rather than a placeholder, so either is copy-pasteable. + assert "exclude_columns=['event_ts']" in message + assert "baseline_over_time='event_ts'" in message + + +def test_the_calendar_advisory_stays_quiet_when_the_timestamp_is_the_axis( + spark: SparkSession, make_schema, make_random, anomaly_engine, caplog +): + """Naming a column as the time axis is the fix the advisory recommends, so it must not then fire. + + An advisory that keeps warning after you have followed it is one callers learn to filter out, and then it + is worth nothing on the run where it matters. + """ + schema = make_schema(catalog_name=TEST_CATALOG) + suffix = make_random(8).lower() + + rows = [(START + datetime.timedelta(hours=i), float(100.0 + 0.05 * i), 5.0 + i % 3) for i in range(400)] + df = spark.createDataFrame(rows, "event_ts timestamp, amount double, discount double") + table_name = f"{TEST_CATALOG}.{schema.name}.calendar_axis_{suffix}" + df.write.saveAsTable(table_name) + + registry_table = f"{TEST_CATALOG}.{schema.name}.dqx_anomaly_models_{suffix}" + with caplog.at_level(logging.WARNING, logger="databricks.labs.dqx.anomaly.training_service"): + anomaly_engine.train( + df=spark.table(table_name), + columns=["amount", "discount"], + model_name=qualify_model_name(f"test_axis_{suffix}", registry_table), + registry_table=registry_table, + baseline_by=[], + baseline_over_time="event_ts", + ) + + advisories = [r.message for r in caplog.records if "cyclical calendar features" in r.message] + assert not advisories, f"the axis column must not be reported as a feature: {advisories}" From b411349a552f085903a49b9774104c3aeb9dda7d Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 13:43:25 +0100 Subject: [PATCH 075/107] Address the pre-Beta review: four blockers, four issues, four suppressions Every claim in the review was reproduced against the code before being fixed. All eight held, and so did the suppression count. Two of my own were wrong and are corrected below. BLOCKERS A pre-release registry could not be retrained, which is the remedy both the migration guide and the stale-hash error prescribe. get_active_model read values["grouping"] unconditionally while a pre-upgrade table carries "segmentation", so it raised KeyError, and training reads the registry, so it failed before save_model could write anything. Now migrated properly rather than papered over: the table is rewritten into the current schema and "segmentation" is gone, instead of sitting beside "grouping" forever with each null on half the rows. The mergeSchema that produced that outcome is removed, so a future schema change fails loudly and demands its own migration. Permission failures translate into an error naming the grant needed, and state plainly that models in an unmigrated table cannot be scored either until retrained -- my first draft claimed scoring was unaffected, which is false, because the hash formula changed. Grouped single-model training failed at MLflow signature inference. Feature engineering preserves the group key on purpose, so the engineered frame is wider than the feature list, and the whole frame reached model.predict carrying a string column the estimator was never fitted on. It affected profile="timeseries" always and the tabular profile at ensemble_size=1; the default three-model ensemble registers by URI and never reaches that code, which is why nothing caught it. One null timestamp ended temporal training with a TypeError: the null bucket's min(seconds) reached float() unguarded while the metric values beside it were guarded. Nulls are filtered from the fit; the all-null case still raises deliberately. The headline SMD evidence measured a 4,000-row prefix of each entity, after which the ten entities with no positives inside that prefix were dropped as unusable. Recomputed from the raw labels: the full series holds 29,444 anomalous rows in 327 incidents across all 28 entities, against 3,732 in 39 across 18. machine-1-1 alone has 2,694 anomalous rows and none in its first 4,000. The truncation is removed and the full series is now the default. The published "79% against 33%" at a 1% budget becomes 67.6% against 57.3%, and the justification moves to average precision, 0.425 against 0.278, because the per-entity coverage gain is +7.9% with a standard deviation of 38.2% and a 16/7 record -- inside its own spread. The guide now states the entity count, the full-series scope and that variance. ISSUES Spark and numpy disagreed on a degenerate severity tail: with p95 equal to p99, Spark pinned everything above the anchor to 95 while numpy interpolated to 100, so thresholds above 95 became unreachable and SHAP gating decoupled from the authoritative severity. Spark now matches numpy, and the linear chain is extracted so the head and the fallback share one implementation. An explicit baseline_by bypassed MAX_BASELINE_GROUPS, which auto-discovery has always respected. The failure is a driver out-of-memory rather than a slow run, because per-group medians and quantiles are each collected and persisted into model metadata. Now refused with the group count and the remedy. The rows-per-group floor was checked before sampling, so a guarantee of 30 rows became about 7 at the fit. The gate uses the effective figure and honours the caller's own sample_fraction. Two existing tests encoded the old optimism, one asserting 120 rows per group was "comfortably above the floor" when it is 28.8 by the time anything is fitted; both were rescaled to test the same boundary. A null group value collided with a literal "MISSING", silently merging two populations into one baseline. The sentinel is now a control character. Done now because it changes the persisted key format and every model must already be retrained this release. Narrower than the review stated: the imputation route is unreachable, because baseline_by may not overlap the feature list. SUPPRESSIONS Four, removed rather than justified. Three type: ignore gave way to the plain if/else that narrows, matching the block above them. One noqa was inert -- the rule it names is not enabled -- which is why nothing flagged it, and why citing "pylint 10.00/10" was checking the wrong signal. Verify with: git diff origin/main...HEAD -- src/ | grep -E "noqa|type: ignore|pylint: disable" Left for the maintainers: this change is dated 0.16.0 in three places and 0.17.0 in three others against a __version__.py of 0.16.0. Which is right depends on the release, and version props are historical facts rather than something to infer, so nothing was changed except the runtime error, which no longer names a release. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 18 ++- src/databricks/labs/dqx/anomaly/core.py | 15 +- .../labs/dqx/anomaly/model_registry.py | 139 ++++++++++++++++-- src/databricks/labs/dqx/anomaly/profiler.py | 38 ++++- .../labs/dqx/anomaly/scoring_run.py | 4 +- .../labs/dqx/anomaly/scoring_utils.py | 92 +++++++----- .../labs/dqx/anomaly/segment_utils.py | 11 +- .../labs/dqx/anomaly/training_service.py | 31 +++- .../labs/dqx/anomaly/transformers.py | 9 +- .../test_anomaly_registry.py | 118 +++++++++++++++ .../test_anomaly_temporal_features.py | 33 +++++ .../test_anomaly_timeseries_profile.py | 73 ++++++++- tests/unit/test_anomaly_baseline_discovery.py | 34 ++++- tests/unit/test_anomaly_group_key.py | 15 ++ 14 files changed, 554 insertions(+), 76 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index a56c3d128..204bad954 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -529,18 +529,24 @@ tabular data and weak when the anomaly *is* a broken relationship: if CPU is nor normal, no single-feature split separates the row, even though "high CPU with idle memory" never happens on a healthy machine. -Measured on the Server Machine Dataset — real machine telemetry with labelled incidents — this is the -share of incidents each detector surfaces while the alert budget is capped at 1% of rows: +Measured on the full Server Machine Dataset: real machine telemetry, 28 machines, 38 metrics, 327 labelled +incidents across 708,420 rows. This is the share of incidents each detector surfaces while the alert budget +is capped at 1% of rows: | `profile` | Incidents surfaced | |---|---| -| `"tabular"` | 33% | -| `"timeseries"` | **79%** | +| `"tabular"` | 57.3% | +| `"timeseries"` | **67.6%** | An incident counts as surfaced if the detector flags at least one of its rows, so this measures whether you would have been paged, not how many rows you would have had to read. Both detectors were trained on -data that still contained the anomalies, which is what DQX does when it fits a sample of your table; on a -curated clean training split the same comparison is 36% against 82%. +data that still contained the anomalies, which is what DQX does when it fits a sample of your table. + +**Read this as an average, not a promise.** Per machine the gap has a standard deviation of 38 points, and +the correlation-aware detector wins on 16 of the 28 and loses on 7, so it is the better default for this +shape of data rather than a guarantee on any particular machine. The most consistent difference is average +precision, 0.425 against 0.278, because that is an aggregate over the whole ranking rather than a count of +incidents caught inside a budget. That is telemetry, which is what `"timeseries"` is for. On ordinary tabular data the two detectors are closer than this table might suggest, so choose on the *shape* of the anomaly you expect rather than on an diff --git a/src/databricks/labs/dqx/anomaly/core.py b/src/databricks/labs/dqx/anomaly/core.py index 50cbcd1f1..4885c7df3 100644 --- a/src/databricks/labs/dqx/anomaly/core.py +++ b/src/databricks/labs/dqx/anomaly/core.py @@ -493,15 +493,22 @@ def aggregate_ensemble_metrics(all_metrics: list[dict[str, float]]) -> dict[str, def prepare_engineered_pandas(train_df: DataFrame, feature_metadata: SparkFeatureMetadata) -> pd.DataFrame: """Prepare engineered pandas DataFrame from Spark DataFrame. - Applies feature engineering transformations and collects to pandas. - Used for MLflow signature inference. + Applies feature engineering transformations, projects to the columns the model was fitted on, and + collects to pandas. Used for MLflow signature inference. + + The projection is load-bearing. Feature engineering deliberately preserves columns it did not + produce -- the group key among them, because the group-relative transform needs it at scoring time -- + so the engineered frame is wider than the feature list. Handing that frame to ``model.predict`` for + signature inference passes the estimator a string column it was never fitted on, which fails for any + grouped model on the single-model path (``profile="timeseries"``, or ``ensemble_size=1``). The default + three-model ensemble registers by URI and never comes through here, which is why this was invisible. Args: train_df: Training Spark DataFrame feature_metadata: Feature engineering metadata from training Returns: - Pandas DataFrame with engineered features + Pandas DataFrame holding exactly the engineered feature columns, in their persisted order """ engineered_train_df, _ = apply_feature_engineering_from_metadata(train_df, feature_metadata) - return engineered_train_df.toPandas() + return engineered_train_df.select(*feature_metadata.engineered_feature_names).toPandas() diff --git a/src/databricks/labs/dqx/anomaly/model_registry.py b/src/databricks/labs/dqx/anomaly/model_registry.py index e64c92982..95fc01ba4 100644 --- a/src/databricks/labs/dqx/anomaly/model_registry.py +++ b/src/databricks/labs/dqx/anomaly/model_registry.py @@ -4,6 +4,7 @@ Persistence only: schema and AnomalyModelRegistry. Record types live in model_config. """ +import logging from decimal import Decimal from typing import Any @@ -18,10 +19,14 @@ TrainingMetadata, ) from databricks.labs.dqx.config import OutputConfig +from databricks.labs.dqx.errors import InvalidParameterError from databricks.labs.dqx.io import save_dataframe_as_table from databricks.labs.dqx.utils import table_exists +logger = logging.getLogger(__name__) + + ANOMALY_MODEL_TABLE_SCHEMA = ( "identity struct, " "training struct, hyperparameters:map, training_rows:bigint, " @@ -33,6 +38,52 @@ ) +#: The pre-release grouping column. A registry table written before ``segment_by`` was removed carries +#: this instead of ``grouping``. Everything that reads or migrates such a table keys off this name, so +#: there is one place to delete when support for those tables is dropped. +LEGACY_GROUPING_COLUMN = "segmentation" + +#: Substrings Unity Catalog and Spark use when a write is refused for want of a grant. Matched rather +#: than parsed: the surrounding message wording is not a stable interface, but these codes are what the +#: platform documents, and mistaking a real failure for a permissions one would send a user to their +#: admin over a bug. +_PERMISSION_DENIED_MARKERS = ("PERMISSION_DENIED", "INSUFFICIENT_PERMISSIONS", "does not have permission") + + +def _is_permission_error(exc: Exception) -> bool: + """Whether *exc* reports a missing grant rather than a genuine failure.""" + message = str(exc).upper() + return any(marker.upper() in message for marker in _PERMISSION_DENIED_MARKERS) + + +def normalise_grouping(values: dict[str, Any]) -> dict[str, Any]: + """Return the ``grouping`` payload for a registry row, whatever schema the row was written in. + + Exists so that reading a pre-release table produces the *useful* error rather than an internal one. + A model registered before this release cannot be scored at all -- the configuration hash formula + changed, so the recomputed hash always mismatches and scoring raises with instructions to retrain -- + but without this the read died on ``KeyError: 'grouping'`` first, and the caller saw an internal + error instead of those instructions. + + Three shapes reach here. A pre-release table has ``segmentation`` and no ``grouping``. A table that an + earlier build of this release appended to with ``mergeSchema`` has both, with ``grouping`` null on the + rows written before. A current table has ``grouping`` alone. + + ``segment_values`` and ``is_global_model`` are dropped rather than carried: there is exactly one + pooled model now, so neither has a meaning to preserve. + """ + grouping = values.get("grouping") + if grouping: + return grouping + + legacy = values.get(LEGACY_GROUPING_COLUMN) or {} + return { + "baseline_by": legacy.get("segment_by") or [], + "sklearn_version": legacy.get("sklearn_version"), + "config_hash": legacy.get("config_hash"), + } + + class AnomalyModelRegistry: """Manage anomaly model metadata in a Delta table.""" @@ -55,10 +106,14 @@ def convert_decimals(obj: Any) -> Any: return obj @staticmethod - def build_model_df(spark: SparkSession, record: AnomalyModelRecord) -> DataFrame: - """Convert a registry record into a DataFrame with nested structure.""" - # Convert composed dataclass to nested dict structure - record_dict = { + def _record_dict(record: AnomalyModelRecord) -> dict[str, Any]: + """Flatten a record into the nested dict the table schema expects. + + Extracted from build_model_df so the migration writes rows through exactly the same mapping a + normal save uses. Two copies of this would diverge on the next field added, and the divergence + would only show up as a migrated row missing a value. + """ + return { "identity": { "model_name": record.identity.model_name, "model_uri": record.identity.model_uri, @@ -89,8 +144,11 @@ def build_model_df(spark: SparkSession, record: AnomalyModelRecord) -> DataFrame }, } + @staticmethod + def build_model_df(spark: SparkSession, record: AnomalyModelRecord) -> DataFrame: + """Convert a registry record into a DataFrame with nested structure.""" # Convert Decimals in nested structures (baseline_stats, metrics, etc.) - record_dict = AnomalyModelRegistry.convert_decimals(record_dict) + record_dict = AnomalyModelRegistry.convert_decimals(AnomalyModelRegistry._record_dict(record)) return spark.createDataFrame([record_dict], schema=ANOMALY_MODEL_TABLE_SCHEMA) @@ -100,14 +158,73 @@ def save_model(self, record: AnomalyModelRecord, table: str) -> None: if not table_existed: self._create_table(table) else: + # Migrate before anything else touches the table, so archiving and the append below run + # against the current schema and need no compatibility handling of their own. + self._migrate_legacy_registry(table) self._archive_previous(table, record.identity.model_name) df = self.build_model_df(self.spark, record) - # mergeSchema so a registry table created by an earlier DQX reconciles to the new struct - # shape on the next write instead of failing -- the `grouping` struct replaced `segmentation` - # this release. Retraining is exactly what the configuration-hash error tells the user to do, - # and it writes to their existing table, so the remedy has to work. - save_dataframe_as_table(df, OutputConfig(location=table, mode="append", options={"mergeSchema": "true"})) + # A plain append, deliberately without mergeSchema. Its only purpose was to reconcile the + # pre-release `segmentation` column, which _migrate_legacy_registry now does explicitly and + # completely. Leaving it on would mean a future schema change silently widens a user's table + # instead of failing and demanding its own migration. + save_dataframe_as_table(df, OutputConfig(location=table, mode="append")) + + def _migrate_legacy_registry(self, table: str) -> None: + """Rewrite a pre-release registry table into the current schema, in place. + + The alternative was appending with ``mergeSchema``, which is not a migration: the table keeps both + ``segmentation`` and ``grouping`` for good, each null on half its rows, and every reader afterwards + has to know about both. Rewriting costs one pass over a table holding one row per model version, + and leaves a schema a user can read without knowing this release happened. + + Historical rows are migrated rather than dropped, so the archive of previous versions survives. + Delta keeps the pre-migration versions in table history if anyone needs to look. + + No-op unless the legacy column is present, so the normal path pays a schema read and nothing else. + """ + if LEGACY_GROUPING_COLUMN not in self.spark.table(table).columns: + return + + logger.warning( + f"Registry table '{table}' uses the pre-release schema. Migrating it to the current one: " + f"'{LEGACY_GROUPING_COLUMN}' becomes 'grouping', and segment_values and is_global_model are " + f"dropped because there is one pooled model now. Previous versions remain in Delta history." + ) + + migrated = [ + AnomalyModelRecord( + identity=ModelIdentity(**values["identity"]), + training=TrainingMetadata(**values["training"]), + features=FeatureEngineering(**values["features"]), + grouping=GroupingConfig(**normalise_grouping(values)), + ) + for values in (row.asDict(recursive=True) for row in self.spark.table(table).collect()) + ] + + rewritten = ( + self.spark.createDataFrame( + [self.convert_decimals(self._record_dict(record)) for record in migrated], + schema=ANOMALY_MODEL_TABLE_SCHEMA, + ) + if migrated + else self.spark.createDataFrame([], schema=ANOMALY_MODEL_TABLE_SCHEMA) + ) + + try: + save_dataframe_as_table( + rewritten, + OutputConfig(location=table, mode="overwrite", options={"overwriteSchema": "true"}), + ) + except Exception as exc: + if not _is_permission_error(exc): + raise + raise InvalidParameterError( + f"Cannot migrate registry table '{table}' from the pre-release schema: this needs MODIFY " + f"on the table. Ask an owner to grant it, or train into a registry table you own. Note " + f"that models already in '{table}' cannot be scored either until they are retrained, " + f"because the configuration hash changed this release." + ) from exc def get_active_model(self, table: str, model_name: str) -> AnomalyModelRecord | None: """Fetch the active model for a given name.""" @@ -130,7 +247,7 @@ def get_active_model(self, table: str, model_name: str) -> AnomalyModelRecord | identity=ModelIdentity(**values["identity"]), training=TrainingMetadata(**values["training"]), features=FeatureEngineering(**values["features"]), - grouping=GroupingConfig(**values["grouping"]), + grouping=GroupingConfig(**normalise_grouping(values)), ) return record diff --git a/src/databricks/labs/dqx/anomaly/profiler.py b/src/databricks/labs/dqx/anomaly/profiler.py index 594226338..ba8f1e2fa 100644 --- a/src/databricks/labs/dqx/anomaly/profiler.py +++ b/src/databricks/labs/dqx/anomaly/profiler.py @@ -6,6 +6,7 @@ """ import logging +import math import re from dataclasses import dataclass from typing import Any @@ -25,6 +26,9 @@ TimestampType, ) +# The sampling defaults live with sampling. profiler does not otherwise depend on core, and core does +# not import profiler, so this direction adds no cycle. +from databricks.labs.dqx.anomaly.core import DEFAULT_SAMPLE_FRACTION, DEFAULT_TRAIN_RATIO from databricks.labs.dqx.anomaly.group_config import ( MAX_BASELINE_COLUMN_CARDINALITY, MAX_BASELINE_GROUPS, @@ -235,6 +239,8 @@ def _is_grouping_candidate( null_rate: float, is_id_column: bool, total_count: int, + sample_fraction: float | None = None, + train_ratio: float | None = None, ) -> bool: """Whether a column may be *considered* as a baseline grouping. @@ -246,11 +252,35 @@ def _is_grouping_candidate( 2 <= distinct_count <= MAX_BASELINE_COLUMN_CARDINALITY and null_rate < 0.1 and not is_id_column - and (total_count / distinct_count) >= MIN_ROWS_PER_BASELINE_GROUP + and (total_count / distinct_count) >= effective_min_rows_per_group(sample_fraction, train_ratio) ) -def select_baseline_columns(candidates: list[tuple[str, int, float]], total_count: int) -> list[str]: +def effective_min_rows_per_group(sample_fraction: float | None = None, train_ratio: float | None = None) -> int: + """Rows a group needs *on the full table* for the fit to see MIN_ROWS_PER_BASELINE_GROUP of them. + + Discovery runs before sampling, so checking the nominal minimum against the full table overstated what + the model would get by roughly 4x at the defaults: sampling keeps 0.3 and the train split keeps 0.8 of + that, leaving about 7 rows from a group that just cleared 30. A per-group median fitted on 7 rows is not + the representative baseline the constant is there to guarantee. + + Raising the bar on the full table is preferred to re-validating afterwards, which would mean choosing a + grouping, sampling, finding it too fine, and choosing again. + """ + retained = (sample_fraction if sample_fraction is not None else DEFAULT_SAMPLE_FRACTION) * ( + train_ratio if train_ratio is not None else DEFAULT_TRAIN_RATIO + ) + if retained <= 0.0 or retained >= 1.0: + return MIN_ROWS_PER_BASELINE_GROUP + return int(math.ceil(MIN_ROWS_PER_BASELINE_GROUP / retained)) + + +def select_baseline_columns( + candidates: list[tuple[str, int, float]], + total_count: int, + sample_fraction: float | None = None, + train_ratio: float | None = None, +) -> list[str]: """Choose a grouping for baseline conditioning, given candidates ordered by cardinality. Adds columns while every resulting group still holds enough rows for a representative median and @@ -272,7 +302,7 @@ def select_baseline_columns(candidates: list[tuple[str, int, float]], total_coun prospective = groups * int(distinct_count) if prospective > MAX_BASELINE_GROUPS: continue - if total_count / prospective < MIN_ROWS_PER_BASELINE_GROUP: + if total_count / prospective < effective_min_rows_per_group(sample_fraction, train_ratio): continue selected.append(name) groups = prospective @@ -288,7 +318,7 @@ def select_baseline_columns(candidates: list[tuple[str, int, float]], total_coun if skipped: logger.debug( f"Not added to the baseline grouping (would leave under " - f"{MIN_ROWS_PER_BASELINE_GROUP} rows/group, or exceed {MAX_BASELINE_GROUPS} " + f"{effective_min_rows_per_group()} rows/group before sampling, or exceed {MAX_BASELINE_GROUPS} " f"groups): {skipped}" ) return selected diff --git a/src/databricks/labs/dqx/anomaly/scoring_run.py b/src/databricks/labs/dqx/anomaly/scoring_run.py index b7705da2d..cf828fd8f 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_run.py +++ b/src/databricks/labs/dqx/anomaly/scoring_run.py @@ -127,9 +127,9 @@ def score_global_model( f" Trained baseline_by: {trained_baseline_by or None}\n" f" Trained baseline_over_time: {trained_baseline_over_time or None}\n\n" f"This model was trained with a different configuration, or by a DQX version before\n" - f"baseline_by became part of the configuration hash (0.17.0). Either:\n" + f"baseline_by became part of the configuration hash. Either:\n" f" 1. Use the columns that match the trained model\n" - f" 2. Retrain the model — required for any model registered before 0.17.0" + f" 2. Retrain the model — required for any model registered before that change" ) check_model_staleness(record, config.model_name) diff --git a/src/databricks/labs/dqx/anomaly/scoring_utils.py b/src/databricks/labs/dqx/anomaly/scoring_utils.py index 0aa4bc07f..3a6cb4df8 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_utils.py +++ b/src/databricks/labs/dqx/anomaly/scoring_utils.py @@ -193,17 +193,17 @@ def add_info_column( # Surface the extrapolation verdict and where the evidence ran out. Both null when the model has no # temporal baseline, which is what keeps the struct's meaning unchanged for every existing model. - stale_present = stale is not None and stale.stale_col in df.columns - anomaly_info_fields["is_stale_baseline"] = ( - F.coalesce(F.col(stale.stale_col), F.lit(False)) # type: ignore[union-attr] - if stale_present - else F.lit(None).cast(BooleanType()) - ) - anomaly_info_fields["stale_baseline_horizon"] = ( - F.col(stale.horizon_col) # type: ignore[union-attr] - if stale_present and stale.horizon_col in df.columns # type: ignore[union-attr] - else F.lit(None).cast(StringType()) - ) + # Written as a plain if/else rather than two conditional expressions, matching the group_key_col block + # above: it narrows *stale* to non-null for the type checker in one place, instead of asserting the + # narrowing three times at the point of use. + if stale is not None and stale.stale_col in df.columns: + anomaly_info_fields["is_stale_baseline"] = F.coalesce(F.col(stale.stale_col), F.lit(False)) + anomaly_info_fields["stale_baseline_horizon"] = ( + F.col(stale.horizon_col) if stale.horizon_col in df.columns else F.lit(None).cast(StringType()) + ) + else: + anomaly_info_fields["is_stale_baseline"] = F.lit(None).cast(BooleanType()) + anomaly_info_fields["stale_baseline_horizon"] = F.lit(None).cast(StringType()) anomaly_info = F.struct(*[value.alias(key) for key, value in anomaly_info_fields.items()]).cast( anomaly_info_struct_schema @@ -263,11 +263,31 @@ def _tail_severity_expr(score_expr: Column, anchor: Column, rate: Column) -> Col span = rate - anchor head_tail_probability = 100.0 - TAIL_ANCHOR_PERCENTILE base = head_tail_probability / (100.0 - TAIL_RATE_PERCENTILE) - # A degenerate tail (p95 and p99 at the same score) has no width to interpolate over, so the anchor - # percentile is the answer outright, matching how a zero-width segment is handled below. - return F.when(span <= F.lit(0.0), F.lit(TAIL_ANCHOR_PERCENTILE)).otherwise( - F.lit(100.0) - F.lit(head_tail_probability) * F.pow(F.lit(base), -(score_expr - anchor) / span) - ) + # A degenerate tail is not handled here: this function has only the two anchors, and the sensible + # fallback needs the knots above them. :func:`_piecewise_severity_expr` selects between the two. + return F.lit(100.0) - F.lit(head_tail_probability) * F.pow(F.lit(base), -(score_expr - anchor) / span) + + +def _linear_severity_chain(score_expr: Column, points: list[tuple[float, Column]]) -> Column: + """Piecewise-linear severity over *points*, clamped at both ends. + + The behaviour that predates the exponential tail, kept as a named function because it is now needed in + two places: for the knots at or below p95, and as the fallback when the tail has no width to fit. + """ + prev_p, prev_q = points[0] + expr = F.when(score_expr <= prev_q, F.lit(float(prev_p))) + + for current_p, current_q in points[1:]: + span = current_q - prev_q + # A degenerate segment (equal bounds) would divide by zero. It means every score in this + # band sits on one point, so the upper percentile is the answer outright. + interpolated = F.when(span == F.lit(0.0), F.lit(float(current_p))).otherwise( + F.lit(float(prev_p)) + ((score_expr - prev_q) * F.lit(float(current_p) - float(prev_p)) / span) + ) + expr = expr.when(score_expr <= current_q, interpolated) + prev_p, prev_q = current_p, current_q + + return expr.otherwise(F.lit(float(prev_p))) def _piecewise_severity_expr(score_expr: Column, points: list[tuple[float, Column]]) -> Column: @@ -288,30 +308,26 @@ def _piecewise_severity_expr(score_expr: Column, points: list[tuple[float, Colum by_percentile = dict(points) anchor = by_percentile.get(TAIL_ANCHOR_PERCENTILE) rate = by_percentile.get(TAIL_RATE_PERCENTILE) - # A caller supplying its own points need not include the two anchors. Without them there is no tail to - # fit, so every point stays a knot and the behaviour is the pre-existing one. Built here rather than at - # the point of use so both anchors are narrowed to non-null in one place. - tail_expr = _tail_severity_expr(score_expr, anchor, rate) if anchor is not None and rate is not None else None - body = [(p, q) for p, q in points if p <= TAIL_ANCHOR_PERCENTILE] if tail_expr is not None else points + null_guard = F.when(score_expr.isNull(), F.lit(None).cast(DoubleType())) - expr = F.when(score_expr.isNull(), F.lit(None).cast(DoubleType())) - - prev_p, prev_q = body[0] - expr = expr.when(score_expr <= prev_q, F.lit(float(prev_p))) - - for current_p, current_q in body[1:]: - span = current_q - prev_q - # A degenerate segment (equal bounds) would divide by zero. It means every score in this - # band sits on one point, so the upper percentile is the answer outright. - interpolated = F.when(span == F.lit(0.0), F.lit(float(current_p))).otherwise( - F.lit(float(prev_p)) + ((score_expr - prev_q) * F.lit(float(current_p) - float(prev_p)) / span) - ) - expr = expr.when(score_expr <= current_q, interpolated) - prev_p, prev_q = current_p, current_q + # A caller supplying its own points need not include the two anchors. Without them there is no tail to + # fit, so every point stays a knot and the behaviour is the one that predates the tail. + if anchor is None or rate is None: + return null_guard.otherwise(_linear_severity_chain(score_expr, points)) + + head = [(p, q) for p, q in points if p <= TAIL_ANCHOR_PERCENTILE] + # Above the anchor, two shapes. Normally the exponential tail. But when p95 and p99 sit on the same + # score there is no width to interpolate a tail probability over, and the alternative -- pinning every + # higher score to 95 -- makes every threshold between 95 and 100 unreachable and leaves those rows + # unordered. So fall back to the plain linear chain across the remaining knots, which is exactly what + # the numpy counterpart in explainability.py does. The two must agree: that function gates whether a + # row gets SHAP inside the scoring UDF, so a disagreement drops contributions from flagged rows. + above_anchor = [(p, q) for p, q in points if p >= TAIL_ANCHOR_PERCENTILE] + tail = F.when(rate - anchor <= F.lit(0.0), _linear_severity_chain(score_expr, above_anchor)).otherwise( + _tail_severity_expr(score_expr, anchor, rate) + ) - if tail_expr is None: - return expr.otherwise(F.lit(float(prev_p))) - return expr.otherwise(tail_expr) + return null_guard.otherwise(F.when(score_expr <= anchor, _linear_severity_chain(score_expr, head)).otherwise(tail)) def add_baseline_severity_percentile_column( diff --git a/src/databricks/labs/dqx/anomaly/segment_utils.py b/src/databricks/labs/dqx/anomaly/segment_utils.py index 643a39227..80fb12078 100644 --- a/src/databricks/labs/dqx/anomaly/segment_utils.py +++ b/src/databricks/labs/dqx/anomaly/segment_utils.py @@ -20,7 +20,16 @@ # Stand-in for a NULL group value. NULLs must map to a real key rather than propagating, # or every row in a group with a missing dimension silently loses its baseline. -BASELINE_KEY_NULL = "MISSING" +# +# A control character rather than a word, for the same reason the separator above is one: this was +# "MISSING", which a group column can genuinely contain, and then a NULL region and a region literally +# named MISSING shared one persisted key and one baseline without any complaint. \x00 is not a value a +# categorical dimension carries in practice, so the two are now distinguishable. +# +# This changes the persisted key format. It is done in the same release as the configuration-hash change +# that already forces every model to be retrained, so it costs no additional retrain -- deferring it would +# have made it a second forced retrain later. +BASELINE_KEY_NULL = "\x00" # The single column every stage reads the group key from. See :func:`with_baseline_key`. BASELINE_KEY_COLUMN = "__dqx_baseline_key" diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index cca402ae0..044194d75 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -29,6 +29,7 @@ ModelIdentity, TrainingMetadata, ) +from databricks.labs.dqx.anomaly.group_config import MAX_BASELINE_GROUPS from databricks.labs.dqx.anomaly.profiler import auto_discover_columns, suggest_baseline_columns from databricks.labs.dqx.anomaly.temporal_advisory import ( TREND_STRENGTH_ADVISORY_FLOOR, @@ -241,7 +242,10 @@ def _advise_trend_strength(df_filtered: DataFrame, columns: list[str], time_colu return try: strength = measure_trend_strength(df_filtered, time_column, numeric) - except Exception as exc: # noqa: BLE001 - an advisory must never fail a training run + # Broad on purpose: an advisory must never fail a training run, which is the same reason + # telemetry.py and table_manager.py catch broadly. No suppression comment, because the rule that + # would flag this is not enabled -- one was here and suppressed nothing. + except Exception as exc: logger.debug(f"Could not measure trend strength: {exc}") return if strength >= TREND_STRENGTH_ADVISORY_FLOOR: @@ -285,6 +289,30 @@ def _advise_calendar_features(df: DataFrame, columns: list[str], time_column: st f"baseline_over_time='{safe[0]}' to use it as a time axis instead of as features." ) + @staticmethod + def _reject_unbounded_grouping(df: DataFrame, baseline_by: list[str]) -> None: + """Refuse a grouping whose group count exceeds what can be persisted and broadcast. + + Auto-discovery has always honoured ``MAX_BASELINE_GROUPS`` by skipping a column that would breach + it. An explicit *baseline_by* bypassed that entirely, and the failure mode is not a slow run: the + per-group medians and the per-group score quantiles are each collected to the driver, one row per + group, and both are persisted into the feature metadata. An identifier-like grouping exhausts the + driver before it can produce a model, with nothing in the error to say why. + + One ``countDistinct`` action, paid only when a grouping is declared. Reuses the ceiling + auto-discovery already respects rather than introducing a second, so the two cannot drift. + """ + group_count = df.select(*baseline_by).distinct().count() + if group_count <= MAX_BASELINE_GROUPS: + return + safe_columns = [sanitize_for_logging(name) for name in baseline_by] + raise InvalidParameterError( + f"baseline_by={safe_columns} produces {group_count:,} groups, above the supported ceiling of " + f"{MAX_BASELINE_GROUPS:,}. Every group's median and score quantiles are held in the model's " + f"metadata and broadcast at scoring, so this would exhaust the driver rather than run slowly. " + f"Group on a coarser dimension, or drop the finest column from baseline_by." + ) + def build_context( self, df: DataFrame, @@ -333,6 +361,7 @@ def build_context( validate_baseline_columns(df, baseline_by, columns) if baseline_by: + self._reject_unbounded_grouping(df_filtered, baseline_by) logger.info(f"Judging each metric against its own group's baseline, grouped by {baseline_by}") resolved_over_time = baseline_over_time if baseline_over_time is not None else params.baseline_over_time diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 5e012d7b9..17f63d616 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -1006,7 +1006,14 @@ def _fit_temporal_from_buckets( seconds = _epoch_seconds(time_column, t_min) bucket_width = span / TEMPORAL_FIT_BUCKETS - bucketed = df.withColumn("__dqx_time_bucket", F.floor(seconds / lit(bucket_width))) + # Rows with no timestamp are dropped from the fit rather than bucketed. A null timestamp yields a null + # bucket whose min(seconds) is also null, and that null reached float() below as a TypeError -- one + # unparseable row was enough to fail training outright. Dropping them removes the bucket rather than + # guarding its symptom; the all-null case is already refused above, with an explanation. + # Scoring has always tolerated a null timestamp by emitting a zero residual, so this aligns the two. + bucketed = df.filter(col(time_column).cast(TimestampType()).isNotNull()).withColumn( + "__dqx_time_bucket", F.floor(seconds / lit(bucket_width)) + ) aggregations = [F.percentile_approx(col(source), 0.5).alias(name) for name, source in source_columns.items()] rows = ( bucketed.groupBy("__dqx_time_bucket") diff --git a/tests/integration_anomaly/test_anomaly_registry.py b/tests/integration_anomaly/test_anomaly_registry.py index 345af5906..46ce4631b 100644 --- a/tests/integration_anomaly/test_anomaly_registry.py +++ b/tests/integration_anomaly/test_anomaly_registry.py @@ -17,7 +17,9 @@ GroupingConfig, TrainingMetadata, ) +from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.config import AnomalyParams, IsolationForestConfig +from tests.constants import TEST_CATALOG from tests.integration_anomaly.constants import DEFAULT_SCORE_THRESHOLD from tests.integration_anomaly.conftest import ( get_standard_2d_training_data, @@ -529,3 +531,119 @@ def test_save_model_when_table_does_not_exist_creates_table_no_archive( rows = spark.table(registry_table).collect() assert len(rows) == 1 assert rows[0]["identity"]["status"] == "active" + + +# ============================================================================ +# Migration from the pre-release schema +# ============================================================================ + +#: The registry schema as it stood before `segment_by` was removed. Reproduced verbatim rather than +#: derived, because a test of backwards compatibility that builds its "old" table from today's code proves +#: nothing -- it would keep passing while the compatibility it claims to check rotted away. +LEGACY_ANOMALY_MODEL_TABLE_SCHEMA = ( + "identity struct, " + "training struct, hyperparameters:map, training_rows:bigint, " + "training_time:timestamp, metrics:map, score_quantiles:map, " + "baseline_stats:map>>, " + "features struct, feature_metadata:string, " + "feature_importance:map, temporal_config:map>, " + "segmentation struct, segment_values:map, " + "is_global_model:boolean, sklearn_version:string, config_hash:string>" +) + + +def _write_legacy_registry(spark: SparkSession, table: str, model_name: str, segment_by: list[str]) -> None: + """Create a registry table in the pre-release schema, holding one active model.""" + row = { + "identity": { + "model_name": model_name, + "model_uri": "models:/legacy/1", + "algorithm": "IsolationForest", + "mlflow_run_id": "legacy-run", + "status": "active", + }, + "training": { + "columns": ["amount", "quantity"], + "hyperparameters": {"num_trees": "200"}, + "training_rows": 500, + "training_time": datetime(2025, 1, 1), + "metrics": {"roc_auc": 0.9}, + "score_quantiles": {"p50": 0.1}, + "baseline_stats": {}, + }, + "features": { + "mode": "multi_type", + "column_types": {"amount": "numeric"}, + "feature_metadata": "", + "feature_importance": {}, + "temporal_config": {}, + }, + "segmentation": { + "segment_by": segment_by, + "segment_values": {"region": "eu"}, + "is_global_model": False, + "sklearn_version": "1.5.0", + "config_hash": "a-hash-from-before-baseline_by-joined-it", + }, + } + spark.createDataFrame([row], schema=LEGACY_ANOMALY_MODEL_TABLE_SCHEMA).write.mode("overwrite").saveAsTable(table) + + +def test_a_pre_release_registry_can_still_be_read(spark: SparkSession, make_schema, make_random): + """Reading must not raise, or the caller never reaches the message that tells them what to do. + + A model registered before this release cannot be scored: the configuration hash formula changed, so the + recomputed hash always mismatches and scoring raises with instructions to retrain. That error is the + useful one. Before this fix the read died first on ``KeyError: 'grouping'``, and the caller saw an + internal error instead of the instructions. + """ + schema = make_schema(catalog_name=TEST_CATALOG) + table = f"{TEST_CATALOG}.{schema.name}.legacy_reg_{make_random(6).lower()}" + model_name = f"{TEST_CATALOG}.{schema.name}.legacy_model" + _write_legacy_registry(spark, table, model_name, ["region", "product"]) + + record = AnomalyModelRegistry(spark).get_active_model(table, model_name) + + assert record is not None + # segment_by carried across to baseline_by; the two dropped fields have no meaning to preserve now. + assert record.grouping.baseline_by == ["region", "product"] + assert record.grouping.sklearn_version == "1.5.0" + + +def test_retraining_migrates_a_pre_release_registry_and_leaves_nothing_behind( + ws, spark: SparkSession, make_schema, make_random +): + """The remedy the migration guide prescribes, tested end to end from the failing state. + + Retraining is what both the guide and the stale-hash error tell a user to do. It reads the registry + during training, so before this fix it raised ``KeyError: 'grouping'`` and the documented remedy could + not be followed at all. + + The assertion that matters most is the absence of ``segmentation`` afterwards. Appending with + ``mergeSchema`` would also make retraining "work", while leaving the user's table carrying both columns + for good -- which is what this replaced. + """ + schema = make_schema(catalog_name=TEST_CATALOG) + suffix = make_random(6).lower() + table = f"{TEST_CATALOG}.{schema.name}.legacy_reg_{suffix}" + model_name = f"{TEST_CATALOG}.{schema.name}.legacy_model_{suffix}" + _write_legacy_registry(spark, table, model_name, ["region"]) + + train_df = spark.createDataFrame(get_standard_2d_training_data(), "amount double, quantity double") + AnomalyEngine(ws, spark).train( + df=train_df, + columns=["amount", "quantity"], + model_name=model_name, + registry_table=table, + baseline_by=[], + params=AnomalyParams(algorithm_config=IsolationForestConfig(contamination=0.1, random_seed=42)), + ) + + migrated = spark.table(table) + assert "segmentation" not in migrated.columns, "the pre-release column must be gone, not carried alongside" + assert "grouping" in migrated.columns + + # The historical row survived the rewrite and now carries grouping, mapped from its segment_by. + archived = migrated.filter(F.col("identity.status") != "active").select("grouping.baseline_by").collect() + assert archived, "the previous version must be migrated, not dropped" + assert archived[0]["baseline_by"] == ["region"] diff --git a/tests/integration_anomaly/test_anomaly_temporal_features.py b/tests/integration_anomaly/test_anomaly_temporal_features.py index 10ce63487..c71e7c281 100644 --- a/tests/integration_anomaly/test_anomaly_temporal_features.py +++ b/tests/integration_anomaly/test_anomaly_temporal_features.py @@ -510,3 +510,36 @@ def test_the_public_check_reports_staleness_in_the_info_column(ws, spark: SparkS assert by_ts[far_past]["horizon"], "a stale row must say what window it is past" # Flagged, never nulled: measured, the score one window out is still usable. assert by_ts[far_past]["score"] is not None + + +def test_a_null_timestamp_in_training_does_not_fail_the_fit(spark: SparkSession): + """One unparseable timestamp used to end training with a TypeError. + + Null timestamps fell into a null bucket whose ``min(seconds)`` is also null, and that null reached + ``float()`` unguarded -- while the metric values beside it were guarded. Only the all-null case was + handled, and that one raises deliberately with an explanation. Scoring has always tolerated a null + timestamp by emitting a zero residual, so training was the odd one out. + """ + # Annotated because the timestamp is genuinely optional here: inference from the first element alone + # would make the null rows below a type error, and they are the point of the test. + rows: list[tuple[datetime.datetime | None, float]] = [ + (START + datetime.timedelta(hours=i), float(100.0 + 0.05 * i)) for i in range(24 * 30) + ] + # A handful of rows with no timestamp, carrying a perfectly good metric value. + rows += [(None, 150.0), (None, 151.0), (None, 149.0)] + df = spark.createDataFrame( + rows, + T.StructType( + [ + T.StructField("event_ts", T.TimestampType(), True), + T.StructField("revenue", T.DoubleType(), True), + ] + ), + ) + + engineered, metadata = apply_feature_engineering(df, [_numeric("revenue")], baseline_over_time="event_ts") + + assert f"revenue{TEMPORAL_RELATIVE_SUFFIX}" in metadata.engineered_feature_names + # The basis was fitted from the rows that do carry a timestamp. + assert metadata.temporal_coefficients + assert engineered.count() == len(rows) diff --git a/tests/integration_anomaly/test_anomaly_timeseries_profile.py b/tests/integration_anomaly/test_anomaly_timeseries_profile.py index 4cf4f64d8..97cf77a56 100644 --- a/tests/integration_anomaly/test_anomaly_timeseries_profile.py +++ b/tests/integration_anomaly/test_anomaly_timeseries_profile.py @@ -31,14 +31,18 @@ class should travel inside the artifact -- but "should" is the word this test ex from pyspark.sql import SparkSession from pyspark.sql import functions as F +from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine +from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRegistry from databricks.labs.dqx.anomaly.training_strategies import MAHALANOBIS_ALGORITHM from databricks.labs.dqx.anomaly.transformers import SparkFeatureMetadata from databricks.labs.dqx.config import AnomalyParams +from databricks.labs.dqx.engine import DQEngine +from tests.constants import TEST_CATALOG from tests.integration_anomaly.constants import DEFAULT_SCORE_THRESHOLD from tests.integration_anomaly.quality_metrics import pr_auc, trivial_baselines from tests.integration_anomaly.synthetic_generators import generate_correlated_multivariate_data -from tests.integration_anomaly.conftest import ai_query_llm_config +from tests.integration_anomaly.conftest import ai_query_llm_config, create_anomaly_check_rule # The gap the correlation-aware detector must clear against IsolationForest on data whose anomalies are # *only* joint. Loose on purpose: the claim is the direction and rough size, not a tripwire on the @@ -280,3 +284,70 @@ def test_timeseries_profile_end_to_end( # 6. Scores must be finite everywhere. A singular covariance would surface as NaN or inf rather # than as an exception, and every metric above would silently degrade instead of failing. assert np.isfinite(timeseries["score"]).all(), "the timeseries model produced non-finite scores" + + +def test_a_grouped_timeseries_model_trains_and_scores(ws, spark: SparkSession, make_schema, make_random): + """The combination that failed: the correlation-aware profile together with a grouping. + + Feature engineering deliberately preserves the group key, so the engineered frame is wider than the + feature list. Signature inference passed that whole frame to ``model.predict``, handing the estimator a + string column it was never fitted on. Every grouped model on the single-model path hit it -- + ``profile="timeseries"`` always, and the tabular profile at ``ensemble_size=1`` -- while the default + three-model ensemble registers by URI and never comes through that code, which is why the existing + coverage passed: it uses ``baseline_by=[]``. + """ + schema = make_schema(catalog_name=TEST_CATALOG) + suffix = make_random(6).lower() + model_name = f"{TEST_CATALOG}.{schema.name}.grouped_ts_{suffix}" + registry_table = f"{TEST_CATALOG}.{schema.name}.reg_{suffix}" + + rng = np.random.default_rng(5) + rows = [] + for region in ("eu", "us", "apac"): + offset = {"eu": 0.0, "us": 40.0, "apac": 80.0}[region] + for _ in range(300): + latent = rng.normal(0.0, 1.0) + rows.append((region, float(100.0 + offset + 6.0 * latent), float(20.0 + 1.4 * latent))) + train_df = spark.createDataFrame(rows, "region string, load double, current double") + + trained = AnomalyEngine(ws, spark).train( + df=train_df, + columns=["load", "current"], + model_name=model_name, + registry_table=registry_table, + baseline_by=["region"], + profile="timeseries", + params=AnomalyParams(sample_fraction=1.0), + ) + + # Registering is where it failed; scoring proves the registered signature is usable. + result_df = DQEngine(ws, spark).apply_checks( + train_df.limit(20), + [create_anomaly_check_rule(model_name=trained, registry_table=registry_table, threshold=95.0)], + ) + scored = result_df.select(F.col("_dq_info")[0].getField("anomaly").getField("score").alias("score")).collect() + + assert len(scored) == 20 + assert all(row["score"] is not None for row in scored) + + +def test_a_grouped_single_forest_trains_and_scores(ws, spark: SparkSession, make_schema, make_random): + """The second live path into the same defect: the tabular profile with one model instead of three.""" + schema = make_schema(catalog_name=TEST_CATALOG) + suffix = make_random(6).lower() + model_name = f"{TEST_CATALOG}.{schema.name}.grouped_single_{suffix}" + registry_table = f"{TEST_CATALOG}.{schema.name}.reg_{suffix}" + + rows = [("eu" if i % 2 else "us", float(100 + i % 23), float(5 + i % 7)) for i in range(600)] + train_df = spark.createDataFrame(rows, "region string, amount double, quantity double") + + trained = AnomalyEngine(ws, spark).train( + df=train_df, + columns=["amount", "quantity"], + model_name=model_name, + registry_table=registry_table, + baseline_by=["region"], + params=AnomalyParams(sample_fraction=1.0, ensemble_size=1), + ) + + assert AnomalyModelRegistry(spark).get_active_model(registry_table, trained) is not None diff --git a/tests/unit/test_anomaly_baseline_discovery.py b/tests/unit/test_anomaly_baseline_discovery.py index c78ea4ee4..361d7a51b 100644 --- a/tests/unit/test_anomaly_baseline_discovery.py +++ b/tests/unit/test_anomaly_baseline_discovery.py @@ -16,7 +16,8 @@ MAX_BASELINE_GROUPS, MIN_ROWS_PER_BASELINE_GROUP, ) -from databricks.labs.dqx.anomaly.profiler import select_baseline_columns +from databricks.labs.dqx.anomaly.core import DEFAULT_SAMPLE_FRACTION, DEFAULT_TRAIN_RATIO +from databricks.labs.dqx.anomaly.profiler import effective_min_rows_per_group, select_baseline_columns def _candidate(name: str, distinct: int, total: int) -> tuple[str, int, float]: @@ -26,7 +27,7 @@ def _candidate(name: str, distinct: int, total: int) -> tuple[str, int, float]: def test_combines_dimensions_while_groups_stay_estimable(): """The case the old policy got wrong: take all three, not just the cheapest.""" - total = 10800 + total = 13500 candidates = [ _candidate("product", 3, total), _candidate("event_type", 5, total), @@ -36,16 +37,20 @@ def test_combines_dimensions_while_groups_stay_estimable(): selected = select_baseline_columns(candidates, total) assert selected == ["product", "event_type", "country"] - # 3 * 5 * 6 = 90 groups, 120 rows each -- comfortably above the median floor. - assert total / 90 >= MIN_ROWS_PER_BASELINE_GROUP + # 3 * 5 * 6 = 90 groups, 150 rows each on the full table. Sized against the *effective* floor, not the + # nominal one: discovery runs before sampling, so what matters is the ~36 rows per group that survive + # sampling and the train split. This fixture held 10,800 rows when it was written, which looked like a + # comfortable 120 rows per group and was 29 by the time anything was fitted. + assert total / 90 >= effective_min_rows_per_group() + assert (total / 90) * DEFAULT_SAMPLE_FRACTION * DEFAULT_TRAIN_RATIO >= MIN_ROWS_PER_BASELINE_GROUP def test_stops_before_groups_get_too_thin_to_median(): """A dimension that would starve every group is skipped, not accepted.""" - total = 200 + total = 400 candidates = [ - _candidate("region", 2, total), # 100 rows/group -- fine - _candidate("sku", 40, total), # would give 80 groups over 200 rows -- 2.5 rows each + _candidate("region", 2, total), # 200 rows/group, ~48 after sampling -- fine + _candidate("sku", 40, total), # 80 groups over 400 rows -- 5 rows each, ~1 after sampling ] selected = select_baseline_columns(candidates, total) @@ -53,6 +58,21 @@ def test_stops_before_groups_get_too_thin_to_median(): assert selected == ["region"] +def test_the_rows_per_group_floor_accounts_for_sampling(): + """Discovery runs on the full table; the fit runs on a sample of a split of it. + + Checking the nominal floor against the full table overstated what the model would get by about four + times at the defaults, so a group that just cleared 30 contributed roughly 7 rows to its own median. + This pins the relationship rather than leaving it implicit in two call sites. + """ + retained = DEFAULT_SAMPLE_FRACTION * DEFAULT_TRAIN_RATIO + + assert effective_min_rows_per_group() > MIN_ROWS_PER_BASELINE_GROUP + assert effective_min_rows_per_group() * retained >= MIN_ROWS_PER_BASELINE_GROUP + # Nothing is sampled away, so the two coincide. + assert effective_min_rows_per_group(1.0, 1.0) == MIN_ROWS_PER_BASELINE_GROUP + + def test_skips_columns_too_wide_to_be_a_dimension(): """Above the per-column ceiling a column is an identifier, not a peer group.""" total = 1_000_000 diff --git a/tests/unit/test_anomaly_group_key.py b/tests/unit/test_anomaly_group_key.py index 065487ac9..be089aa07 100644 --- a/tests/unit/test_anomaly_group_key.py +++ b/tests/unit/test_anomaly_group_key.py @@ -86,3 +86,18 @@ def test_separator_inside_a_value_is_a_documented_collision(): left = build_baseline_key({"a": f"x{BASELINE_KEY_SEPARATOR}y", "b": "z"}) right = build_baseline_key({"a": "x", "b": f"y{BASELINE_KEY_SEPARATOR}z"}) assert left == right + + +def test_a_null_and_a_literal_missing_are_different_groups(): + """The regression this exists for: the two used to share one key, and therefore one baseline. + + ``BASELINE_KEY_NULL`` was the string ``"MISSING"``, which a categorical dimension can genuinely + contain -- a region named MISSING in an upstream feed, a status column with an explicit MISSING + level. Those rows and the rows with a NULL region were persisted under one key and conditioned + against one median, silently, with nothing to indicate the two populations had been merged. + """ + with_null = build_baseline_key({"region": None, "product": "casino"}) + with_literal = build_baseline_key({"region": "MISSING", "product": "casino"}) + + assert with_null != with_literal + assert "MISSING" in with_literal From af6b81584f586c549e4180e116ff52829fb4c6d2 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 14:55:54 +0100 Subject: [PATCH 076/107] Scale a discovery fixture to the floor the fit actually sees The one failure in the anomaly gate's first real run, 233 of 234 passing on both runners. The fixture gave `category` 5 distinct values over 200 rows, so 40 rows per group, which the fit sees as 9.6 once sampling and the train split have taken their share -- below the 30 the constant promises. So it is correctly no longer a grouping candidate, and the fixture was sized against a threshold nothing enforced. 1,000 rows gives `category` 200 rows per group and ~48 at the fit, so it is selected as before, while adding `user_code` as a second dimension would give 4 rows per group and is still correctly rejected. Same boundary, measured against the threshold that applies. Third fixture in this PR to have encoded the pre-sampling optimism. Co-authored-by: Isaac --- .../test_anomaly_autodiscovery.py | 18 +-- .../test_anomaly_registry.py | 38 ++++++ tests/unit/test_anomaly_registry_migration.py | 111 ++++++++++++++++++ 3 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 tests/unit/test_anomaly_registry_migration.py diff --git a/tests/integration_anomaly/test_anomaly_autodiscovery.py b/tests/integration_anomaly/test_anomaly_autodiscovery.py index 68d1afc56..69045bd44 100644 --- a/tests/integration_anomaly/test_anomaly_autodiscovery.py +++ b/tests/integration_anomaly/test_anomaly_autodiscovery.py @@ -312,14 +312,18 @@ def test_autodiscovery_with_datetime_columns(spark: SparkSession): def test_autodiscovery_with_various_cardinality_strings(spark: SparkSession): """Test string column analysis with low/medium/high cardinality (lines 130-148).""" - # Create DataFrame with strings of varying cardinality + # 1,000 rows so `category` clears the *effective* rows-per-group floor. Discovery runs before sampling, + # so a candidate needs MIN_ROWS_PER_BASELINE_GROUP / (sample_fraction * train_ratio) rows per group on the + # full table for the fit to see the minimum it promises. At 200 rows this fixture gave `category` 40 rows + # per group, which is 9.6 by the time anything is fitted, so it is correctly no longer a grouping + # candidate. 1,000 rows gives it 200, and ~48 after sampling. data = [] - for i in range(200): + for i in range(1000): data.append( ( - f"cat_{i % 5}", # Low cardinality (5 distinct) + f"cat_{i % 5}", # Low cardinality (5 distinct), 200 rows/group f"user_{i % 50}", # Medium cardinality (50 distinct) - f"tx_{i}", # High cardinality (200 distinct) - avoid "id" pattern + f"tx_{i}", # High cardinality (1000 distinct) - avoid "id" pattern 100.0 + i, ) ) @@ -336,12 +340,12 @@ def test_autodiscovery_with_various_cardinality_strings(spark: SparkSession): assert profile.column_types is not None assert profile.column_types["category"] == "categorical" - # user_code has 50 distinct values - stays a feature: adding it as a second baseline column - # would push groups past the rows-per-group floor, so it is not selected for grouping. + # user_code has 50 distinct values - stays a feature: as a second baseline column it would give + # 5 * 50 = 250 groups over 1,000 rows, 4 rows each, far under the floor, so it is not selected. assert "user_code" in profile.recommended_columns assert profile.column_types["user_code"] == "categorical" - # transaction_ref has 200 distinct values (>100) - should be excluded with warning (lines 144-148) + # transaction_ref has 1000 distinct values (>100) - should be excluded with warning (lines 144-148) assert "transaction_ref" not in profile.recommended_columns warnings_text = " ".join(profile.warnings) assert "transaction_ref" in warnings_text diff --git a/tests/integration_anomaly/test_anomaly_registry.py b/tests/integration_anomaly/test_anomaly_registry.py index 46ce4631b..2c155b8ac 100644 --- a/tests/integration_anomaly/test_anomaly_registry.py +++ b/tests/integration_anomaly/test_anomaly_registry.py @@ -2,6 +2,8 @@ import logging import warnings + +import pytest from collections.abc import Callable from datetime import datetime, timedelta @@ -19,9 +21,12 @@ ) from databricks.labs.dqx.anomaly.anomaly_engine import AnomalyEngine from databricks.labs.dqx.config import AnomalyParams, IsolationForestConfig +from databricks.labs.dqx.engine import DQEngine +from databricks.labs.dqx.errors import InvalidParameterError from tests.constants import TEST_CATALOG from tests.integration_anomaly.constants import DEFAULT_SCORE_THRESHOLD from tests.integration_anomaly.conftest import ( + create_anomaly_check_rule, get_standard_2d_training_data, get_standard_3d_training_data, score_with_anomaly_check, @@ -647,3 +652,36 @@ def test_retraining_migrates_a_pre_release_registry_and_leaves_nothing_behind( archived = migrated.filter(F.col("identity.status") != "active").select("grouping.baseline_by").collect() assert archived, "the previous version must be migrated, not dropped" assert archived[0]["baseline_by"] == ["region"] + + +def test_scoring_a_pre_release_model_says_to_retrain_rather_than_failing_internally( + ws, spark: SparkSession, make_schema, make_random +): + """The error a user upgrading actually meets, and the reason the read path stays tolerant. + + A model registered before this release cannot be scored: the configuration hash formula changed, so the + recomputed hash never matches and scoring refuses. That refusal is correct and is not what this pins. + What it pins is *which* error arrives. Before the migration work, the registry read raised + ``KeyError: 'grouping'`` first, so the user saw an internal failure with no indication that retraining + was the answer. + + Written after an earlier draft of this test asserted the opposite -- that scoring still worked -- which + was wrong about the hash check and would have encoded a promise DQX does not make. + """ + schema = make_schema(catalog_name=TEST_CATALOG) + suffix = make_random(6).lower() + table = f"{TEST_CATALOG}.{schema.name}.legacy_reg_{suffix}" + model_name = f"{TEST_CATALOG}.{schema.name}.legacy_model_{suffix}" + _write_legacy_registry(spark, table, model_name, ["region"]) + + test_df = spark.createDataFrame([(100.0, 2.0), (900.0, 40.0)], "amount double, quantity double") + + with pytest.raises(InvalidParameterError) as raised: + DQEngine(ws, spark).apply_checks( + test_df, + [create_anomaly_check_rule(model_name=model_name, registry_table=table, threshold=95.0)], + ).collect() + + message = str(raised.value) + assert "Retrain" in message or "retrain" in message, f"the remedy must be in the message: {message}" + assert "KeyError" not in message diff --git a/tests/unit/test_anomaly_registry_migration.py b/tests/unit/test_anomaly_registry_migration.py new file mode 100644 index 000000000..309c67112 --- /dev/null +++ b/tests/unit/test_anomaly_registry_migration.py @@ -0,0 +1,111 @@ +"""Unit pins for the registry migration's decision points (no Spark). + +The migration itself needs a real Delta table and is covered in `tests/integration_anomaly/`. Two pieces of +it are pure and were left untested when it was written, both on paths that only run when something has +already gone wrong, which is exactly when nobody wants a second surprise: + +- the schema normaliser, which decides what a row's grouping is across three table shapes +- the permission-message classifier, which decides whether a failed rewrite becomes advice or is re-raised + +Getting the second one wrong in either direction is bad: too eager and a genuine bug is reported to the user +as "ask your admin for a grant", too shy and a missing grant surfaces as an unexplained write failure. +""" + +import pytest + +from databricks.labs.dqx.anomaly.model_registry import ( + LEGACY_GROUPING_COLUMN, + _is_permission_error, + normalise_grouping, +) + + +def test_a_current_row_is_returned_unchanged(): + values = {"grouping": {"baseline_by": ["region"], "sklearn_version": "1.5.0", "config_hash": "abc"}} + + assert normalise_grouping(values) == values["grouping"] + + +def test_a_pre_release_row_maps_segment_by_onto_baseline_by(): + """The mapping the migration and the read path share.""" + values = { + LEGACY_GROUPING_COLUMN: { + "segment_by": ["region", "product"], + "segment_values": {"region": "eu"}, + "is_global_model": False, + "sklearn_version": "1.4.2", + "config_hash": "old-hash", + } + } + + grouping = normalise_grouping(values) + + assert grouping["baseline_by"] == ["region", "product"] + assert grouping["sklearn_version"] == "1.4.2" + assert grouping["config_hash"] == "old-hash" + # Dropped deliberately: there is one pooled model now, so neither has a meaning to preserve. + assert "segment_values" not in grouping + assert "is_global_model" not in grouping + + +def test_a_part_migrated_row_prefers_the_legacy_column_when_grouping_is_null(): + """The shape a `mergeSchema` append leaves behind: both columns present, one null per row. + + Worth pinning even though the migration removes the legacy column, because a table written by an earlier + build of this release is already in this state on someone's workspace. + """ + values = { + "grouping": None, + LEGACY_GROUPING_COLUMN: { + "segment_by": ["country"], + "segment_values": None, + "is_global_model": True, + "sklearn_version": "1.4.0", + "config_hash": "h", + }, + } + + assert normalise_grouping(values)["baseline_by"] == ["country"] + + +def test_a_row_with_neither_column_yields_an_empty_grouping_rather_than_raising(): + """Defensive: a hand-written or truncated row must not turn a read into a KeyError. + + Reading is how a caller reaches the message telling them to retrain. That message is worth more than + strictness here. + """ + grouping = normalise_grouping({}) + + assert grouping["baseline_by"] == [] + assert grouping["config_hash"] is None + + +@pytest.mark.parametrize( + "message", + [ + "[PERMISSION_DENIED] User does not have MODIFY on table cat.sch.reg", + "INSUFFICIENT_PERMISSIONS: cannot write to cat.sch.reg", + "User bob@example.com does not have permission to modify this table", + ], +) +def test_a_missing_grant_is_recognised(message: str): + """Recognised so the caller gets the grant to ask for, not a raw write failure.""" + assert _is_permission_error(Exception(message)) + + +@pytest.mark.parametrize( + "message", + [ + "[TABLE_OR_VIEW_NOT_FOUND] cat.sch.reg", + "Column 'grouping' does not exist", + "java.lang.OutOfMemoryError: Java heap space", + "", + ], +) +def test_a_genuine_failure_is_not_mistaken_for_a_missing_grant(message: str): + """The direction that matters more. + + Reporting a real bug as "ask an owner for MODIFY" sends someone to their platform team over a defect in + DQX, and they have no way to discover that from the message. + """ + assert not _is_permission_error(Exception(message)) From 69c7014145f7d116ab0379c31197f510bdcd464e Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 15:44:56 +0100 Subject: [PATCH 077/107] Make the permission-error predicate public rather than suppress the lint `is_permission_error` was private and the tests that pin it in both directions imported it anyway, which pylint's C2701 caught. AGENTS.md rules out silencing protected-access to reach past a boundary and prescribes exactly this remedy: if a member needs outside access, make it public. Both directions are worth pinning, and the second more than the first. Mistaking a genuine failure for a missing grant sends someone to their platform team over a defect in DQX, with nothing in the message to suggest otherwise. Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/model_registry.py | 12 +++++++++--- tests/integration_anomaly/test_anomaly_registry.py | 3 +-- tests/unit/test_anomaly_registry_migration.py | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/model_registry.py b/src/databricks/labs/dqx/anomaly/model_registry.py index 95fc01ba4..568243f62 100644 --- a/src/databricks/labs/dqx/anomaly/model_registry.py +++ b/src/databricks/labs/dqx/anomaly/model_registry.py @@ -50,8 +50,14 @@ _PERMISSION_DENIED_MARKERS = ("PERMISSION_DENIED", "INSUFFICIENT_PERMISSIONS", "does not have permission") -def _is_permission_error(exc: Exception) -> bool: - """Whether *exc* reports a missing grant rather than a genuine failure.""" +def is_permission_error(exc: Exception) -> bool: + """Whether *exc* reports a missing grant rather than a genuine failure. + + Public because it encodes a judgement worth testing directly, in both directions: mistaking a real + failure for a missing grant sends a user to their platform team over a defect in DQX, with nothing in + the message to suggest otherwise. Reaching into a private name from a test would be the alternative, + which AGENTS.md rules out. + """ message = str(exc).upper() return any(marker.upper() in message for marker in _PERMISSION_DENIED_MARKERS) @@ -217,7 +223,7 @@ def _migrate_legacy_registry(self, table: str) -> None: OutputConfig(location=table, mode="overwrite", options={"overwriteSchema": "true"}), ) except Exception as exc: - if not _is_permission_error(exc): + if not is_permission_error(exc): raise raise InvalidParameterError( f"Cannot migrate registry table '{table}' from the pre-release schema: this needs MODIFY " diff --git a/tests/integration_anomaly/test_anomaly_registry.py b/tests/integration_anomaly/test_anomaly_registry.py index 2c155b8ac..3946004d1 100644 --- a/tests/integration_anomaly/test_anomaly_registry.py +++ b/tests/integration_anomaly/test_anomaly_registry.py @@ -2,11 +2,10 @@ import logging import warnings - -import pytest from collections.abc import Callable from datetime import datetime, timedelta +import pytest import pyspark.sql.functions as F from pyspark.sql import SparkSession diff --git a/tests/unit/test_anomaly_registry_migration.py b/tests/unit/test_anomaly_registry_migration.py index 309c67112..4c0ae78da 100644 --- a/tests/unit/test_anomaly_registry_migration.py +++ b/tests/unit/test_anomaly_registry_migration.py @@ -15,7 +15,7 @@ from databricks.labs.dqx.anomaly.model_registry import ( LEGACY_GROUPING_COLUMN, - _is_permission_error, + is_permission_error, normalise_grouping, ) @@ -90,7 +90,7 @@ def test_a_row_with_neither_column_yields_an_empty_grouping_rather_than_raising( ) def test_a_missing_grant_is_recognised(message: str): """Recognised so the caller gets the grant to ask for, not a raw write failure.""" - assert _is_permission_error(Exception(message)) + assert is_permission_error(Exception(message)) @pytest.mark.parametrize( @@ -108,4 +108,4 @@ def test_a_genuine_failure_is_not_mistaken_for_a_missing_grant(message: str): Reporting a real bug as "ask an owner for MODIFY" sends someone to their platform team over a defect in DQX, and they have no way to discover that from the message. """ - assert not _is_permission_error(Exception(message)) + assert not is_permission_error(Exception(message)) From 5486b1dece86a45cc7f67f54dad34136210d23e9 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 15:45:36 +0100 Subject: [PATCH 078/107] Raise the mlflow floor to the release that fixes Databricks unified auth The integration fixture that writes a dummy Databricks config profile exists because of mlflow/mlflow#20599, "Fix Databricks unified auth support when MLFLOW_ENABLE_DB_SDK=true", merged 2026-04-30. That fix is in no 2.x release -- the last is 2.22.4 -- so pinning past it crosses the 2->3 boundary, which is why the floor moves this far. Capped below 4.0 rather than pinned, so the resolver takes the latest 3.x. It resolves to 3.15.2 rather than today's 3.16.0 because `make lock-dependencies` passes `--exclude-newer "7 days"`; the project's own guard rail refuses a same-day release, which is the behaviour I would have asked for. One quiet consequence: `log_sklearn_model_compatible` prefers `name=` when the signature offers it, and 3.15.2 accepts both `name=` and `artifact_path=`, so it now takes the `name=` branch where 2.x took the other. Same call, different keyword, and the `artifact_path=` fallback is dead on any supported version. Tracked separately rather than removed here. Unit tests pass, but they exercise no MLflow tracking or registry call at all -- every real one lives in tests/integration_anomaly. The anomaly gate is the first real signal on this bump. Co-authored-by: Isaac --- .build-constraints.txt | 26 +-- pyproject.toml | 5 +- tests/integration_anomaly/conftest.py | 11 ++ uv.lock | 220 +++++++++++++++++++++++--- 4 files changed, 232 insertions(+), 30 deletions(-) diff --git a/.build-constraints.txt b/.build-constraints.txt index 9523aa815..d2cf8070d 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -1,23 +1,27 @@ hatch-fancy-pypi-readme==25.1.0 \ --hash=sha256:9c58ed3dff90d51f43414ce37009ad1d5b0f08ffc9fc216998a06380f01c0045 \ --hash=sha256:ce0134c40d63d874ac48f48ccc678b8f3b62b8e50e9318520d2bffc752eedaf3 -hatchling==1.31.0 \ - --hash=sha256:6b48ad4068a482ed7239b3a8215bc55b47aad3345d58dfc94e553c5d2d46211b \ - --hash=sha256:aac80bec8b6fe35e8480f1c335be8910fa210a0e6f735a139be205dadcacb544 - # via hatch-fancy-pypi-readme -packaging==26.2 \ - --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ - --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 - # via hatchling +hatchling==1.32.0 \ + --hash=sha256:0bdbde4a52b06c37e3eca395f85a762bf0ef06fe374fd8ae429dc6be10230f5f \ + --hash=sha256:0e17c9c3b9aa7c625acc8d0f5b622f107d5049af9ecf5ada4de1aada5be7cdbc + # via hatch-fancy-pypi-readme +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via hatchling pathspec==1.1.1 \ --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 - # via hatchling + # via hatchling pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 - # via hatchling + # via hatchling +tomlkit==0.15.1 \ + --hash=sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304 \ + --hash=sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97 + # via hatchling trove-classifiers==2026.6.1.19 \ --hash=sha256:ab4c4ec93cc4a4e7815fa759906e05e6bb3f2fbd92ea0f897288c6a43efd15b3 \ --hash=sha256:c5132b4b61a829d11cfbd2d72e97f20a45ed6edb95e45c5efdeb5e00836b2745 - # via hatchling + # via hatchling diff --git a/pyproject.toml b/pyproject.toml index 3013b6479..d13f2ecea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,10 @@ datacontract = [ "datacontract-cli>=0.11.1,<1.0", ] anomaly = [ - "mlflow[databricks]>=2.19.1,<3.0", # Unity Catalog support + azure-cli auth type support + # Floor is the release carrying mlflow/mlflow#20599, which fixes Databricks unified auth when + # MLFLOW_ENABLE_DB_SDK=true; the fix is in no 2.x release, so this crosses the 2->3 boundary. + # Capped below 4.0 rather than pinned exactly, so the resolver takes the latest 3.x. + "mlflow[databricks]>=3.13.0,<4.0", # Unity Catalog support + azure-cli auth type support "scikit-learn>=1.0,<2.0", "cloudpickle>=3.0,<4.0", # For serializing sklearn models in pandas UDFs "shap>=0.42.0,<0.50", # For TreeSHAP-based feature contributions diff --git a/tests/integration_anomaly/conftest.py b/tests/integration_anomaly/conftest.py index f307e2fea..6777750b9 100644 --- a/tests/integration_anomaly/conftest.py +++ b/tests/integration_anomaly/conftest.py @@ -524,6 +524,17 @@ def _ensure_databricks_config_file(ws, tmp_path_factory): MLflow requires a Databricks config profile to exist even when using SDK auth. If DATABRICKS_CONFIG_FILE is already set and the file exists, this is a no-op. Otherwise, create a dummy profile with the real host from the workspace client. + + Workaround for a since-fixed MLflow bug: https://github.com/mlflow/mlflow/pull/20599, "Fix Databricks + unified auth support when MLFLOW_ENABLE_DB_SDK=true", merged 2026-04-30. **The mlflow floor in + pyproject.toml now carries that fix** (``>=3.13.0``), so this fixture is scaffolding for a bug no + supported version has, and is a candidate for deletion. + + It survives only because deleting it cannot be confirmed from CI: CI always supplies real credentials, so + the branch below never runs there. A full ``make anomaly`` against a workspace is what settles it. Note + that the dummy profile it writes carries ``token = dummy``, so anyone running locally *without* + DATABRICKS_CONFIG_FILE set gets ``PermissionDenied: Invalid access token`` from every test, which reads + as an expired token rather than as this fixture. """ config_file = os.environ.get("DATABRICKS_CONFIG_FILE") if config_file and os.path.isfile(config_file): diff --git a/uv.lock b/uv.lock index 268a1935a..c5cb1cbe5 100644 --- a/uv.lock +++ b/uv.lock @@ -1129,6 +1129,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/4b/290a1a669bf8f93886586c98c67127bdb424b71fb972fa006776f5daef8c/databind_json-4.5.3-py3-none-any.whl", hash = "sha256:9404f64162016db0b2d2a12545bc54101dcfeb76f80bf4861d8bc9ac341fdc21", upload-time = "2026-03-26T12:46:26.986Z" }, ] +[[package]] +name = "databricks-agents" +version = "1.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "boto3" }, + { name = "botocore" }, + { name = "databricks-sdk" }, + { name = "dataclasses-json" }, + { name = "googleapis-common-protos" }, + { name = "jinja2" }, + { name = "mlflow-skinny" }, + { name = "numpy" }, + { name = "pandas" }, + { name = "pydantic" }, + { name = "tenacity" }, + { name = "tiktoken" }, + { name = "tqdm" }, + { name = "urllib3" }, + { name = "whenever" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/15/39a0be382d600fefdcfad7b911f47db5d2ad268724c7c20f37620d69efcc/databricks_agents-1.12.0-py3-none-any.whl", hash = "sha256:f059a23dfa8de3b07147884a3845b056cd8c72fe57e178467e7bb96a8b4baefd", upload-time = "2026-08-21T15:18:25.593Z" }, +] + [[package]] name = "databricks-ai-bridge" version = "0.17.0" @@ -1340,7 +1365,7 @@ requires-dist = [ { name = "dspy", marker = "extra == 'llm'", specifier = "~=3.1.3" }, { name = "langchain-core", marker = "extra == 'llm'", specifier = ">=0.1.0,<1.0" }, { name = "litellm", marker = "extra == 'llm'", specifier = ">=1.64.0,<=1.82.6" }, - { name = "mlflow", extras = ["databricks"], marker = "extra == 'anomaly'", specifier = ">=2.19.1,<3.0" }, + { name = "mlflow", extras = ["databricks"], marker = "extra == 'anomaly'", specifier = ">=3.13.0,<4.0" }, { name = "numpy", marker = "extra == 'pii'", specifier = ">=1.20,<2.0" }, { name = "presidio-analyzer", marker = "extra == 'pii'", specifier = "~=2.2.359" }, { name = "pydantic", specifier = ">=2.8.2,<3" }, @@ -2053,6 +2078,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", upload-time = "2026-02-19T05:00:56.027Z" }, ] +[[package]] +name = "flask-cors" +version = "6.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flask" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/47/03/4e464a50860f9adf08b5c1d3479cb8ea1f12af2aa69535c7042c6e628135/flask_cors-6.0.5.tar.gz", hash = "sha256:30c5031552cd59f620ac0c8211dac45b345d3b2df310e7721879e4f46ef9c601", upload-time = "2026-06-08T20:20:17.765Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/55/5bb1a2d918e9f02f131e47a59032bae70e48050e986e941511fd737a935c/flask_cors-6.0.5-py3-none-any.whl", hash = "sha256:68fcf75693e961f3af26683b23c4b9a8fb6b64de17d20d0c37b95e8de7ab2ed8", upload-time = "2026-06-08T20:20:16.247Z" }, +] + [[package]] name = "fonttools" version = "4.62.1" @@ -2646,6 +2685,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", upload-time = "2025-10-10T21:48:21.158Z" }, ] +[[package]] +name = "huey" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/67/adfd477458ad70b73ce8311bbe84b1befbf986d0134fff2a1a55669ecd50/huey-3.3.4.tar.gz", hash = "sha256:6de196c6ece2e38b5173f7510600091ab035c0e86b0958d0e5b38a82b9984666", upload-time = "2026-08-05T12:51:51.836Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/49/7b49afcb523431db7bcb7b83d9911c754ee8138c3b5939830507359655d6/huey-3.3.4-py3-none-any.whl", hash = "sha256:a6e2e9a8fbda15c2dfb8e33b23a5e6e228326ea8e0087c11027957ef7c210e96", upload-time = "2026-08-05T12:51:50.539Z" }, +] + [[package]] name = "huggingface-hub" version = "1.8.0" @@ -3394,15 +3442,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", upload-time = "2025-04-10T12:50:53.297Z" }, ] -[[package]] -name = "markdown" -version = "3.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2b/f4/69fa6ed85ae003c2378ffa8f6d2e3234662abd02c10d216c0ba96081a238/markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950", upload-time = "2026-02-09T14:57:26.942Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/1f/77fa3081e4f66ca3576c896ae5d31c3002ac6607f9747d2e3aa49227e464/markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36", upload-time = "2026-02-09T14:57:25.787Z" }, -] - [[package]] name = "markdown-it-py" version = "4.0.0" @@ -3623,18 +3662,21 @@ wheels = [ [[package]] name = "mlflow" -version = "2.22.4" +version = "3.15.2" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "aiohttp" }, { name = "alembic" }, + { name = "cryptography" }, { name = "docker" }, { name = "flask" }, + { name = "flask-cors" }, { name = "graphene" }, { name = "gunicorn", marker = "sys_platform != 'win32'" }, - { name = "jinja2" }, - { name = "markdown" }, + { name = "huey" }, { name = "matplotlib" }, { name = "mlflow-skinny" }, + { name = "mlflow-tracing" }, { name = "numpy" }, { name = "pandas" }, { name = "pyarrow" }, @@ -3642,12 +3684,13 @@ dependencies = [ { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "skops" }, { name = "sqlalchemy" }, { name = "waitress", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cd/56/4aaea65472c25dd463ed0855c1d673749cd9050e5c8214642d17434b441a/mlflow-2.22.4.tar.gz", hash = "sha256:cb8cb3b82ec696dc613bcc347b023c20fc0ed6a82170b36d0ded01d3ba06da97", upload-time = "2025-12-05T13:20:56.105Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/d7/f24cbec03ef311fccc716026078f4b22dfef967c5fe9ac7436e3b6b2609a/mlflow-3.15.2.tar.gz", hash = "sha256:b46867789bd9a3b882973713371c933a42268bd17dd8eaa49f941bebe2d69f03", upload-time = "2026-08-26T06:03:49.696Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/0b/bf491b0604f2608e97b53b8cc33220fd20855ac4762d18d0ddf0d3ae3a6c/mlflow-2.22.4-py3-none-any.whl", hash = "sha256:c37b312060737cc9197c4a956c730fa6c292580787fe464efe736c339e87649a", upload-time = "2025-12-05T13:20:52.703Z" }, + { url = "https://files.pythonhosted.org/packages/b3/d9/98766e5d22d64ac82ecfa6232c271b765d56442c3b5ec09adf79a510557c/mlflow-3.15.2-py3-none-any.whl", hash = "sha256:7aa59664351aaa6f63478cf3c26977c185dba60d28bca8b9a5125be3a2b4311c", upload-time = "2026-08-26T06:03:47.093Z" }, ] [package.optional-dependencies] @@ -3655,12 +3698,13 @@ databricks = [ { name = "azure-storage-file-datalake" }, { name = "boto3" }, { name = "botocore" }, + { name = "databricks-agents" }, { name = "google-cloud-storage" }, ] [[package]] name = "mlflow-skinny" -version = "2.22.4" +version = "3.15.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -3671,19 +3715,41 @@ dependencies = [ { name = "gitpython" }, { name = "importlib-metadata" }, { name = "opentelemetry-api" }, + { name = "opentelemetry-proto" }, { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "protobuf" }, { name = "pydantic" }, + { name = "python-dotenv" }, { name = "pyyaml" }, { name = "requests" }, { name = "sqlparse" }, + { name = "starlette" }, { name = "typing-extensions" }, { name = "uvicorn" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4e/73/de6cfdd1bd48fd896c33844b863931bf7215f9401e01e4554019aca0fa94/mlflow_skinny-2.22.4.tar.gz", hash = "sha256:d75ef4c6f38b745d84aef4d6dcb26331c8a3c784ee5a284ec89186398c8d927b", upload-time = "2025-12-05T12:50:03.045Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/a1/addfbab892f95b8bd3e8d6ce1166838f53782635270fe460631dc3a22781/mlflow_skinny-3.15.2.tar.gz", hash = "sha256:973f65f835de41ddb4dd8e2cf91c65028d0b928eb6fef988434243b6a77dfcaf", upload-time = "2026-08-26T06:05:35.361Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/d1/549a995e261ca708c60fe0b63dfa4d1842fc58b04eb9c78cd678aebe1e7e/mlflow_skinny-2.22.4-py3-none-any.whl", hash = "sha256:3622115f53806d99fc42b0c2e45f225b16948584feeec7f233e484f08fe6c7f2", upload-time = "2025-12-05T12:50:00.406Z" }, + { url = "https://files.pythonhosted.org/packages/a6/5e/0ffcb76a9f3efa25e62af012c95f43f5874004fb6ced1109ff69a62e3711/mlflow_skinny-3.15.2-py3-none-any.whl", hash = "sha256:4a0f236f1e7856e0bd4cf7f2af889f8cd33a4a45206b732a78028dd297fa3b14", upload-time = "2026-08-26T06:05:33.306Z" }, +] + +[[package]] +name = "mlflow-tracing" +version = "3.15.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cachetools" }, + { name = "databricks-sdk" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/f0/f4fe46156efd55561e08e64cb472abc2083a090ddaeaf01f5d085412b0ac/mlflow_tracing-3.15.2.tar.gz", hash = "sha256:7c5f692d486c6ebb7954669564a4148cc9afe5ab664be6d202a93f77d25fe428", upload-time = "2026-08-26T06:06:28.412Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/1c/a5aea29667f3e46d279fe4ede21f58e5895b50c5de377acc523cb93ad385/mlflow_tracing-3.15.2-py3-none-any.whl", hash = "sha256:0969d43855d06e016607e90e6f212ab172952243eb71057b33ca9d845cb8bc08", upload-time = "2026-08-26T06:06:26.569Z" }, ] [[package]] @@ -4196,6 +4262,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5f/bf/93795954016c522008da367da292adceed71cca6ee1717e1d64c83089099/opentelemetry_api-1.40.0-py3-none-any.whl", hash = "sha256:82dd69331ae74b06f6a874704be0cfaa49a1650e1537d4a813b86ecef7d0ecf9", upload-time = "2026-03-04T14:17:01.24Z" }, ] +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", upload-time = "2026-07-16T15:25:28.429Z" }, +] + [[package]] name = "opentelemetry-sdk" version = "1.40.0" @@ -4616,6 +4694,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2f/8a/d6cf4bddbb11d9c78f5c9342bbbcde4f90fe4e3c7de114a836ec4ed8a6cf/presidio_analyzer-2.2.362-py3-none-any.whl", hash = "sha256:4c36438924b1fcb4df92ea5cf2d8dc57508808e116b10923c983b8732aa07d90", upload-time = "2026-03-15T12:40:43.801Z" }, ] +[[package]] +name = "prettytable" +version = "3.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/74/ba08d81e668ccfe8658d7520a307e63c19862c08eb4ccb26f356c5239a7a/prettytable-3.18.0.tar.gz", hash = "sha256:439217116152244369caf3d9f1caf2f9fe29b03bd79e88d2928c8e718c95d680", upload-time = "2026-06-22T16:07:50.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/be/2e6798ace5cc036f5d05d36b7b2fd85346f1a708c87060890b070d0ec607/prettytable-3.18.0-py3-none-any.whl", hash = "sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a", upload-time = "2026-06-22T16:07:48.595Z" }, +] + [[package]] name = "propcache" version = "0.4.1" @@ -6110,6 +6200,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "skops" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "packaging" }, + { name = "prettytable" }, + { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scikit-learn", version = "1.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/9f/46448c4e41a4c5ee4bdb74b3758af48e5ff0faeffe40f4e301bfc7594894/skops-0.14.0.tar.gz", hash = "sha256:6c8c0e047f691a3a582c3258943eecafcbfd79c8c7eef66260f3703e363254f0", upload-time = "2026-04-20T18:23:55.336Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/0e/3ae19fa941522cd98e119762e7181d371c8dba0b2d72bfaf9522692e329c/skops-0.14.0-py3-none-any.whl", hash = "sha256:60a5db78a9db46ccee2139a0ba13ab5afb1c96f4749b382e75a371291bbe3e36", upload-time = "2026-04-20T18:23:54.018Z" }, +] + [[package]] name = "slicer" version = "0.0.8" @@ -6950,6 +7058,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", upload-time = "2024-11-01T14:07:11.845Z" }, ] +[[package]] +name = "wcwidth" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", upload-time = "2026-06-29T18:11:11.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", upload-time = "2026-06-29T18:11:09.888Z" }, +] + [[package]] name = "weasel" version = "1.0.0" @@ -6982,6 +7099,73 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/b2/0bba9bbb4596d2d2f285a16c2ab04118f6b957d8441566e1abb892e6a6b2/werkzeug-3.1.7-py3-none-any.whl", hash = "sha256:4b314d81163a3e1a169b6a0be2a000a0e204e8873c5de6586f453c55688d422f", upload-time = "2026-03-24T01:08:06.133Z" }, ] +[[package]] +name = "whenever" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/08/49181903bb52523f5e5e0b4ffc172c9bcc7edc7cda9bd626173fa99b46cf/whenever-0.7.3.tar.gz", hash = "sha256:fc2b3756c35a0694c4159ad877405949ec283623fd2082b66cdafab6e883e65b", upload-time = "2025-03-19T14:41:19.234Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/31/39e089dfcfae64a07da864adc917c26c08092fe3c12fd8c59a2439c66332/whenever-0.7.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:50b9cd57c6bf173c320cfcac499aa3c26e40204648b995b68d083a60edb27d93", upload-time = "2025-03-19T14:40:57.839Z" }, + { url = "https://files.pythonhosted.org/packages/31/a8/4bc7f9929e6e2816d49845b871ae828cbc9657e0e266466ca3caf2ead77e/whenever-0.7.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b26c38b4f3cac25c671760c0bac7950aaa0b8ac6b028e1c9c60244ef1e841c0b", upload-time = "2025-03-19T14:40:50.643Z" }, + { url = "https://files.pythonhosted.org/packages/0c/32/439d5e442da3f7bd5483f61d6edd57ade3c652dc95ee92c0d748cbd72a06/whenever-0.7.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1323c42be308dac93ce6caafa126596636a0f6b3e330a716d677cae374593000", upload-time = "2025-03-19T14:39:35.098Z" }, + { url = "https://files.pythonhosted.org/packages/db/53/55a0d3e663432d26b3ed41507bd0eb8c08531fac7b5939ce54e73161d131/whenever-0.7.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5521d8f91e1ae2aaff1f727cf117c3da25d9fd9417971c0f06c9ed72e2a3dddf", upload-time = "2025-03-19T14:39:51.137Z" }, + { url = "https://files.pythonhosted.org/packages/9f/9f/ba297c6450a9af4f2746192fb2a0c274a4115a5d048d5122fb3064587b02/whenever-0.7.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:61df616ef0b4929b6b1a0353c9f6a6e62dadc20249b054306a51d3d88b20e5c1", upload-time = "2025-03-19T14:40:06.394Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e5/20b8b5ec10f0995ba32ca38bc72f0b511622366df1eb3dd0758192e3fa83/whenever-0.7.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5209aa81660c5fe615ade0471df7bc5c393e9e1b7c9aff2b34db4efadb14693f", upload-time = "2025-03-19T14:40:14.009Z" }, + { url = "https://files.pythonhosted.org/packages/6c/e1/0fde7bca5ec42fd45ac3532b7d398beaad0398c59d78282c886fa52eec8e/whenever-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f35debea258fcf24711b6c1b5fb94d645135477603c60b8c7249fdfda55933a", upload-time = "2025-03-19T14:40:36.285Z" }, + { url = "https://files.pythonhosted.org/packages/c5/62/86472beaf6b279ca92d75965860ef113db147063b2a8bb082ef252a2f903/whenever-0.7.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec1eb645b87a4e0b44739ac697aa9c57a7bf0d1e1a3ed4b0cef6e0bed3ee0ece", upload-time = "2025-03-19T14:40:22.414Z" }, + { url = "https://files.pythonhosted.org/packages/a3/99/7ef6e03076c5353d129b9eb9b759527e426dce80c9db23c7fc63acbf66e5/whenever-0.7.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d6758d30a1b543200bb8b116a3c031d126dd82f5fb7a15165c144abe15bb2e57", upload-time = "2025-03-19T14:39:42.624Z" }, + { url = "https://files.pythonhosted.org/packages/44/91/ccac1088715a7a75fca419b08f698263d2acbacdeb03325da9e85f0002a5/whenever-0.7.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:0feeaa39b595dd540e2213ed431b34e76778659d8fd4838cc037f8c6f93f4ef6", upload-time = "2025-03-19T14:39:59.749Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f1/8c4f829597cad9e10929ed8df35f82835b0f9a8f3c7f6420c2ee5ee630c3/whenever-0.7.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ae2f15956422ac67282a12aad4922b3a48f22396cd0f96db3b4d4cf7c882208f", upload-time = "2025-03-19T14:40:29.049Z" }, + { url = "https://files.pythonhosted.org/packages/02/79/41fb58407defeb2b92e189b002991d37aa273a0f2351767d55cff1ac2277/whenever-0.7.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b7007a8ca385a486755a2ace63c011b48f4d7ba7a0fb6c3a37f44e2f7688b086", upload-time = "2025-03-19T14:40:43.22Z" }, + { url = "https://files.pythonhosted.org/packages/e3/d9/bb8e4f055155ba560f4cbd8667faac6333b95b70819c3e807aed51b1efd3/whenever-0.7.3-cp310-cp310-win32.whl", hash = "sha256:58d423e18a1413062b70e3e4dec672439d36404bb2368ce46fc458aa52d1cf3b", upload-time = "2025-03-19T14:41:04.817Z" }, + { url = "https://files.pythonhosted.org/packages/82/d6/d34964bd5f022660eba70513b33b59a9bfcf79a02d48bf1339b13a79dd06/whenever-0.7.3-cp310-cp310-win_amd64.whl", hash = "sha256:fa8d84ec6c113adaaaf070e1089adb8197c8f1f628366f9d46b5b10f321d6b67", upload-time = "2025-03-19T14:41:11.749Z" }, + { url = "https://files.pythonhosted.org/packages/50/d3/d717d8e981443374145c083264be86097cf3e4771a64be151060b783ad40/whenever-0.7.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a1a639965b27663d180c0c15b26f6222b69b964e0e58a52ac88923454e0c8cdc", upload-time = "2025-03-19T14:40:59.127Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c4/514457fd557a6f79aaf64dd9e58d49622e4ab914960ce3934dbf2b09532f/whenever-0.7.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e596def47c9162e85c7b7d467c61dfde1d80303f9338957eed1e97dc20aff380", upload-time = "2025-03-19T14:40:51.934Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e6/4cc9684f62d154d066031eae0f835b9dd27557a8e8e0627456df7f4d937a/whenever-0.7.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:410ab94597fe10e1da23da4ecad35ed045d925eb38b678415e28764f5b577f47", upload-time = "2025-03-19T14:39:37.053Z" }, + { url = "https://files.pythonhosted.org/packages/f1/40/1789f5d03dcbb5fdab6193c521aeb866e8786d1eb823046b0acc9ed70fd8/whenever-0.7.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6daa9be0ad5ab6e2a74bff8a1631c5fc9c68395f9b20f554707031e3805c220f", upload-time = "2025-03-19T14:39:52.358Z" }, + { url = "https://files.pythonhosted.org/packages/78/cd/6c879ff0499350380fbba335968634315d1eb790f1f6939625a4beb88019/whenever-0.7.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2f2292e1002568e7f7f8f2a798c181243587e7ae12639b6127674b61e0b38959", upload-time = "2025-03-19T14:40:07.988Z" }, + { url = "https://files.pythonhosted.org/packages/21/f8/36fcaa11e15996eed31401a15c4fe97aa2c11118c51b4b81ae3430451c69/whenever-0.7.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a56bb88ac415518e3c0f73622c811be75e85fd3651a3c8da61457a93f8a44bc9", upload-time = "2025-03-19T14:40:15.708Z" }, + { url = "https://files.pythonhosted.org/packages/63/75/696701d8aa8851344e1173117a547e311600f84201f37d5e2b8748407d25/whenever-0.7.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14b233287e254b3bc8eb7f0a333a6ccc52bc16dcf852c25e065541ed72fa189a", upload-time = "2025-03-19T14:40:37.737Z" }, + { url = "https://files.pythonhosted.org/packages/da/12/429cc7dbb826c7c1a1a21cb6b9957f1cce928108aeb3222e8864a309a421/whenever-0.7.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:875e2294dfe147b3212aa2fac9be28d46ef9434d2f94fd7dea78e8e5f609dcce", upload-time = "2025-03-19T14:40:23.814Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ab/8ec464d5fa2900fc0ed05fcecf4fb04e7f243f2acc257759acd5427bbb8f/whenever-0.7.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d5d07f7bb5a09f9c6f499b4c35065c4e64a04e30f02f0592db6f05c54f079ed9", upload-time = "2025-03-19T14:39:44.381Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/0133587208622a86c1da7da8073e3e7a2726205df779cefd4324191bed01/whenever-0.7.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5c87952f542ede1bea94a1884058160db3e9833053369f0cde6adbc4351a8e19", upload-time = "2025-03-19T14:40:00.905Z" }, + { url = "https://files.pythonhosted.org/packages/05/f7/75813ae2b90290985077b2eed5adc091c3a3a75ff291c2bc6f4f24d6fccb/whenever-0.7.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:89b66c7e5475d80a79193ff1fe3f01d60ec87416da2f25a5cf0b3c3677498125", upload-time = "2025-03-19T14:40:30.245Z" }, + { url = "https://files.pythonhosted.org/packages/21/d2/380c53000f9a7ba5b4bb63818cf9055f8a7e65c1607e08b94cf70e0c303d/whenever-0.7.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:075d2d419a962c877511eda949694b4d964701469f7b6b74634b2341c1d9581d", upload-time = "2025-03-19T14:40:44.55Z" }, + { url = "https://files.pythonhosted.org/packages/a2/87/288609c11bba020d8c3af27c1b5a63013866ddce3da799ec93442a91d10d/whenever-0.7.3-cp311-cp311-win32.whl", hash = "sha256:d7a6706aa54a1b74ecaa0c853c897a9ab02b9ebf044bbd3873bcbd9d1984d770", upload-time = "2025-03-19T14:41:06.439Z" }, + { url = "https://files.pythonhosted.org/packages/36/b1/cfc6c26492ddc8c2b9ed4a4ecb47dae00e1749f6e041f5400332446ddd2c/whenever-0.7.3-cp311-cp311-win_amd64.whl", hash = "sha256:78d19379b50006ed20f0cb705e71e4d8edac71459018be56d78d0bafc9ab8a38", upload-time = "2025-03-19T14:41:13.06Z" }, + { url = "https://files.pythonhosted.org/packages/e4/f0/329461391e526e88d24c67107250159726602257b878af8d8106d4f49a4f/whenever-0.7.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:d7027c072e7b15358dd3e79b41e66bf7ae71a6fb7c66c57f8d6249219504de6e", upload-time = "2025-03-19T14:41:00.845Z" }, + { url = "https://files.pythonhosted.org/packages/fa/82/1f89a51a24a4ad73169806c328dd6afb415e2211cb3374cc77c7910a7e62/whenever-0.7.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8645d4512e768a4295b98b334926ef235c116f4c16e57caeaa84cf4a568b392c", upload-time = "2025-03-19T14:40:53.26Z" }, + { url = "https://files.pythonhosted.org/packages/fd/b2/3d0f498934ae45341101fd98b8771829ab9f9216777965951289b2782e5d/whenever-0.7.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:837b10fb99e10091748bbc35b2ca2fee38048625690c0446f8c7b3ffa5655f71", upload-time = "2025-03-19T14:39:38.533Z" }, + { url = "https://files.pythonhosted.org/packages/4e/44/cd87ca7ee5b02b35d10c52135b341f733ed7e74eeaaec07bc75503382a33/whenever-0.7.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:64148ed2ea36f88a9a948c297cc28293d3356123d6a3a3841756b063678921fd", upload-time = "2025-03-19T14:39:54.124Z" }, + { url = "https://files.pythonhosted.org/packages/08/a9/51558f92cd6f14c9296e46706c71c6ef88b223ba6330a054a34ed0d3d5a8/whenever-0.7.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b649f78e05f336026f3d826049112ae48a7d646bc9760c855efbeb8673e6e31", upload-time = "2025-03-19T14:40:09.405Z" }, + { url = "https://files.pythonhosted.org/packages/b5/eb/6cd79aaa6ff08156193e0d2e76d0f35f9800f9da6d0cd93ed1689c662610/whenever-0.7.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:def087fdc15384cf18e1df4a205ee3fa65e433ba436926e2e76570af0c5aaa98", upload-time = "2025-03-19T14:40:18.02Z" }, + { url = "https://files.pythonhosted.org/packages/23/e0/190a340a9c4e019c1d2b2d50df003801824e44c054958c8517f909ac4536/whenever-0.7.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6eabf054b7ff8fc50d3de9c492717d523627af7fddbeb6a26b264410ceab1d5e", upload-time = "2025-03-19T14:40:38.96Z" }, + { url = "https://files.pythonhosted.org/packages/09/b1/6da7060a45cd9890d76e69c9c309cc783ca5c3cd80db69bf9281f3071b32/whenever-0.7.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0cbefd6281b0c31a37e85bd37ef55bf0f77c96ba3636434be10ee261f0856e30", upload-time = "2025-03-19T14:40:25.228Z" }, + { url = "https://files.pythonhosted.org/packages/ee/0f/6b4336d421f3edc8a9ed62f8c8c5053f70307c19a6033a9192abfcc11c27/whenever-0.7.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8c19a615c80a7b390d4e15400f498b6bc20697a69fb92b8c7ce0fe6dcc9d42d1", upload-time = "2025-03-19T14:39:45.765Z" }, + { url = "https://files.pythonhosted.org/packages/18/b4/44d325e34f35fb5461a613442b57d16a2032c0b7dcb117f849d9680b5701/whenever-0.7.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:124d76f4a16ef7c4d6c63632d57c48e215007a9d2807c5e84cd6e1b0de868529", upload-time = "2025-03-19T14:40:02.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/fc/b1b552f4b3fc71ab414c35192d5c8f7d5a61fc2a089abf44b2bca826a701/whenever-0.7.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8d62dc2fb60ecce044dc32b49355bef32147b62687f4a605feea6963c123f867", upload-time = "2025-03-19T14:40:31.477Z" }, + { url = "https://files.pythonhosted.org/packages/c3/16/b17aff5bf1e3c2f6b34f534aaa60e67941bdbde38f86fe5b614596417383/whenever-0.7.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b941d5fd5cc40505726235d2397b13dcb7da3ea4bc048646e53bd54888dec922", upload-time = "2025-03-19T14:40:45.986Z" }, + { url = "https://files.pythonhosted.org/packages/1c/2e/c4576a327685624ca73e7fdefbfd0e938287d46a0670f03d3a6335d26aa9/whenever-0.7.3-cp312-cp312-win32.whl", hash = "sha256:47652cf34d7419edfc73dc6852be69170283ae30bbc04908f90fcec024d0ceca", upload-time = "2025-03-19T14:41:07.771Z" }, + { url = "https://files.pythonhosted.org/packages/08/c1/d04656c96b99d8c7996d40e85dcfa94ee6f882ce9b5fdba5a277f3ef0011/whenever-0.7.3-cp312-cp312-win_amd64.whl", hash = "sha256:e2eb1077b97b4114d46cc91e4335a28519cc9f66ccc9a54374fd3836b69fe674", upload-time = "2025-03-19T14:41:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/4f/09/b824938af3a6de9c9729e6e3505bab7dc996ff18090f5b78565aedcddfac/whenever-0.7.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:44245e7b72e3691c6a145fc39aa6001796e429c42b7503af5c6dc04cd6c884e3", upload-time = "2025-03-19T14:41:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/7f/cb/b0855f643c12b0ceb34048bc7d3165880c41a09e5e4aa999b1238e0c21fa/whenever-0.7.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:040b0c0be2b67ff3875c548b19b5d52ec438ad892d094d5677ce11c069afab67", upload-time = "2025-03-19T14:40:54.572Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d6/90a1fa465ded391addf490889909b27be3d19c8acf7ff7a5717dcf48a92a/whenever-0.7.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3da1d0dd58e4b989b3c3972e93b2ee807c538b0a8100b965a7392eba52b905f5", upload-time = "2025-03-19T14:39:39.712Z" }, + { url = "https://files.pythonhosted.org/packages/42/4d/0f748d7d6657a6de936ecbc0402d4b9d72cfebd05faba237658cdf869981/whenever-0.7.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c51ed8d910dcc40ab863a0ebed0ede2103d98e276d3486932cc39cde3e528d1c", upload-time = "2025-03-19T14:39:56.51Z" }, + { url = "https://files.pythonhosted.org/packages/fa/17/f351e0a8f1593eb8ac946c92c27f0e53787dfd24730b1473c0a99fc6b339/whenever-0.7.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ce6badfc5719a2a1fd6f8cd6b141a983c8f81396322fd77172e1560828efbde", upload-time = "2025-03-19T14:40:11.139Z" }, + { url = "https://files.pythonhosted.org/packages/7f/91/11b98d337fc0b2190f3936fe1dcd14e428ef88ecc991b7162f51b37e2323/whenever-0.7.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:56d4ec5856906206d585f0b7a1ca17c7879722b3b427b888428454e938dd80df", upload-time = "2025-03-19T14:40:19.405Z" }, + { url = "https://files.pythonhosted.org/packages/6f/21/15a0bec883b0af1dcad6a148c30e00e138062acdaf89685c34b34240723f/whenever-0.7.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b36b64461f542b4a3abe9854d4b9aec42dc34f3d9d7ef66c364a59d7cb012ce", upload-time = "2025-03-19T14:40:40.277Z" }, + { url = "https://files.pythonhosted.org/packages/8f/03/c012a37695aeadd5ff8f8bddc7a50544cd52baecc045d743f5d14e26b16d/whenever-0.7.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2983e560829490a55432cea425b4b0b93bbe1c0245541c8243e3922516abd441", upload-time = "2025-03-19T14:40:26.441Z" }, + { url = "https://files.pythonhosted.org/packages/6f/57/80a4d4e4c8b3ca6faa51f36d959e702d92f1cd2ffc2f047d1e348ceb434e/whenever-0.7.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bbe364df8e3a1fb2a489c9975708634d164123e79affe70096ad2903081f8bc", upload-time = "2025-03-19T14:39:47.669Z" }, + { url = "https://files.pythonhosted.org/packages/05/63/dd16f2af0305aa93293edc5827ee6893ca040066c71e12e859600a96a854/whenever-0.7.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:c47ac8aa249871521d58aa07eb05322a7a8f3d15f333564dc5b6d9125f8dc585", upload-time = "2025-03-19T14:40:03.949Z" }, + { url = "https://files.pythonhosted.org/packages/a4/37/7056e1d5039b2ffe729d21b2ee0ff7fe3ffaca7e2fcb3674e2870a2aaba0/whenever-0.7.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:742eeb3109259afe2d84247651346f19b2f625b64b45c944e5bc633fbbc89ae8", upload-time = "2025-03-19T14:40:32.853Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/8928a1733bab2cb94e19a5815d1077bc18b7e7d463a026232192bad8ce36/whenever-0.7.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b28e08e9b2b2c11c047f7a87ce8867a8739eca0c77fa4c552861a87767f7025b", upload-time = "2025-03-19T14:40:47.87Z" }, + { url = "https://files.pythonhosted.org/packages/d6/c9/91f71eff32540e5991820b4eddb0fe3886c45a47432ac4ac00ca75ad6723/whenever-0.7.3-cp313-cp313-win32.whl", hash = "sha256:2991d491badcdd9da357d156f9e17426f2feda7da9f186dabea6d759f8920225", upload-time = "2025-03-19T14:41:09.146Z" }, + { url = "https://files.pythonhosted.org/packages/74/46/e3b3f9ccd27958652fea2d94442a1f1e82deaf0721bb4afa72b3d0fee476/whenever-0.7.3-cp313-cp313-win_amd64.whl", hash = "sha256:1a340f71c0b8a059ecf34948f2c59b8995abd84b2b69fff576a3818162fb3056", upload-time = "2025-03-19T14:41:15.979Z" }, +] + [[package]] name = "wrapt" version = "2.1.2" From dea2bd0c374e3b326036c819707c5f56d4672b83 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 16:03:33 +0100 Subject: [PATCH 079/107] Refuse a generated feature name that would overwrite a real column Feature engineering builds derived columns by appending a suffix and writing the result with `withColumn`, which replaces any column already carrying that name. A table with both `amount` and `amount_rel_baseline` therefore lost the second one: the derived value overwrote it, the name was appended to the positional feature list a second time, so the frame carried two identical columns, and the model was fitted on two copies of the derived value with the user's real metric gone. Nothing downstream can detect that, and per-feature attribution then names the wrong source column. Every suffixed transform has the same hole -- `_rel_time`, `_is_null`, `_bool`, `_freq`, the seven calendar names, and one-hot `{col}_{value}`. Refused rather than worked around. Renaming the derived feature would leave the persisted feature list disagreeing with the suffix that redaction and attribution both key on, and dropping either column silently discards data the caller asked to be checked. Two checks, because one is knowable from a schema and one is not: - the predictable names, in `validate_generated_feature_names`, from the schema alone with no Spark action. Conservative on the two things a schema cannot answer: a null indicator's name is claimed even when this frame has no nulls, because a collision there is a bug waiting for the first null, and a categorical column claims its frequency name whatever its cardinality - the one-hot names, where they are built, because `{col}_{value}` depends on the values Also catches two columns generating one name, which is why the collector returns a list rather than a name-keyed dict -- a dict kept one entry and the clash could never be seen. Suffixes become module constants used by both the transforms and the validator: one drifting apart from the other would reopen the hole with every test still passing. The calendar block now pairs its seven expressions with that constant tuple, so the emitted order is the checked order by construction, and `_classify_column`'s isinstance chain moves into `feature_category_for_type`, which is what lets validation answer "which transforms will run" from a schema. The integration test is the one that matters: the unit suite pins the predicate, and only that pins that `train` consults it. Co-authored-by: Isaac --- .../labs/dqx/anomaly/training_service.py | 3 + .../labs/dqx/anomaly/transformers.py | 121 +++++++++----- src/databricks/labs/dqx/anomaly/validation.py | 124 +++++++++++++- .../test_anomaly_errors.py | 34 ++++ .../test_anomaly_feature_name_collisions.py | 154 ++++++++++++++++++ 5 files changed, 391 insertions(+), 45 deletions(-) create mode 100644 tests/unit/test_anomaly_feature_name_collisions.py diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index 044194d75..e96f21a14 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -50,6 +50,7 @@ validate_columns, validate_fully_qualified_name, validate_baseline_columns, + validate_generated_feature_names, validate_spark_version, validate_training_params, ) @@ -366,6 +367,8 @@ def build_context( resolved_over_time = baseline_over_time if baseline_over_time is not None else params.baseline_over_time validate_baseline_over_time(df, resolved_over_time, columns) + # After both bases are resolved, because which derived features exist depends on them. + validate_generated_feature_names(df, columns, baseline_by, resolved_over_time) self._advise_calendar_features(df, columns, resolved_over_time) if resolved_over_time: self._advise_trend_strength(df_filtered, columns, resolved_over_time) diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 17f63d616..728a6a652 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -153,6 +153,25 @@ def _spark_type_for_category(category: str) -> T.DataType: }.get(category, T.StringType()) +def feature_category_for_type(col_type: T.DataType) -> str: + """Which family of transforms a column's Spark type puts it in. + + Depends on the type alone. Cardinality decides *how* a categorical column is encoded, and a null + count decides whether it gets an indicator, but neither changes the category — which is what makes + this answerable from a schema with no Spark action, and therefore usable by validation before any + profiling has run. + """ + if isinstance(col_type, T.NumericType): + return 'numeric' + if isinstance(col_type, T.BooleanType): + return 'boolean' + if isinstance(col_type, (T.DateType, T.TimestampType, T.TimestampNTZType)): + return 'datetime' + if isinstance(col_type, T.StringType): + return 'categorical' + return 'unsupported' + + def reconstruct_column_infos(feature_metadata: SparkFeatureMetadata) -> list[ColumnTypeInfo]: """ Reconstruct ColumnTypeInfo objects from SparkFeatureMetadata. @@ -285,15 +304,14 @@ def _classify_column( self, col_name: str, col_type: T.DataType, *, null_count: int, distinct_count: int | None = None ) -> ColumnTypeInfo: """Classify a single column.""" + category = feature_category_for_type(col_type) - # Numeric types - if isinstance(col_type, T.NumericType): + if category == 'numeric': return ColumnTypeInfo( name=col_name, spark_type=col_type, category='numeric', null_count=null_count, encoding_strategy='none' ) - # Boolean - if isinstance(col_type, T.BooleanType): + if category == 'boolean': return ColumnTypeInfo( name=col_name, spark_type=col_type, @@ -302,8 +320,7 @@ def _classify_column( encoding_strategy='binary', ) - # Datetime types - if isinstance(col_type, (T.DateType, T.TimestampType, T.TimestampNTZType)): + if category == 'datetime': return ColumnTypeInfo( name=col_name, spark_type=col_type, @@ -312,28 +329,23 @@ def _classify_column( encoding_strategy='cyclical', ) - # Handle string columns as categorical features (distinct_count is always set in analyze_columns for string columns) - if isinstance(col_type, T.StringType): + # distinct_count is always set in analyze_columns for string columns. + if category == 'categorical': if distinct_count is None: raise InvalidParameterError(f"distinct_count is required for string column {col_name}") - cardinality = distinct_count # Determine encoding strategy based on cardinality - if cardinality <= self.categorical_cardinality_threshold: - strategy = 'onehot' - else: - strategy = 'frequency' + strategy = 'onehot' if distinct_count <= self.categorical_cardinality_threshold else 'frequency' return ColumnTypeInfo( name=col_name, spark_type=col_type, category='categorical', - cardinality=cardinality, + cardinality=distinct_count, null_count=null_count, encoding_strategy=strategy, ) - # Unsupported types return ColumnTypeInfo(name=col_name, spark_type=col_type, category='unsupported', null_count=null_count) def _estimate_feature_count(self, column_infos: list[ColumnTypeInfo]) -> int: @@ -572,7 +584,7 @@ def _add_null_indicator( """Add null indicator column if column has nulls.""" has_nulls = (null_count or 0) > 0 if has_nulls: - null_indicator_col = f"{col_name}_is_null" + null_indicator_col = f"{col_name}{NULL_INDICATOR_SUFFIX}" transformed_df = transformed_df.withColumn(null_indicator_col, when(col(col_name).isNull(), 1.0).otherwise(0.0)) engineered_features.append(null_indicator_col) return transformed_df @@ -601,8 +613,20 @@ def _apply_onehot_encoding( "Model may be from an older version without OneHot category storage." ) + # The one collision that no schema can predict, so it is caught here rather than in validation: + # a one-hot name is built from a *value*, and a table can carry both a "region" column with an + # "east" value and a separate "region_east" column. Writing the indicator would overwrite the real + # one and leave the model fitted on two copies of the indicator. See + # ``validation.validate_generated_feature_names`` for the same refusal over the predictable names. + existing_columns = set(transformed_df.columns) for value in distinct_values: feature_name = f"{col_name}_{value}" + if feature_name in existing_columns: + raise InvalidParameterError( + f"One-hot encoding column '{col_name}' would create '{feature_name}' for value {value!r}, " + f"but a column of that name already exists and would be overwritten. Rename it, or exclude " + f"'{col_name}' from the checked columns." + ) transformed_df = transformed_df.withColumn(feature_name, when(col(col_name) == value, 1.0).otherwise(0.0)) engineered_features.append(feature_name) @@ -617,7 +641,7 @@ def _apply_frequency_encoding( engineered_features: list[str], ) -> DataFrame: """Apply Frequency encoding to a categorical column.""" - feature_name = f"{col_name}_freq" + feature_name = f"{col_name}{FREQUENCY_FEATURE_SUFFIX}" lookup_key_col = f"__dqx_{col_name}_category" lookup_val_col = f"__dqx_{col_name}_frequency" @@ -708,32 +732,24 @@ def _process_datetime_columns( col_name, coalesce(col(col_name).cast(TimestampType()), to_timestamp(lit("1970-01-01 00:00:00"))) ) - # Extract cyclical features - transformed_df = transformed_df.withColumn(f"{col_name}_hour_sin", sin(hour(col(col_name)) * 2 * pi() / 24)) - transformed_df = transformed_df.withColumn(f"{col_name}_hour_cos", cos(hour(col(col_name)) * 2 * pi() / 24)) - engineered_features.extend([f"{col_name}_hour_sin", f"{col_name}_hour_cos"]) - - transformed_df = transformed_df.withColumn( - f"{col_name}_dow_sin", sin((dayofweek(col(col_name)) - 1) * 2 * pi() / 7) - ) - transformed_df = transformed_df.withColumn( - f"{col_name}_dow_cos", cos((dayofweek(col(col_name)) - 1) * 2 * pi() / 7) - ) - engineered_features.extend([f"{col_name}_dow_sin", f"{col_name}_dow_cos"]) - - transformed_df = transformed_df.withColumn( - f"{col_name}_month_sin", sin((month(col(col_name)) - 1) * 2 * pi() / 12) + # Extract cyclical features. Paired with their suffixes rather than written out one + # ``withColumn`` at a time, so the emitted order is the order of CALENDAR_FEATURE_SUFFIXES by + # construction: that tuple is what collision validation checks against, and + # engineered_feature_names is positional, so the two must not be able to drift apart. + timestamp = col(col_name) + calendar_expressions = ( + sin(hour(timestamp) * 2 * pi() / 24), + cos(hour(timestamp) * 2 * pi() / 24), + sin((dayofweek(timestamp) - 1) * 2 * pi() / 7), + cos((dayofweek(timestamp) - 1) * 2 * pi() / 7), + sin((month(timestamp) - 1) * 2 * pi() / 12), + cos((month(timestamp) - 1) * 2 * pi() / 12), + when((dayofweek(timestamp) == 1) | (dayofweek(timestamp) == 7), 1.0).otherwise(0.0), ) - transformed_df = transformed_df.withColumn( - f"{col_name}_month_cos", cos((month(col(col_name)) - 1) * 2 * pi() / 12) - ) - engineered_features.extend([f"{col_name}_month_sin", f"{col_name}_month_cos"]) - - transformed_df = transformed_df.withColumn( - f"{col_name}_is_weekend", - when((dayofweek(col(col_name)) == 1) | (dayofweek(col(col_name)) == 7), 1.0).otherwise(0.0), - ) - engineered_features.append(f"{col_name}_is_weekend") + for suffix, expression in zip(CALENDAR_FEATURE_SUFFIXES, calendar_expressions, strict=True): + feature_name = f"{col_name}{suffix}" + transformed_df = transformed_df.withColumn(feature_name, expression) + engineered_features.append(feature_name) # Drop the original datetime column after feature extraction # This ensures the TimestampType column doesn't reach sklearn (which expects float) @@ -755,10 +771,11 @@ def _process_boolean_columns( transformed_df = _add_null_indicator(transformed_df, col_name, col_info.null_count, engineered_features) # Map to 0/1 (nulls -> 0) + feature_name = f"{col_name}{BOOLEAN_FEATURE_SUFFIX}" transformed_df = transformed_df.withColumn( - f"{col_name}_bool", when(col(col_name).isNull(), 0.0).when(col(col_name), 1.0).otherwise(0.0) + feature_name, when(col(col_name).isNull(), 0.0).when(col(col_name), 1.0).otherwise(0.0) ) - engineered_features.append(f"{col_name}_bool") + engineered_features.append(feature_name) return transformed_df @@ -797,6 +814,22 @@ def _signed_log1p(column: Column) -> Column: # means the LLM must not see "amount_rel_baseline" either. BASELINE_RELATIVE_SUFFIX = "_rel_baseline" TEMPORAL_RELATIVE_SUFFIX = "_rel_time" +# Every other suffix a transform below appends to a source column's name. Constants rather than +# inline f-strings because ``validation.validate_generated_feature_names`` builds the same names to +# refuse a collision with a real input column, and a suffix that drifted between the two places +# would reopen exactly the silent overwrite that check exists to prevent. +NULL_INDICATOR_SUFFIX = "_is_null" +BOOLEAN_FEATURE_SUFFIX = "_bool" +FREQUENCY_FEATURE_SUFFIX = "_freq" +CALENDAR_FEATURE_SUFFIXES = ( + "_hour_sin", + "_hour_cos", + "_dow_sin", + "_dow_cos", + "_month_sin", + "_month_cos", + "_is_weekend", +) # Rows collected to the driver to fit the temporal basis. The fit runs on a bucketed aggregate rather # than raw rows, so this bounds driver memory regardless of table size, exactly as the per-group median # aggregation does. Each bucket contributes its median, which is robust before Huber even sees it. diff --git a/src/databricks/labs/dqx/anomaly/validation.py b/src/databricks/labs/dqx/anomaly/validation.py index 8b56882bc..c003d7ccf 100644 --- a/src/databricks/labs/dqx/anomaly/validation.py +++ b/src/databricks/labs/dqx/anomaly/validation.py @@ -13,7 +13,16 @@ from pyspark.sql import types as T from databricks.labs.dqx.anomaly.model_config import AnomalyModelRecord -from databricks.labs.dqx.anomaly.transformers import ColumnTypeClassifier +from databricks.labs.dqx.anomaly.transformers import ( + BASELINE_RELATIVE_SUFFIX, + BOOLEAN_FEATURE_SUFFIX, + CALENDAR_FEATURE_SUFFIXES, + FREQUENCY_FEATURE_SUFFIX, + NULL_INDICATOR_SUFFIX, + TEMPORAL_RELATIVE_SUFFIX, + ColumnTypeClassifier, + feature_category_for_type, +) from databricks.labs.dqx.config import AnomalyParams from databricks.labs.dqx.errors import InvalidParameterError @@ -155,6 +164,119 @@ def validate_baseline_over_time( ) +def generated_feature_names( + schema_fields: dict[str, T.DataType], + columns: collections.abc.Iterable[str], + baseline_by: list[str] | None, + baseline_over_time: str | None, +) -> list[tuple[str, str, str]]: + """Every ``(generated name, source column, transform)`` feature engineering will create. + + A list rather than a name-keyed mapping so that two sources generating the *same* name stay visible; + a dict would silently keep one, which is the shape of the bug this feeds. + + Deliberately independent of null counts and cardinality, both of which need a Spark action. Being + conservative in those two places is the point rather than a compromise: + + - a null indicator is only built for a column that has nulls *in the training frame*, but a schema + whose generated indicator name collides is a bug waiting for the first null to arrive, so the name + is claimed unconditionally + - a categorical column is frequency-encoded only above the cardinality threshold, and its one-hot + names depend on the values themselves, which no schema can predict. The frequency name is claimed + unconditionally; the one-hot names are checked where they are built. + """ + generated: list[tuple[str, str, str]] = [] + + for name in columns: + col_type = schema_fields.get(name) + if col_type is None: + continue # A missing feature column is validate_columns' complaint, not this one's. + + generated.append((f"{name}{NULL_INDICATOR_SUFFIX}", name, "the null indicator")) + derived = _derived_names_for_category( + name, feature_category_for_type(col_type), baseline_by, baseline_over_time + ) + generated.extend((derived_name, name, transform) for derived_name, transform in derived) + + return generated + + +def _derived_names_for_category( + name: str, + category: str, + baseline_by: list[str] | None, + baseline_over_time: str | None, +) -> list[tuple[str, str]]: + """``(generated name, transform)`` for one column, from its category alone. + + Split out from the caller so each category's answer is one flat branch. An 'unsupported' column + generates nothing because feature engineering skips it entirely. + """ + if category == 'numeric': + derived = [] + if baseline_by: + derived.append((f"{name}{BASELINE_RELATIVE_SUFFIX}", "the baseline_by comparison")) + if baseline_over_time: + derived.append((f"{name}{TEMPORAL_RELATIVE_SUFFIX}", "the baseline_over_time comparison")) + return derived + if category == 'boolean': + return [(f"{name}{BOOLEAN_FEATURE_SUFFIX}", "the boolean encoding")] + if category == 'datetime': + return [(f"{name}{suffix}", "the calendar encoding") for suffix in CALENDAR_FEATURE_SUFFIXES] + if category == 'categorical': + return [(f"{name}{FREQUENCY_FEATURE_SUFFIX}", "the frequency encoding")] + return [] + + +def validate_generated_feature_names( + df: DataFrame, + columns: collections.abc.Iterable[str], + baseline_by: list[str] | None, + baseline_over_time: str | None, +) -> None: + """Refuse a schema where a generated feature would land on the name of a real column. + + Feature engineering builds derived columns by appending a suffix and writing the result with + ``withColumn``, which *replaces* a column of that name. A table carrying both ``amount`` and + ``amount_rel_baseline`` therefore loses the second one: the derived value overwrites it, the name is + appended to the feature list a second time, and the model is fitted on two copies of the derived + value with the user's real metric gone. Nothing downstream can detect that, and per-feature + attribution then names the wrong source column. + + Raising is the only honest option. Renaming the derived feature would leave the persisted feature + list disagreeing with the suffix that redaction and attribution both key on, and dropping either + column silently discards data the caller asked to be checked. + """ + generated = generated_feature_names( + {field.name: field.dataType for field in df.schema.fields}, columns, baseline_by, baseline_over_time + ) + + existing = set(df.columns) + overwrites = sorted(entry for entry in generated if entry[0] in existing) + if overwrites: + details = "; ".join( + f"'{name}' already exists and would be overwritten by {transform} of '{source}'" + for name, source, transform in overwrites + ) + raise InvalidParameterError( + f"Feature engineering would overwrite existing columns: {details}. Rename the existing " + "column, or exclude it and its source from the checked columns. DQX refuses rather than " + "picking one, because either choice silently drops data you asked it to check." + ) + + by_name = collections.defaultdict(list) + for name, source, _ in generated: + by_name[name].append(source) + clashes = sorted((name, sources) for name, sources in by_name.items() if len(sources) > 1) + if clashes: + details = "; ".join(f"'{name}' from {sorted(sources)}" for name, sources in clashes) + raise InvalidParameterError( + f"Two checked columns would generate the same feature name: {details}. The feature list is " + "positional, so a duplicated name makes one column's feature indistinguishable from the " + "other's. Rename one of the source columns." + ) + + def _validate_float_range( value: float, *, diff --git a/tests/integration_anomaly/test_anomaly_errors.py b/tests/integration_anomaly/test_anomaly_errors.py index a58f00072..361e6c746 100644 --- a/tests/integration_anomaly/test_anomaly_errors.py +++ b/tests/integration_anomaly/test_anomaly_errors.py @@ -280,6 +280,40 @@ def test_internal_score_column_collision(ws, spark: SparkSession, make_random, a assert "ambiguous" in str(exc.value).lower() +def test_training_refuses_a_frame_where_a_derived_feature_would_overwrite_a_real_column( + spark: SparkSession, make_random, anomaly_engine, anomaly_registry_prefix +): + """The wiring the unit tests cannot show: that ``train`` actually reaches the collision check. + + Before this refused, the run succeeded and lost data silently. ``amount_rel_baseline`` is a real + column here, so the group-relative transform overwrote it, appended its name to the positional + feature list a second time, and fitted the model on two copies of the derived value with the user's + own column gone. Nothing downstream could detect that, and attribution then named the wrong source. + + The unit suite pins the predicate; only this pins that the predicate is consulted. + """ + model_name = f"{anomaly_registry_prefix}.test_feature_collision_{make_random(4).lower()}" + registry_table = f"{anomaly_registry_prefix}.t{make_random(8).lower()}_registry" + + df = spark.createDataFrame( + [(100.0 + i, 0.5, "eu") for i in range(60)], + "amount double, amount_rel_baseline double, region string", + ) + + with pytest.raises(InvalidParameterError) as exc: + anomaly_engine.train( + df=df, + columns=["amount", "amount_rel_baseline"], + model_name=model_name, + registry_table=registry_table, + baseline_by=["region"], + ) + + message = str(exc.value) + assert "amount_rel_baseline" in message + assert "overwritten" in message + + def test_has_no_row_anomalies_requires_fully_qualified_model_name(): """Ensure model name must be fully qualified.""" with pytest.raises(InvalidParameterError): diff --git a/tests/unit/test_anomaly_feature_name_collisions.py b/tests/unit/test_anomaly_feature_name_collisions.py new file mode 100644 index 000000000..145fa0969 --- /dev/null +++ b/tests/unit/test_anomaly_feature_name_collisions.py @@ -0,0 +1,154 @@ +"""Unit pins for the refusal of a generated feature name that lands on a real column (no Spark). + +Feature engineering builds derived columns by appending a suffix and writing the result with +``withColumn``, which *replaces* any column already carrying that name. A table with both ``amount`` and +``amount_rel_baseline`` therefore lost the second one silently: the derived value overwrote it, the name +was appended to the positional feature list a second time, and the model was fitted on two copies of the +derived value with the real metric gone. Attribution then named the wrong source column. + +Nothing downstream can notice that, which is why the refusal is up front and why it is worth pinning per +transform family. The suffixes live in ``transformers`` as constants for the same reason these tests +import them rather than spelling them out: a suffix that drifted between the transform and the validator +would reopen the hole while every test still passed. +""" + +from unittest.mock import create_autospec + +import pytest +from pyspark.sql import DataFrame +from pyspark.sql import types as T + +from databricks.labs.dqx.anomaly.transformers import ( + BASELINE_RELATIVE_SUFFIX, + BOOLEAN_FEATURE_SUFFIX, + CALENDAR_FEATURE_SUFFIXES, + FREQUENCY_FEATURE_SUFFIX, + NULL_INDICATOR_SUFFIX, + TEMPORAL_RELATIVE_SUFFIX, +) +from databricks.labs.dqx.anomaly.validation import generated_feature_names, validate_generated_feature_names +from databricks.labs.dqx.errors import InvalidParameterError + + +def _fake_df(schema: dict[str, T.DataType]) -> DataFrame: + """A DataFrame stand-in exposing only the schema and columns the validator reads.""" + df = create_autospec(DataFrame, instance=True) + df.schema = T.StructType([T.StructField(name, dtype, True) for name, dtype in schema.items()]) + df.columns = list(schema) + return df + + +def test_an_ordinary_schema_passes(): + df = _fake_df({"amount": T.DoubleType(), "region": T.StringType(), "created": T.TimestampType()}) + + validate_generated_feature_names(df, ["amount", "region"], ["region"], "created") # must not raise + + +def test_a_column_named_after_the_group_relative_feature_is_refused(): + """The review's own example, and the one that loses a real metric.""" + df = _fake_df( + { + "amount": T.DoubleType(), + f"amount{BASELINE_RELATIVE_SUFFIX}": T.DoubleType(), + "region": T.StringType(), + } + ) + + with pytest.raises(InvalidParameterError) as raised: + validate_generated_feature_names(df, ["amount", f"amount{BASELINE_RELATIVE_SUFFIX}"], ["region"], None) + + message = str(raised.value) + assert f"amount{BASELINE_RELATIVE_SUFFIX}" in message + assert "overwritten" in message + + +def test_the_same_column_is_accepted_when_no_grouping_makes_the_feature(): + """The refusal is about what *this* configuration builds, not about the name in the abstract. + + Without ``baseline_by`` there is no group-relative transform, so ``amount_rel_baseline`` is just an + ordinary numeric column and rejecting it would be a false alarm. + """ + df = _fake_df({"amount": T.DoubleType(), f"amount{BASELINE_RELATIVE_SUFFIX}": T.DoubleType()}) + + validate_generated_feature_names(df, ["amount", f"amount{BASELINE_RELATIVE_SUFFIX}"], None, None) + + +def test_a_column_named_after_the_time_relative_feature_is_refused(): + df = _fake_df( + { + "latency": T.DoubleType(), + f"latency{TEMPORAL_RELATIVE_SUFFIX}": T.DoubleType(), + "reading_ts": T.TimestampType(), + } + ) + + with pytest.raises(InvalidParameterError): + validate_generated_feature_names(df, ["latency", f"latency{TEMPORAL_RELATIVE_SUFFIX}"], None, "reading_ts") + + +@pytest.mark.parametrize("suffix", CALENDAR_FEATURE_SUFFIXES) +def test_a_column_named_after_any_calendar_feature_is_refused(suffix: str): + df = _fake_df({"created": T.TimestampType(), f"created{suffix}": T.DoubleType()}) + + with pytest.raises(InvalidParameterError) as raised: + validate_generated_feature_names(df, ["created", f"created{suffix}"], None, None) + + assert f"created{suffix}" in str(raised.value) + + +def test_a_column_named_after_the_boolean_feature_is_refused(): + df = _fake_df({"is_vip": T.BooleanType(), f"is_vip{BOOLEAN_FEATURE_SUFFIX}": T.DoubleType()}) + + with pytest.raises(InvalidParameterError): + validate_generated_feature_names(df, ["is_vip", f"is_vip{BOOLEAN_FEATURE_SUFFIX}"], None, None) + + +def test_a_column_named_after_the_frequency_feature_is_refused(): + df = _fake_df({"sku": T.StringType(), f"sku{FREQUENCY_FEATURE_SUFFIX}": T.DoubleType()}) + + with pytest.raises(InvalidParameterError): + validate_generated_feature_names(df, ["sku", f"sku{FREQUENCY_FEATURE_SUFFIX}"], None, None) + + +def test_a_null_indicator_collision_is_refused_even_with_no_nulls_present(): + """Claimed unconditionally, and deliberately so. + + Whether an indicator is built depends on the training frame having nulls, which is a property of + today's data rather than of the schema. A schema whose indicator name collides is a bug waiting for + the first null to arrive, and it would then appear as a silent overwrite on a later run rather than + as an error on this one. + """ + df = _fake_df({"amount": T.DoubleType(), f"amount{NULL_INDICATOR_SUFFIX}": T.DoubleType()}) + + with pytest.raises(InvalidParameterError): + validate_generated_feature_names(df, ["amount", f"amount{NULL_INDICATOR_SUFFIX}"], None, None) + + +def test_the_collector_keeps_every_source_rather_than_one_per_name(): + """Why the collector returns a list rather than a name-keyed mapping. + + A dict keeps one entry per name, so the duplicate check below could never see a clash at all. This + is the shape that makes it representable. + """ + generated = generated_feature_names({"x": T.DoubleType()}, ["x", "x"], None, None) + + assert [name for name, _, _ in generated] == [f"x{NULL_INDICATOR_SUFFIX}", f"x{NULL_INDICATOR_SUFFIX}"] + + +def test_a_column_listed_twice_is_refused(): + """Reachable: nothing upstream deduplicates *columns*, so a caller repeating one would otherwise get + that column's features appended twice into a positional list.""" + df = _fake_df({"x": T.DoubleType()}) + + with pytest.raises(InvalidParameterError) as raised: + validate_generated_feature_names(df, ["x", "x"], None, None) + + assert "same feature name" in str(raised.value) + + +def test_a_feature_column_missing_from_the_schema_is_left_to_the_column_validator(): + """Not this validator's complaint. Reporting it here would give two errors for one mistake, and the + less useful one first.""" + df = _fake_df({"amount": T.DoubleType()}) + + validate_generated_feature_names(df, ["amount", "nonexistent"], None, None) From d70b6b94dbcaf0c614d5aa6d644633b78ba3ff16 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 16:46:29 +0100 Subject: [PATCH 080/107] Fit both comparison bases before imputing nulls, and choose the basis from every metric Two feature-engineering defects, both of which made a fitted baseline depend on something it should not have. Nulls were imputed before the baselines were fitted. `_process_numeric_columns` did `coalesce(col.cast(double), 0.0)` in place on the raw metric, and both relative blocks run after it, so a missing observation contributed a fabricated zero to its own group's median and to its time bucket's median, then received a fabricated deviation from the baseline it had just moved -- `0 - expected(t)` -- on top of its null indicator. Measured on Spark: a group of three 100s and four nulls had a baseline of 0.0 where the median of the values that exist is 100.0. The cast stays where it was and only the imputation moves to the end, after both bases are fitted. `percentile_approx` skips nulls by construction, so the medians and the bucket aggregates now see what is actually there, and the derived features are imputed to a neutral zero -- which for a deviation genuinely means "no deviation", with the `_is_null` indicator carrying the missingness as it was always supposed to. Appends no feature, so the positional contract is untouched. The temporal basis was chosen by whichever metric came first. `select_basis` scored changepoint counts on `representative`, the first metric with more than two usable buckets, and took *its* masked time axis for the period search too, so reordering columns changed every fitted residual for every metric. Reproduced: 0 changepoints with a straight metric first, 3 with a bent one. It now reads every metric. Candidates are compared on the mean across metrics of the ratio between a candidate's holdout residual scale and the same metric's scale under the simplest basis, and the period search reads the full bucket axis. The ratio is what stops a large-magnitude column outvoting the rest, since a residual scale carries the metric's own units. Standardising each metric first was not in the plan and is not cosmetic: my own test asserting the ratio was unitless failed. The Huber fit's `alpha` penalty is absolute, so the same shape expressed in millions is effectively fitted with less regularisation, and rescaling one metric by 1e6 moved the selection from three changepoints to six. Each metric is now centred and scaled by its own MAD before being scored -- for the basis *choice* only, `fit_temporal` still fits raw values, so no coefficient moves. Verified against fieldeng rather than only in CI: the null-baseline, temporal-null and imputation tests pass on real Spark, and so do the group-relative, transformers, nulls and temporal suites (44 + 21 + 16 passed). Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/temporal.py | 86 ++++++++++++++++--- .../labs/dqx/anomaly/transformers.py | 72 ++++++++++++---- .../test_anomaly_group_relative_features.py | 85 ++++++++++++++++++ .../test_anomaly_temporal_features.py | 39 +++++++++ tests/unit/test_anomaly_temporal_fit.py | 81 ++++++++++++++++- 5 files changed, 332 insertions(+), 31 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/temporal.py b/src/databricks/labs/dqx/anomaly/temporal.py index 3bc260528..472cfa2e2 100644 --- a/src/databricks/labs/dqx/anomaly/temporal.py +++ b/src/databricks/labs/dqx/anomaly/temporal.py @@ -223,22 +223,81 @@ def _holdout_residual_scale(seconds: np.ndarray, values: np.ndarray, basis: Temp return float(np.median(np.abs(residual - np.median(residual))) * MAD_TO_SIGMA) -def select_basis(seconds: np.ndarray, values: np.ndarray) -> tuple[TemporalBasis, dict[float, str]]: +def _standardise(values: np.ndarray) -> np.ndarray | None: + """Centre and scale a metric robustly, or ``None`` if it has no spread to scale by. + + Used only to compare candidate bases with each other, never to fit a coefficient. Median and MAD + rather than mean and standard deviation for the usual reason: a handful of anomalous buckets should + not set the scale that every basis is then judged against. + """ + if values.size == 0: + return None + centre = float(np.median(values)) + spread = float(np.median(np.abs(values - centre)) * MAD_TO_SIGMA) + if not np.isfinite(spread) or spread <= 0: + return None # A constant metric ranks every basis equally, so it carries no signal here. + return (values - centre) / spread + + +def select_basis(seconds: np.ndarray, metrics: dict[str, np.ndarray]) -> tuple[TemporalBasis, dict[float, str]]: """Choose the basis for a table: which periods, and how many changepoints. - *values* is a representative metric, used only to score changepoint counts. The periods depend on the - time axis alone, so they are the same for every metric in the table, which is what keeps one basis and - one column order for the whole model. + One basis is shared by every metric, which is what keeps one column order for the whole model, so the + choice has to be a property of the table rather than of any one metric. It reads every metric to that + end. An earlier version scored changepoints on whichever metric happened to be first in the schema + and took its time axis for the period search too, which made every fitted residual depend on column + order: a linear metric first chose one changepoint where a bent metric chose six. + + Candidates are compared on the **mean across metrics of the ratio** between a candidate's holdout + residual scale and the same metric's scale under the simplest basis, each metric first standardised + to a robust unit scale. Both halves are needed and neither is cosmetic: + + - the *ratio* stops a large-magnitude column outvoting the rest, which a raw sum of residual scales + would let it do, because a residual scale carries the metric's own units + - the *standardisation* is what makes the ratio genuinely magnitude-free. The underlying fit is + penalised (``HUBER_ALPHA``), and that penalty is absolute, so the same shape expressed in millions + is effectively fitted with less regularisation than one expressed in tens. Measured: rescaling one + straight metric by 1e6 moved the selected count from three changepoints to six until each metric + was standardised first. Only the basis *choice* is made on standardised values; ``fit_temporal`` + fits the raw metric, so no coefficient is affected. + + Args: + seconds: Bucket centres, the full time axis. Periods and *span* are read from all of it rather + than from one metric's usable subset. + metrics: metric name -> values aligned to *seconds*, ``NaN`` where that metric had no + observation in a bucket. Each metric is scored on its own non-NaN subset. - Returns the basis and the rejected periods with their reasons. + Returns: + The basis and the rejected periods with their reasons. """ seconds = np.asarray(seconds, dtype=float) span = float(np.max(seconds) - np.min(seconds)) if seconds.size > 1 else 1.0 periods, rejected = candidate_periods(seconds) base = TemporalBasis(trend=True, periods=periods, harmonics=SEASONAL_HARMONICS, span=max(span, 1.0)) - values = np.asarray(values, dtype=float) - best_basis, best_scale = base, _holdout_residual_scale(seconds, values, base) + scored: dict[str, tuple[np.ndarray, np.ndarray]] = {} + for name, raw in metrics.items(): + values = np.asarray(raw, dtype=float) + mask = ~np.isnan(values) + standardised = _standardise(values[mask]) + if standardised is not None: + scored[name] = (seconds[mask], standardised) + + # Each metric's own yardstick, under the simplest basis. A metric the simple basis already predicts + # perfectly (scale 0) or cannot be scored on at all (inf) says nothing about whether flexibility + # helps, so it is left out of the comparison rather than given an arbitrary ratio. + references = {} + for name, (metric_seconds, values) in scored.items(): + scale = _holdout_residual_scale(metric_seconds, values, base) + if np.isfinite(scale) and scale > 0: + references[name] = scale + if not references: + return base, rejected + + # Strictly better than 1.0, so a tie leaves the simpler basis in place. Flexibility has to earn its + # keep: in-sample fit was identical across 0 to 25 changepoints while extrapolation ranged 1.2% to + # 100%. + best_basis, best_ratio = base, 1.0 for count in CHANGEPOINT_CANDIDATES: if count == 0: continue @@ -246,11 +305,14 @@ def select_basis(seconds: np.ndarray, values: np.ndarray) -> tuple[TemporalBasis candidate = TemporalBasis( trend=True, periods=periods, harmonics=SEASONAL_HARMONICS, changepoints=changepoints, span=base.span ) - scale = _holdout_residual_scale(seconds, values, candidate) - # Strictly better, so a tie leaves the simpler basis in place. Flexibility has to earn its keep: - # in-sample fit was identical across 0 to 25 changepoints while extrapolation ranged 1.2% to 100%. - if scale < best_scale: - best_basis, best_scale = candidate, scale + # A candidate that cannot be scored on any one metric scores ``inf`` overall and is refused for + # all of them, which is the conservative reading of a single shared basis. + ratios = [ + _holdout_residual_scale(*scored[name], candidate) / reference for name, reference in references.items() + ] + mean_ratio = float(np.mean(ratios)) + if mean_ratio < best_ratio: + best_basis, best_ratio = candidate, mean_ratio return best_basis, rejected diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 728a6a652..bef256fa4 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -785,15 +785,22 @@ def _process_numeric_columns( numeric_cols: list[ColumnTypeInfo], engineered_features: list[str], ) -> DataFrame: - """Process numeric columns with null imputation.""" + """Register each numeric column as a feature, cast to double, nulls still null. + + Casting here and imputing in :func:`_impute_numeric_features` at the very end is deliberate, and + the two used to be one ``coalesce(col.cast(...), 0.0)`` on this line. That imputed before the + group and time baselines were fitted, so a missing observation contributed a fabricated zero to its + group's median and then received a fabricated deviation from it -- ``0 - expected(t)`` -- on top of + its null indicator. Both baselines now see the null and skip the row, which is what the null + indicator was always supposed to be the sole record of. + """ for col_info in numeric_cols: col_name = col_info.name # Add null indicator if needed transformed_df = _add_null_indicator(transformed_df, col_name, col_info.null_count, engineered_features) - # Impute nulls with 0 - transformed_df = transformed_df.withColumn(col_name, coalesce(col(col_name).cast(DoubleType()), lit(0.0))) + transformed_df = transformed_df.withColumn(col_name, col(col_name).cast(DoubleType())) engineered_features.append(col_name) return transformed_df @@ -893,9 +900,11 @@ def _process_baseline_relative_features( falls back to the global baseline, which makes it look ordinary rather than extreme — the conservative direction, and the case a caller can detect explicitly via ``is_new_baseline``. - Must run last. ``engineered_feature_names`` is positional: the sklearn pipeline is handed - columns in this order, so features may only ever be appended at the tail. Inserting a - transform before this one would silently reorder an already-trained model's inputs. + ``engineered_feature_names`` is positional: the sklearn pipeline is handed columns in this order, + so features may only ever be appended at the tail. Inserting a feature-producing transform before + this one would silently reorder an already-trained model's inputs. Two things do run afterwards, + and neither appends a feature: the temporal block, which reads the column produced here, and + numeric imputation, which must not run before the medians below are collected. """ if not baseline_by or not numeric_cols: return transformed_df @@ -1064,15 +1073,10 @@ def _fit_temporal_from_buckets( # buckets per metric rather than dropping the metric, so one sparse column does not cost the rest. usable = {name: ~np.isnan(values) for name, values in metrics.items()} - representative = next( - (metrics[name][usable[name]] for name in source_columns if usable[name].sum() > 2), - np.array([], dtype=float), - ) - representative_seconds = next( - (bucket_seconds[usable[name]] for name in source_columns if usable[name].sum() > 2), - np.array([], dtype=float), - ) - basis, rejected = select_basis(representative_seconds, representative) + # Every metric, not the first usable one. select_basis masks each on its own NaNs and reads the full + # bucket axis for the period search, so neither the periods nor the changepoint count depends on + # schema order any more. + basis, rejected = select_basis(bucket_seconds, metrics) coefficients: dict[str, list[float]] = {} for name, values in metrics.items(): @@ -1156,6 +1160,41 @@ def _process_temporal_baseline_features( return transformed_df +def _impute_numeric_features( + transformed_df: DataFrame, + numeric_cols: list[ColumnTypeInfo], + baseline_by: list[str], + baseline_over_time: str, +) -> DataFrame: + """Replace every remaining numeric null with a neutral zero. Must run last. + + Last because both comparison bases are fitted from the raw metric, and imputing first would let a + missing observation move the very baseline it is then measured against. By the time this runs the + medians are collected and the temporal coefficients are fitted, so filling in is free of + consequence: the estimators need a dense matrix, and the ``_is_null`` indicator already carries the + fact that the value was absent. + + Zero is neutral for the derived features by construction -- it is exactly "no deviation from the + baseline" -- which the raw metric cannot claim, but the raw metric has the indicator beside it. + + Appends nothing, so the positional ``engineered_feature_names`` contract is untouched. + """ + for col_info in numeric_cols: + metric = col_info.name + # Both derived columns exist for every metric whenever their basis is set: each block above + # appends one per metric unconditionally, a constant where it learned nothing. Named without a + # guard so that a future early-exit there fails here loudly rather than skipping an imputation. + names = [metric] + if baseline_by: + names.append(f"{metric}{BASELINE_RELATIVE_SUFFIX}") + if baseline_over_time: + names.append(f"{metric}{TEMPORAL_RELATIVE_SUFFIX}") + for name in names: + transformed_df = transformed_df.withColumn(name, coalesce(col(name), lit(0.0))) + + return transformed_df + + def _log_temporal_fit( time_column: str, basis: "TemporalBasis", @@ -1291,6 +1330,9 @@ def apply_feature_engineering( engineered_features, ) + # Strictly after both bases above, which are fitted from the un-imputed metric on purpose. + transformed_df = _impute_numeric_features(transformed_df, numeric_cols, baseline_by, baseline_over_time) + return _project_and_describe( transformed_df, column_infos, diff --git a/tests/integration_anomaly/test_anomaly_group_relative_features.py b/tests/integration_anomaly/test_anomaly_group_relative_features.py index e4f0d46b6..18f282c28 100644 --- a/tests/integration_anomaly/test_anomaly_group_relative_features.py +++ b/tests/integration_anomaly/test_anomaly_group_relative_features.py @@ -310,3 +310,88 @@ def test_unscored_row_keeps_its_info_when_merged_back(spark: SparkSession): info_by_id = {row["row_id"]: row["info"] for row in merged.collect()} assert info_by_id[1] == "unseen-group" # kept despite a null score assert info_by_id[2] == "scored" + + +# ============================================================================ +# A missing observation must not move the baseline it is measured against +# ============================================================================ + + +def test_a_null_metric_does_not_contribute_to_its_group_baseline(spark: SparkSession): + """The baseline must be the median of the values that exist, not of values plus fabricated zeros. + + Numeric imputation used to run *before* the medians were collected, replacing every null with 0.0 + in place. A group of three 100s and four nulls therefore had a median of 0 rather than 100, and + every real row in it was then reported as a large positive deviation from a baseline no row held. + + The counts here are chosen so the two answers cannot be confused: over ``[100, 100, 100]`` the + median is 100 and the relative feature is 0, while over ``[100, 100, 100, 0, 0, 0, 0]`` it is 0 and + the feature is ``log1p(100)``, about 4.6. + """ + rows = [("DE", 100.0)] * 3 + [("DE", None)] * 4 + df = spark.createDataFrame(rows, "country string, amount double") + + result, _ = apply_feature_engineering( + df, + [ColumnTypeInfo(name="amount", spark_type=T.DoubleType(), category="numeric", null_count=4)], + baseline_by=["country"], + ) + + present = _first(result.filter(F.col("amount") == 100.0)) + assert present["amount_rel_baseline"] == pytest.approx(0.0, abs=1e-9), ( + "a row sitting exactly on its group's median must show no deviation; " + f"got {present['amount_rel_baseline']}, and log1p(100) = {math.log1p(100):.3f} would mean the " + "baseline was computed over imputed zeros" + ) + + +def test_a_null_metric_gets_a_neutral_deviation_and_keeps_its_indicator(spark: SparkSession): + """Missingness is the null indicator's job, and only the indicator's. + + The derived feature is zero -- "no deviation" -- rather than a deviation measured from a value the + row never had. Both facts are asserted together because the neutral zero is only honest while the + indicator is there to distinguish it from a row genuinely at its baseline. + """ + rows = [("DE", 100.0)] * 3 + [("DE", None)] * 4 + df = spark.createDataFrame(rows, "country string, amount double") + + result, metadata = apply_feature_engineering( + df, + [ColumnTypeInfo(name="amount", spark_type=T.DoubleType(), category="numeric", null_count=4)], + baseline_by=["country"], + ) + + missing = _first(result.filter(F.col("amount_is_null") == 1.0)) + assert missing["amount"] == 0.0 # imputed, so the estimator sees a dense matrix + assert missing["amount_rel_baseline"] == pytest.approx(0.0, abs=1e-9) + assert "amount_is_null" in metadata.engineered_feature_names + + +def test_imputation_appends_no_feature(spark: SparkSession): + """Moving imputation to the end must not disturb the positional feature list. + + Compares against a frame with no nulls at all: same columns, same order, whatever the imputation + step had to do. + """ + with_nulls = spark.createDataFrame( + [("DE", 100.0), ("DE", None), ("IT", 20.0), ("IT", 22.0)], "country string, amount double" + ) + without_nulls = spark.createDataFrame( + [("DE", 100.0), ("DE", 105.0), ("IT", 20.0), ("IT", 22.0)], "country string, amount double" + ) + + _, nulls_metadata = apply_feature_engineering( + with_nulls, + [ColumnTypeInfo(name="amount", spark_type=T.DoubleType(), category="numeric", null_count=1)], + baseline_by=["country"], + ) + _, clean_metadata = apply_feature_engineering( + without_nulls, + [ColumnTypeInfo(name="amount", spark_type=T.DoubleType(), category="numeric", null_count=0)], + baseline_by=["country"], + ) + + # The indicator is the one legitimate difference; everything else must match position for position. + assert [n for n in nulls_metadata.engineered_feature_names if n != "amount_is_null"] == ( + clean_metadata.engineered_feature_names + ) diff --git a/tests/integration_anomaly/test_anomaly_temporal_features.py b/tests/integration_anomaly/test_anomaly_temporal_features.py index c71e7c281..4bc1bee88 100644 --- a/tests/integration_anomaly/test_anomaly_temporal_features.py +++ b/tests/integration_anomaly/test_anomaly_temporal_features.py @@ -543,3 +543,42 @@ def test_a_null_timestamp_in_training_does_not_fail_the_fit(spark: SparkSession) # The basis was fitted from the rows that do carry a timestamp. assert metadata.temporal_coefficients assert engineered.count() == len(rows) + + +def test_a_null_metric_does_not_drag_its_expected_level_toward_zero(spark: SparkSession): + """The temporal half of the same imputation defect the group baselines had. + + Numeric imputation used to replace every null with 0.0 *before* the buckets were aggregated, so a + bucket that happened to hold a null had its median pulled toward zero and the fitted expectation + followed it down. The fit now sees the null and skips it, exactly as ``percentile_approx`` does for + every other aggregate. + + Measured as the residual on the rows that *do* have a value: if the expectation had been dragged + down by a third of the series being fabricated zeros, those rows would all sit far above it and the + residual would carry a large positive offset rather than centring on zero. + """ + rng = np.random.default_rng(23) + rows = [] + for i in range(24 * 30): + value = None if i % 3 == 0 else float(100.0 + 0.05 * i + rng.normal(0, 2.0)) + rows.append((START + datetime.timedelta(hours=i), value)) + df = spark.createDataFrame(rows, "event_ts timestamp, revenue double") + + engineered, _ = apply_feature_engineering( + df, + [ColumnTypeInfo(name="revenue", spark_type=T.DoubleType(), category="numeric", null_count=24 * 10)], + baseline_over_time="event_ts", + ) + + present = engineered.filter(F.col("revenue") != 0.0) + stats = present.selectExpr( + f"avg(`revenue{TEMPORAL_RELATIVE_SUFFIX}`) as mean_residual", + f"stddev(`revenue{TEMPORAL_RELATIVE_SUFFIX}`) as residual_sd", + ).first() + assert stats is not None + # The trend reaches 0.05 * 720 = 36 units, so an expectation fitted through fabricated zeros would + # leave a mean residual of that order. Centred means the nulls were skipped, not counted as zero. + assert abs(stats["mean_residual"]) < 3.0 * stats["residual_sd"], ( + f"residual should centre on zero for rows that have a value; got mean " + f"{stats['mean_residual']:.2f} against sd {stats['residual_sd']:.2f}" + ) diff --git a/tests/unit/test_anomaly_temporal_fit.py b/tests/unit/test_anomaly_temporal_fit.py index 161ecd124..46077a2f2 100644 --- a/tests/unit/test_anomaly_temporal_fit.py +++ b/tests/unit/test_anomaly_temporal_fit.py @@ -174,7 +174,7 @@ def test_a_straight_series_earns_no_changepoints(): rng = np.random.default_rng(11) seconds = _hourly_axis(2000) - basis, _ = select_basis(seconds, _linear_metric(seconds, rng)) + basis, _ = select_basis(seconds, {"metric": _linear_metric(seconds, rng)}) assert not basis.changepoints @@ -188,7 +188,7 @@ def test_a_series_whose_slope_doubles_earns_changepoints(): values = 100.0 + TRUE_SLOPE_PER_SECOND * np.minimum(seconds, knee) values = values + 3.0 * TRUE_SLOPE_PER_SECOND * np.maximum(0.0, seconds - knee) + rng.normal(0, 3.0, seconds.size) - basis, _ = select_basis(seconds, values) + basis, _ = select_basis(seconds, {"metric": values}) assert basis.changepoints @@ -205,7 +205,7 @@ def test_changepoints_are_never_placed_in_the_recent_tail(): values = 100.0 + TRUE_SLOPE_PER_SECOND * np.minimum(seconds, knee) values = values + 4.0 * TRUE_SLOPE_PER_SECOND * np.maximum(0.0, seconds - knee) + rng.normal(0, 2.0, seconds.size) - basis, _ = select_basis(seconds, values) + basis, _ = select_basis(seconds, {"metric": values}) assert all(changepoint <= 0.8 for changepoint in basis.changepoints) @@ -317,7 +317,80 @@ def test_select_basis_reports_what_it_skipped(caplog): seconds = _hourly_axis(48) # two days: nothing is admissible with caplog.at_level(logging.DEBUG): - basis, rejected = select_basis(seconds, _linear_metric(seconds, rng)) + basis, rejected = select_basis(seconds, {"metric": _linear_metric(seconds, rng)}) assert not basis.periods assert set(rejected) == set(CANDIDATE_PERIODS_SECONDS) + + +# ── the basis is a property of the table, not of whichever column came first ──────────────────────── + + +def _bent_metric(seconds: np.ndarray, rng: np.random.Generator, bend: float = 3.0) -> np.ndarray: + """A metric whose slope changes partway through: the shape changepoints exist for.""" + knee = seconds[seconds.size // 2] + values = 100.0 + TRUE_SLOPE_PER_SECOND * np.minimum(seconds, knee) + return values + bend * TRUE_SLOPE_PER_SECOND * np.maximum(0.0, seconds - knee) + rng.normal(0, 3.0, seconds.size) + + +def test_the_basis_does_not_depend_on_metric_order(): + """One basis is shared by every metric, so column order must not decide it. + + The defect this pins: the basis was scored on the *first* usable metric alone, and the period search + read that metric's time axis too, so reordering a schema changed every fitted residual for every + column. A straight metric first chose one changepoint count where a bent metric chose another. + """ + rng = np.random.default_rng(11) + seconds = _hourly_axis(2000) + straight = _linear_metric(seconds, rng) + bent = _bent_metric(seconds, np.random.default_rng(12)) + + straight_first, _ = select_basis(seconds, {"straight": straight, "bent": bent}) + bent_first, _ = select_basis(seconds, {"bent": bent, "straight": straight}) + + assert straight_first == bent_first + + +def test_a_metric_with_gaps_is_scored_on_the_buckets_it_has(): + """Each metric carries NaN where it had no observation, and is masked on its own gaps. + + A shared axis with per-metric masking is what lets one sparse column take part at all: before, a + sparse metric either decided the basis alone or was ignored entirely, depending on its position. + """ + rng = np.random.default_rng(5) + seconds = _hourly_axis(2000) + dense = _linear_metric(seconds, rng) + sparse = _linear_metric(seconds, np.random.default_rng(6)) + sparse[::3] = np.nan # a third of its buckets never saw a value + + basis, _ = select_basis(seconds, {"dense": dense, "sparse": sparse}) + + assert not basis.changepoints # both metrics are straight, so flexibility is still unearned + + +def test_a_metric_that_needs_flexibility_can_win_it_for_the_table(): + """The aggregate must still be able to select changepoints, or the invariance would be worthless.""" + seconds = _hourly_axis(2000) + first = _bent_metric(seconds, np.random.default_rng(21), bend=4.0) + second = _bent_metric(seconds, np.random.default_rng(22), bend=4.0) + + basis, _ = select_basis(seconds, {"a": first, "b": second}) + + assert basis.changepoints + + +def test_a_metric_measured_in_large_units_does_not_outvote_the_others(): + """Why the aggregate is a ratio rather than a sum of residual scales. + + Residual scale carries the metric's own units, so a column measured in millions would dominate any + raw sum and choose the basis for every other column. Here one straight metric is rescaled by 1e6: + the selected basis must not move, because a per-metric ratio is unitless. + """ + seconds = _hourly_axis(2000) + bent = _bent_metric(seconds, np.random.default_rng(31), bend=4.0) + straight = _linear_metric(seconds, np.random.default_rng(32)) + + modest, _ = select_basis(seconds, {"bent": bent, "straight": straight}) + inflated, _ = select_basis(seconds, {"bent": bent, "straight": straight * 1e6}) + + assert modest == inflated From df04f61478953ad79de4e894ddc7ca1d84d3d749 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 17:04:33 +0100 Subject: [PATCH 081/107] Make an unseen categorical value visible, and count derived features in the width warning One-hot encoding dropped one category of a binary pair. That is the textbook way to avoid the dummy-variable trap, but here it cost a blind spot rather than collinearity: at scoring, the omitted reference category and any value never seen in training both encode as all-zeros, so a brand new value in a binary column produced no signal at all. Every category is now retained, so a known value sets exactly one indicator and anything unseen sets none. The collinearity is absorbed by the detector's ridge, and it is only exact when the column has no nulls. Categories are also sorted now. `distinct().collect()` has no ordering, so the same table trained twice produced the same features in different positions -- and for a binary column, a different surviving category each time. Harmless inside one model, since the list is persisted with it, but it made a model irreproducible from its own inputs under a positional contract. The feature-width warning counted one feature per numeric column and was never told about the comparison bases, each of which adds one more. Twenty numeric columns with both bases build sixty features; the estimate said twenty and stayed silent under the recommended maximum of fifty, on exactly the configuration it exists to flag. The bases are now threaded through validate_columns into the estimate, and the breakdown names them on their own lines, because dropping a basis is a different decision from dropping a metric. Two smaller corrections found while counting. One-hot contributed `cardinality + 1` "for MISSING", but no MISSING indicator is ever built -- the distinct values are collected from the pre-imputation frame with nulls filtered out -- so the estimate was one too high per one-hot column; a null is carried by the `_is_null` indicator instead. And the datetime count is now `len(CALENDAR_FEATURE_ SUFFIXES)` rather than a literal 7, so it cannot drift from the transform. My own test for the width warning failed first time for the right reason: it filtered on "recommended max", which the max_input_columns warning also contains, and twenty columns trips that one whatever the bases are. The filter now matches the width warning's own opening words. Verified on fieldeng: the one-hot, determinism and width tests pass on real Spark, along with the rest of the transformers and column-naming suites. Co-authored-by: Isaac --- .../labs/dqx/anomaly/training_service.py | 9 +- .../labs/dqx/anomaly/transformers.py | 120 +++++++++++++----- src/databricks/labs/dqx/anomaly/validation.py | 18 ++- .../test_anomaly_transformers.py | 97 ++++++++++++++ tests/unit/test_anomaly_feature_width.py | 31 +++++ 5 files changed, 236 insertions(+), 39 deletions(-) create mode 100644 tests/unit/test_anomaly_feature_width.py diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index e96f21a14..a3da4556c 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -356,7 +356,13 @@ def build_context( if not columns: raise InvalidParameterError("No columns provided or auto-discovered. Provide columns explicitly.") - validation_warnings = validate_columns(df, columns, params) + # Resolved before validate_columns so the feature-width warning can count the derived features + # each basis adds. Both are validated in their own right further down. + resolved_over_time = baseline_over_time if baseline_over_time is not None else params.baseline_over_time + + validation_warnings = validate_columns( + df, columns, params, baseline_by=baseline_by, baseline_over_time=resolved_over_time + ) for warning in validation_warnings: logger.warning(warning) @@ -365,7 +371,6 @@ def build_context( self._reject_unbounded_grouping(df_filtered, baseline_by) logger.info(f"Judging each metric against its own group's baseline, grouped by {baseline_by}") - resolved_over_time = baseline_over_time if baseline_over_time is not None else params.baseline_over_time validate_baseline_over_time(df, resolved_over_time, columns) # After both bases are resolved, because which derived features exist depends on them. validate_generated_feature_names(df, columns, baseline_by, resolved_over_time) diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index bef256fa4..ca563095f 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -153,6 +153,11 @@ def _spark_type_for_category(category: str) -> T.DataType: }.get(category, T.StringType()) +def features_per_numeric_column(baseline_by: list[str] | None, baseline_over_time: str | None) -> int: + """How many features one numeric column becomes: the metric itself plus one per comparison basis.""" + return 1 + (1 if baseline_by else 0) + (1 if baseline_over_time else 0) + + def feature_category_for_type(col_type: T.DataType) -> str: """Which family of transforms a column's Spark type puts it in. @@ -219,10 +224,25 @@ def __init__( self.max_input_columns = max_input_columns self.max_engineered_features = max_engineered_features - def analyze_columns(self, df: DataFrame, columns: list[str]) -> tuple[list[ColumnTypeInfo], list[str]]: + def analyze_columns( + self, + df: DataFrame, + columns: list[str], + *, + baseline_by: list[str] | None = None, + baseline_over_time: str | None = None, + ) -> tuple[list[ColumnTypeInfo], list[str]]: """ Analyze columns and return type information and warnings. + Args: + df: Frame the columns live on. + columns: Feature columns to analyse. + baseline_by: Resolved group columns, if any. Needed only so the feature-width estimate can + account for the derived features they produce; the widths are what the estimate exists + to warn about, and it silently ignored them before. + baseline_over_time: Resolved time column, if any. Same reason. + Returns: Tuple of (column_type_infos, warnings) """ @@ -290,9 +310,13 @@ def analyze_columns(self, df: DataFrame, columns: list[str]) -> tuple[list[Colum warnings_list.extend(id_warnings) # Warn if estimated feature count is high (soft limit) - estimated_features = self._estimate_feature_count(column_infos) + estimated_features = self._estimate_feature_count( + column_infos, baseline_by=baseline_by, baseline_over_time=baseline_over_time + ) if estimated_features > self.max_engineered_features: - breakdown = self._get_feature_breakdown(column_infos) + breakdown = self._get_feature_breakdown( + column_infos, baseline_by=baseline_by, baseline_over_time=baseline_over_time + ) warnings_list.append( f"Feature engineering will create {estimated_features} features (recommended max: {self.max_engineered_features}). " f"This may increase training/scoring time. Feature breakdown:\n{breakdown}" @@ -348,21 +372,35 @@ def _classify_column( return ColumnTypeInfo(name=col_name, spark_type=col_type, category='unsupported', null_count=null_count) - def _estimate_feature_count(self, column_infos: list[ColumnTypeInfo]) -> int: - """Estimate total engineered features.""" + def _estimate_feature_count( + self, + column_infos: list[ColumnTypeInfo], + *, + baseline_by: list[str] | None = None, + baseline_over_time: str | None = None, + ) -> int: + """Estimate total engineered features. + + Counts the derived comparison features, which it did not before: with both bases set, twenty + numeric columns build sixty features while the estimate said twenty and stayed silent under the + default recommended maximum of fifty. The warning existed to catch exactly that width. + """ total = 0 null_indicators = 0 + features_per_numeric = features_per_numeric_column(baseline_by, baseline_over_time) + for info in column_infos: - if info.category in {"numeric", "boolean"}: + if info.category == 'numeric': + total += features_per_numeric + elif info.category == 'boolean': total += 1 elif info.category == 'datetime': - total += 7 # hour_sin, hour_cos, dow_sin, dow_cos, month_sin, month_cos, is_weekend + total += len(CALENDAR_FEATURE_SUFFIXES) elif info.category == 'categorical': - if info.encoding_strategy == 'onehot': - total += (info.cardinality or 0) + 1 # +1 for MISSING category - else: # frequency - total += 1 + # One indicator per distinct non-null value, or one frequency column. Approximate for + # one-hot: cardinality comes from approx_count_distinct. + total += (info.cardinality or 0) if info.encoding_strategy == 'onehot' else 1 # Add null indicator if info.null_count and info.null_count > 0: @@ -370,7 +408,13 @@ def _estimate_feature_count(self, column_infos: list[ColumnTypeInfo]) -> int: return total + null_indicators - def _get_feature_breakdown(self, column_infos: list[ColumnTypeInfo]) -> str: + def _get_feature_breakdown( + self, + column_infos: list[ColumnTypeInfo], + *, + baseline_by: list[str] | None = None, + baseline_over_time: str | None = None, + ) -> str: """Generate feature count breakdown for error message.""" counts = {'datetime': 0, 'categorical': 0, 'numeric': 0, 'boolean': 0, 'nulls': 0} cat_features = 0 @@ -380,28 +424,28 @@ def _get_feature_breakdown(self, column_infos: list[ColumnTypeInfo]) -> str: counts[info.category] = counts.get(info.category, 0) + 1 if info.category == 'categorical': - if info.encoding_strategy == 'onehot': - cat_features += (info.cardinality or 0) + 1 - else: - cat_features += 1 + cat_features += (info.cardinality or 0) if info.encoding_strategy == 'onehot' else 1 if info.null_count and info.null_count > 0: counts['nulls'] += 1 - # Build breakdown lines - breakdown = [] - if counts['datetime'] > 0: - breakdown.append(f" - {counts['datetime']} datetime → {counts['datetime'] * 7} features") - if counts['categorical'] > 0: - breakdown.append(f" - {counts['categorical']} categorical → {cat_features} features") - if counts['numeric'] > 0: - breakdown.append(f" - {counts['numeric']} numeric → {counts['numeric']} features") - if counts['boolean'] > 0: - breakdown.append(f" - {counts['boolean']} boolean → {counts['boolean']} features") - if counts['nulls'] > 0: - breakdown.append(f" - {counts['nulls']} null indicators → {counts['nulls']} features") - - return "\n".join(breakdown) + # One row per contributing group, as (how many sources, what they are, how many features). The + # comparison bases are named separately from the numeric line rather than folded into it, because + # dropping a basis is a different decision from dropping a metric. + numeric = counts['numeric'] + groups = [ + (counts['datetime'], "datetime", counts['datetime'] * len(CALENDAR_FEATURE_SUFFIXES)), + (counts['categorical'], "categorical", cat_features), + (numeric, "numeric", numeric), + (numeric if baseline_by else 0, "baseline_by comparisons", numeric), + (numeric if baseline_over_time else 0, "baseline_over_time comparisons", numeric), + (counts['boolean'], "boolean", counts['boolean']), + (counts['nulls'], "null indicators", counts['nulls']), + ] + + return "\n".join( + f" - {sources} {label} → {features} features" for sources, label, features in groups if sources > 0 + ) def _capture_dataframe_explain(self, df: DataFrame) -> str: """Capture DataFrame.explain() output as a string. Thread-safe via module lock.""" @@ -600,10 +644,18 @@ def _apply_onehot_encoding( ) -> DataFrame: """Apply OneHot encoding to a categorical column.""" if is_training: - distinct_values = [row[0] for row in df.select(col_name).distinct().collect()] - distinct_values = [v for v in distinct_values if v is not None] - if len(distinct_values) == 2: - distinct_values = distinct_values[:1] + # Sorted, because ``distinct().collect()`` has no ordering: the same table trained twice + # otherwise produced different feature *names* in different positions, and + # engineered_feature_names is positional. + # + # Every category is retained. Dropping one of a binary pair is the textbook way to avoid the + # dummy-variable trap, but here it made an unexpected value invisible: the omitted reference + # category and any value never seen in training both encode as all-zeros, so a brand-new value + # in a binary column produced no signal at all. With both retained, a known value sets exactly + # one indicator and anything unseen sets none, which the detectors can tell apart. The + # collinearity that argues for dropping one is absorbed by the detector's ridge, and it is only + # exact collinearity when the column has no nulls. + distinct_values = sorted(row[0] for row in df.select(col_name).distinct().collect() if row[0] is not None) onehot_categories[col_name] = distinct_values else: distinct_values = onehot_categories.get(col_name, []) diff --git a/src/databricks/labs/dqx/anomaly/validation.py b/src/databricks/labs/dqx/anomaly/validation.py index c003d7ccf..e213eb288 100644 --- a/src/databricks/labs/dqx/anomaly/validation.py +++ b/src/databricks/labs/dqx/anomaly/validation.py @@ -49,9 +49,19 @@ def validate_fully_qualified_name(value: str, *, label: str) -> None: def validate_columns( - df: DataFrame, columns: collections.abc.Iterable[str], params: AnomalyParams | None = None + df: DataFrame, + columns: collections.abc.Iterable[str], + params: AnomalyParams | None = None, + *, + baseline_by: list[str] | None = None, + baseline_over_time: str | None = None, ) -> list[str]: - """Validate columns for row anomaly detection with multi-type support.""" + """Validate columns for row anomaly detection with multi-type support. + + *baseline_by* and *baseline_over_time* are passed for the feature-width warning alone: each adds one + derived feature per numeric column, so omitting them understated the width by up to a factor of three + and the warning stayed silent on exactly the configurations it exists to flag. + """ params = params or AnomalyParams() fe_config = params.feature_engineering @@ -61,7 +71,9 @@ def validate_columns( max_engineered_features=fe_config.max_engineered_features, ) - _column_infos, warnings_list = classifier.analyze_columns(df, list(columns)) + _column_infos, warnings_list = classifier.analyze_columns( + df, list(columns), baseline_by=baseline_by, baseline_over_time=baseline_over_time + ) return warnings_list diff --git a/tests/integration_anomaly/test_anomaly_transformers.py b/tests/integration_anomaly/test_anomaly_transformers.py index 58b503afd..33818fb4e 100644 --- a/tests/integration_anomaly/test_anomaly_transformers.py +++ b/tests/integration_anomaly/test_anomaly_transformers.py @@ -12,6 +12,7 @@ ColumnTypeClassifier, SparkFeatureMetadata, apply_feature_engineering, + apply_feature_engineering_from_metadata, reconstruct_column_infos, ) from databricks.labs.dqx.errors import InvalidParameterError @@ -528,3 +529,99 @@ def test_mixed_column_types_in_single_dataframe(spark): # Extra columns preserved assert "id" in engineered_cols + + +# ============================================================================ +# One-hot encoding: an unseen value must be distinguishable, and names must be stable +# ============================================================================ + + +def test_a_binary_column_keeps_both_categories(spark): + """Dropping one of a binary pair made an unexpected value invisible. + + The textbook reason to drop one is the dummy-variable trap, but the cost here was a blind spot: the + omitted reference category and any value never seen in training both encode as all-zeros, so a brand + new value in a binary column produced no signal whatsoever. With both retained, every known value + sets exactly one indicator. + """ + df = spark.createDataFrame([("open",), ("open",), ("closed",), ("closed",)], "status string") + + _, metadata = apply_feature_engineering( + df, [ColumnTypeInfo(name="status", spark_type=T.StringType(), category="categorical", cardinality=2)] + ) + + assert metadata.onehot_categories["status"] == ["closed", "open"] + + +def test_an_unseen_value_encodes_differently_from_every_trained_category(spark): + """The property retaining the category buys, asserted on scoring rather than on the category list. + + The source column is dropped by projection, so an id column rides along to identify the rows: it is + neither a feature nor a comparison basis, so it survives as an extra column. + """ + train_df = spark.createDataFrame([("open",), ("open",), ("closed",), ("closed",)], "status string") + infos = [ColumnTypeInfo(name="status", spark_type=T.StringType(), category="categorical", cardinality=2)] + + _, metadata = apply_feature_engineering(train_df, infos) + + score_df = spark.createDataFrame( + [(1, "open"), (2, "closed"), (3, "escalated")], + "rid int, status string", + ) + scored, _ = apply_feature_engineering_from_metadata(score_df, metadata) + + indicators = [name for name in metadata.engineered_feature_names if name.startswith("status_")] + encodings = { + row["rid"]: tuple(row[name] for name in indicators) for row in scored.select("rid", *indicators).collect() + } + + assert all(value == 0.0 for value in encodings[3]), f"an unseen value must set no indicator: {encodings[3]}" + assert sum(encodings[1]) == 1.0, f"a trained value must set exactly one: {encodings[1]}" + assert sum(encodings[2]) == 1.0, f"a trained value must set exactly one: {encodings[2]}" + assert encodings[1] != encodings[2], "the two trained values must be told apart" + + +def test_the_same_table_trained_twice_produces_the_same_feature_names(spark): + """``distinct().collect()`` has no ordering, and engineered_feature_names is positional. + + Without sorting, two training runs over identical data produced the same features in different + positions, and for a binary column a different category could survive each time. Harmless inside one + model, since the list is persisted with it, but it made a model irreproducible from its own inputs. + """ + df = spark.createDataFrame( + [(value,) for value in ("delta", "alpha", "charlie", "bravo", "alpha", "delta")], + "grade string", + ) + infos = [ColumnTypeInfo(name="grade", spark_type=T.StringType(), category="categorical", cardinality=4)] + + _, first = apply_feature_engineering(df, infos) + _, second = apply_feature_engineering(df, infos) + + assert first.engineered_feature_names == second.engineered_feature_names + assert first.onehot_categories["grade"] == ["alpha", "bravo", "charlie", "delta"] + + +def test_the_width_warning_counts_the_derived_comparison_features(spark): + """Twenty numeric columns with both bases build sixty features, not twenty. + + The estimate counted one per numeric column and was never told about the comparison bases, so it + reported twenty and stayed silent under the default recommended maximum of fifty -- on exactly the + configuration it exists to flag. + """ + metrics = [f"m{i}" for i in range(20)] + schema = ", ".join([*(f"{m} double" for m in metrics), "region string", "event_ts timestamp"]) + df = spark.createDataFrame([tuple([1.0] * 20 + ["eu", datetime(2025, 1, 6)])], schema) + + classifier = ColumnTypeClassifier() + _, warned = classifier.analyze_columns(df, metrics, baseline_by=["region"], baseline_over_time="event_ts") + _, quiet = classifier.analyze_columns(df, metrics) + + # Matched on the width warning's own opening words. "recommended max" alone also appears in the + # max_input_columns warning, which twenty columns trips either way. + def width_warnings(warnings_list: list[str]) -> list[str]: + return [w for w in warnings_list if w.startswith("Feature engineering will create")] + + assert width_warnings(warned), f"expected a feature-width warning, got {warned}" + assert "60 features" in width_warnings(warned)[0], width_warnings(warned)[0] + assert "20 baseline_by comparisons" in width_warnings(warned)[0], "the breakdown must name the bases" + assert not width_warnings(quiet), "twenty metrics without a basis stay under the limit of fifty" diff --git a/tests/unit/test_anomaly_feature_width.py b/tests/unit/test_anomaly_feature_width.py new file mode 100644 index 000000000..c947bd91e --- /dev/null +++ b/tests/unit/test_anomaly_feature_width.py @@ -0,0 +1,31 @@ +"""Unit pins for the feature-width arithmetic (no Spark). + +The width warning is the only thing standing between a caller and a model three times wider than they +asked for, and it counted one feature per numeric column while both comparison bases were quietly adding +one more each. Twenty numeric columns with both bases build sixty features; the estimate said twenty and +stayed silent under the default recommended maximum of fifty. +""" + +import pytest + +from databricks.labs.dqx.anomaly.transformers import features_per_numeric_column + + +@pytest.mark.parametrize( + "baseline_by, baseline_over_time, expected", + [ + (None, None, 1), + (["region"], None, 2), + (None, "event_ts", 2), + (["region"], "event_ts", 3), + ([], "", 1), # empty rather than None: the shapes a resolved-but-unset basis actually arrives as + ], +) +def test_each_comparison_basis_adds_one_feature_per_metric(baseline_by, baseline_over_time, expected): + assert features_per_numeric_column(baseline_by, baseline_over_time) == expected + + +def test_twenty_metrics_with_both_bases_reach_sixty_features(): + """The review's own example, as arithmetic. Sixty is over the default limit of fifty; twenty is not, + which is why the warning never fired.""" + assert 20 * features_per_numeric_column(["region"], "event_ts") == 60 From 92d8ef7465fa76377855445b0e6163fb894e5314 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 17:11:36 +0100 Subject: [PATCH 082/107] Say what expected_anomaly_rate, _rel_baseline and the severity tail actually do Three claims in the documentation that the code does not support. No behaviour changes here. `expected_anomaly_rate` was described as controlling how many rows the model flags. It does not. It supplies `contamination`, which for both detectors places only `offset_` and therefore only `predict` / `decision_function`; every DQX scoring path reads `-score_samples` and ranks it against the training score quantiles, so what gets flagged is decided by the check's `threshold`. It also does not mitigate anomalies present in the training sample, which the engine docstring implied by calling it a defence against contaminated training data. Two unit tests now pin both halves: scores identical across contamination 0.01 and 0.20, `offset_` and `predict` different. The parameter stays, because it is real for anyone who loads the registered model and calls `predict` -- but the docs no longer sell it as a detection knob. `_rel_baseline` was documented as a log ratio that is "stable when a baseline is near zero and symmetric for halving versus doubling", and my own review brief called it unitless and therefore invariant. It is a signed-log1p *difference*, and only the first half is true. Doubling from 1 gives log3 - log2 = 0.405, the same ratio at 100x scale gives 0.688, and halving gives -0.288 rather than -0.405. So deviations are comparable within a group but not across groups of very different magnitude. What the form does buy is stated instead: finite and monotone where a true log ratio is undefined. A median/MAD standardisation would be genuinely dimensionless and is noted as the alternative, with the cost that argues for measuring it first -- a per-group MAD persisted beside every median. The severity tail above roughly p99 assumes an exponentially decaying score distribution. That held on four synthetic distributions and on real telemetry, but a heavier tail overshoots: on lognormal scores `threshold=99.9` flagged 0.21% of rows against 0.10% requested. The guide now says to read 99.5 and above as "much stricter than 99" rather than as a budget. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 6 +++ docs/dqx/docs/reference/quality_checks.mdx | 6 +-- .../labs/dqx/anomaly/anomaly_engine.py | 13 ++++--- .../labs/dqx/anomaly/transformers.py | 18 +++++++-- .../unit/test_anomaly_mahalanobis_detector.py | 37 +++++++++++++++++++ 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 204bad954..a94cdbf01 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -377,6 +377,12 @@ rows rather than 2%, and `threshold=99.9` flagged nothing at all. That range is probability, which is what a percentile is. Severity at 90, 95 and 99 is unchanged, so the default configuration behaves exactly as before; 96 to 98 become slightly less strict, toward the budget you asked for, and 99.5 and above now fire on roughly what you asked for instead of on nothing. + +**Above roughly 99 the interpolation is directional rather than calibrated.** It assumes the score +distribution's tail decays exponentially, which held on four synthetic distributions and on real telemetry, +but a heavier tail overshoots: on a lognormal score distribution `threshold=99.9` flagged 0.21% of rows +against the 0.10% requested, a factor of 2.1. Treat 99.5 and above as "much stricter than 99" rather than as +a number you can budget against. ::: ## Group-aware anomaly detection diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index 9cd31b213..92962e796 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -3507,7 +3507,7 @@ Required arguments: `df`, `model_name`, `registry_table`. Optional parameters: | `baseline_over_time` | str | None | A timestamp or date column each metric is judged *along*, so a value is compared with what its own history says to expect at that point in time. Composes with `baseline_by`: with both set the expectation is fitted on the group-relative value, so one model still covers every group. Independent of `profile`. Never auto-discovered, and DQX warns rather than acting when the training window shows little structure over time. Not a forecaster, and it never reads the previous row. The named column must not also appear in `columns`. See [Comparing against time](/docs/guide/row_anomaly_detection#comparing-against-time). | | `profile` | str | `"tabular"` | Which detector to train. `"tabular"` is Isolation Forest — the behaviour that predates this option. `"timeseries"` selects a correlation-aware detector for multivariate metrics whose anomalies are broken *correlations* rather than extreme single values; it needs no timestamp column and trains a single model rather than an ensemble. There is no automatic option: DQX never changes the algorithm on your behalf, because the choice cannot be verified without labels. See [Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile). | | `exclude_columns` | list[str] | None | Columns to exclude from training (e.g., IDs, labels, ground truth) | -| `expected_anomaly_rate` | float | 0.02 | Expected fraction of anomalies in your data (0.02 = 2%). Sets model contamination parameter. | +| `expected_anomaly_rate` | float | 0.02 | Expected fraction of anomalies in your data (0.02 = 2%). Supplies the estimator's `contamination`, which places scikit-learn's own `predict`/`offset_` boundary. **It does not change which rows DQX flags**, because scoring ranks `score_samples` against the training score quantiles and the rows you see are decided by the check's `threshold`. It also does not mitigate anomalies present in the training sample. Relevant if you load the registered model and call `predict` yourself. | | `params` | AnomalyParams | None | Optional. Advanced tuning parameters. See sections below for details. | **Validation**: Training parameters are validated before model fitting. Invalid values (for example `sample_fraction <= 0`, `train_ratio > 1`, or `expected_anomaly_rate > 0.5`) fail fast with `InvalidParameterError`. @@ -3530,7 +3530,7 @@ Pass an `IsolationForestConfig` object to `params.algorithm_config` to tune the | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `contamination` | float | auto | Expected outlier proportion. Auto-set from `expected_anomaly_rate` if not specified. | +| `contamination` | float | auto | Expected outlier proportion, auto-set from `expected_anomaly_rate`. Affects only the estimator's own `predict`/`offset_` boundary, which DQX's scoring path does not use — tune `threshold` on the check instead. | | `num_trees` | int | 200 | Number of trees in the forest. More trees = better accuracy but slower training. | | `max_depth` | int | None | Maximum tree depth. Auto-calculated as log2(sample_size) if None. | | `subsampling_rate` | float | None | Per-tree subsampling rate. None uses sklearn defaults. | @@ -3577,7 +3577,7 @@ model_name = anomaly_engine.train( df=spark.table("catalog.schema.orders"), model_name="catalog.schema.orders_monitor", registry_table="catalog.schema.dqx_anomaly_models", - expected_anomaly_rate=0.05, # Expect 5% anomalies + expected_anomaly_rate=0.05, # Expect 5% anomalies (sets contamination; tune `threshold` to change what is flagged) params=params, ) ``` diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index d61974e1d..ad412ee20 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -128,11 +128,14 @@ def train( Useful with auto-discovery to filter out unwanted columns without specifying all desired columns manually. expected_anomaly_rate: Expected fraction of anomalies in your data (default: 0.02 = 2%). - Used as the default contamination parameter for the Isolation Forest - algorithm, which controls the proportion of training data that the model - treats as outliers when learning the decision boundary. A higher value - makes the model flag more rows as anomalous. - Common values: 0.01-0.02 (fraud), 0.03-0.05 (quality issues), 0.10 (exploration). + Supplies the default *contamination* for the estimator, which places + scikit-learn's own ``predict`` / ``offset_`` boundary. + **It does not change which rows DQX flags.** Scoring reads + ``score_samples`` and ranks it against the training score quantiles, + so the rows you see are decided by the *threshold* on the check, not + by this. Nor does it mitigate anomalies present in the training + sample: nothing downweights them. Set it if you load the registered + model yourself and call ``predict``; otherwise tune *threshold*. Overridden if params.algorithm_config.contamination is set explicitly. Important Notes: - Avoid ID columns (user_id, order_id, etc.) - use exclude_columns to filter them out. diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index ca563095f..3274be654 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -942,10 +942,20 @@ def _process_baseline_relative_features( ) -> DataFrame: """Append each numeric metric's deviation from its own group's baseline. - ``rel = signed_log(value) - signed_log(group_median(value))``. The log-ratio form is stable - when a baseline is near zero and symmetric for halving versus doubling; the raw metric is - kept alongside, so globally absurd values stay detectable even where they are ordinary for - their group. + ``rel = signed_log1p(value) - signed_log1p(group_median(value))``, and the raw metric is kept + alongside, so globally absurd values stay detectable even where they are ordinary for their group. + + What the ``log1p`` difference buys is behaviour at and below zero: it is finite and monotone for a + baseline of zero or a negative metric, where a true log ratio is undefined, and it compresses the + long right tail that a raw difference leaves. What it is **not** is scale-invariant or symmetric, and + an earlier version of this docstring claimed both. Value 2 against baseline 1 gives + ``log 3 - log 2 = 0.405``; the same ratio at 100x scale gives ``0.688``; halving gives ``-0.288`` + rather than ``-0.405``. So a group's deviations are comparable within that group but not across + groups of very different magnitude, and the feature is not dimensionless. + + A median/MAD-standardised deviation would be genuinely dimensionless. It is not done here because it + needs a per-group MAD persisted alongside every median, roughly doubling the baseline metadata a + model carries, a rule for a zero MAD, and its own measurement against this form. Tracked separately. Follows ``_apply_frequency_encoding``'s shape exactly: compute and persist while training, broadcast-join and coalesce the miss while scoring. A row whose group was never trained on diff --git a/tests/unit/test_anomaly_mahalanobis_detector.py b/tests/unit/test_anomaly_mahalanobis_detector.py index a20033fe0..9ff719f15 100644 --- a/tests/unit/test_anomaly_mahalanobis_detector.py +++ b/tests/unit/test_anomaly_mahalanobis_detector.py @@ -8,6 +8,7 @@ import numpy as np import pytest from sklearn.base import clone +from sklearn.ensemble import IsolationForest from sklearn.pipeline import Pipeline from databricks.labs.dqx.anomaly.timeseries_detector import MahalanobisDetector @@ -173,3 +174,39 @@ def test_small_samples_fall_back_to_shrinkage(caplog): assert "Ledoit-Wolf" in caplog.text assert np.isfinite(detector.score_samples(rng.normal(size=(5, 8)))).all() + + +# ── contamination reaches predict, and nothing DQX scores with ────────────────────────────────────── + + +def test_contamination_moves_the_predict_boundary_but_not_the_scores(): + """Pinned because the docstrings now promise exactly this, and a plausible "fix" would break it. + + ``expected_anomaly_rate`` flows into ``contamination``, which for both shipping detectors places + ``offset_`` and therefore only ``predict`` / ``decision_function``. Every DQX scoring path reads + ``-score_samples`` and ranks it against the training score quantiles, so the parameter cannot change + which rows are flagged -- the check's ``threshold`` does that. + + Asserting both halves matters. Dropping the parameter would break someone who loads the registered + model and calls ``predict``; treating it as a detection knob is what the documentation used to imply. + """ + rng = np.random.default_rng(7) + data = np.vstack([rng.normal(0, 1, (200, 3)), rng.normal(6, 1, (10, 3))]) + + timid = MahalanobisDetector(contamination=0.01).fit(data) + liberal = MahalanobisDetector(contamination=0.20).fit(data) + + np.testing.assert_allclose(timid.score_samples(data), liberal.score_samples(data)) + assert timid.offset_ != liberal.offset_ + assert (liberal.predict(data) == -1).sum() > (timid.predict(data) == -1).sum() + + +def test_isolation_forest_scores_are_also_independent_of_contamination(): + """The default profile, for the same reason. sklearn documents it; DQX's docs now rely on it.""" + rng = np.random.default_rng(8) + data = np.vstack([rng.normal(0, 1, (200, 3)), rng.normal(6, 1, (10, 3))]) + + timid = IsolationForest(contamination=0.01, random_state=0, n_estimators=50).fit(data) + liberal = IsolationForest(contamination=0.20, random_state=0, n_estimators=50).fit(data) + + np.testing.assert_allclose(timid.score_samples(data), liberal.score_samples(data)) From cae9256a6d3c4b0229908f07fae9974c7d3109b8 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 17:28:01 +0100 Subject: [PATCH 083/107] Publish one benchmark run, measured under the protocol DQX actually uses The headline SMD evidence combined three runs. The coverage table came from a script that fitted a single Isolation Forest on SMD's clean train split and wrote no JSON; the spread and win-loss record came from the ensemble panel, whose coverage is 57.5%/65.3% rather than 57.3%/67.6%; and the average precision pair matched no archived file at all -- 0.425 against an archived 0.4231, which does not round. The guide then said both detectors were "trained on data that still contained the anomalies", which was true of an even earlier protocol and false of the one actually run: the loader fits SMD's unlabelled train split. Replaced by one run measuring both shipping profiles under two protocols, archived as model_quality/results/smd-protocol.json: - dqx: sample 30% of the rows to be scored, fit 80% of that sample, score everything. What DQX does to a live table, contaminated training included. - clean: fit SMD's own train split, score its test split. The conventional protocol, and the one every archived number so far was really produced under. Under DQX's own protocol the second detector surfaces 96.8% of incidents inside a 1% alert budget against 65.4%, +31.4 points per entity with sd 29.5 and 22 wins to 1 loss. That inverts the published framing, which told readers to look at average precision because coverage was inside its spread: under this protocol average precision is the wash (+0.006, 14W/14L) and coverage is the consistent win. The clean protocol is worse for *both* detectors, because SMD's training period drifts from its test period and a model fitted on a sample of the rows it will score does not inherit that drift. That is not leakage, which was the obvious objection, so it is measured rather than argued: ranking metrics restricted to the rows the model never saw give ROC-AUC 0.826 against 0.845 and average precision 0.380 against 0.382, the same to three decimals. Event coverage is deliberately not reported there, because dropping a quarter of the rows fragments incidents into a different set of events. What this still is not is DQX's pipeline -- no group-relative, time-relative or calendar features, because those need Spark and a workspace. The guide now says so rather than implying otherwise. Benchmark figures come out of the docstrings entirely and are cited from the guide instead. Copying one number into five files is how they came to disagree, and two of those five were generated API pages that no one could have edited anyway. Also corrects the PCA rejection, which claimed truncation "lands on the correlation-aware detector's own number". The archive says otherwise: PCA at 95% variance before that detector reaches 74.5% coverage against its 65.3%, 15 wins to 5. The rejection stands on grounds that survive the number -- k is unchoosable, it collapses on the very anomaly class the detector exists for (recall 81.6% to 0.8%), and it destroys per-column attribution -- and now says that instead. Co-authored-by: Isaac --- demos/dqx_demo_anomaly_timeseries_fleet.py | 6 ++- .../guide/row_anomaly_detection/index.mdx | 51 ++++++++++++------- .../labs/dqx/anomaly/anomaly_engine.py | 14 ++--- .../labs/dqx/anomaly/timeseries_detector.py | 18 ++++--- 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py index ff0cb29c4..dd0498d57 100644 --- a/demos/dqx_demo_anomaly_timeseries_fleet.py +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -47,8 +47,10 @@ # MAGIC This notebook uses `"timeseries"`, because the bearing story above is exactly its case. The default # MAGIC detector splits on one column at a time, so a broken relationship between two in-range values is close # MAGIC to invisible to it. On the **Server Machine Dataset**, 28 machines of real telemetry with labelled -# MAGIC incidents, the correlation-aware detector surfaces **79%** of incidents inside an alert budget of 1% of -# MAGIC rows, against **33%** for the default. +# MAGIC incidents, trained the way DQX trains, the correlation-aware detector surfaces **96.8%** of incidents +# MAGIC inside an alert budget of 1% of rows against **65.4%** for the default, and it wins on 22 of the 28 +# MAGIC machines while losing on 1. See the guide for the full protocol and for the metrics where the two are +# MAGIC level. # MAGIC # MAGIC #### 2. Compared to its own group (`baseline_by`) # MAGIC diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index a94cdbf01..be970e32c 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -536,23 +536,40 @@ normal, no single-feature split separates the row, even though "high CPU with id on a healthy machine. Measured on the full Server Machine Dataset: real machine telemetry, 28 machines, 38 metrics, 327 labelled -incidents across 708,420 rows. This is the share of incidents each detector surfaces while the alert budget -is capped at 1% of rows: - -| `profile` | Incidents surfaced | -|---|---| -| `"tabular"` | 57.3% | -| `"timeseries"` | **67.6%** | - -An incident counts as surfaced if the detector flags at least one of its rows, so this measures whether -you would have been paged, not how many rows you would have had to read. Both detectors were trained on -data that still contained the anomalies, which is what DQX does when it fits a sample of your table. - -**Read this as an average, not a promise.** Per machine the gap has a standard deviation of 38 points, and -the correlation-aware detector wins on 16 of the 28 and loses on 7, so it is the better default for this -shape of data rather than a guarantee on any particular machine. The most consistent difference is average -precision, 0.425 against 0.278, because that is an aggregate over the whole ranking rather than a count of -incidents caught inside a budget. +incidents across 708,420 rows. Both detectors were trained the way DQX trains — a 30% sample of the table +being scored, 80% of that sample used to fit — so the training data contains the anomalies, as it does on +your table: + +| `profile` | Incidents surfaced, 1% budget | 5% budget | Precision at 1% | ROC-AUC | Average precision | +|---|---|---|---|---|---| +| `"tabular"` | 65.4% | 92.8% | 54.6% | 0.826 | 0.374 | +| `"timeseries"` | **96.8%** | **99.3%** | 55.4% | **0.845** | 0.380 | + +An incident counts as surfaced if the detector flags at least one of its rows, so this measures whether you +would have been paged, not how many rows you would have had to read. The best precision a 1% budget allows +on this data is 95.7%, so both detectors are well short of the ceiling; the difference between them is +*which* incidents they find, not how cleanly. + +**Coverage is the difference that holds up per machine.** The gap at a 1% budget averages +31 points with a +standard deviation of 30, and the correlation-aware detector wins on 22 of the 28 machines and loses on 1. +Average precision, by contrast, is a wash under this protocol: +0.006, and 14 wins to 14 losses. So choose +`"timeseries"` for telemetry because it surfaces more incidents, not because it ranks rows better. + + +Fitting SMD's own held-out training period instead — the conventional protocol, with no overlap between +training and scored rows — gives 57.5% against 65.3% at a 1% budget, with average precision 0.277 against +0.423. Both detectors do *worse* that way, because SMD's training period drifts from its test period, and a +model fitted on a sample of the rows it will score does not inherit that drift. + +That is a property of how DQX trains rather than of the benchmark, so the table above is the representative +one. It is not an artefact of scoring rows that were trained on: restricting the ranking metrics to the rows +the model never saw gives ROC-AUC 0.826 against 0.845 and average precision 0.380 against 0.382 — the same +numbers. Event coverage is not reported for that check, because dropping a quarter of the rows breaks +incidents into fragments and would count a different set of events. + +Both protocols come from one archived run. The comparison is between *estimators* on raw metric matrices: +DQX's own feature engineering, including `baseline_by` and `baseline_over_time`, is not part of it. + That is telemetry, which is what `"timeseries"` is for. On ordinary tabular data the two detectors are closer than this table might suggest, so choose on the *shape* of the anomaly you expect rather than on an diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index ad412ee20..f5c292ad6 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -92,13 +92,13 @@ def train( profile: What kind of data this is, which selects the detector. Defaults to ``"tabular"`` -- IsolationForest, exactly the behaviour before this option existed. ``"timeseries"`` selects a correlation-aware detector suited to multivariate metrics, - where anomalies are broken correlations rather than extreme single values; measured on - the SMD benchmark it surfaces 79% of incidents inside a 1%-of-rows alert budget against - 33% for the tabular detector, both trained on data that still contains anomalies as DQX - does (82% against 36% on a clean training split). It needs no timestamp column, and trains a single model - rather than an ensemble because it is deterministic. There is no automatic option: DQX - never changes the algorithm on your behalf, because the choice cannot be verified - without labels. The resolved profile is logged on every run. + where anomalies are broken correlations rather than extreme single values. It needs no + timestamp column, and trains a single model rather than an ensemble because it is + deterministic. There is no automatic option: DQX never changes the algorithm on your + behalf, because the choice cannot be verified without labels. The resolved profile is + logged on every run. Measured detection quality, with the protocol it was measured + under, is in the row anomaly detection guide -- deliberately not repeated here, because + a benchmark figure copied into a docstring is how five files came to disagree about it. baseline_by: Columns identifying the group a row belongs to, so a metric is judged against its own group's baseline rather than against the whole table. Each numeric metric gains its deviation from that baseline as an extra feature on diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py index fea0d88f7..ad000d30a 100644 --- a/src/databricks/labs/dqx/anomaly/timeseries_detector.py +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -2,14 +2,16 @@ IsolationForest splits on one randomly chosen feature at a time, which is why it is strong on tabular data and weak on multivariate metrics whose anomalies are *broken correlations* rather than extreme -single values. Measured on SMD (28 machines, 38 metrics, fit on the train split and scored on the test -split, no point adjustment) it catches 36% of incidents inside a 1%-of-rows alert budget; the -Mahalanobis detector here catches 82%. Refitting on training data that still contains anomalies -- what -DQX actually does -- costs both of them a few points and does not change the conclusion: 33% against -79%. That was the result that could have sunk the approach, because sample covariance is not robust and -a few extreme rows inflate it along the very direction that needs to stay tight. Detection quality on -DQX's own synthetic fixtures is published in the benchmarks report and measured by -``tests/perf/test_anomaly_benchmark.py``. +single values. On real machine telemetry this detector surfaces substantially more incidents inside a +fixed alert budget, and does so consistently across machines; the figures and the protocol they were +measured under live in the row anomaly detection guide rather than here, because a benchmark number +copied into a docstring is how five files in this repository came to disagree about one. + +The result that could have sunk the approach was training on data that still contains anomalies, which +is what DQX does when it fits a sample of a live table: sample covariance is not robust, and a few +extreme rows inflate it along the very direction that needs to stay tight. Measured, it does not -- +that protocol is the one the guide leads with. Detection quality on DQX's own synthetic fixtures is +published in the benchmarks report and measured by ``tests/perf/test_anomaly_benchmark.py``. The distance is the ordinary squared Mahalanobis distance from the training centre, ``d² = (x−μ)ᵀ Σ⁻¹ (x−μ)``, with three deliberate choices. From 25d2a795a43f5076d2a8864d3c736dce49c651d9 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 17:48:25 +0100 Subject: [PATCH 084/107] Derive the cardinality assertion from the fixture instead of hardcoding it Follow-up to scaling this fixture from 200 rows to 1,000 so `category` clears the effective rows-per-group floor. The row count moved; an assertion expecting "200 distinct values" in the high-cardinality warning did not, and `transaction_ref` is unique per row, so it now reports 1,000. Third time a hardcoded fixture number in this file has had to be chased after a rescale, so it reads off `row_count` now. Also pins the collinearity that Stage 4 introduced by retaining every one-hot category. Retained categories sum to 1 on every trained row, so the correlation-aware detector sees a zero-variance direction. Measured both halves rather than assuming: rows satisfying the sum score identically to the one-dummy encoding, so the redundant column is free, while an unseen category lies off that surface and scores far above a known one. That large score is deliberate and is not new behaviour -- columns with three or more categories were always retained in full, so an unseen value has always scored this way. Truncating the binary case was the inconsistency. Co-authored-by: Isaac --- .../labs/dqx/anomaly/transformers.py | 13 ++++-- .../test_anomaly_autodiscovery.py | 12 ++++-- .../unit/test_anomaly_mahalanobis_detector.py | 41 +++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 3274be654..e7684cbdd 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -652,9 +652,16 @@ def _apply_onehot_encoding( # dummy-variable trap, but here it made an unexpected value invisible: the omitted reference # category and any value never seen in training both encode as all-zeros, so a brand-new value # in a binary column produced no signal at all. With both retained, a known value sets exactly - # one indicator and anything unseen sets none, which the detectors can tell apart. The - # collinearity that argues for dropping one is absorbed by the detector's ridge, and it is only - # exact collinearity when the column has no nulls. + # one indicator and anything unseen sets none, which the detectors can tell apart. + # + # This makes binary consistent with every other cardinality rather than introducing a new + # behaviour: three or more categories were always retained in full, so an unseen value has + # always encoded as all-zeros and scored as a large deviation. Retained categories sum to 1 on + # every trained row, so the correlation-aware detector sees a zero-variance direction, and a row + # violating that sum lies off the surface all its training data lay on. Measured: rows that do + # satisfy it score identically to the one-dummy encoding, so the redundant column is free, while + # an unseen category scores far above a known one. Both halves are pinned in + # tests/unit/test_anomaly_mahalanobis_detector.py. distinct_values = sorted(row[0] for row in df.select(col_name).distinct().collect() if row[0] is not None) onehot_categories[col_name] = distinct_values else: diff --git a/tests/integration_anomaly/test_anomaly_autodiscovery.py b/tests/integration_anomaly/test_anomaly_autodiscovery.py index 69045bd44..e60b8b069 100644 --- a/tests/integration_anomaly/test_anomaly_autodiscovery.py +++ b/tests/integration_anomaly/test_anomaly_autodiscovery.py @@ -317,13 +317,14 @@ def test_autodiscovery_with_various_cardinality_strings(spark: SparkSession): # full table for the fit to see the minimum it promises. At 200 rows this fixture gave `category` 40 rows # per group, which is 9.6 by the time anything is fitted, so it is correctly no longer a grouping # candidate. 1,000 rows gives it 200, and ~48 after sampling. + row_count = 1000 data = [] - for i in range(1000): + for i in range(row_count): data.append( ( f"cat_{i % 5}", # Low cardinality (5 distinct), 200 rows/group f"user_{i % 50}", # Medium cardinality (50 distinct) - f"tx_{i}", # High cardinality (1000 distinct) - avoid "id" pattern + f"tx_{i}", # One per row, so cardinality == row_count - avoid the "id" name pattern 100.0 + i, ) ) @@ -345,11 +346,14 @@ def test_autodiscovery_with_various_cardinality_strings(spark: SparkSession): assert "user_code" in profile.recommended_columns assert profile.column_types["user_code"] == "categorical" - # transaction_ref has 1000 distinct values (>100) - should be excluded with warning (lines 144-148) + # transaction_ref is unique per row, so its cardinality is above the threshold and it is excluded + # with a warning naming the count. Derived from row_count rather than written as a literal: the + # fixture was rescaled from 200 rows to 1,000 to clear the effective rows-per-group floor, and this + # assertion kept expecting "200 distinct values" -- a hardcoded number the fixture no longer had. assert "transaction_ref" not in profile.recommended_columns warnings_text = " ".join(profile.warnings) assert "transaction_ref" in warnings_text - assert "200 distinct values" in warnings_text + assert f"{row_count} distinct values" in warnings_text assert "too high cardinality" in warnings_text # amount should be selected as numeric diff --git a/tests/unit/test_anomaly_mahalanobis_detector.py b/tests/unit/test_anomaly_mahalanobis_detector.py index 9ff719f15..539ee92c2 100644 --- a/tests/unit/test_anomaly_mahalanobis_detector.py +++ b/tests/unit/test_anomaly_mahalanobis_detector.py @@ -210,3 +210,44 @@ def test_isolation_forest_scores_are_also_independent_of_contamination(): liberal = IsolationForest(contamination=0.20, random_state=0, n_estimators=50).fit(data) np.testing.assert_allclose(timid.score_samples(data), liberal.score_samples(data)) + + +# ── retaining every one-hot category, and what that means for a singular direction ─────────────────── + + +def test_a_redundant_dummy_costs_nothing_and_an_unseen_category_scores_high(): + """Retaining every one-hot category induces an exactly collinear pair. Both halves are checked. + + A binary column encoded as both indicators satisfies ``d_a + d_b == 1`` on every trained row, so the + covariance is singular along that direction. Two things follow, and only one of them is obvious: + + - for rows that *do* satisfy the constraint, the redundant dummy changes nothing: the score is + identical to the same data encoded with one dummy, because the ridged pseudo-inverse gives the + zero-variance direction no weight + - a row that violates it -- an unseen category, encoded all-zeros -- sits off the surface every + training row lay on, and scores enormously + + The second is deliberate rather than accidental, and it is not new: columns with three or more + categories always retained all of them, so an unseen value has always scored this way. Truncating + the binary case was the inconsistency, and it is what made an unexpected value in a binary column + invisible instead. + """ + rng = np.random.default_rng(3) + metric = rng.normal(10.0, 1.0, 400) + indicator = (rng.random(400) < 0.5).astype(float) + + both = MahalanobisDetector().fit(np.column_stack([metric, indicator, 1.0 - indicator])) + one = MahalanobisDetector().fit(np.column_stack([metric, indicator])) + + on_surface = np.array([[40.0, 1.0, 0.0], [10.0, 1.0, 0.0]]) + scores_both = -both.score_samples(on_surface) + scores_one = -one.score_samples(np.array([[40.0, 1.0], [10.0, 1.0]])) + + assert np.all(np.isfinite(scores_both)) + np.testing.assert_allclose(scores_both, scores_one, rtol=1e-6) + assert scores_both[0] > scores_both[1] # the extreme metric still ranks above the ordinary one + + unseen = -both.score_samples(np.array([[10.0, 0.0, 0.0]]))[0] + known = -both.score_samples(np.array([[10.0, 1.0, 0.0]]))[0] + assert np.isfinite(unseen) + assert unseen > known From a5922b41e08c2970ec1232a9907a2f4db7dcf23d Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 18:01:40 +0100 Subject: [PATCH 085/107] Strip ANSI colour codes I committed into .build-constraints.txt `make build` failed with `Unexpected '', expected '-c', '-e', '-r' or the start of a requirement at .build-constraints.txt:7`. Line 7 is a `# via` comment wrapped in escape sequences. `make lock-dependencies` pipes `uv pip compile` into the file. uv suppresses colour when its output is not a terminal, but this shell exports FORCE_COLOR=3, which overrides that, so regenerating the file while bumping the mlflow floor wrote twelve escape sequences into it and uv's own requirements parser then rejected the result. That breaks the release wheel, not just a local build. The file is now byte-identical to main: the build-system requires are hatchling and hatch-fancy-pypi-readme, which an mlflow floor cannot affect, so it should never have changed in that commit at all. Stripped rather than re-resolved, so the pins cannot drift against a different --exclude-newer window; all seven pinned requirements are unchanged. `make build` exits 0 and produces both the wheel and the sdist. Found by building the wheel to re-run the demo notebooks, which is not a step CI performs on a pull request -- so nothing would have reported this before the release. Co-authored-by: Isaac --- .build-constraints.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.build-constraints.txt b/.build-constraints.txt index d2cf8070d..24290ae20 100644 --- a/.build-constraints.txt +++ b/.build-constraints.txt @@ -4,24 +4,24 @@ hatch-fancy-pypi-readme==25.1.0 \ hatchling==1.32.0 \ --hash=sha256:0bdbde4a52b06c37e3eca395f85a762bf0ef06fe374fd8ae429dc6be10230f5f \ --hash=sha256:0e17c9c3b9aa7c625acc8d0f5b622f107d5049af9ecf5ada4de1aada5be7cdbc - # via hatch-fancy-pypi-readme + # via hatch-fancy-pypi-readme packaging==26.3 \ --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c - # via hatchling + # via hatchling pathspec==1.1.1 \ --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 - # via hatchling + # via hatchling pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 - # via hatchling + # via hatchling tomlkit==0.15.1 \ --hash=sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304 \ --hash=sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97 - # via hatchling + # via hatchling trove-classifiers==2026.6.1.19 \ --hash=sha256:ab4c4ec93cc4a4e7815fa759906e05e6bb3f2fbd92ea0f897288c6a43efd15b3 \ --hash=sha256:c5132b4b61a829d11cfbd2d72e97f20a45ed6edb95e45c5efdeb5e00836b2745 - # via hatchling + # via hatchling From 1af981788c789fa53e8367282463b291ab6fab28 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Fri, 4 Sep 2026 18:28:08 +0100 Subject: [PATCH 086/107] Name cloudpickle as the sklearn serialization format, which MLflow 3 needs for our own detector Raising the mlflow floor to 3.x broke `profile="timeseries"` outright. MLflow 3 validates a saved sklearn model against skops' set of trusted types and refuses anything it does not recognise: MlflowException: The saved sklearn model references untrusted types. Untrusted types found in the file: ['...timeseries_detector.MahalanobisDetector'] `IsolationForest` is recognised, so the default profile was unaffected and every unit test still passed. The fleet demo failed on a cluster while the tabular demo succeeded, which is exactly the split the diagnosis predicts, and it reproduces locally against mlflow 3.15.2. Naming cloudpickle restores what MLflow 2 did by default, so this is the format every DQX model has always been written with rather than a new one, and it works identically on both major versions. `skops_trusted_types` would be narrower but exists only on MLflow 3, so the shim would need a second version branch. The trust model is unchanged and already documented: DQX loads models only from the Unity Catalog registry the caller owns. The regression test round-trips rather than only saving, because scoring loads the model back and a format that writes but cannot read would move the failure from training to the first scored batch. Scores and feature contributions must both come back identical. Two existing tests stubbed `log_model` with fakes that did not accept the new argument; they now assert it is passed, on both branches. Fixing them surfaced a second problem worth recording: on MLflow 3 `mlflow.sklearn` is a LazyLoader until something touches it, and patching an attribute on the loader materialises the real module underneath, so the patch landed on the loader while the code under test called the real `log_model` and tried to reach a tracking server. Importing `mlflow.sklearn` explicitly makes the patch target and the call target the same object. Co-authored-by: Isaac --- .../labs/dqx/anomaly/mlflow_registry.py | 18 +++++++++ .../unit/test_anomaly_mahalanobis_detector.py | 37 +++++++++++++++++++ tests/unit/test_anomaly_mlflow_registry.py | 27 ++++++++++++-- 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/mlflow_registry.py b/src/databricks/labs/dqx/anomaly/mlflow_registry.py index e4d9ce7c1..5c6c93875 100644 --- a/src/databricks/labs/dqx/anomaly/mlflow_registry.py +++ b/src/databricks/labs/dqx/anomaly/mlflow_registry.py @@ -164,6 +164,22 @@ def _flatten_hyperparams(hyperparams: dict[str, Any]) -> dict[str, Any]: return {f"hyperparam_{k}": v for k, v in hyperparams.items() if v is not None} +#: Serialization format for logged sklearn models, stated rather than defaulted. +#: +#: MLflow 3 validates a saved sklearn model against skops' set of trusted types and refuses anything it +#: does not recognise. ``IsolationForest`` is recognised; :class:`MahalanobisDetector` is DQX's own class, +#: so ``profile="timeseries"`` failed outright at registration with "The saved sklearn model references +#: untrusted types". MLflow 2 defaulted to cloudpickle and never ran that check, so naming cloudpickle +#: here restores the behaviour every DQX model has always been written with rather than introducing new +#: behaviour, and it works identically on both major versions -- unlike ``skops_trusted_types``, which +#: exists only on MLflow 3 and would need a second version branch below. +#: +#: The trust model is unchanged and is documented: DQX loads models only from the Unity Catalog registry +#: the caller owns, and cloudpickle deserialization executes code, so a model URI is as trusted as the +#: registry it names. +SKLEARN_SERIALIZATION_FORMAT = "cloudpickle" + + def log_sklearn_model_compatible( *, model: TrainedModel, @@ -181,12 +197,14 @@ def log_sklearn_model_compatible( name="model", registered_model_name=model_name, signature=signature, + serialization_format=SKLEARN_SERIALIZATION_FORMAT, ) return mlflow.sklearn.log_model( sk_model=model, artifact_path="model", registered_model_name=model_name, signature=signature, + serialization_format=SKLEARN_SERIALIZATION_FORMAT, ) diff --git a/tests/unit/test_anomaly_mahalanobis_detector.py b/tests/unit/test_anomaly_mahalanobis_detector.py index 539ee92c2..41dcf0624 100644 --- a/tests/unit/test_anomaly_mahalanobis_detector.py +++ b/tests/unit/test_anomaly_mahalanobis_detector.py @@ -5,12 +5,16 @@ narrative, so the rejected alternative is asserted explicitly rather than described in a comment. """ +import tempfile + +import mlflow import numpy as np import pytest from sklearn.base import clone from sklearn.ensemble import IsolationForest from sklearn.pipeline import Pipeline +from databricks.labs.dqx.anomaly.mlflow_registry import SKLEARN_SERIALIZATION_FORMAT from databricks.labs.dqx.anomaly.timeseries_detector import MahalanobisDetector # The correlated 2x2 case used throughout: rho = 0.9, so the off-diagonal precision terms are large @@ -251,3 +255,36 @@ def test_a_redundant_dummy_costs_nothing_and_an_unseen_category_scores_high(): known = -both.score_samples(np.array([[10.0, 1.0, 0.0]]))[0] assert np.isfinite(unseen) assert unseen > known + + +# ── the serialization format DQX's own estimator needs ─────────────────────────────────────────────── + + +def test_the_detector_round_trips_through_mlflow_in_the_format_dqx_declares(): + """MLflow 3 refuses to save an sklearn model referencing types skops does not trust. + + ``IsolationForest`` is trusted; :class:`MahalanobisDetector` is DQX's own class, so raising the + mlflow floor to 3.x broke ``profile="timeseries"`` at registration with "The saved sklearn model + references untrusted types", after every unit test still passed. Naming cloudpickle restores what + MLflow 2 did by default. + + Round-tripping rather than only saving, because scoring loads the model back: a format that writes + but does not read would move the failure from training to the first scored batch. Scores must be + identical, not merely finite -- a persisted model that scores differently from the fitted one is + the same defect wearing a different hat. + """ + rng = np.random.default_rng(0) + train = rng.normal(0, 1, (300, 4)) + probe = np.vstack([rng.normal(0, 1, (3, 4)), rng.normal(8, 1, (1, 4))]) + + detector = MahalanobisDetector().fit(train) + + with tempfile.TemporaryDirectory() as directory: + path = f"{directory}/model" + mlflow.sklearn.save_model(sk_model=detector, path=path, serialization_format=SKLEARN_SERIALIZATION_FORMAT) + reloaded = mlflow.sklearn.load_model(path) + + np.testing.assert_allclose(reloaded.score_samples(probe), detector.score_samples(probe), rtol=1e-12) + # Attribution is the reason this detector exists in DQX rather than raw scipy, so it has to survive + # the round trip too. + np.testing.assert_allclose(reloaded.feature_contributions(probe), detector.feature_contributions(probe)) diff --git a/tests/unit/test_anomaly_mlflow_registry.py b/tests/unit/test_anomaly_mlflow_registry.py index df5f32169..08e181ce6 100644 --- a/tests/unit/test_anomaly_mlflow_registry.py +++ b/tests/unit/test_anomaly_mlflow_registry.py @@ -1,10 +1,22 @@ -"""Unit tests for MLflow anomaly model registry compatibility.""" +"""Unit tests for MLflow anomaly model registry compatibility. + +``mlflow.sklearn`` is imported explicitly, and that import is load-bearing rather than tidiness. On +MLflow 3 ``mlflow.sklearn`` is a ``LazyLoader`` until something touches it, and patching an attribute on +the loader materialises the real module underneath, so the patch lands on the loader while the code under +test reads the freshly materialised module and calls the real ``log_model`` -- which then tries to reach +a tracking server. Importing the submodule first means the patch target and the call target are the same +object. +""" from types import SimpleNamespace import mlflow +import mlflow.sklearn # noqa: F401 -- see below -from databricks.labs.dqx.anomaly.mlflow_registry import log_sklearn_model_compatible +from databricks.labs.dqx.anomaly.mlflow_registry import ( + SKLEARN_SERIALIZATION_FORMAT, + log_sklearn_model_compatible, +) class _DummyModel: @@ -20,12 +32,13 @@ class _DummySignature: def test_log_model_uses_name_when_supported(monkeypatch): captured = {} - def fake_log_model(*, sk_model, name, registered_model_name, signature): + def fake_log_model(*, sk_model, name, registered_model_name, signature, serialization_format): captured["kwargs"] = { "sk_model": sk_model, "name": name, "registered_model_name": registered_model_name, "signature": signature, + "serialization_format": serialization_format, } return SimpleNamespace(registered_model_version="1") @@ -40,17 +53,22 @@ def fake_log_model(*, sk_model, name, registered_model_name, signature): assert info.registered_model_version == "1" assert captured["kwargs"]["name"] == "model" assert captured["kwargs"]["registered_model_name"] == "catalog.schema.model" + # Named rather than defaulted, and asserted on both branches: MLflow 3 validates a saved sklearn + # model against skops' trusted types and refuses MahalanobisDetector, which is DQX's own class, so + # omitting this breaks profile="timeseries" at registration while every other test still passes. + assert captured["kwargs"]["serialization_format"] == SKLEARN_SERIALIZATION_FORMAT def test_log_model_uses_artifact_path_when_name_not_supported(monkeypatch): captured = {} - def fake_log_model(*, sk_model, artifact_path, registered_model_name, signature): + def fake_log_model(*, sk_model, artifact_path, registered_model_name, signature, serialization_format): captured["kwargs"] = { "sk_model": sk_model, "artifact_path": artifact_path, "registered_model_name": registered_model_name, "signature": signature, + "serialization_format": serialization_format, } return SimpleNamespace(registered_model_version="2") @@ -65,3 +83,4 @@ def fake_log_model(*, sk_model, artifact_path, registered_model_name, signature) assert info.registered_model_version == "2" assert captured["kwargs"]["artifact_path"] == "model" assert captured["kwargs"]["registered_model_name"] == "catalog.schema.model" + assert captured["kwargs"]["serialization_format"] == SKLEARN_SERIALIZATION_FORMAT From 8689f8013803c81ad5b6631e72416e38d16edcf4 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 09:46:54 +0100 Subject: [PATCH 087/107] Standardise the response before the temporal fit, so a metric's units stop deciding its trend Found by reading the demo's cell output rather than its exit code: the run logged `ConvergenceWarning: lbfgs failed to converge ... max_iter=400` from the Huber fit, with sklearn's own advice to raise the cap or scale the data. The design columns were already scaled (seconds / span, harmonics in [-1, 1]) but the response was not, and HuberRegressor solves for the coefficients and a scale parameter jointly, so a metric in its own large units is badly conditioned. The warning turned out to be the mild symptom. Measured against a known slope across 20 seeds: a metric in billions (bytes) 99.89% wrong (sd 0.00%) against 0.13%, new wins 20/20 a contaminated sample, 5% at 6x 1.32% against 0.14%, new wins 20/20 millions, and negatives 8.37% and 0.89% against 0.26% heavy-tailed (lognormal) noise 0.31% against 0.28%, new wins 10/20 -- a coin flip That error went straight into the residual `baseline_over_time` exists to produce: a metric measured in billions had essentially no trend removed at all. So this generalises rather than fitting the demo. Rates near 1e-6, small magnitudes, integer counts, zero-crossing metrics and a large offset with tiny variation are all a wash to within a tenth of a point. A single seed appeared to regress on heavy-tailed noise, which is why that case was repeated across 20 -- it is a coin flip, and reporting the n=1 result would have been wrong. Exactly invertible for the unpenalised problem, because the model is linear, so persisted coefficients stay in the metric's own units and scoring is untouched. The one real change is that HUBER_ALPHA's effective regularisation was magnitude-dependent and is now scale-free, which is the behaviour worth having but is not a strict no-op. Median and MAD rather than mean and standard deviation, for the same reason the loss is Huber at all, with a documented fallback when more than half the values are identical. make fmt exit 0, unit 2713 passing. Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/temporal.py | 47 ++++++++++++++- tests/unit/test_anomaly_temporal_fit.py | 66 +++++++++++++++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/temporal.py b/src/databricks/labs/dqx/anomaly/temporal.py index 472cfa2e2..b15460186 100644 --- a/src/databricks/labs/dqx/anomaly/temporal.py +++ b/src/databricks/labs/dqx/anomaly/temporal.py @@ -194,13 +194,56 @@ def _fit_one(design: np.ndarray, values: np.ndarray) -> list[float] | None: if float(np.std(values)) == 0.0: # A constant metric has no expectation to learn beyond its own level. return [float(values[0])] + [0.0] * (design.shape[1] - 1) + + # The response is standardised before fitting and the coefficients mapped straight back. The design + # columns are already scaled (seconds / span, harmonics in [-1, 1]) but the metric is in its own + # units, and HuberRegressor solves for the coefficients and a scale parameter jointly, so a metric + # measured in millions is badly conditioned: lbfgs hits max_iter, and the slope it reaches is wrong. + # + # Measured against a known slope across the shapes a user might pass, 20 seeds each: + # + # a metric in billions (bytes) raw is 99.89% out (sd 0.00%) against 0.13%, new wins 20/20 + # a contaminated sample, 5% at 6x raw 1.32% against 0.14%, new wins 20/20 + # negatives, integer counts, rates near 1e-6, a large offset with tiny variation, zero-crossing, + # and small magnitudes are all a wash, to within a tenth of a point + # heavy-tailed (lognormal) noise is 0.31% against 0.28%, new winning 10 of 20 -- a coin flip. A + # single seed appeared to regress here, which is why the check was repeated. + # + # So this is a general conditioning fix rather than a fit to one dataset: it is a large win where + # magnitude is large, a win on exactly the contaminated case the Huber loss is here for, and neutral + # elsewhere. The error it removes goes straight into the residual this feature exists to produce. + # + # Exactly invertible for the unpenalised problem, because the model is linear, so persisted + # coefficients stay in the metric's own units and scoring needs to know nothing about this. The one + # real change is HUBER_ALPHA: an L2 penalty on coefficients was previously magnitude-dependent and is + # now scale-free, which is the behaviour worth having but is a change rather than a strict no-op. + centre, spread = _response_scale(values) try: model = HuberRegressor(epsilon=HUBER_EPSILON, alpha=HUBER_ALPHA, max_iter=HUBER_MAX_ITER) - model.fit(design[:, 1:], values) + model.fit(design[:, 1:], (values - centre) / spread) except (ValueError, FloatingPointError) as exc: logger.debug(f"Temporal fit failed, falling back to no temporal feature for this metric: {exc}") return None - return [float(model.intercept_), *(float(c) for c in model.coef_)] + return [ + float(model.intercept_) * spread + centre, + *(float(c) * spread for c in model.coef_), + ] + + +def _response_scale(values: np.ndarray) -> tuple[float, float]: + """Robust centre and spread for the fitted response. + + Median and MAD rather than mean and standard deviation, for the same reason the fit is Huber at all: + DQX trains on a sample that still contains anomalies, and a handful of extreme rows should not set + the scale the whole fit is conditioned on. Falls back to the standard deviation when the MAD is zero + (more than half the values identical), and to 1.0 when that is zero too, which leaves the fit exactly + as it was rather than dividing by nothing. + """ + centre = float(np.median(values)) + spread = float(np.median(np.abs(values - centre)) * MAD_TO_SIGMA) + if spread <= 0.0: + spread = float(np.std(values)) + return centre, (spread if spread > 0.0 else 1.0) def _holdout_residual_scale(seconds: np.ndarray, values: np.ndarray, basis: TemporalBasis) -> float: diff --git a/tests/unit/test_anomaly_temporal_fit.py b/tests/unit/test_anomaly_temporal_fit.py index 46077a2f2..760d07cbc 100644 --- a/tests/unit/test_anomaly_temporal_fit.py +++ b/tests/unit/test_anomaly_temporal_fit.py @@ -16,6 +16,7 @@ from databricks.labs.dqx.anomaly.temporal import ( CANDIDATE_PERIODS_SECONDS, MIN_SEASONAL_CYCLES, + SEASONAL_HARMONICS, TemporalBasis, candidate_periods, design_matrix, @@ -394,3 +395,68 @@ def test_a_metric_measured_in_large_units_does_not_outvote_the_others(): inflated, _ = select_basis(seconds, {"bent": bent, "straight": straight * 1e6}) assert modest == inflated + + +# ── the response is standardised before fitting, and it matters at scale ───────────────────────────── + + +@pytest.mark.parametrize("magnitude", [1.0, 1e3, 1e5, 1e7]) +def test_the_slope_is_recovered_whatever_units_the_metric_is_in(magnitude: float): + """A metric measured in millions must not get a worse trend than one measured in tens. + + The design columns were already scaled, but the response was not, and HuberRegressor solves for the + coefficients and a scale parameter jointly. Measured before the fix across 20 seeds: a metric in + billions had its slope **99.89% wrong (sd 0.00%)** against 0.13% once the response is standardised, + winning 20 of 20, and a contaminated sample went from 1.32% to 0.14%, also 20 of 20. Negatives, + integer counts, rates near 1e-6 and zero-crossing metrics are a wash. The demo logged + `ConvergenceWarning: lbfgs failed to converge ... max_iter=400` from the same cause. + + Parametrised across four orders of magnitude rather than asserted at one, because the failure is + invisible at the magnitudes a test would naturally pick. + """ + seconds = _hourly_axis(2000) + span = float(seconds.max() - seconds.min()) + basis = TemporalBasis(trend=True, periods=(DAY, WEEK), harmonics=SEASONAL_HARMONICS, span=span) + + slope = TRUE_SLOPE_PER_SECOND * magnitude + rng = np.random.default_rng(11) + values = 100.0 * magnitude + slope * seconds + rng.normal(0, 2.0 * magnitude, seconds.size) + + fitted = fit_temporal(seconds, {"metric": values}, basis) + + # Column 1 is the trend, expressed against seconds / span. + recovered = fitted["metric"][1] / span + assert recovered == pytest.approx( + slope, rel=0.05 + ), f"slope at magnitude {magnitude:g} recovered as {recovered:.4g} against a true {slope:.4g}" + + +def test_standardising_the_response_leaves_a_small_metric_unchanged(): + """The transform is exactly invertible, so it must not move a fit that was already fine. + + Same 2,000-point axis as the parametrised test above rather than a shorter one: at 600 points the + noise alone moves the recovered slope by about 2%, which says nothing about the transform. + """ + seconds = _hourly_axis(2000) + span = float(seconds.max() - seconds.min()) + basis = TemporalBasis(trend=True, periods=(DAY,), harmonics=SEASONAL_HARMONICS, span=span) + values = _linear_metric(seconds, np.random.default_rng(3)) + + fitted = fit_temporal(seconds, {"metric": values}, basis) + recovered = fitted["metric"][1] / span + + assert recovered == pytest.approx(TRUE_SLOPE_PER_SECOND, rel=0.02) + + +def test_a_metric_with_more_than_half_identical_values_still_fits(): + """The MAD is zero there, so the fallback to the standard deviation is what keeps it fittable.""" + seconds = _hourly_axis(400) + span = float(seconds.max() - seconds.min()) + basis = TemporalBasis(trend=True, periods=(), harmonics=SEASONAL_HARMONICS, span=span) + values = np.full(seconds.size, 50.0) + values[:120] = 50.0 + TRUE_SLOPE_PER_SECOND * seconds[:120] # a minority that moves + + fitted = fit_temporal(seconds, {"metric": values}, basis) + + assert "metric" in fitted + assert all(np.isfinite(fitted["metric"])) From 9269c02e31463d6a5bc9706cb28bf306ffa932e6 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 10:26:12 +0100 Subject: [PATCH 088/107] Name the mechanism that absorbs a collinear one-hot pair correctly The docstring said "the ridged pseudo-inverse gives the zero-variance direction no weight". There is no pseudo-inverse in the detector and it never forms an inverse at all: a ridge floor is added as a fraction of the average variance, the result is factored by Cholesky, and diag(inverse) comes from the squared column norms of L-inverse. The ridge is what makes a singular covariance factorable in the first place, so it is doing the work the comment credited elsewhere. Also corrects an implication: both dummies of a retained pair vary, so neither is dropped by the constant-feature mask, which keys on per-feature spread rather than on directions in feature space. The measurement the test makes is unchanged and still passes -- scores identical to the one-dummy encoding, an unseen category far above a known one. Only the explanation was wrong, from confusing this detector with the benchmark harness's Mahalanobis, which does use np.linalg.pinv. Found by an agent fact-checking a teaching deck against the source, which is a better review than re-reading my own comment would have been. Co-authored-by: Isaac --- tests/unit/test_anomaly_mahalanobis_detector.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_anomaly_mahalanobis_detector.py b/tests/unit/test_anomaly_mahalanobis_detector.py index 41dcf0624..b132f18f6 100644 --- a/tests/unit/test_anomaly_mahalanobis_detector.py +++ b/tests/unit/test_anomaly_mahalanobis_detector.py @@ -226,8 +226,11 @@ def test_a_redundant_dummy_costs_nothing_and_an_unseen_category_scores_high(): covariance is singular along that direction. Two things follow, and only one of them is obvious: - for rows that *do* satisfy the constraint, the redundant dummy changes nothing: the score is - identical to the same data encoded with one dummy, because the ridged pseudo-inverse gives the - zero-variance direction no weight + identical to the same data encoded with one dummy. The mechanism is the **ridge floor**, which is + added as a fraction of the average variance and is what leaves the singular covariance factorable + by Cholesky at all; it is not a pseudo-inverse, and the detector never forms one. Both dummies vary, + so neither is dropped by the constant-feature mask, which keys on per-feature spread rather than on + directions in feature space. - a row that violates it -- an unseen category, encoded all-zeros -- sits off the surface every training row lay on, and scores enormously From 1c868b94cc25b5ef52a2ee0925f2d74a29f67d9d Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 10:31:04 +0100 Subject: [PATCH 089/107] Report the order-dependence figure we measured, not the one the review reported The docstring said "a linear metric first chose one changepoint where a bent metric chose six". That is the reviewer's number, quoted from their report. Our own reproduction on the straight-and-bent pair the unit test uses gives **0 changepoints against 3**, which is what the commit that fixed this said, so the docstring and the commit message contradicted each other and the docstring carried a figure nobody here had measured. Re-confirmed just now: straight-first 0, bent-first 3, and both 3 once every metric is read. The docstring now names the test that reproduces it, so the number has somewhere to be checked against. Both figures demonstrate the same defect, so nothing about the fix changes. The point is that publishing a measurement we did not take is the exact habit this review round was about, and it survived into a docstring while the correct number sat in the commit message beside it. The second figure in that docstring (rescaling one metric by 1e6 moving the count from three to six) is ours, from the scale-invariance test that failed before the response was standardised, and stands. Found by an agent cross-checking a code walkthrough against the source. Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/temporal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/databricks/labs/dqx/anomaly/temporal.py b/src/databricks/labs/dqx/anomaly/temporal.py index b15460186..f9da7fe4b 100644 --- a/src/databricks/labs/dqx/anomaly/temporal.py +++ b/src/databricks/labs/dqx/anomaly/temporal.py @@ -289,7 +289,9 @@ def select_basis(seconds: np.ndarray, metrics: dict[str, np.ndarray]) -> tuple[T choice has to be a property of the table rather than of any one metric. It reads every metric to that end. An earlier version scored changepoints on whichever metric happened to be first in the schema and took its time axis for the period search too, which made every fitted residual depend on column - order: a linear metric first chose one changepoint where a bent metric chose six. + order. Reproduced on the straight-and-bent pair that + ``tests/unit/test_anomaly_temporal_fit.py::test_the_basis_does_not_depend_on_metric_order`` uses: the + straight metric first chose **no** changepoints where the bent metric chose **three**. Candidates are compared on the **mean across metrics of the ratio** between a candidate's holdout residual scale and the same metric's scale under the simplest basis, each metric first standardised From 4d95cceee57e5de8b7c170273cbdb25f29dc13fa Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 11:04:51 +0100 Subject: [PATCH 090/107] Stop claiming the bucket median is what makes the temporal fit robust Prompted by the question "does our sampling harm the temporal fit, or sample insensitively to the trend?" -- worth measuring rather than reasoning about, and the measurement says the comment overstated one thing while the design is otherwise sound. Sampling does not harm the trend, and the reason is worth writing down because it is not obvious: `DataFrame.sample` is a uniform Bernoulli draw and the train split is `randomSplit`, neither ordered nor chronological, so training rows spread across the whole history and the slope estimator stays unbiased. Measured slope error, full table against the 24% DQX actually fits: 0.00% vs 0.00% at 200k rows, 0.01% vs 0.01% at 20k, 0.13% vs 0.27% at 2k. The fitted window pulls in by under 0.1% of span, so staleness boundaries are unaffected. What sampling does erode is bucket occupancy, and the comment leaned on exactly that: "each bucket contributes its median, which is robust before Huber even sees it". Rows per bucket, full against sampled, are 50 -> 12 at 200k rows, 5 -> 1.6 at 20k, and 1 -> 1 at 2k, where a median of one row is the row. So below roughly 100k rows that sentence describes nothing. It does not matter, because Huber is carrying the robustness: with 5% of rows at 6x normal the recovered slope error is 0.02% sampled against 0.01% full. But the comment credited the wrong mechanism, which is how someone later "optimises away" the thing that was actually load-bearing. Also records a bucketing consequence unrelated to sampling: width is span/4000, so at 200k hourly rows the width is 50h and a daily period is correctly rejected as unresolvable. Sub-bucket-width seasonality is unavailable on very long spans, by construction. Measurement reuses the shipped `candidate_periods`, `fit_temporal` and TEMPORAL_FIT_BUCKETS over 8 seeds per size, reproducing `_fit_temporal_from_buckets`' bucketing exactly. Co-authored-by: Isaac --- .../labs/dqx/anomaly/transformers.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index e7684cbdd..2e7a5cc49 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -898,7 +898,25 @@ def _signed_log1p(column: Column) -> Column: ) # Rows collected to the driver to fit the temporal basis. The fit runs on a bucketed aggregate rather # than raw rows, so this bounds driver memory regardless of table size, exactly as the per-group median -# aggregation does. Each bucket contributes its median, which is robust before Huber even sees it. +# aggregation does. +# +# Each bucket contributes its median, but do not lean on that for robustness: it only attenuates outliers +# while buckets hold enough rows to have a meaningful median, and training sees ~24% of the table +# (sample_fraction x train_ratio). Measured rows per bucket, full table against sampled: 50 -> 12 at +# 200k rows, 5 -> 1.6 at 20k, and 1 -> 1 at 2k, where a "median" is just the row. **Huber is what +# actually carries robustness here** -- with 5% of rows at 6x normal, the recovered slope error is 0.02% +# sampled against 0.01% on the full table, so contamination survives the thinning intact. +# +# The sampling is otherwise harmless to the trend, which is worth stating because it is not obvious: +# `DataFrame.sample` is a uniform Bernoulli draw and the train split is `randomSplit`, neither ordered +# nor chronological, so training rows spread over the whole history and the slope estimator stays +# unbiased. Measured slope error, full against sampled: 0.00% vs 0.00% at 200k rows, 0.01% vs 0.01% at +# 20k, 0.13% vs 0.27% at 2k. The fitted window pulls in by under 0.1% of span. +# +# One consequence of bucketing that is unrelated to sampling: bucket width is span/4000, so a very long +# history coarsens the axis. At 200k hourly rows the width is 50h and a daily period is correctly +# rejected as unresolvable by `candidate_periods`. Sub-bucket-width seasonality is not available on very +# long spans, by construction. TEMPORAL_FIT_BUCKETS = 4000 From 95fa777200f079c07282704d8941b142d47bc766 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 11:46:44 +0100 Subject: [PATCH 091/107] Remove expected_anomaly_rate, which never changed what DQX flags It read as the knob for how much gets flagged, and it was not one. It set the estimator's `contamination`, which places only scikit-learn's own `predict`/`offset_` boundary; every DQX scoring path reads `-score_samples` and ranks it against training-score quantiles, so what gets flagged is decided by the check's `threshold`. A parameter whose name promises the opposite of what it does is worse than no parameter, and this release already carries breaking changes, so it goes now rather than being documented around. Behaviour is unchanged. `IsolationForestConfig.contamination` now defaults to 0.02 directly, which is exactly what the removed parameter filled in when unset, so a caller who passed nothing gets what they got before. Anyone who was setting it can set `params.algorithm_config.contamination` instead -- still honest, because that name promises only what it delivers, and it still matters to anyone loading the registered model and calling `predict` themselves. Gone with it: `apply_expected_anomaly_rate_if_default_contamination`, the parameter on `AnomalyEngine.train` and `build_context`, the field on `AnomalyTrainingContext`, and its range check in `validate_training_params`. Not in `AnomalyConfig` or the workflow, so no run-config or YAML break. The two docstring examples that used it to "adjust expected anomaly rate for specific use cases" now show `threshold` on the check, which is the thing those examples were reaching for. Tests: the three unit tests for the removed mapping helper become two for the config default and the explicit escape hatch; the two range-check tests become one asserting contamination is still validated, which matters more now that it is the only route. The integration test that asserted the parameter reached the persisted hyperparameters is kept and rewritten against the default, because it is the only coverage that the algorithm config flows into `training.hyperparameters` at all. Verified: fmt exit 0, unit 2709 passing, and the nine training-validation integration tests pass against a live workspace. Co-authored-by: Isaac --- docs/dqx/docs/reference/quality_checks.mdx | 8 ++- .../labs/dqx/anomaly/anomaly_engine.py | 26 ++-------- .../labs/dqx/anomaly/timeseries_detector.py | 8 +-- .../labs/dqx/anomaly/training_service.py | 26 ++-------- src/databricks/labs/dqx/anomaly/types.py | 1 - src/databricks/labs/dqx/anomaly/validation.py | 9 +--- src/databricks/labs/dqx/config.py | 5 +- tests/integration_anomaly/conftest.py | 2 - .../test_anomaly_registry.py | 24 ++++----- .../test_anomaly_training_validation.py | 21 -------- tests/unit/test_anomaly_configs.py | 4 +- ...test_anomaly_isolation_forest_inertness.py | 6 +-- .../unit/test_anomaly_mahalanobis_detector.py | 9 ++-- tests/unit/test_anomaly_utils_functions.py | 49 +++++++------------ tests/unit/test_anomaly_validation.py | 35 ++++++------- 15 files changed, 75 insertions(+), 158 deletions(-) diff --git a/docs/dqx/docs/reference/quality_checks.mdx b/docs/dqx/docs/reference/quality_checks.mdx index d1481e763..87da0746f 100644 --- a/docs/dqx/docs/reference/quality_checks.mdx +++ b/docs/dqx/docs/reference/quality_checks.mdx @@ -3536,10 +3536,9 @@ Required arguments: `df`, `model_name`, `registry_table`. Optional parameters: | `baseline_over_time` | str | None | A timestamp or date column each metric is judged *along*, so a value is compared with what its own history says to expect at that point in time. Composes with `baseline_by`: with both set the expectation is fitted on the group-relative value, so one model still covers every group. Independent of `profile`. Never auto-discovered, and DQX warns rather than acting when the training window shows little structure over time. Not a forecaster, and it never reads the previous row. The named column must not also appear in `columns`. See [Comparing against time](/docs/guide/row_anomaly_detection#comparing-against-time). | | `profile` | str | `"tabular"` | Which detector to train. `"tabular"` is Isolation Forest — the behaviour that predates this option. `"timeseries"` selects a correlation-aware detector for multivariate metrics whose anomalies are broken *correlations* rather than extreme single values; it needs no timestamp column and trains a single model rather than an ensemble. There is no automatic option: DQX never changes the algorithm on your behalf, because the choice cannot be verified without labels. See [Choosing a profile](/docs/guide/row_anomaly_detection#choosing-a-profile). | | `exclude_columns` | list[str] | None | Columns to exclude from training (e.g., IDs, labels, ground truth) | -| `expected_anomaly_rate` | float | 0.02 | Expected fraction of anomalies in your data (0.02 = 2%). Supplies the estimator's `contamination`, which places scikit-learn's own `predict`/`offset_` boundary. **It does not change which rows DQX flags**, because scoring ranks `score_samples` against the training score quantiles and the rows you see are decided by the check's `threshold`. It also does not mitigate anomalies present in the training sample. Relevant if you load the registered model and call `predict` yourself. | | `params` | AnomalyParams | None | Optional. Advanced tuning parameters. See sections below for details. | -**Validation**: Training parameters are validated before model fitting. Invalid values (for example `sample_fraction <= 0`, `train_ratio > 1`, or `expected_anomaly_rate > 0.5`) fail fast with `InvalidParameterError`. +**Validation**: Training parameters are validated before model fitting. Invalid values (for example `sample_fraction <= 0`, `train_ratio > 1`, or `algorithm_config.contamination > 0.5`) fail fast with `InvalidParameterError`. #### AnomalyParams (Advanced Tuning) @@ -3559,7 +3558,7 @@ Pass an `IsolationForestConfig` object to `params.algorithm_config` to tune the | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `contamination` | float | auto | Expected outlier proportion, auto-set from `expected_anomaly_rate`. Affects only the estimator's own `predict`/`offset_` boundary, which DQX's scoring path does not use — tune `threshold` on the check instead. | +| `contamination` | float | 0.02 | Expected outlier proportion. Affects only the estimator's own `predict`/`offset_` boundary, which DQX's scoring path does not use, so it does **not** change which rows DQX flags — tune `threshold` on the check for that. Relevant if you load the registered model and call `predict` yourself. | | `num_trees` | int | 200 | Number of trees in the forest. More trees = better accuracy but slower training. | | `max_depth` | int | None | Maximum tree depth. Auto-calculated as log2(sample_size) if None. | | `subsampling_rate` | float | None | Per-tree subsampling rate. None uses sklearn defaults. | @@ -3606,7 +3605,6 @@ model_name = anomaly_engine.train( df=spark.table("catalog.schema.orders"), model_name="catalog.schema.orders_monitor", registry_table="catalog.schema.dqx_anomaly_models", - expected_anomaly_rate=0.05, # Expect 5% anomalies (sets contamination; tune `threshold` to change what is flagged) params=params, ) ``` @@ -3614,7 +3612,7 @@ model_name = anomaly_engine.train( - **Too many false positives?** Increase `threshold` (95 → 98) in scoring, or increase `ensemble_size` (3 → 5) - **Training too slow?** Decrease `sample_fraction`, `max_rows`, or `num_trees`; disable SHAP explainability with `enable_contributions=False` -- **Missing real anomalies?** Decrease `threshold` (95 → 90) in scoring, or increase `expected_anomaly_rate` +- **Missing real anomalies?** Decrease `threshold` (95 → 90) on the check - **Reproducible results needed?** Set `random_seed` in `IsolationForestConfig` - **High-cardinality categoricals slow?** Increase `categorical_cardinality_threshold` to use Frequency encoding diff --git a/src/databricks/labs/dqx/anomaly/anomaly_engine.py b/src/databricks/labs/dqx/anomaly/anomaly_engine.py index f5c292ad6..ffe227f00 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_engine.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_engine.py @@ -62,7 +62,6 @@ def train( columns: list[str] | None = None, params: AnomalyParams | None = None, exclude_columns: list[str] | None = None, - expected_anomaly_rate: float = 0.02, baseline_by: list[str] | None = None, profile: str | None = None, baseline_over_time: str | None = None, @@ -127,16 +126,6 @@ def train( Exclusions always take precedence over `columns` if both are provided. Useful with auto-discovery to filter out unwanted columns without specifying all desired columns manually. - expected_anomaly_rate: Expected fraction of anomalies in your data (default: 0.02 = 2%). - Supplies the default *contamination* for the estimator, which places - scikit-learn's own ``predict`` / ``offset_`` boundary. - **It does not change which rows DQX flags.** Scoring reads - ``score_samples`` and ranks it against the training score quantiles, - so the rows you see are decided by the *threshold* on the check, not - by this. Nor does it mitigate anomalies present in the training - sample: nothing downweights them. Set it if you load the registered - model yourself and call ``predict``; otherwise tune *threshold*. - Overridden if params.algorithm_config.contamination is set explicitly. Important Notes: - Avoid ID columns (user_id, order_id, etc.) - use exclude_columns to filter them out. - Choose behavioral columns, not identifiers. Good: amount, quantity. Bad: user_id. @@ -161,18 +150,12 @@ def train( exclude_columns=["user_id", "order_id"], ) - # Adjust expected anomaly rate for specific use cases - anomaly_engine.train( - df, + # How much gets flagged is set on the *check*, not at training time: threshold is an + # alert budget over training severity, so 95 flags the top 5%. + check = has_no_row_anomalies( model_name="catalog.schema.fraud_detector", registry_table="catalog.schema.dqx_anomaly_models", - expected_anomaly_rate=0.01, # 1% fraud - ) - anomaly_engine.train( - df, - model_name="catalog.schema.quality_monitor", - registry_table="catalog.schema.dqx_anomaly_models", - expected_anomaly_rate=0.10, # 10% defects + threshold=99.0, # a tighter budget for a rare-event table ) # Explicit columns @@ -201,7 +184,6 @@ def train( columns=columns, params=params, exclude_columns=exclude_columns, - expected_anomaly_rate=expected_anomaly_rate, baseline_by=baseline_by, profile=profile, baseline_over_time=baseline_over_time, diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py index ad000d30a..cf0a94b34 100644 --- a/src/databricks/labs/dqx/anomaly/timeseries_detector.py +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -81,8 +81,8 @@ CONSTANT_FEATURE_TOLERANCE = 1e-12 # Ridge added to the covariance diagonal, as a fraction of the average variance so it is scale-free. DEFAULT_RIDGE = 1e-6 -# Fallback expected anomaly rate, matching AnomalyEngine.train's expected_anomaly_rate default. Only -# used if contamination somehow reached this point unset; production fills it in before training. +# Fallback used only if contamination reached this point unset. `IsolationForestConfig.contamination` +# defaults to the same value, so production never relies on this. DEFAULT_CONTAMINATION = 0.02 @@ -261,8 +261,8 @@ def fit_mahalanobis_model(train_pandas: pd.DataFrame, params: AnomalyParams) -> the estimator — so ``named_steps["model"]`` resolves identically for both algorithms. *contamination* is read from ``algorithm_config``, which is where - ``training_service.apply_expected_anomaly_rate_if_default_contamination`` puts - *expected_anomaly_rate*. Reusing that field rather than adding a parallel one keeps one source of + ``IsolationForestConfig.contamination`` carries it. + Reusing that field rather than adding a parallel one keeps one source of truth for "how many anomalies do we expect"; the genuinely IsolationForest-specific fields beside it (tree count, subsampling) are simply not read here. """ diff --git a/src/databricks/labs/dqx/anomaly/training_service.py b/src/databricks/labs/dqx/anomaly/training_service.py index a3da4556c..1155399e8 100644 --- a/src/databricks/labs/dqx/anomaly/training_service.py +++ b/src/databricks/labs/dqx/anomaly/training_service.py @@ -107,24 +107,6 @@ def _perform_auto_discovery(df_filtered: DataFrame) -> tuple[list[str], list[str logger.warning(warning) return profile.recommended_columns, profile.recommended_segments or None - @staticmethod - def apply_expected_anomaly_rate_if_default_contamination( - params: AnomalyParams | None, expected_anomaly_rate: float - ) -> AnomalyParams: - """Apply expected_anomaly_rate to params if contamination is not explicitly set.""" - if params is None: - params = AnomalyParams() - params = deepcopy(params) - if params.algorithm_config.contamination is None: - params.algorithm_config.contamination = expected_anomaly_rate - logger.info(f"Using expected_anomaly_rate={expected_anomaly_rate:.2%} for model training") - else: - logger.info( - f"Using explicitly set contamination={params.algorithm_config.contamination:.2%} " - f"(expected_anomaly_rate={expected_anomaly_rate:.2%} ignored)" - ) - return params - @staticmethod def _model_exists_in_uc(model_name: str) -> bool: """Check if a model exists in Unity Catalog using MLflow API.""" @@ -323,7 +305,6 @@ def build_context( columns: list[str] | None, params: AnomalyParams | None, exclude_columns: list[str] | None, - expected_anomaly_rate: float, baseline_by: list[str] | None = None, profile: str | None = None, baseline_over_time: str | None = None, @@ -346,7 +327,7 @@ def build_context( raise InvalidParameterError(f"exclude_columns contains columns not in DataFrame: {invalid}") params = AnomalyParams() if params is None else params - validate_training_params(params, expected_anomaly_rate) + validate_training_params(params) declared_baseline_by = baseline_by if baseline_by is not None else params.baseline_by columns, df_filtered = self._resolve_columns_and_filtered_df(df, columns, exclude_list) @@ -390,8 +371,8 @@ def build_context( baseline_by=baseline_by, ) - params = self.apply_expected_anomaly_rate_if_default_contamination(params, expected_anomaly_rate) - # Already a deepcopy, so recording the resolved grouping here cannot leak back to the + params = deepcopy(params) if params is not None else AnomalyParams() + # A deepcopy, so recording the resolved grouping here cannot leak back to the # caller's params. Downstream feature engineering reads baseline_by off params, because # every narrowing select is already handed params and nothing else. params.baseline_by = baseline_by @@ -405,7 +386,6 @@ def build_context( registry_table=registry_table, columns=columns, params=params, - expected_anomaly_rate=expected_anomaly_rate, exclude_columns=exclude_columns, auto_discovery_used=auto_discovery_used, baseline_by=baseline_by, diff --git a/src/databricks/labs/dqx/anomaly/types.py b/src/databricks/labs/dqx/anomaly/types.py index 96fe3b199..d96f0cdf0 100644 --- a/src/databricks/labs/dqx/anomaly/types.py +++ b/src/databricks/labs/dqx/anomaly/types.py @@ -84,7 +84,6 @@ class AnomalyTrainingContext: registry_table: str columns: list[str] params: AnomalyParams - expected_anomaly_rate: float exclude_columns: list[str] | None auto_discovery_used: bool baseline_by: list[str] | None = None diff --git a/src/databricks/labs/dqx/anomaly/validation.py b/src/databricks/labs/dqx/anomaly/validation.py index e213eb288..947f54083 100644 --- a/src/databricks/labs/dqx/anomaly/validation.py +++ b/src/databricks/labs/dqx/anomaly/validation.py @@ -312,7 +312,7 @@ def _validate_int_min(value: int, *, label: str, min_value: int) -> None: raise InvalidParameterError(f"{label} must be >= {min_value}. Got {value}.") -def validate_training_params(params: AnomalyParams, expected_anomaly_rate: float) -> None: +def validate_training_params(params: AnomalyParams) -> None: """Validate training parameters with strict fail-fast checks.""" _validate_float_range(params.sample_fraction, label="params.sample_fraction", min_exclusive=0.0, max_inclusive=1.0) _validate_float_range(params.train_ratio, label="params.train_ratio", min_exclusive=0.0, max_inclusive=1.0) @@ -322,13 +322,6 @@ def validate_training_params(params: AnomalyParams, expected_anomaly_rate: float if params.ensemble_size is not None: _validate_int_min(params.ensemble_size, label="params.ensemble_size", min_value=1) - _validate_float_range( - expected_anomaly_rate, - label="expected_anomaly_rate", - min_exclusive=0.0, - max_inclusive=0.5, - ) - algo_cfg = params.algorithm_config if algo_cfg.contamination is not None: _validate_float_range( diff --git a/src/databricks/labs/dqx/config.py b/src/databricks/labs/dqx/config.py index 50ea91792..ce3b5fdb2 100644 --- a/src/databricks/labs/dqx/config.py +++ b/src/databricks/labs/dqx/config.py @@ -159,7 +159,10 @@ class ProfilerConfig: class IsolationForestConfig: """Algorithm parameters for Spark ML IsolationForest.""" - contamination: float | None = None + # 0.02 is the value `AnomalyEngine.train`'s removed `expected_anomaly_rate` used to fill in, so the + # effective default is unchanged. It places only the estimator's own `predict`/`offset_` boundary, + # which DQX's scoring path does not read -- what DQX flags is decided by the check's `threshold`. + contamination: float | None = 0.02 num_trees: int = 200 max_depth: int | None = None subsampling_rate: float | None = None diff --git a/tests/integration_anomaly/conftest.py b/tests/integration_anomaly/conftest.py index 6777750b9..c4494023c 100644 --- a/tests/integration_anomaly/conftest.py +++ b/tests/integration_anomaly/conftest.py @@ -188,7 +188,6 @@ def train_model_with_params( registry_table: str, columns: list[str], params: AnomalyParams, - expected_anomaly_rate: float = 0.02, baseline_by: list[str] | None = None, baseline_over_time: str | None = None, profile: str | None = None, @@ -202,7 +201,6 @@ def train_model_with_params( baseline_by=baseline_by, baseline_over_time=baseline_over_time, params=params, - expected_anomaly_rate=expected_anomaly_rate, profile=profile, ) diff --git a/tests/integration_anomaly/test_anomaly_registry.py b/tests/integration_anomaly/test_anomaly_registry.py index 3946004d1..34c833f86 100644 --- a/tests/integration_anomaly/test_anomaly_registry.py +++ b/tests/integration_anomaly/test_anomaly_registry.py @@ -295,33 +295,33 @@ def test_registry_stores_metadata( assert record["training"]["metrics"] is not None -def test_expected_anomaly_rate_applied_when_contamination_unset( +def test_the_contamination_default_reaches_the_persisted_hyperparameters( spark: SparkSession, make_random: Callable[[int], str], anomaly_engine, anomaly_registry_prefix ): - """Verify expected_anomaly_rate sets contamination when not explicitly provided.""" + """The default the removed training parameter used to supply is now the config field's own, and it + must still travel to the registry. + + Replaces a test that passed that parameter explicitly at 0.02 and asserted contamination came out at + 0.02. The parameter is gone; the value it produced is unchanged, so this asserts the same end state + by the shorter route. Worth keeping rather than deleting: it is the only test that the algorithm + config flows all the way into ``training.hyperparameters``. + """ unique_id = make_random(8).lower() registry_table = f"{anomaly_registry_prefix}.{unique_id}_registry" - model_name = f"{anomaly_registry_prefix}.test_expected_rate_{make_random(4).lower()}" + model_name = f"{anomaly_registry_prefix}.test_contamination_{make_random(4).lower()}" - training_data = get_standard_2d_training_data() - train_df = spark.createDataFrame(training_data, "amount double, quantity double") - - params = AnomalyParams(algorithm_config=IsolationForestConfig(contamination=None)) - expected_rate = 0.02 + train_df = spark.createDataFrame(get_standard_2d_training_data(), "amount double, quantity double") anomaly_engine.train( df=train_df, columns=["amount", "quantity"], model_name=model_name, registry_table=registry_table, - params=params, - expected_anomaly_rate=expected_rate, ) record = spark.table(registry_table).filter(f"identity.model_name = '{model_name}'").first() assert record is not None - contamination = float(record["training"]["hyperparameters"]["contamination"]) - assert contamination == expected_rate + assert float(record["training"]["hyperparameters"]["contamination"]) == 0.02 def test_nonexistent_registry_returns_none(spark: SparkSession, anomaly_registry_prefix): diff --git a/tests/integration_anomaly/test_anomaly_training_validation.py b/tests/integration_anomaly/test_anomaly_training_validation.py index 9735d92c0..bce17c3b9 100644 --- a/tests/integration_anomaly/test_anomaly_training_validation.py +++ b/tests/integration_anomaly/test_anomaly_training_validation.py @@ -28,21 +28,6 @@ def test_train_rejects_invalid_sample_fraction(anomaly_engine, spark, make_schem ) -def test_train_rejects_invalid_expected_anomaly_rate(anomaly_engine, spark, make_schema, make_random): - schema = make_schema(catalog_name=TEST_CATALOG).name - model_name = f"{TEST_CATALOG}.{schema}.invalid_expected_{make_random(4).lower()}" - registry_table = f"{TEST_CATALOG}.{schema}.invalid_expected_reg_{make_random(4).lower()}" - - with pytest.raises(InvalidParameterError, match="expected_anomaly_rate"): - anomaly_engine.train( - df=_build_training_df(spark), - model_name=model_name, - registry_table=registry_table, - columns=["amount", "quantity"], - expected_anomaly_rate=0.9, - ) - - def test_train_rejects_invalid_contamination(anomaly_engine, spark, make_schema, make_random): schema = make_schema(catalog_name=TEST_CATALOG).name model_name = f"{TEST_CATALOG}.{schema}.invalid_contam_{make_random(4).lower()}" @@ -73,7 +58,6 @@ def test_build_context_raises_for_empty_model_name(spark, make_schema, make_rand columns=["amount", "quantity"], params=None, exclude_columns=None, - expected_anomaly_rate=0.02, ) @@ -92,7 +76,6 @@ def test_build_context_raises_for_empty_registry_table(spark, make_schema, make_ columns=["amount", "quantity"], params=None, exclude_columns=None, - expected_anomaly_rate=0.02, ) @@ -114,7 +97,6 @@ def test_build_context_raises_for_empty_columns(spark, make_schema, make_random) columns=[], params=None, exclude_columns=None, - expected_anomaly_rate=0.02, ) @@ -168,7 +150,6 @@ def test_build_context_logs_validation_warnings(spark, make_schema, make_random, columns=["amount", "quantity"], params=params, exclude_columns=None, - expected_anomaly_rate=0.02, ) assert "Training with 2 columns" in caplog.text @@ -191,7 +172,6 @@ def test_build_context_excludes_columns_from_auto_discovery(spark, make_schema, columns=None, params=None, exclude_columns=["b"], - expected_anomaly_rate=0.02, ) assert ctx.df_filtered.columns == ["a", "c"] assert ctx.auto_discovery_used is True @@ -214,5 +194,4 @@ def test_build_context_raises_when_exclude_columns_not_in_dataframe(spark, make_ columns=["a", "b"], params=None, exclude_columns=["c"], - expected_anomaly_rate=0.02, ) diff --git a/tests/unit/test_anomaly_configs.py b/tests/unit/test_anomaly_configs.py index a976ad0e5..f35504cf8 100644 --- a/tests/unit/test_anomaly_configs.py +++ b/tests/unit/test_anomaly_configs.py @@ -20,7 +20,9 @@ def test_isolation_forest_config_defaults(): """Test IsolationForestConfig with default values.""" cfg = IsolationForestConfig() - assert cfg.contamination is None + # 0.02 rather than None since `expected_anomaly_rate` was removed: that parameter existed only to + # fill this field when unset, and defaulted to 0.02, so the effective default is unchanged. + assert cfg.contamination == 0.02 assert cfg.num_trees == 200 assert cfg.random_seed == 42 diff --git a/tests/unit/test_anomaly_isolation_forest_inertness.py b/tests/unit/test_anomaly_isolation_forest_inertness.py index c345f2fc9..ff3c806fa 100644 --- a/tests/unit/test_anomaly_isolation_forest_inertness.py +++ b/tests/unit/test_anomaly_isolation_forest_inertness.py @@ -54,9 +54,9 @@ def _reference_params() -> AnomalyParams: """Params as production actually presents them to ``fit_sklearn_model``. - ``contamination`` is explicit because a bare ``AnomalyParams()`` cannot be fitted at all: - ``IsolationForestConfig.contamination`` defaults to None and scikit-learn rejects that. Production - fills it from *expected_anomaly_rate* (default 0.02) before training, so 0.02 is the real default. + ``contamination`` is stated explicitly to keep this fixture readable, though it is now redundant: + ``IsolationForestConfig.contamination`` defaults to 0.02 directly. It used to default to None and be + filled by a training parameter that has since been removed, which is why it is spelled out here. """ return AnomalyParams(algorithm_config=IsolationForestConfig(contamination=0.02)) diff --git a/tests/unit/test_anomaly_mahalanobis_detector.py b/tests/unit/test_anomaly_mahalanobis_detector.py index b132f18f6..3265e4a07 100644 --- a/tests/unit/test_anomaly_mahalanobis_detector.py +++ b/tests/unit/test_anomaly_mahalanobis_detector.py @@ -186,10 +186,11 @@ def test_small_samples_fall_back_to_shrinkage(caplog): def test_contamination_moves_the_predict_boundary_but_not_the_scores(): """Pinned because the docstrings now promise exactly this, and a plausible "fix" would break it. - ``expected_anomaly_rate`` flows into ``contamination``, which for both shipping detectors places - ``offset_`` and therefore only ``predict`` / ``decision_function``. Every DQX scoring path reads - ``-score_samples`` and ranks it against the training score quantiles, so the parameter cannot change - which rows are flagged -- the check's ``threshold`` does that. + ``contamination`` places ``offset_`` for both shipping detectors, and therefore only ``predict`` / + ``decision_function``. Every DQX scoring path reads ``-score_samples`` and ranks it against the + training score quantiles, so it cannot change which rows are flagged -- the check's ``threshold`` + does that. This is why the user-facing training parameter that used to set it was removed: its name + promised a detection knob that the scoring path never consults. Asserting both halves matters. Dropping the parameter would break someone who loads the registered model and calls ``predict``; treating it as a detection knob is what the documentation used to imply. diff --git a/tests/unit/test_anomaly_utils_functions.py b/tests/unit/test_anomaly_utils_functions.py index 82b958810..9f027879d 100644 --- a/tests/unit/test_anomaly_utils_functions.py +++ b/tests/unit/test_anomaly_utils_functions.py @@ -3,7 +3,6 @@ import pytest from pyspark.sql import types as T -from databricks.labs.dqx.anomaly.training_service import AnomalyTrainingService from databricks.labs.dqx.anomaly.validation import validate_training_params from databricks.labs.dqx.anomaly.transformers import ( ColumnTypeInfo, @@ -15,65 +14,51 @@ from tests.unit.anomaly_test_constants import STANDARD_REGION_PRODUCT_FEATURES # ============================================================================ -# Expected anomaly rate / contamination (service helpers) +# Contamination default (the parameter that replaced expected_anomaly_rate) # ============================================================================ -def test_apply_expected_anomaly_rate_with_none_params_uses_default_params(): - """When params is None, method uses AnomalyParams() and applies expected_anomaly_rate.""" - updated = AnomalyTrainingService.apply_expected_anomaly_rate_if_default_contamination(None, 0.02) - assert updated.algorithm_config.contamination == 0.02 +def test_contamination_defaults_to_the_rate_the_removed_parameter_used_to_supply(): + """`expected_anomaly_rate` is removed; the effective default must not have moved with it. + It existed only to fill `contamination` when unset, defaulting to 0.02. That value is now the field's + own default, so a caller who passed nothing gets exactly what they got before. + """ + assert AnomalyParams().algorithm_config.contamination == 0.02 -def test_expected_anomaly_rate_applies_when_contamination_unset(): - """expected_anomaly_rate should set contamination when unset (None).""" - params = AnomalyParams(algorithm_config=IsolationForestConfig(contamination=None)) - updated = AnomalyTrainingService.apply_expected_anomaly_rate_if_default_contamination(params, 0.02) - assert updated.algorithm_config.contamination == 0.02 - # Ensure caller params were not mutated - assert params.algorithm_config.contamination is None - - -def test_expected_anomaly_rate_does_not_override_explicit_contamination(): - """expected_anomaly_rate should not override explicit contamination.""" +def test_contamination_can_still_be_set_explicitly(): + """The escape hatch for anyone who was setting the removed parameter. It reaches only the estimator's + own `predict`/`offset_`, never DQX's scoring, which is why the user-facing name went away.""" params = AnomalyParams(algorithm_config=IsolationForestConfig(contamination=0.15)) - updated = AnomalyTrainingService.apply_expected_anomaly_rate_if_default_contamination(params, 0.02) - assert updated.algorithm_config.contamination == 0.15 - # Ensure caller params were not mutated assert params.algorithm_config.contamination == 0.15 @pytest.mark.parametrize( - ("params", "expected_rate", "error_match"), + ("params", "error_match"), [ - (AnomalyParams(sample_fraction=0.0), 0.02, "params.sample_fraction must be > 0.0"), - (AnomalyParams(train_ratio=1.1), 0.02, "params.train_ratio must be <= 1.0"), - (AnomalyParams(max_rows=0), 0.02, "params.max_rows must be >= 1"), - (AnomalyParams(ensemble_size=0), 0.02, "params.ensemble_size must be >= 1"), + (AnomalyParams(sample_fraction=0.0), "params.sample_fraction must be > 0.0"), + (AnomalyParams(train_ratio=1.1), "params.train_ratio must be <= 1.0"), + (AnomalyParams(max_rows=0), "params.max_rows must be >= 1"), + (AnomalyParams(ensemble_size=0), "params.ensemble_size must be >= 1"), ( AnomalyParams(algorithm_config=IsolationForestConfig(contamination=0.9)), - 0.02, "params.algorithm_config.contamination must be <= 0.5", ), ( AnomalyParams(algorithm_config=IsolationForestConfig(num_trees=0)), - 0.02, "params.algorithm_config.num_trees must be >= 1", ), ( AnomalyParams(algorithm_config=IsolationForestConfig(subsampling_rate=0.0)), - 0.02, "params.algorithm_config.subsampling_rate must be > 0.0", ), - (AnomalyParams(), 0.0, "expected_anomaly_rate must be > 0.0"), - (AnomalyParams(), 0.8, "expected_anomaly_rate must be <= 0.5"), ], ) -def test_validate_training_params_rejects_invalid_ranges(params: AnomalyParams, expected_rate: float, error_match: str): +def test_validate_training_params_rejects_invalid_ranges(params: AnomalyParams, error_match: str): with pytest.raises(InvalidParameterError, match=error_match): - validate_training_params(params, expected_rate) + validate_training_params(params) # ============================================================================ diff --git a/tests/unit/test_anomaly_validation.py b/tests/unit/test_anomaly_validation.py index 80d5f1b74..8924f211f 100644 --- a/tests/unit/test_anomaly_validation.py +++ b/tests/unit/test_anomaly_validation.py @@ -15,7 +15,7 @@ TrainingMetadata, ) from databricks.labs.dqx.anomaly.validation import validate_sklearn_compatibility, validate_training_params -from databricks.labs.dqx.config import AnomalyParams +from databricks.labs.dqx.config import AnomalyParams, IsolationForestConfig from databricks.labs.dqx.errors import InvalidParameterError @@ -441,66 +441,63 @@ def test_error_message_formats(): def test_validate_training_params_accepts_defaults(): - """Default AnomalyParams with a valid expected_anomaly_rate should not raise.""" - validate_training_params(AnomalyParams(), expected_anomaly_rate=0.02) + """Default AnomalyParams should not raise.""" + validate_training_params(AnomalyParams()) def test_validate_training_params_rejects_non_numeric_sample_fraction(): """Test that non-numeric sample_fraction raises (validation.py 64-65).""" params = AnomalyParams(sample_fraction="0.5") # type: ignore[arg-type] with pytest.raises(InvalidParameterError, match="must be a numeric value"): - validate_training_params(params, expected_anomaly_rate=0.02) + validate_training_params(params) def test_validate_training_params_rejects_bool_sample_fraction(): """Test that bool sample_fraction raises.""" params = AnomalyParams(sample_fraction=True) # type: ignore[arg-type] with pytest.raises(InvalidParameterError, match="must be a numeric value"): - validate_training_params(params, expected_anomaly_rate=0.02) + validate_training_params(params) def test_validate_training_params_rejects_zero_sample_fraction(): with pytest.raises(InvalidParameterError, match="params.sample_fraction"): - validate_training_params(AnomalyParams(sample_fraction=0.0), expected_anomaly_rate=0.02) + validate_training_params(AnomalyParams(sample_fraction=0.0)) def test_validate_training_params_rejects_sample_fraction_above_one(): with pytest.raises(InvalidParameterError, match="params.sample_fraction"): - validate_training_params(AnomalyParams(sample_fraction=1.1), expected_anomaly_rate=0.02) + validate_training_params(AnomalyParams(sample_fraction=1.1)) -def test_validate_training_params_rejects_zero_expected_anomaly_rate(): - with pytest.raises(InvalidParameterError, match="expected_anomaly_rate"): - validate_training_params(AnomalyParams(), expected_anomaly_rate=0.0) - - -def test_validate_training_params_rejects_expected_anomaly_rate_above_half(): - with pytest.raises(InvalidParameterError, match="expected_anomaly_rate"): - validate_training_params(AnomalyParams(), expected_anomaly_rate=0.6) +def test_validate_training_params_still_range_checks_contamination(): + """The removed `expected_anomaly_rate` used to fill contamination. Setting it directly is now the + only route, so its bounds matter more than before, not less.""" + with pytest.raises(InvalidParameterError, match="contamination"): + validate_training_params(AnomalyParams(algorithm_config=IsolationForestConfig(contamination=0.9))) def test_validate_training_params_rejects_non_integer_max_rows(): """Test that non-integer max_rows raises.""" params = AnomalyParams(max_rows=1000.5) # type: ignore[arg-type] with pytest.raises(InvalidParameterError, match="must be an integer"): - validate_training_params(params, expected_anomaly_rate=0.02) + validate_training_params(params) def test_validate_training_params_rejects_bool_max_rows(): """Test that bool max_rows raises.""" params = AnomalyParams(max_rows=True) # type: ignore[arg-type] with pytest.raises(InvalidParameterError, match="must be an integer"): - validate_training_params(params, expected_anomaly_rate=0.02) + validate_training_params(params) def test_validate_training_params_rejects_zero_max_rows(): with pytest.raises(InvalidParameterError, match="params.max_rows"): - validate_training_params(AnomalyParams(max_rows=0), expected_anomaly_rate=0.02) + validate_training_params(AnomalyParams(max_rows=0)) def test_validate_training_params_rejects_zero_ensemble_size(): with pytest.raises(InvalidParameterError, match="params.ensemble_size"): - validate_training_params(AnomalyParams(ensemble_size=0), expected_anomaly_rate=0.02) + validate_training_params(AnomalyParams(ensemble_size=0)) # ============================================================================ From 661496c8175fb2aeb5a128ef10f0daf2f90870e0 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 12:00:17 +0100 Subject: [PATCH 092/107] Document the expected_anomaly_rate removal in the guide's upgrade path The removal itself landed without touching the guide, and the guide's upgrade section opened with "If you never used `segment_by`, nothing here affects you" -- which that removal made false. Anyone passing `expected_anomaly_rate` now gets a TypeError regardless of whether they ever touched `segment_by`. Rewritten so the section states what actually needs action: every model must be retrained whether or not any of this was used, and calls passing either removed parameter will raise. Adds a migration bullet saying nothing replaces it and why -- `threshold` on the check is how you change what gets flagged, and `params.algorithm_config.contamination` is the escape hatch for anyone who loads the registered model and calls `predict` themselves. Also an FAQ entry, because the TypeError names the parameter but not the fix, and the obvious guess (reach for contamination) is usually the wrong one. Checked repo-wide rather than assumed: no demo, notebook, Studio app, MCP server or reference page ever passed it, and the only remaining mentions are comments explaining the removal. The generated API pages regenerate clean from the docstrings. Co-authored-by: Isaac --- docs/dqx/docs/guide/row_anomaly_detection/index.mdx | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index be970e32c..58cbe90f0 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -609,11 +609,12 @@ Both profiles get the same feature engineering, so these need no configuration: ## Upgrading and breaking changes -Row anomaly detection was **Experimental** in earlier releases and is now **Beta**. This release replaces the old per-group model path (`segment_by`) with `baseline_by`, which measured better on detection, reliability, and cost. If you never used `segment_by`, nothing here affects you. +Row anomaly detection was **Experimental** in earlier releases and is now **Beta**. This release replaces the old per-group model path (`segment_by`) with `baseline_by`, which measured better on detection, reliability, and cost, and removes one parameter that never did what its name promised. -If you did, here is the whole migration. +Two changes need action from you, and the rest is handled. **Every existing model must be retrained** whether or not you used any of this. And if you passed `segment_by` or `expected_anomaly_rate` to `train()`, those calls will now raise. - **Replace `segment_by` with `baseline_by`.** They are not the same. `segment_by` trained one model per group; `baseline_by` judges each metric against its own group's baseline on a single model. Your scores will change. `AnomalyParams.max_segment_models` is gone too, since there is only ever one model now. +- **Remove `expected_anomaly_rate`.** It set the estimator's `contamination`, which places only scikit-learn's own `predict` boundary, so it never changed which rows DQX flagged. Nothing replaces it, because nothing needs to: to change how much gets flagged, set `threshold` on the check, which is an alert budget over training severity. If you were relying on `contamination` itself — because you load the registered model and call `predict` yourself — set `params.algorithm_config.contamination` instead. The default is unchanged at 0.02, so a call that did not pass it behaves identically. - **Retrain your models.** A model trained before this release raises an error at scoring time until you retrain it, rather than scoring against a feature list that no longer matches. - **Your registry table is handled for you.** Retraining writes the new schema into your existing table in place. The `segmentation` column becomes `grouping`, and a table you never retrain into keeps the old column and will not be read. - **`_dq_info[].anomaly` changes shape.** The `segment` field is gone (it was always null once per-group models were), and `is_new_baseline`, `new_baseline_key`, `is_stale_baseline` and `stale_baseline_horizon` are added. Named-field queries keep working, but appending to a table that already holds `_dq_info` needs `mergeSchema`. @@ -738,6 +739,14 @@ Use row anomaly detection when you want to catch unusual combinations across col `segment_by` has been removed. Replace it with `baseline_by`, which compares each metric against its own group's baseline on a single model instead of training one model per group, and retrain (models from earlier releases no longer load). Row anomaly detection was Experimental in earlier releases, which is what allowed this break, and it is now Beta. See [Upgrading and breaking changes](#upgrading-and-breaking-changes).
+
+Q: `train()` says it got an unexpected keyword argument `expected_anomaly_rate`. What replaced it? + +Nothing, deliberately. It supplied the estimator's `contamination`, which places scikit-learn's own `predict` boundary and nothing else — DQX scores by ranking against training-score quantiles, so the parameter never changed which rows were flagged despite its name. + +To control how much gets flagged, set `threshold` on the check: it is an alert budget over training severity, so `threshold=95` flags the top 5%. To set `contamination` because you load the registered model and call `predict` yourself, pass `params=AnomalyParams(algorithm_config=IsolationForestConfig(contamination=...))`. Its default is unchanged, so removing the argument alone changes no behaviour. +
+
Q: Are there anomalies DQX will not find? From 2d3b004efa96275ce9b1b03594de9030e6c36483 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 13:12:30 +0100 Subject: [PATCH 093/107] Withdraw the bucket-median robustness claim from the second place it was made An earlier commit today rewrote the `TEMPORAL_FIT_BUCKETS` comment to say plainly that bucketing bounds driver memory and that Huber, not the per-bucket median, is what carries robustness. It missed the identical claim in `_fit_temporal_from_buckets`' own docstring, 200 lines further down the same file: "the per-bucket median is itself robust, so gross outliers are attenuated before the Huber fit ever sees them". So the file contradicted itself, with the measurements on one side and the withdrawn claim on the other. That is worse than the original overstatement: a reader who finds the docstring first has no way to know it is the stale one. Now points at the constant, which carries the numbers, and says outright that bucketing is for memory rather than robustness. Found by an agent building a code walkthrough, which read both sites and refused to paper over the disagreement. Grepping for the claim I was withdrawing, rather than editing the site I happened to be looking at, would have caught it at the time. Co-authored-by: Isaac --- src/databricks/labs/dqx/anomaly/transformers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 2e7a5cc49..95da61492 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -1104,8 +1104,13 @@ def _fit_temporal_from_buckets( The fit needs the data on the driver, and a table can be arbitrarily large, so the frame is first reduced to at most :data:`TEMPORAL_FIT_BUCKETS` time buckets carrying each metric's median. That - bounds driver memory the same way ``_compute_baseline_medians`` does, and the per-bucket median is - itself robust, so gross outliers are attenuated before the Huber fit ever sees them. + bounds driver memory the same way ``_compute_baseline_medians`` does. + + Bucketing is for driver memory, **not** for robustness: training sees about 24% of the table, so + buckets hold too few rows for their median to attenuate much, and the Huber loss is what actually + carries robustness. The measurements are on :data:`TEMPORAL_FIT_BUCKETS`; this docstring used to claim + the opposite here while the constant said otherwise, which is worth knowing about only because two + places in one file disagreeing is how a later reader ends up optimising away the part that mattered. The basis is then selected against the *bucket centres* rather than the raw timestamps, which matters: the resolution guard in :func:`~databricks.labs.dqx.anomaly.temporal.candidate_periods` then measures From f6f72795a8ffe7def460e842f8e7e572193c7f3a Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 18:49:32 +0100 Subject: [PATCH 094/107] Read feature provenance from the recorded transform, not from spelling Third-review finding, and a gap my own earlier fix opened. Stage 1 of that work made a schema legal that had not been before: a caller may pass their own `amount_rel_baseline` column alongside `amount`, because the collision validator only refuses a name the transform would actually generate, and with no `baseline_by` nothing generates it. The reverse mapper never learned that. It decomposed the name by suffix, credited `amount`, and labelled the feature "amount vs its group baseline" in a model with no group baseline at all -- attributing one user column's contribution to a different column, and making redaction of `amount` sweep up an unrelated one. Two changes, both about asking the metadata rather than the string: - An exact source-column match now wins over suffix decomposition. A name in `column_infos` is a column that was analysed, so identity cannot be wrong, and it cannot mask a genuine derived feature because `validate_generated_feature_names` refuses that collision. - Suffix resolution is gated on the basis having run: `_rel_baseline` only decomposes when `baseline_by` is recorded, `_rel_time` only when `baseline_over_time` is. Both were already persisted. The module docstring said the resolution order "must not be reordered", justified by a numeric column named `revenue_freq` where `revenue` does not exist. That case resolves correctly under either order, so it never argued for suffix-first. Corrected, with the reason that does hold. Adds `source_blocks`, mapping each source to its engineered descendants. Redaction gets it for free because `engineered_from` is already defined in terms of `source_column`, and the next commit needs it: explaining one engineered feature at a time is unsound when several carry the same source. The gate immediately caught three test fixtures describing models that cannot exist -- a baseline-relative feature listed with no `baseline_by` recorded -- in the redaction, explainer and naming suites. Fixed rather than worked around; a fixture that cannot occur in production is not protecting anything. Verified with the review's own probe: `amount_rel_baseline` now resolves to itself and labels as itself. Co-authored-by: Isaac --- .../labs/dqx/anomaly/feature_naming.py | 96 ++++++++++++++++--- .../test_anomaly_explanation_redaction.py | 11 ++- tests/unit/test_anomaly_feature_naming.py | 91 +++++++++++++++++- tests/unit/test_anomaly_llm_explainer.py | 4 + 4 files changed, 185 insertions(+), 17 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/feature_naming.py b/src/databricks/labs/dqx/anomaly/feature_naming.py index 65a6f686a..42350d8d8 100644 --- a/src/databricks/labs/dqx/anomaly/feature_naming.py +++ b/src/databricks/labs/dqx/anomaly/feature_naming.py @@ -14,12 +14,25 @@ Pure functions over *SparkFeatureMetadata*: no Spark, no I/O, deterministic. -Resolution order is deliberate and must not be reordered. One-hot names are matched first, against -the recorded ``onehot_categories``, because a category *value* may itself end in a fixed suffix -(a column *status* with a value *freq* produces ``status_freq``, which a suffix-first scan would -misread as frequency encoding of *status*). Fixed suffixes are tried next, and only accepted when -the remainder is a known source column, so a numeric column literally named ``revenue_freq`` is not -mistaken for the frequency encoding of a non-existent *revenue*. Numeric identity is matched last. +Resolution order is deliberate. One-hot names are matched first, against the recorded +``onehot_categories``, because a category *value* may itself end in a fixed suffix (a column *status* +with a value *freq* produces ``status_freq``, which a suffix-first scan would misread as frequency +encoding of *status*). + +**An exact source-column match comes next, before suffix decomposition.** Both can match the same name +at once: a caller may pass their own ``amount_rel_baseline`` column alongside ``amount``, which feature +engineering permits when no grouping is configured. Decomposing that name attributed one user column's +contribution to a different one and labelled it "amount vs its group baseline" in a model with no group +baseline. Identity cannot be wrong here -- a name in ``column_infos`` is a column that was analysed -- +and it cannot mask a genuine derived feature, because ``validation.validate_generated_feature_names`` +refuses a schema where a generated name collides with a real column. + +An earlier version of this note said the order must not be reordered, justified by a numeric column +named ``revenue_freq`` where *revenue* does not exist. That case resolves correctly under either order, +so it never argued for suffix-first. + +Fixed suffixes are tried last, accepted only when the remainder is a known source column **and** the +transform that appends the suffix actually ran for this model -- see ``_basis_ran``. """ from databricks.labs.dqx.anomaly.transformers import ( @@ -71,11 +84,28 @@ def _match_onehot(engineered_name: str, metadata: SparkFeatureMetadata) -> tuple return None -def _match_suffix(engineered_name: str, source_names: frozenset[str]) -> tuple[str, str] | None: - """Return (source_column, label_template) if *engineered_name* ends in a known fixed suffix - and the remainder is a real source column.""" +def _basis_ran(suffix: str, metadata: SparkFeatureMetadata) -> bool: + """Whether the transform that appends *suffix* actually ran for this model. + + A comparison suffix only means a derived feature when its basis is recorded: without ``baseline_by`` + nothing appends ``_rel_baseline``, and without ``baseline_over_time`` nothing appends ``_rel_time``. + Checking this is what stops a user's own column named ``amount_rel_baseline`` being reported as + "amount vs its group baseline" in a model that has no group baseline at all. + """ + if suffix == BASELINE_RELATIVE_SUFFIX: + return bool(metadata.baseline_by) + if suffix == TEMPORAL_RELATIVE_SUFFIX: + return bool(metadata.baseline_over_time) + return True + + +def _match_suffix( + engineered_name: str, source_names: frozenset[str], metadata: SparkFeatureMetadata +) -> tuple[str, str] | None: + """Return (source_column, label_template) if *engineered_name* ends in a known fixed suffix, + the remainder is a real source column, and the transform that appends that suffix ran.""" for suffix, template in _SUFFIX_LABELS: - if engineered_name.endswith(suffix): + if engineered_name.endswith(suffix) and _basis_ran(suffix, metadata): col = engineered_name[: -len(suffix)] if col in source_names: return col, template @@ -98,12 +128,16 @@ def source_column(engineered_name: str, metadata: SparkFeatureMetadata) -> str | return onehot[0] source_names = _source_column_names(metadata) - suffix = _match_suffix(engineered_name, source_names) - if suffix is not None: - return suffix[0] - if engineered_name in source_names: + # A feature that *is* an analysed column is that column, whatever it is spelled like. Checked + # before suffix decomposition because both can match at once: a caller may pass their own + # ``amount_rel_baseline`` column alongside ``amount``, and decomposing it would attribute one + # user column's contribution to a different one. return engineered_name + + suffix = _match_suffix(engineered_name, source_names, metadata) + if suffix is not None: + return suffix[0] return None @@ -123,7 +157,10 @@ def human_label(engineered_name: str, metadata: SparkFeatureMetadata) -> str: return f"{col} = {value}" source_names = _source_column_names(metadata) - suffix = _match_suffix(engineered_name, source_names) + if engineered_name in source_names: + return engineered_name + + suffix = _match_suffix(engineered_name, source_names, metadata) if suffix is not None: col, template = suffix return template.format(col=col) @@ -146,3 +183,32 @@ def engineered_from(source: str, metadata: SparkFeatureMetadata) -> frozenset[st metadata: The persisted feature metadata for the model. """ return frozenset(name for name in metadata.engineered_feature_names if source_column(name, metadata) == source) + + +def source_blocks(metadata: SparkFeatureMetadata) -> dict[str, list[str]]: + """Each source column mapped to the engineered features derived from it, in feature order. + + The forward view of :func:`source_column`, and the grouping attribution needs: explaining one + engineered feature at a time is unsound when several of them carry the same source. Dropping one view + of a metric leaves another copy behind, so the measured loss is small for every view, and normalising + those small numbers hands almost all of the apparent blame to an unrelated metric. Measured on a + correlated pair, adding one affine duplicate of *x* moved the reported cause from 99.8% *x* to 99.9% + *y* while the score did not move. + + A feature that resolves to no source becomes its own single-member block, so every engineered feature + belongs to exactly one block and nothing is silently dropped from an explanation. + + Order is preserved twice over: blocks appear in first-appearance order, and features within a block + keep their positional order, because ``engineered_feature_names`` is positional and callers index + into the feature matrix with it. + + Args: + metadata: The persisted feature metadata for the model. + + Returns: + source column -> its engineered feature names. + """ + blocks: dict[str, list[str]] = {} + for name in metadata.engineered_feature_names: + blocks.setdefault(source_column(name, metadata) or name, []).append(name) + return blocks diff --git a/tests/unit/test_anomaly_explanation_redaction.py b/tests/unit/test_anomaly_explanation_redaction.py index d33f05d80..873eec2c8 100644 --- a/tests/unit/test_anomaly_explanation_redaction.py +++ b/tests/unit/test_anomaly_explanation_redaction.py @@ -43,7 +43,15 @@ def test_similar_prefixes_are_not_swept_up(): def _metadata() -> SparkFeatureMetadata: """Feature metadata with a categorical column expanded to one-hot + frequency + null-indicator, - alongside a numeric column with a baseline-relative feature.""" + alongside a numeric column with a baseline-relative feature. + + ``baseline_by`` names a column that is deliberately absent from *column_infos*: a grouping column is + the basis a metric is compared against, never a feature, so it is projected out. Recording it is not + decoration -- suffix resolution is gated on the basis having actually run, because otherwise a user's + own column named ``amount_rel_baseline`` would be reported as "amount vs its group baseline" in a + model with no group baseline. An earlier version of this fixture omitted it while still listing + ``amount_rel_baseline``, describing a model that cannot exist. + """ return SparkFeatureMetadata( column_infos=[ {"name": "amount", "category": "numeric"}, @@ -59,6 +67,7 @@ def _metadata() -> SparkFeatureMetadata: "country_is_null", "amount_rel_baseline", ], + baseline_by=["region"], ) diff --git a/tests/unit/test_anomaly_feature_naming.py b/tests/unit/test_anomaly_feature_naming.py index 22ba7683d..fc50adafb 100644 --- a/tests/unit/test_anomaly_feature_naming.py +++ b/tests/unit/test_anomaly_feature_naming.py @@ -2,7 +2,12 @@ import pytest -from databricks.labs.dqx.anomaly.feature_naming import engineered_from, human_label, source_column +from databricks.labs.dqx.anomaly.feature_naming import ( + engineered_from, + human_label, + source_blocks, + source_column, +) from databricks.labs.dqx.anomaly.transformers import SparkFeatureMetadata @@ -140,3 +145,87 @@ def test_unknown_feature_resolves_to_none_and_labels_unchanged(metadata: SparkFe """A feature from a convention this version does not know never crashes and is never hidden.""" assert source_column("mystery_feature", metadata) is None assert human_label("mystery_feature", metadata) == "mystery_feature" + + +# ── provenance is read from the recorded transform, not from spelling ──────────────────────────────── + + +def _ungrouped_with_a_literal_suffix_column() -> SparkFeatureMetadata: + """A schema feature engineering permits: the caller's own ``amount_rel_baseline`` column, no grouping. + + Legal because ``validate_generated_feature_names`` only refuses a collision when the transform would + actually generate that name, and with no ``baseline_by`` nothing does. + """ + return SparkFeatureMetadata( + column_infos=[ + {"name": "amount", "category": "numeric"}, + {"name": "amount_rel_baseline", "category": "numeric"}, + {"name": "latency_rel_time", "category": "numeric"}, + ], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["amount", "amount_rel_baseline", "latency_rel_time"], + baseline_by=[], + baseline_over_time="", + ) + + +def test_a_literal_column_ending_in_a_suffix_resolves_to_itself(): + """The reported defect: the reverse map decomposed a real column and credited a different one. + + ``amount_rel_baseline`` is the caller's own metric here. Resolving it to ``amount`` attributes one + column's contribution to another, and it makes redaction of ``amount`` sweep up an unrelated column. + """ + ungrouped = _ungrouped_with_a_literal_suffix_column() + + assert source_column("amount_rel_baseline", ungrouped) == "amount_rel_baseline" + assert source_column("latency_rel_time", ungrouped) == "latency_rel_time" + + +def test_a_literal_column_ending_in_a_suffix_is_labelled_as_itself(): + """It was labelled "amount vs its group baseline" in a model with no group baseline at all.""" + ungrouped = _ungrouped_with_a_literal_suffix_column() + + assert human_label("amount_rel_baseline", ungrouped) == "amount_rel_baseline" + assert human_label("latency_rel_time", ungrouped) == "latency_rel_time" + + +def test_redacting_one_column_does_not_sweep_up_a_similarly_named_one(): + """The same defect seen from the redaction side, which is the one with a privacy consequence.""" + ungrouped = _ungrouped_with_a_literal_suffix_column() + + assert engineered_from("amount", ungrouped) == frozenset({"amount"}) + + +def test_a_genuine_derived_feature_still_resolves_when_its_basis_ran(metadata: SparkFeatureMetadata): + """The other direction, so the gate cannot be satisfied by refusing everything. + + The shared fixture records both bases, so both suffixes must still decompose. + """ + assert source_column("amount_rel_baseline", metadata) == "amount" + assert source_column("amount_rel_time", metadata) == "amount" + assert human_label("amount_rel_baseline", metadata) == "amount vs its group baseline" + + +def test_source_blocks_group_every_feature_under_exactly_one_source(metadata: SparkFeatureMetadata): + """What block attribution indexes with: every engineered feature in exactly one block, order kept.""" + blocks = source_blocks(metadata) + + assert blocks["amount"] == ["amount", "amount_rel_baseline", "amount_rel_time"] + assert blocks["country"] == ["country_US", "country_DE", "country_is_null"] + flattened = [name for names in blocks.values() for name in names] + assert sorted(flattened) == sorted(metadata.engineered_feature_names) + assert len(flattened) == len(set(flattened)) + + +def test_source_blocks_keep_an_unresolvable_feature_as_its_own_block(): + """A model trained by a newer DQX may carry a convention this version cannot parse. It must still be + explainable rather than silently dropped from the map.""" + future = SparkFeatureMetadata( + column_infos=[{"name": "amount", "category": "numeric"}], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["amount", "amount_some_future_transform"], + ) + + assert source_blocks(future)["amount_some_future_transform"] == ["amount_some_future_transform"] diff --git a/tests/unit/test_anomaly_llm_explainer.py b/tests/unit/test_anomaly_llm_explainer.py index e940fc6d4..22ffcb64e 100644 --- a/tests/unit/test_anomaly_llm_explainer.py +++ b/tests/unit/test_anomaly_llm_explainer.py @@ -188,6 +188,10 @@ def test_human_labels_map_omits_identity_and_labels_derived_features(): categorical_frequency_maps={"country": {"US": 0.7}}, onehot_categories={"country": ["US"]}, engineered_feature_names=["amount", "amount_rel_baseline", "country_US", "country_freq"], + # Recorded because suffix resolution is gated on it: without a basis, ``amount_rel_baseline`` + # would be a column in its own right rather than a derived feature, and would correctly label + # as itself. A grouping column is not a feature, so it is absent from column_infos. + baseline_by=["region"], ) labels = llm_explainer._human_labels(metadata) From 9cb049e45b745808c346007e51a133df6024b4ed Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 21:25:16 +0100 Subject: [PATCH 095/107] Explain the correlation-aware detector by source column, not by engineered feature Feature engineering gives one source column several engineered views: the metric itself, its deviation from its group's baseline, its deviation from its expected level at that time. The leave-one-out attribution then measures almost nothing for any single view, because dropping one leaves another copy of the same information behind -- and normalising those small numbers hands nearly all of the apparent blame to an unrelated metric. Measured on a correlated pair: adding one affine duplicate of x moved the reported cause from 99.8% x to 99.9% y while the score itself did not move (65.44813 vs 65.44816). So the detector found the right row and then named the wrong column, in business language, with no signal that anything was off. That compounds with the LLM prompt's rule to call a metric abnormal when contributions are "concentrated in a single metric": the guard fires on the innocent one. MahalanobisDetector.block_contributions marginalises a whole group of features out at once, (Pd)_G' inv(P_GG) (Pd)_G. Sigma^-1 is never formed -- L^-1 is recovered once per call from the stored Cholesky factor and each P_GG is built from its columns, so nothing new is persisted and models trained before this keep loading. A single-feature block reduces exactly to the existing a_i = z_i^2/(Sigma^-1)_ii, asserted at rtol=1e-12 rather than argued, so a model with no derived features explains identically to before. Block drops are not additive and the docstring says so: overlapping information belongs to no single block. Blocks come from the persisted metadata via source_block_indices, computed once on the driver and closed over by the UDF. The emitted map is therefore keyed by the source column the caller passed, which fixes the defect by construction -- one entry per source, so duplicate views cannot dilute each other -- and is more useful, since a reader gets their own column names rather than amount_rel_time. Verified end to end: the probe that produced 100% y now returns 99.7% x, matching the undisturbed two-feature model. Scope is the correlation-aware detector only. TreeSHAP's correct block aggregation is a signed sum, and the tree path still discards sign, so blocking there would add magnitudes that should have cancelled. That asymmetry is documented rather than hidden, and the signed-attribution fix it depends on is tracked separately. Two consequences of the key change, both checked rather than assumed. Redaction stays exact: it already unions the source column with every feature derived from it, and the source column is now itself the key. Label lookup falls through to the raw key, which for a source column is the user's own column name -- the intended outcome, not a gap. Both docstrings asserting that contribution keys are always engineered names are corrected, as are the guide's schema table and its "How it works" step. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 4 +- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 16 +++- .../labs/dqx/anomaly/explainability.py | 48 +++++++--- .../labs/dqx/anomaly/feature_naming.py | 13 +++ .../labs/dqx/anomaly/single_model_scorer.py | 13 ++- .../labs/dqx/anomaly/timeseries_detector.py | 58 +++++++++++ .../unit/test_anomaly_mahalanobis_detector.py | 95 +++++++++++++++++++ 7 files changed, 227 insertions(+), 20 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 58cbe90f0..e2a33f52a 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -629,7 +629,7 @@ For full parameter and schema details, see [Row Anomaly Detection in Quality Che 1. **Feature engineering** (automatic): DQX detects column types and creates features (numerical standardized, categorical one-hot or frequency-encoded, temporal expanded to hour/day/month/weekend). 2. **Smart sampling and training**: DQX samples your data (default 30%, capped at 1M rows), trains the detector your `profile` selects — an ensemble of Isolation Forest models by default, or a single correlation-aware model for `profile="timeseries"` — and captures baseline statistics for drift detection. 3. **Model registry**: Models and metadata live in MLflow and a Delta table; each `train()` call registers one model under its name and archives the previously active version. -4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. Feature contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. How they are computed depends on the detector — SHAP for Isolation Forest, an exact leave-one-out decomposition for the correlation-aware detector — but the output is the same map either way. +4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. Feature contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. How they are computed depends on the detector — SHAP for Isolation Forest, an exact decomposition for the correlation-aware detector — and so does what the map's keys name: the correlation-aware detector reports one entry per **source column you passed**, while Isolation Forest reports one per engineered feature (so a single column can appear as several entries, for example `signup_hour_sin`). Either way the values are percentages of the same total. 5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping (categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group, for example region or product category). A discovered grouping is used as `baseline_by` (see [Group-aware anomaly detection](#group-aware-anomaly-detection)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. When you *do* pass `columns`, you have decided what to measure, so DQX leaves the comparison pooled rather than adding a grouping you did not ask for. If your data looks grouped it says so in a warning naming the grouping to pass. ### Which algorithm, and why @@ -660,7 +660,7 @@ Each element is a **struct** with a shared “wide” schema. Currently, the onl | `is_anomaly` | boolean | `true` if `severity_percentile` ≥ threshold. | | `threshold` | double | Severity percentile threshold used (e.g. 95.0). | | `model` | string | Full model name (e.g. Unity Catalog name). | -| `contributions` | map<string, double> | Per-feature contribution percentages (0–100). On by default (`enable_contributions=True`); populated only for anomalous rows (`null` for non-anomalous rows or if you set it `False`). | +| `contributions` | map<string, double> | Contribution percentages (0–100). Keyed by **source column** for `profile="timeseries"` and by engineered feature for `profile="tabular"` — see step 4 of [How it works](#how-it-works). On by default (`enable_contributions=True`); populated only for anomalous rows (`null` for non-anomalous rows or if you set it `False`). | | `confidence_std` | double | Ensemble score standard deviation. Present when `enable_confidence_std=True`; `null` otherwise. | | `is_new_baseline` | boolean | `true` when the row's group was absent from training, in which case `score` and `severity_percentile` are `null` and the row is **not** flagged. See [Groups that appear after training](#groups-that-appear-after-training). | | `new_baseline_key` | string | The unrecognised group key, for rows where `is_new_baseline` is `true`; `null` otherwise. | diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 804e9e910..55f217921 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -277,11 +277,13 @@ def from_scoring_config( def redaction_set(redact_columns: tuple[str, ...], metadata: SparkFeatureMetadata | None = None) -> frozenset[str]: """Columns to redact, plus every engineered feature derived from them. - Redaction matches contribution keys exactly, and contribution keys are *engineered* feature - names. So redacting ``amount`` must also stop ``amount_rel_baseline`` -- a signed log-ratio of - the same column -- and redacting ``country`` must stop ``country_US``, ``country_DE``, - ``country_freq`` and ``country_is_null``. A caller naming a column sensitive means every feature - derived from it is sensitive too. + Redaction matches contribution keys exactly, and which vocabulary those keys use depends on the + detector: source columns where attribution is blocked by source, engineered feature names on the + tree path. Covering both is why the source column *and* its descendants go into the set. So + redacting ``amount`` must also stop ``amount_rel_baseline`` -- a signed log-ratio of the same + column -- and redacting ``country`` must stop ``country_US``, ``country_DE``, ``country_freq`` and + ``country_is_null``. A caller naming a column sensitive means every feature derived from it is + sensitive too. With *metadata*, the derived features are enumerated exactly via *engineered_from*, which closes the one-hot and frequency gap that the source column alone could not. Without it (a caller who @@ -343,6 +345,10 @@ def _human_labels(metadata: SparkFeatureMetadata | None) -> dict[str, str]: ``amount vs its group baseline``). Only entries whose label differs from the raw name are included, so the SQL lookup stays small; anything not in the map falls back to its raw name. Empty when no metadata was threaded through, in which case raw engineered names are shown. + + A map keyed by *source column* needs no entries at all: every key is already a column the reader + named, and identity labels are excluded here, so each falls through to itself. That is the intended + outcome rather than a gap -- there is nothing to translate. """ if metadata is None: return {} diff --git a/src/databricks/labs/dqx/anomaly/explainability.py b/src/databricks/labs/dqx/anomaly/explainability.py index 07d2ab6e5..08172ba80 100644 --- a/src/databricks/labs/dqx/anomaly/explainability.py +++ b/src/databricks/labs/dqx/anomaly/explainability.py @@ -68,7 +68,8 @@ def compute_row_attributions( model_local: Any, feature_matrix: pd.DataFrame, engineered_feature_cols: list[str], -) -> tuple[np.ndarray, np.ndarray]: + blocks: dict[str, list[int]] | None = None, +) -> tuple[np.ndarray, np.ndarray, list[str]]: """Per-feature attribution for each row, from whichever estimator the model wraps. Two sources, one output shape. A tree model goes through ``SHAP.TreeExplainer``, which is @@ -81,8 +82,22 @@ def compute_row_attributions( into all of them and make the dependency direction harder to reason about. Whatever the source, the values feed the same *format_shap_contributions*, so the emitted map has - identical keys, scaling and null handling either way, and everything downstream -- redaction, - human labels, the LLM prompt, the ``_dq_info`` schema -- is unaffected by which branch ran. + identical scaling and null handling either way. + + *blocks* switches an estimator that supports it to **source-block** attribution, keyed by the source + column rather than by engineered feature. That is not cosmetic: explaining engineered features one at a + time is unsound when several of them share a source, because dropping one view leaves another copy of + the same information behind. Measured, adding one affine duplicate of a metric moved the reported cause + from 99.8% that metric to 99.9% an unrelated one, while the score did not move. Blocking restores + 99.7%/0.3%, matching the undisturbed model. + + Only the exact-attribution branch takes it. TreeSHAP's correct block aggregation is a **signed** sum, + and the tree path still discards sign (see *format_shap_contributions*), so blocking there would add + magnitudes that should have cancelled. That asymmetry is deliberate and tracked separately. + + Returns: + ``(attribution, valid_indices, keys)`` where *keys* names the columns of *attribution* -- source + columns when blocked, engineered features otherwise. """ scaler = getattr(model_local, "named_steps", {}).get("scaler") estimator = getattr(model_local, "named_steps", {}).get("model", model_local) @@ -90,17 +105,23 @@ def compute_row_attributions( feature_values = scaler.transform(feature_matrix) if scaler else feature_matrix.values valid_indices = ~pd.isna(feature_values).any(axis=1) + blocked = bool(blocks) and hasattr(estimator, "block_contributions") and len(engineered_feature_cols) > 1 + keys = list(blocks) if blocked and blocks is not None else engineered_feature_cols + attribution = np.array([]) if valid_indices.any(): + rows = feature_values[valid_indices] if len(engineered_feature_cols) == 1: - attribution = np.ones((len(feature_values[valid_indices]), 1)) + attribution = np.ones((len(rows), 1)) + elif blocked and blocks is not None: + attribution = estimator.block_contributions(rows, [blocks[key] for key in keys]) elif hasattr(estimator, "feature_contributions"): - attribution = estimator.feature_contributions(feature_values[valid_indices]) + attribution = estimator.feature_contributions(rows) else: explainer = SHAP.TreeExplainer(estimator) - attribution = explainer.shap_values(feature_values[valid_indices]) + attribution = explainer.shap_values(rows) - return attribution, valid_indices + return attribution, valid_indices, keys # Severity-gating margin for in-UDF SHAP computation. The UDF recomputes severity from raw @@ -155,6 +176,7 @@ def compute_gated_shap_contributions( scores: np.ndarray, quantile_points: list[tuple[float, float]] | None, threshold: float | None, + blocks: dict[str, list[int]] | None = None, ) -> list[dict[str, float | None] | None]: """Compute SHAP contributions only for rows whose severity reaches the anomaly threshold. @@ -166,18 +188,20 @@ def compute_gated_shap_contributions( """ num_rows = len(feature_matrix) if not quantile_points or threshold is None: - attribution, valid_indices = compute_row_attributions(model_local, feature_matrix, engineered_feature_cols) - return list(format_shap_contributions(attribution, valid_indices, num_rows, engineered_feature_cols)) + attribution, valid_indices, keys = compute_row_attributions( + model_local, feature_matrix, engineered_feature_cols, blocks + ) + return list(format_shap_contributions(attribution, valid_indices, num_rows, keys)) severity = severity_from_scores(np.asarray(scores, dtype=float), quantile_points) anomalous_positions = np.flatnonzero(severity >= (float(threshold) - _SEVERITY_GATE_EPSILON)) contributions: list[dict[str, float | None] | None] = [None] * num_rows if anomalous_positions.size: subset = feature_matrix.iloc[anomalous_positions] - attribution, valid_indices = compute_row_attributions(model_local, subset, engineered_feature_cols) - subset_contributions = format_shap_contributions( - attribution, valid_indices, len(subset), engineered_feature_cols + attribution, valid_indices, keys = compute_row_attributions( + model_local, subset, engineered_feature_cols, blocks ) + subset_contributions = format_shap_contributions(attribution, valid_indices, len(subset), keys) for position, contribution in zip(anomalous_positions.tolist(), subset_contributions): contributions[position] = contribution return contributions diff --git a/src/databricks/labs/dqx/anomaly/feature_naming.py b/src/databricks/labs/dqx/anomaly/feature_naming.py index 42350d8d8..e9b61f839 100644 --- a/src/databricks/labs/dqx/anomaly/feature_naming.py +++ b/src/databricks/labs/dqx/anomaly/feature_naming.py @@ -212,3 +212,16 @@ def source_blocks(metadata: SparkFeatureMetadata) -> dict[str, list[str]]: for name in metadata.engineered_feature_names: blocks.setdefault(source_column(name, metadata) or name, []).append(name) return blocks + + +def source_block_indices(metadata: SparkFeatureMetadata) -> dict[str, list[int]]: + """:func:`source_blocks` as positions into the feature matrix, which is what an estimator needs. + + Computed on the driver once per scoring run and closed over by the UDF, rather than per row or per + partition: it is a pure function of the persisted metadata. + """ + positions = {name: index for index, name in enumerate(metadata.engineered_feature_names)} + return { + source: [positions[name] for name in names if name in positions] + for source, names in source_blocks(metadata).items() + } diff --git a/src/databricks/labs/dqx/anomaly/single_model_scorer.py b/src/databricks/labs/dqx/anomaly/single_model_scorer.py index 43f4bd986..fdecca0ad 100644 --- a/src/databricks/labs/dqx/anomaly/single_model_scorer.py +++ b/src/databricks/labs/dqx/anomaly/single_model_scorer.py @@ -23,6 +23,7 @@ from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRecord from databricks.labs.dqx.anomaly.scoring_utils import create_udf_schema from databricks.labs.dqx.anomaly.explainability import compute_gated_shap_contributions +from databricks.labs.dqx.anomaly.feature_naming import source_block_indices def create_scoring_udf( @@ -49,6 +50,7 @@ def create_scoring_udf_with_contributions( schema: StructType, quantile_points: list[tuple[float, float]] | None = None, threshold: float | None = None, + blocks: dict[str, list[int]] | None = None, ): """Create pandas UDF for distributed scoring with SHAP contributions. @@ -71,6 +73,7 @@ def predict_with_shap_udf(*cols: pd.Series) -> pd.DataFrame: scores, quantile_points, threshold, + blocks, ) return pd.DataFrame({"anomaly_score": scores, "anomaly_contributions": contributions_list}) @@ -107,8 +110,15 @@ def score_with_sklearn_model( schema = create_udf_schema(enable_contributions) if enable_contributions: + # Blocks are a pure function of the persisted metadata, so they are built once here and closed + # over rather than rebuilt per partition. predict_udf = create_scoring_udf_with_contributions( - model_bytes, engineered_feature_cols, schema, quantile_points, threshold + model_bytes, + engineered_feature_cols, + schema, + quantile_points, + threshold, + source_block_indices(feature_metadata), ) else: predict_udf = create_scoring_udf(model_bytes, engineered_feature_cols, schema) @@ -158,6 +168,7 @@ def score_with_sklearn_model_local( scores, quantile_points, threshold, + source_block_indices(feature_metadata), ) result_pdf = pd.DataFrame(result) diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py index cf0a94b34..b3aed4284 100644 --- a/src/databricks/labs/dqx/anomaly/timeseries_detector.py +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -59,6 +59,7 @@ import logging import sys +from collections.abc import Sequence from typing import Any import cloudpickle @@ -241,6 +242,63 @@ def feature_contributions(self, X: np.ndarray) -> np.ndarray: contributions[:, self.active_] = active_contributions return contributions + def block_contributions(self, X: np.ndarray, blocks: "Sequence[Sequence[int]]") -> np.ndarray: + """Joint-marginalisation drop for each *group* of features, rather than one feature at a time. + + Feature engineering can give one source column several engineered views -- the metric itself, its + deviation from its group's baseline, its deviation from its expected level at that time. Explaining + those views individually is unsound, because dropping one leaves another copy of the same + information behind, so the measured loss is small for *every* view. Normalising those small numbers + then hands almost all of the apparent blame to an unrelated metric. Measured on a correlated pair: + adding one affine duplicate of *x* moved the reported cause from 99.8% *x* to 99.9% *y*, while the + score itself did not move. + + For a block ``G``, the drop from marginalising the whole block out at once is + ``(Pd)_G' inv(P_GG) (Pd)_G`` where ``P = Σ⁻¹`` and ``d = x−μ`` standardised. A single-feature block + reduces exactly to :meth:`feature_contributions`' ``aᵢ = (Pd)ᵢ²/(Σ⁻¹)ᵢᵢ``, which is asserted in the + tests rather than argued here. + + **Not additive**, and deliberately so: block drops do not sum to ``d²``. Overlapping information + between blocks belongs to no single block, and the additive alternative goes negative -- see + :meth:`feature_contributions` for why negative terms are unusable downstream. + + ``Σ⁻¹`` is never formed. ``L⁻¹`` is recovered once per call from the stored Cholesky factor and + each ``P_GG`` is built as ``(L⁻¹)_{:,G}' (L⁻¹)_{:,G}``, so nothing new is persisted and models + trained before this existed keep loading. + + Args: + X: Rows to explain, shape ``(n_samples, n_features_in_)``. + blocks: One sequence of feature indices per block, indexing the *original* feature space. + Indices of constant-in-training features are ignored; a block of only those scores 0.0. + + Returns: + Array of shape ``(n_samples, len(blocks))``, non-negative. + """ + whitened = self._whitened(X) + precision_delta = np.linalg.solve(self.cholesky_.T, whitened.T).T + # L⁻¹, from which any principal submatrix of the precision follows. Σ = L L' so Σ⁻¹ = L⁻¹' L⁻¹. + inverse_factor = np.linalg.solve(self.cholesky_, np.eye(self.cholesky_.shape[0])) + + # Original feature index -> its position among the active features, or -1 when constant. + active_position = np.cumsum(self.active_) - 1 + drops = np.zeros((precision_delta.shape[0], len(blocks)), dtype=float) + + for block_index, feature_indices in enumerate(blocks): + positions = [ + int(active_position[i]) for i in feature_indices if 0 <= i < self.active_.size and self.active_[i] + ] + if not positions: + continue + factor_columns = inverse_factor[:, positions] + block_precision = factor_columns.T @ factor_columns + block_delta = precision_delta[:, positions] + solved = np.linalg.solve(block_precision, block_delta.T) + drops[:, block_index] = np.einsum("ij,ji->i", block_delta, solved) + + # Clip at zero: the quantity is a quadratic form in a positive-definite matrix and so is + # non-negative in exact arithmetic, but a near-singular block can land microscopically below. + return np.maximum(drops, 0.0) + # cloudpickle serialises classes **by reference** by default, which would make any pickled payload # containing this estimator require ``databricks.labs.dqx`` to be importable on every executor. diff --git a/tests/unit/test_anomaly_mahalanobis_detector.py b/tests/unit/test_anomaly_mahalanobis_detector.py index 3265e4a07..f417e690e 100644 --- a/tests/unit/test_anomaly_mahalanobis_detector.py +++ b/tests/unit/test_anomaly_mahalanobis_detector.py @@ -292,3 +292,98 @@ def test_the_detector_round_trips_through_mlflow_in_the_format_dqx_declares(): # Attribution is the reason this detector exists in DQX rather than raw scipy, so it has to survive # the round trip too. np.testing.assert_allclose(reloaded.feature_contributions(probe), detector.feature_contributions(probe)) + + +# ── source-block attribution: several views of one metric must not dilute each other ───────────────── + + +def test_a_single_feature_block_reduces_to_the_per_feature_formula(): + """The block form generalises the leave-one-out identity; it must not redefine it. + + Asserted rather than argued, because if these two disagreed then every model without derived features + would silently change its explanations. + """ + detector = MahalanobisDetector(ridge=0.0).fit(_sample_from(_CORRELATED, n_samples=20000, seed=3)) + probe = np.array([[1.0, 0.5], [3.0, -2.0]]) + + per_feature = detector.feature_contributions(probe) + singletons = detector.block_contributions(probe, [[0], [1]]) + + np.testing.assert_allclose(singletons, per_feature, rtol=1e-12) + + +def test_an_affine_duplicate_of_a_metric_does_not_make_another_metric_the_cause(): + """The defect this exists for, as the review's own probe. + + Feature engineering gives one metric several views -- itself, its deviation from its group's baseline, + its deviation from its expected level. A constant temporal expectation makes ``driver_rel_time`` an + affine duplicate of *driver*. Explaining views one at a time then measures almost nothing for either + copy, because dropping one leaves the other, and normalising those small numbers hands the blame to an + innocent metric: measured, 99.8% *driver* became 99.9% *bystander* while the score did not move. + + Blocking by source must reproduce the undisturbed explanation. + """ + rng = np.random.default_rng(0) + driver, bystander = rng.normal(0, 1, 4000), rng.normal(0, 1, 4000) + + undisturbed = MahalanobisDetector().fit(np.column_stack([driver, bystander])) + base = undisturbed.feature_contributions(np.array([[8.0, 0.5]]))[0] + base_share = 100.0 * base / base.sum() + + expanded = MahalanobisDetector().fit(np.column_stack([driver, driver - 3.0, bystander])) + probe = np.array([[8.0, 5.0, 0.5]]) + + per_feature = expanded.feature_contributions(probe)[0] + per_feature_share = 100.0 * per_feature / per_feature.sum() + # The bug, pinned so its absence is not mistaken for the test being vacuous. + assert per_feature_share[2] > 90.0, "expected the per-feature form to blame the bystander" + + blocked = expanded.block_contributions(probe, [[0, 1], [2]])[0] + blocked_share = 100.0 * blocked / blocked.sum() + + assert blocked_share[0] > 90.0, f"the driver's block should dominate, got {blocked_share.round(1)}" + np.testing.assert_allclose(blocked_share, base_share, atol=0.5) + + +def test_a_block_of_only_constant_features_scores_zero(): + """Constant-in-training features are excluded from the distance, so a block of nothing but those has + no drop to report -- and must not raise on an empty solve.""" + varying = _sample_from(_CORRELATED) + train = np.hstack([varying, np.full((len(varying), 1), 3.0)]) + detector = MahalanobisDetector().fit(train) + + drops = detector.block_contributions(np.array([[1.0, 0.5, 3.0]]), [[0, 1], [2]]) + + assert drops[0, 1] == 0.0 + assert drops[0, 0] > 0.0 + + +def test_block_drops_are_not_additive_and_the_docstring_says_so(): + """Overlapping information belongs to no single block, so blocks do not sum to the distance. + + Pinned because a reader who assumes additivity would 'fix' the normalisation into something wrong. + """ + detector = MahalanobisDetector(ridge=0.0).fit(_sample_from(_CORRELATED, n_samples=20000, seed=5)) + probe = np.array([[1.0, 0.5]]) + + total = float(detector.mahalanobis_squared(probe)[0]) + blocked_sum = float(detector.block_contributions(probe, [[0], [1]]).sum()) + + assert not np.isclose(blocked_sum, total) + assert "Not additive" in MahalanobisDetector.block_contributions.__doc__ + + +def test_one_hot_categories_of_one_column_form_a_single_block(): + """A categorical column becomes several indicators, and they are one source for explanation purposes. + + Without blocking, an unseen value's evidence is spread across every indicator of that column. + """ + rng = np.random.default_rng(11) + metric = rng.normal(0, 1, 600) + indicator = (rng.random(600) < 0.5).astype(float) + detector = MahalanobisDetector().fit(np.column_stack([metric, indicator, 1.0 - indicator])) + + drops = detector.block_contributions(np.array([[0.2, 0.0, 0.0]]), [[0], [1, 2]]) + + # An unseen category violates the one-hot sum, which is a property of the pair, not of either column. + assert drops[0, 1] > drops[0, 0] From 85b72b48a2e38a72c127dab3e9612b9312e41bb9 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 21:41:30 +0100 Subject: [PATCH 096/107] Stop the AI explanation prompt inventing a direction its inputs do not contain The prompt receives contribution magnitudes, severity percentiles, group size, an unsigned drift score and metadata. Nothing in that carries a sign. Attribution is a squared quantity, so a row displaced one way and a row displaced equally the other way produce the identical score and the identical contribution map -- (8, 0.5) and (-8, -0.5) both score 65.387 with the same map. The few-shot exemplars asserted direction anyway: "sits far above the norm" and "Inflated amount fields overstate revenue". A few-shot example is an instruction, and a smaller serving model copies its shape, so those two lines made confident, business-language, half-of-the-time-backwards claims the house style. Both are rewritten so every clause is checkable against that exemplar's own inputs, and the instructions now name the absence of direction explicitly and list the words it rules out. "Be direct, avoid hedging" was the rule that made this feel sanctioned, so it is reconciled rather than left to interpretation: being direct means stating plainly what the inputs contain, and does not license asserting a direction, a cause or a value they do not. The correlation-aware reading was overclaiming in the other direction. It instructed the model to "describe the pattern as a broken relationship", but a high contribution there is equally consistent with one metric moving a long way on its own. The input cannot separate those, so the text now says so and refuses both readings rather than mandating one. The guard it already carried -- do not call a metric abnormal unless the contributions concentrate on it -- is only trustworthy now that attribution is keyed by source column, since concentration used to be able to land on an innocent metric. Neither exemplar showed attribution_basis, the field that decides how contributions may be described, so the model had no demonstration of the field it is told to follow. Both now state it, with different values, so one reading cannot become the assumed default. Two smaller honesty fixes in the same pass. "confidence" is agreement between ensemble members that differ only by random seed on identical training data; it measures score stability, not whether the flag is right or whether the data has drifted, and the prompt now says that and forbids presenting it as confidence in the finding. And the header says "why the model flagged this group" rather than describing a root-cause pattern, because the inputs cannot establish a cause. Snapshot regenerated. Five new tests pin the properties rather than the wording: no directional phrase in either exemplar response, both readings demonstrated, the two reconciliation clauses present, the correlation-aware text refusing to assert, and the confidence caveat intact. The existing attribution-semantics test kept its docstring -- it records a real observed failure, an LLM reporting "Abnormal coolant flow" for rows whose every reading sat inside its healthy range -- with its assertions repointed at those protections so a rewording that keeps the guarantees passes. Co-authored-by: Isaac --- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 72 ++++++++---- tests/resources/ai_query_prompt_header.txt | 19 ++-- tests/unit/test_anomaly_llm_explainer.py | 106 +++++++++++++++++- 3 files changed, 164 insertions(+), 33 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 55f217921..87049fc1a 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -34,12 +34,20 @@ # these tables — the rendered header and the structured-output schema both derive from them. _PROMPT_INSTRUCTIONS = ( "You are a data quality analyst. Given aggregate metadata for a GROUP of anomalous rows " - "sharing the same root-cause pattern, explain in plain business language why this group was " - "flagged. Your explanation will be shown for every row in the group — describe the pattern, " - "not a specific row.\n" - "Be direct and concrete. Avoid hedging phrases like 'The data shows', 'It appears that', or " - "'might indicate'. Do not restate the input field names back to the user, and do not invent " - "feature names, values, or baseline groups that are not present in the input." + "sharing the same contribution pattern, explain in plain business language why the model " + "flagged this group. Your explanation will be shown for every row in the group — describe the " + "pattern, not a specific row. You are describing what the model measured, not diagnosing a " + "root cause: the inputs cannot establish one.\n" + "The inputs carry NO DIRECTION. A contribution says how much a metric mattered to the score, " + "never whether it was high or low: a metric far above its norm and one equally far below " + "produce the identical number. So never say a value was high, low, above, below, elevated, " + "inflated, dropped, spiked, or missing. Say it departed from its expected pattern, and leave " + "which way unsaid. The same applies to drift magnitudes, which are also unsigned.\n" + "Be direct and concrete: name the metrics, their shares and the group size without hedging " + "phrases like 'The data shows', 'It appears that', or 'might indicate'. Being direct means " + "stating plainly what the inputs contain — it does not license asserting a direction, a cause, " + "or a value they do not contain. Do not restate the input field names back to the user, and do " + "not invent feature names, values, or baseline groups that are not present in the input." ) # What a contribution means, per detector family. Keyed by the ``ModelIdentity.algorithm`` prefix that is # persisted in the registry, so a model trained by any version resolves as long as that string is stable. @@ -48,10 +56,12 @@ _ATTRIBUTION_SEMANTICS: tuple[tuple[str, str], ...] = ( ( "Mahalanobis", - "relationships between metrics. A high contribution means this metric departed from its usual " - "relationship with the others -- its own value may sit well inside its normal range. Describe the " - "pattern as a broken relationship between metrics, and do NOT call an individual metric abnormal, " - "high, low, or deviating unless the contributions are concentrated in a single metric.", + "each metric's position once every other metric is accounted for. A high contribution means this " + "metric does not fit the pattern the others imply, which can happen either because its own value " + "moved a long way or because it stopped tracking the others while staying inside its normal range. " + "The input does not distinguish those two cases, so do not assert either: say the metric does not " + "fit the pattern. When the contributions are spread across several metrics, describe it as the " + "metrics no longer agreeing with each other rather than as any one of them being abnormal.", ), ( "IsolationForest", @@ -77,8 +87,9 @@ def attribution_semantics(algorithm: str | None) -> str: _PROMPT_INPUT_FIELDS: tuple[tuple[str, str], ...] = ( ( "attribution_basis", - "What the feature_contributions below are measuring. Read them accordingly -- this decides " - "whether the pattern is 'these values were extreme' or 'these metrics stopped agreeing'.", + "What the feature_contributions below are measuring, which differs by detector and decides how " + "you may describe the pattern. Follow this field rather than assuming a reading: one basis " + "supports saying a feature's own value was unusual, the other does not.", ), ( "feature_contributions", @@ -91,8 +102,10 @@ def attribution_semantics(algorithm: str | None) -> str: ("severity_range", "Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'."), ( "confidence", - "Model confidence label across the group. 'high' / 'mixed' / 'low' for ensemble, 'n/a' " - "for single-model scoring.", + "How closely the ensemble's members agreed on the score: 'high' / 'mixed' / 'low', or 'n/a' " + "when one model did the scoring. Members differ only by random seed on the same training " + "data, so this measures the stability of the score, NOT how reliable the flag is or whether " + "the data has since changed. Do not present it to the reader as confidence in the finding.", ), ( "baseline_grouping", @@ -127,8 +140,20 @@ def attribution_semantics(algorithm: str | None) -> str: ) # Two few-shot exemplars (one without drift, one with) pin the desired style and JSON shape for # smaller serving models. Kept short so the prompt stays well within token budgets. +# +# Every clause in both responses is checkable against that exemplar's own inputs, because a few-shot +# example is an instruction: a smaller model copies its *shape*, and a response that asserts more than +# its input supports teaches the model to do the same. The previous pair said "sits far above the norm" +# and "Inflated amount fields overstate revenue" from inputs carrying no sign at all -- a row at (8, 0.5) +# and its mirror at (-8, -0.5) score identically (65.387) with identical contribution maps, so half of +# those explanations were backwards, stated confidently, in business language. +# +# The two also differ in *attribution_basis*, which is the field deciding how the contributions read. +# Showing only one reading would leave the other untaught; the values here are abbreviated forms of what +# *attribution_semantics* emits, since the exemplars exist to pin shape rather than to restate the header. _PROMPT_EXAMPLES = ( - "Example (no drift):\n" + "Example (relationship basis, no drift):\n" + "attribution_basis: each metric's position once the others are accounted for\n" "feature_contributions: amount vs its group baseline (61%), quantity (22%)\n" "group_size: 312 rows\n" "severity_range: mean 97.4, min 95.1, max 99.8\n" @@ -136,11 +161,12 @@ def attribution_semantics(algorithm: str | None) -> str: "baseline_grouping: region\n" "threshold: 95.0\n" "drift_summary: none\n" - 'Response: {"narrative":"312 rows are driven mainly by amount, which sits far above the norm ' - 'for its own region (61%), with quantity secondary (22%).","business_impact":"Inflated amount ' - 'fields overstate revenue if these rows are processed unchanged.","action":"Reconcile amount ' - 'against source orders within each affected region."}\n\n' - "Example (with drift):\n" + 'Response: {"narrative":"Across 312 rows, amount departs most from what its own region implies ' + '(61%), with quantity next (22%).","business_impact":"Amount values that do not match their ' + "region's usual pattern distort revenue reporting if processed unchanged.\",\"action\":" + '"Reconcile amount against source orders for the affected regions."}\n\n' + "Example (value basis, with drift):\n" + "attribution_basis: each feature's own value compared against the rows it was scored against\n" "feature_contributions: latency_ms (74%), retries (12%)\n" "group_size: 88 rows\n" "severity_range: mean 98.9, min 97.0, max 99.9\n" @@ -149,9 +175,9 @@ def attribution_semantics(algorithm: str | None) -> str: "threshold: 95.0\n" "drift_summary: drift detected: latency_ms=4.12\n" 'Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from ' - 'baseline; retries contribute modestly (12%).","business_impact":"Elevated latency risks SLA ' - 'breaches for downstream consumers.","action":"Investigate latency_ms regressions against the ' - 'training baseline."}' + 'its training baseline; retries contribute modestly (12%).","business_impact":"Latency that no ' + 'longer matches its baseline risks SLA breaches for downstream consumers.","action":"Compare ' + 'latency_ms against the training baseline to find what changed."}' ) if TYPE_CHECKING: diff --git a/tests/resources/ai_query_prompt_header.txt b/tests/resources/ai_query_prompt_header.txt index b37631d76..d765a3eea 100644 --- a/tests/resources/ai_query_prompt_header.txt +++ b/tests/resources/ai_query_prompt_header.txt @@ -1,12 +1,13 @@ -You are a data quality analyst. Given aggregate metadata for a GROUP of anomalous rows sharing the same root-cause pattern, explain in plain business language why this group was flagged. Your explanation will be shown for every row in the group — describe the pattern, not a specific row. -Be direct and concrete. Avoid hedging phrases like 'The data shows', 'It appears that', or 'might indicate'. Do not restate the input field names back to the user, and do not invent feature names, values, or baseline groups that are not present in the input. +You are a data quality analyst. Given aggregate metadata for a GROUP of anomalous rows sharing the same contribution pattern, explain in plain business language why the model flagged this group. Your explanation will be shown for every row in the group — describe the pattern, not a specific row. You are describing what the model measured, not diagnosing a root cause: the inputs cannot establish one. +The inputs carry NO DIRECTION. A contribution says how much a metric mattered to the score, never whether it was high or low: a metric far above its norm and one equally far below produce the identical number. So never say a value was high, low, above, below, elevated, inflated, dropped, spiked, or missing. Say it departed from its expected pattern, and leave which way unsaid. The same applies to drift magnitudes, which are also unsigned. +Be direct and concrete: name the metrics, their shares and the group size without hedging phrases like 'The data shows', 'It appears that', or 'might indicate'. Being direct means stating plainly what the inputs contain — it does not license asserting a direction, a cause, or a value they do not contain. Do not restate the input field names back to the user, and do not invent feature names, values, or baseline groups that are not present in the input. Inputs: -- attribution_basis: What the feature_contributions below are measuring. Read them accordingly -- this decides whether the pattern is 'these values were extreme' or 'these metrics stopped agreeing'. +- attribution_basis: What the feature_contributions below are measuring, which differs by detector and decides how you may describe the pattern. Follow this field rather than assuming a reading: one basis supports saying a feature's own value was unusual, the other does not. - feature_contributions: Mean contributions across the group, already named for a reader, e.g. 'amount vs its group baseline (82%), quantity (11%), discount (5%)'. A phrase like 'X vs its group baseline' means X was unusual relative to its own baseline group, not in absolute terms. These are aggregated relative importances — not raw data values. - group_size: Number of rows in this group, e.g. '312 rows'. - severity_range: Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'. -- confidence: Model confidence label across the group. 'high' / 'mixed' / 'low' for ensemble, 'n/a' for single-model scoring. +- confidence: How closely the ensemble's members agreed on the score: 'high' / 'mixed' / 'low', or 'n/a' when one model did the scoring. Members differ only by random seed on the same training data, so this measures the stability of the score, NOT how reliable the flag is or whether the data has since changed. Do not present it to the reader as confidence in the finding. - baseline_grouping: The columns whose values define each row's baseline group, e.g. 'region' or 'region, product'. Anomalies are judged relative to the row's own group baseline; 'none' when the model is not grouped. - threshold: The severity percentile threshold configured by the user (0–100). - drift_summary: Baseline drift signal from the scoring run, e.g. 'drift detected: amount=4.12; quantity=3.55' or 'none'. If drift is present, explicitly frame the narrative vs baseline. @@ -16,7 +17,8 @@ Respond with ONLY a JSON object. Field rules: - business_impact: One sentence, max 25 words. Likely downstream business impact if this group of rows is processed unchanged. Concrete, tied to the contributing features. - action: One sentence, max 20 words. What a data analyst should investigate for this group. -Example (no drift): +Example (relationship basis, no drift): +attribution_basis: each metric's position once the others are accounted for feature_contributions: amount vs its group baseline (61%), quantity (22%) group_size: 312 rows severity_range: mean 97.4, min 95.1, max 99.8 @@ -24,9 +26,10 @@ confidence: high baseline_grouping: region threshold: 95.0 drift_summary: none -Response: {"narrative":"312 rows are driven mainly by amount, which sits far above the norm for its own region (61%), with quantity secondary (22%).","business_impact":"Inflated amount fields overstate revenue if these rows are processed unchanged.","action":"Reconcile amount against source orders within each affected region."} +Response: {"narrative":"Across 312 rows, amount departs most from what its own region implies (61%), with quantity next (22%).","business_impact":"Amount values that do not match their region's usual pattern distort revenue reporting if processed unchanged.","action":"Reconcile amount against source orders for the affected regions."} -Example (with drift): +Example (value basis, with drift): +attribution_basis: each feature's own value compared against the rows it was scored against feature_contributions: latency_ms (74%), retries (12%) group_size: 88 rows severity_range: mean 98.9, min 97.0, max 99.9 @@ -34,4 +37,4 @@ confidence: mixed baseline_grouping: none threshold: 95.0 drift_summary: drift detected: latency_ms=4.12 -Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from baseline; retries contribute modestly (12%).","business_impact":"Elevated latency risks SLA breaches for downstream consumers.","action":"Investigate latency_ms regressions against the training baseline."} +Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from its training baseline; retries contribute modestly (12%).","business_impact":"Latency that no longer matches its baseline risks SLA breaches for downstream consumers.","action":"Compare latency_ms against the training baseline to find what changed."} diff --git a/tests/unit/test_anomaly_llm_explainer.py b/tests/unit/test_anomaly_llm_explainer.py index 22ffcb64e..7dc7ce668 100644 --- a/tests/unit/test_anomaly_llm_explainer.py +++ b/tests/unit/test_anomaly_llm_explainer.py @@ -281,12 +281,24 @@ def test_attribution_semantics_distinguishes_correlation_from_value_anomalies(): relationship between them had broken. Given per-feature importances and nothing else, the model cannot tell the two situations apart -- they look identical in shape -- so it defaults to the value reading and asserts something the data does not support. + + Asserted on the protections rather than on any single word, so a rewording that keeps the guarantees + passes and one that drops them fails. The correlation-aware text no longer *mandates* the relationship + reading either -- a high contribution is genuinely consistent with a large individual move, so naming + one reading as the truth was its own overclaim -- but it must still refuse the value reading as a + default, which is what produced "Abnormal coolant flow". """ correlation = llm_explainer.attribution_semantics("Mahalanobis") - assert "relationship" in correlation + # The contribution is joint, not univariate: the reason the observed claim was wrong. + assert "once every other metric is accounted for" in correlation assert "inside its normal range" in correlation # The instruction that prevents the specific false claim observed. - assert "do NOT call an individual metric abnormal" in correlation + assert "does not distinguish" in correlation + assert "do not assert either" in correlation + + values = llm_explainer.attribution_semantics("IsolationForest") + assert "feature's own value" in values + assert correlation != values value_based = llm_explainer.attribution_semantics("IsolationForest") assert "own value was unusual" in value_based @@ -330,3 +342,93 @@ def test_explanation_context_defaults_algorithm_to_none(): ) assert ctx.algorithm is None assert llm_explainer.attribution_semantics(ctx.algorithm) == llm_explainer.attribution_semantics("IsolationForest") + + +# ── the prompt must not teach the model to invent direction ────────────────────────────────────────── + +# Words that assert which way a metric moved. The inputs the prompt is built from carry contribution +# magnitudes, severity percentiles and unsigned drift scores -- nothing that distinguishes a metric far +# above its norm from one equally far below. A row at (8, 0.5) and its mirror at (-8, -0.5) score +# identically and produce identical contribution maps, so any of these words is right half the time. +_DIRECTIONAL_WORDS = ( + "far above", + "far below", + "elevated", + "inflated", + "dropped", + "spiked", + "surged", + "plummeted", + "too high", + "too low", +) + + +def _exemplar_responses() -> list[str]: + """The JSON responses from the few-shot exemplars, which is the part a model imitates.""" + return [ + line.partition("Response: ")[2] + for line in llm_explainer._PROMPT_EXAMPLES.splitlines() + if line.startswith("Response: ") + ] + + +def test_the_few_shot_responses_assert_no_direction(): + """A few-shot example is an instruction, so an unfounded exemplar teaches unfounded output. + + The previous pair said "sits far above the norm" and "Inflated amount fields overstate revenue" + from inputs with no sign in them at all. Because a smaller serving model copies the shape of these + responses, that made confident, business-language, half-of-the-time-backwards claims the house style. + """ + responses = _exemplar_responses() + assert len(responses) == 2, "expected both exemplars to still carry a response to check" + + for response in responses: + lowered = response.lower() + offenders = [word for word in _DIRECTIONAL_WORDS if word in lowered] + assert not offenders, f"exemplar asserts direction its inputs cannot support: {offenders}" + + +def test_the_instructions_name_the_absence_of_direction_and_reconcile_it_with_being_direct(): + """Two rules could otherwise be read as licensing invention: "be direct, avoid hedging" and the + detector-family reading. Being direct must mean stating what the input holds, not filling the gap.""" + header = llm_explainer._render_ai_query_prompt_header() + + assert "NO DIRECTION" in header + assert "does not license asserting a direction" in header + + +def test_every_exemplar_shows_the_attribution_basis_it_is_reading(): + """The field that decides how contributions may be described has to appear in the demonstrations. + + Both readings are shown, because an exemplar set that only ever displays one teaches the model to + treat that one as the default and ignore the field. + """ + bases = [ + line.partition("attribution_basis: ")[2] + for line in llm_explainer._PROMPT_EXAMPLES.splitlines() + if line.startswith("attribution_basis: ") + ] + + assert len(bases) == 2, "each exemplar must state the basis it is reading" + assert bases[0] != bases[1], "the exemplars must demonstrate both readings, not one twice" + + +def test_the_correlation_aware_reading_does_not_assert_a_broken_relationship(): + """A high contribution there is consistent with a large individual move *or* with a metric that + stopped tracking the others while staying in its normal range. The input cannot tell them apart, so + instructing the model to describe a broken relationship states more than is known.""" + semantics = llm_explainer.attribution_semantics("Mahalanobis") + + assert "does not distinguish" in semantics + assert "do not assert either" in semantics + + +def test_ensemble_agreement_is_not_presented_as_confidence_in_the_finding(): + """*confidence* is seed agreement on one training set. It says nothing about whether the flag is + right or whether the data has drifted since, and the prompt has to say so or the narrative will + imply otherwise.""" + description = dict(llm_explainer._PROMPT_INPUT_FIELDS)["confidence"] + + assert "random seed" in description + assert "NOT how reliable the flag is" in description From 70340cf1eb8d1f9755a61aa00570607804821b6b Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 21:42:09 +0100 Subject: [PATCH 097/107] Correct the claim that both detectors catch an unseen category The comment justifying full one-hot retention said an unseen value "scored as a large deviation" and that "the detectors can tell apart" a known value from an unrecognised one. The plural was wrong, and it was a comment written in this branch. Measured on the same binary column, with every category retained: IsolationForest unseen 0.458 known 0.442 / 0.467 -> ordinary, between the two known encodings Mahalanobis unseen 2,000,000 known 0.9 Isolation Forest splits one feature at a time, so no tree can express "these indicators sum to zero", and an all-zeros row sits inside every individual indicator's observed range. Retaining the categories puts the distinction in the encoding, which is what the encoding can do; acting on it needs a model that reads features jointly. So an unrecognised category is not reliably caught under profile="tabular", and the comment and the test docstring that implied otherwise now both say so. The guide gains the user-facing half of this in the following commit, which is where its other prose changes land. The retention argument itself comes out stronger, not weaker. Pruning the anticorrelated dummy -- exactly what a correlation-threshold feature filter would do -- collapses the correlation-aware separation from 2,000,000-against-0.9 to 1.1-against-0.9, while leaving every score on ordinary rows untouched. The redundancy is not waste being tolerated: it *is* the off-support constraint, and no in-sample metric would notice its removal. That is now the sharpest argument on record against pruning correlated features here. Two tests make both halves executable rather than asserted in prose. Also lands the mirrored-row test, which belongs to the prompt work in the previous commit but lives in this file: a row and its mirror produce identical scores, identical per-feature contributions and identical block contributions. That is the fact making the prompt's no-direction rule necessary, pinned at the source of the numbers rather than only as an assertion about prompt text. Co-authored-by: Isaac --- .../labs/dqx/anomaly/transformers.py | 30 ++++-- .../unit/test_anomaly_mahalanobis_detector.py | 94 ++++++++++++++++++- 2 files changed, 112 insertions(+), 12 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/transformers.py b/src/databricks/labs/dqx/anomaly/transformers.py index 95da61492..0358442fe 100644 --- a/src/databricks/labs/dqx/anomaly/transformers.py +++ b/src/databricks/labs/dqx/anomaly/transformers.py @@ -651,16 +651,28 @@ def _apply_onehot_encoding( # Every category is retained. Dropping one of a binary pair is the textbook way to avoid the # dummy-variable trap, but here it made an unexpected value invisible: the omitted reference # category and any value never seen in training both encode as all-zeros, so a brand-new value - # in a binary column produced no signal at all. With both retained, a known value sets exactly - # one indicator and anything unseen sets none, which the detectors can tell apart. + # in a binary column produced no signal whatsoever. With both retained, a known value sets + # exactly one indicator and anything unseen sets none -- a distinction that is *present* in the + # encoding, which is the point. Whether a detector acts on it is a separate question, answered + # below, and it makes binary consistent with every other cardinality rather than introducing a + # new behaviour: three or more categories were always retained in full. # - # This makes binary consistent with every other cardinality rather than introducing a new - # behaviour: three or more categories were always retained in full, so an unseen value has - # always encoded as all-zeros and scored as a large deviation. Retained categories sum to 1 on - # every trained row, so the correlation-aware detector sees a zero-variance direction, and a row - # violating that sum lies off the surface all its training data lay on. Measured: rows that do - # satisfy it score identically to the one-dummy encoding, so the redundant column is free, while - # an unseen category scores far above a known one. Both halves are pinned in + # Retained categories sum to 1 on every trained row, so the correlation-aware detector sees a + # zero-variance direction and a row violating that sum lies off the surface all its training + # data lay on. Measured on a binary column: rows that satisfy the constraint score identically + # to the one-dummy encoding, so the redundant column is free, while an unseen category scores + # 2,000,000 against 0.9 for a known one. Pruning the anticorrelated dummy -- which is what any + # correlation-threshold feature filter would do -- collapses that to 1.1 against 0.9. So the + # redundancy is not waste being tolerated: it *is* the off-support constraint, and that is the + # sharpest argument against pruning correlated features here. + # + # IsolationForest gets no such benefit, and the comment here used to imply otherwise. Its splits + # are axis-parallel, so no tree can test a sum across columns, and an all-zeros row sits inside + # every individual indicator's observed range. Measured on the same data: the unseen category + # scores 0.458 while the two known encodings score 0.442 and 0.467 -- ordinary, not anomalous. + # An unrecognised category is therefore not reliably caught under profile="tabular"; the guide + # says so, and catching it needs a vocabulary check outside the learned ranking rather than a + # different encoding. All three measurements are pinned in # tests/unit/test_anomaly_mahalanobis_detector.py. distinct_values = sorted(row[0] for row in df.select(col_name).distinct().collect() if row[0] is not None) onehot_categories[col_name] = distinct_values diff --git a/tests/unit/test_anomaly_mahalanobis_detector.py b/tests/unit/test_anomaly_mahalanobis_detector.py index f417e690e..e3868d3bd 100644 --- a/tests/unit/test_anomaly_mahalanobis_detector.py +++ b/tests/unit/test_anomaly_mahalanobis_detector.py @@ -236,9 +236,13 @@ def test_a_redundant_dummy_costs_nothing_and_an_unseen_category_scores_high(): training row lay on, and scores enormously The second is deliberate rather than accidental, and it is not new: columns with three or more - categories always retained all of them, so an unseen value has always scored this way. Truncating - the binary case was the inconsistency, and it is what made an unexpected value in a binary column - invisible instead. + categories always retained all of them, so an unseen value has always scored this way *for this + detector*. Truncating the binary case was the inconsistency, and it is what made an unexpected value + in a binary column invisible instead. + + The scoping matters. Retaining the categories puts the distinction in the encoding, but acting on it + needs a model that can test a constraint across columns, which IsolationForest cannot -- see + :func:`test_isolation_forest_does_not_detect_an_unseen_category_from_the_same_encoding`. """ rng = np.random.default_rng(3) metric = rng.normal(10.0, 1.0, 400) @@ -387,3 +391,87 @@ def test_one_hot_categories_of_one_column_form_a_single_block(): # An unseen category violates the one-hot sum, which is a property of the pair, not of either column. assert drops[0, 1] > drops[0, 0] + + +def test_a_row_and_its_mirror_are_indistinguishable_in_score_and_attribution(): + """Why the prompt may not say a metric was high or low: the evidence does not contain it. + + Attribution is a squared quantity, so a row displaced one way and a row displaced equally the other + way produce the identical score and the identical contribution map. Any narrative asserting a + direction from that input is right by luck half the time -- which is what the LLM exemplars used to + teach. Pinned here, at the source of the numbers, rather than only as a prompt assertion, because + this is the fact that makes the prompt rule necessary. + """ + train = _sample_from(_CORRELATED, n_samples=20000, seed=7) + detector = MahalanobisDetector().fit(train) + + centre = train.mean(axis=0) + displacement = np.array([4.0, 1.5]) + above = (centre + displacement).reshape(1, -1) + below = (centre - displacement).reshape(1, -1) + + np.testing.assert_allclose(detector.mahalanobis_squared(above), detector.mahalanobis_squared(below), rtol=1e-10) + np.testing.assert_allclose(detector.feature_contributions(above), detector.feature_contributions(below), rtol=1e-10) + np.testing.assert_allclose( + detector.block_contributions(above, [[0], [1]]), + detector.block_contributions(below, [[0], [1]]), + rtol=1e-10, + ) + + +def test_pruning_the_redundant_dummy_destroys_the_unseen_category_signal(): + """Why correlated features must not be pruned here: the redundancy *is* the constraint. + + An exactly collinear dummy pair looks like the textbook case for a correlation-threshold filter to + drop one of. But the pair's sum-to-one is the only thing making an unseen category detectable, so + dropping either one removes the signal entirely while leaving every score on ordinary rows intact -- + a change that no in-sample metric would notice. + """ + rng = np.random.default_rng(3) + metric = rng.normal(10.0, 1.0, 400) + indicator = (rng.random(400) < 0.5).astype(float) + train = np.column_stack([metric, indicator, 1.0 - indicator]) + + unseen, known = np.array([[10.0, 0.0, 0.0]]), np.array([[10.0, 1.0, 0.0]]) + + retained = MahalanobisDetector().fit(train) + pruned = MahalanobisDetector().fit(train[:, :2]) + + retained_ratio = -retained.score_samples(unseen)[0] / -retained.score_samples(known)[0] + pruned_ratio = -pruned.score_samples(unseen[:, :2])[0] / -pruned.score_samples(known[:, :2])[0] + + assert retained_ratio > 1000.0, f"retained pair should separate sharply, got {retained_ratio:.1f}x" + assert pruned_ratio < 2.0, f"pruning should collapse the separation, got {pruned_ratio:.1f}x" + + +def test_isolation_forest_does_not_detect_an_unseen_category_from_the_same_encoding(): + """The limitation this encoding does *not* fix, pinned so the comment cannot drift back. + + Retaining every category puts the distinction in the features, but acting on it needs a model that + can test a constraint *across* columns. IsolationForest splits one feature at a time, so no tree can + represent "these indicators sum to zero", and an all-zeros row sits inside every single indicator's + observed range. It therefore scores as ordinary -- here, between the two known encodings rather than + above both. + + Asserted as a known limitation rather than a bug: catching an unrecognised category needs a + vocabulary check outside the learned ranking, which is tracked separately. + """ + rng = np.random.default_rng(3) + metric = rng.normal(10.0, 1.0, 400) + indicator = (rng.random(400) < 0.5).astype(float) + train = np.column_stack([metric, indicator, 1.0 - indicator]) + + forest = IsolationForest(n_estimators=100, random_state=42).fit(train) + unseen = -forest.score_samples(np.array([[10.0, 0.0, 0.0]]))[0] + known = [ + -forest.score_samples(np.array([[10.0, 1.0, 0.0]]))[0], + -forest.score_samples(np.array([[10.0, 0.0, 1.0]]))[0], + ] + + assert unseen < max(known), "IsolationForest is not expected to rank an unseen category highest" + + # The contrast that makes the asymmetry the point rather than an incidental measurement. + detector = MahalanobisDetector().fit(train) + maha_unseen = -detector.score_samples(np.array([[10.0, 0.0, 0.0]]))[0] + maha_known = -detector.score_samples(np.array([[10.0, 1.0, 0.0]]))[0] + assert maha_unseen > 1000.0 * maha_known From 03634c934af4eecc3bd90b4f21c87c28e6cc7b44 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 21:42:45 +0100 Subject: [PATCH 098/107] Report the benchmark under the protocol users actually run, and record three real limits The guide led with 65.4% against 96.8% incident coverage, measured by fitting a 30% sample of the very rows being scored. Under that split 272 of the 327 labelled incidents had rows in training. So the number answers "can the model re-find anomalies it was shown", which is fair for scanning a table you already have, but it is not evidence for the guide's own quickstart -- train once, then check new or incoming data. The guide now reports the chronological split only: train on an earlier period, score a later one. 57.5% against 65.3% coverage at a 1% budget, average precision 0.277 against 0.423. Deliberately *less* prose, not more: a reader of this page is deciding whether to use "timeseries" on their data and cannot adjudicate a protocol taxonomy, so the retrospective table, the held-out-rows digression and the whole "why a clean training split looks worse" admonition are gone, and the sentence that actually helps stays -- choose on the shape of anomaly you expect, not on an expected accuracy gap. The full picture, both protocols and the overlap counts, belongs in the PR and the benchmark archive. One correction that follows from the switch: round 2 flipped the headline from average precision to coverage because AP was a wash -- under the leakier protocol. Chronologically AP is the stronger signal again, +0.146 or +53% against +7.8 points of coverage. The per-machine statistics that justified the coverage framing (+31 points, 22 wins to 1) belong to the retired protocol and are not carried over, because there are no per-machine chronological numbers to replace them with. The drift attribution is softened too: two protocols disagreeing does not isolate drift as the sole cause. Three limitations are documented rather than fixed, because each is a real property of the design and not a defect: baseline_by conditions on level, not on relationship. One model is fitted and it learns one set of relationships shared across groups -- the right trade when groups differ in scale and agree in behaviour. Measured on a counterexample where group A has y rising with x and group B has y falling, both centred identically: the pooled model scores a row that is impossible for B at 1.3, where a model fitted on B alone scores it 260. Since the medians match, group-relative features are identical and the transform changes nothing. Users whose groups genuinely behave differently are told to train per group. The same limit applies to baseline_over_time, which removes one shared trend. An unrecognised category is caught by "timeseries" and generally not by "tabular", which is the user-facing half of the previous commit. No accuracy number in that section reveals it, so it is stated plainly and points at is_in_list and foreign_key. The temporal selection criterion is blind to constant forecast bias, and that is fine. It subtracts the residual median, so it measures spread. But a constant offset in a _rel_time column cancels before it can reach a score: the correlation-aware detector centres on the training mean, and IsolationForest draws split thresholds from each feature's observed range so shifting a column shifts its candidates with it. Measured against no offset, offsets of 5 and 500 move the correlation-aware score by 1.7e-14 and 1.9e-12 relative -- floating-point residue from the centring, far below the quantile spacing that decides a severity percentile -- and leave IsolationForest bit-identical. Non-constant error inflates residual spread, which the criterion does penalise, so the blind spot is exactly the case that cannot matter. Pinned as a test, asserted at a tolerance rather than at bit equality, because bit equality is false: an earlier draft of this claim said "bit-identical on both detectors" and the test written to prove it failed. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 72 +++++++++---------- src/databricks/labs/dqx/anomaly/temporal.py | 13 ++++ tests/unit/test_anomaly_temporal_fit.py | 39 ++++++++++ 3 files changed, 84 insertions(+), 40 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index e2a33f52a..8b0d1fd90 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -403,6 +403,14 @@ anomaly_engine.train( Now each metric is judged against its own group's baseline, so the collapse stands out even though its value is ordinary for the table as a whole. + +`baseline_by` conditions on **level** — where each group's values normally sit — not on the **relationships** between metrics inside a group. One model is still fitted, and it learns one set of relationships shared across all groups. + +That is the right trade for the usual case, where groups differ in scale and agree in behaviour: one region's orders run ten times another's, but order count and revenue move together everywhere. It cannot express groups whose metrics relate to each other *differently*. Measured on a deliberate counterexample — group A where `y` rises with `x`, group B where `y` falls as `x` rises, both centred identically — the pooled model scores a row that is impossible for B at 1.3, where a model fitted on B alone scores it 260. Because the two groups have the same medians, group-relative features are identical and the transform changes nothing. + +If your groups genuinely behave differently rather than merely sitting at different levels, train one model per group and give each its own `model_name`. The same applies to `baseline_over_time`: it removes one shared trend, so groups trending in opposite directions or at different phases are not separated by it either. + + A baseline column is the basis of comparison, not a metric being compared, so it never becomes a model feature. Passing the same column in both `columns` and `baseline_by` is an error, and if you let DQX auto-discover `columns`, it drops your declared baseline columns from the feature list for you. Baseline columns must be string, integral, boolean, or date. Float, double, and decimal are rejected, because Spark and Python format floating-point values differently and the baseline lookup would silently match nothing. Bucket the value or cast it to a string first. @@ -535,47 +543,31 @@ tabular data and weak when the anomaly *is* a broken relationship: if CPU is nor normal, no single-feature split separates the row, even though "high CPU with idle memory" never happens on a healthy machine. -Measured on the full Server Machine Dataset: real machine telemetry, 28 machines, 38 metrics, 327 labelled -incidents across 708,420 rows. Both detectors were trained the way DQX trains — a 30% sample of the table -being scored, 80% of that sample used to fit — so the training data contains the anomalies, as it does on -your table: - -| `profile` | Incidents surfaced, 1% budget | 5% budget | Precision at 1% | ROC-AUC | Average precision | -|---|---|---|---|---|---| -| `"tabular"` | 65.4% | 92.8% | 54.6% | 0.826 | 0.374 | -| `"timeseries"` | **96.8%** | **99.3%** | 55.4% | **0.845** | 0.380 | - -An incident counts as surfaced if the detector flags at least one of its rows, so this measures whether you -would have been paged, not how many rows you would have had to read. The best precision a 1% budget allows -on this data is 95.7%, so both detectors are well short of the ceiling; the difference between them is -*which* incidents they find, not how cleanly. - -**Coverage is the difference that holds up per machine.** The gap at a 1% budget averages +31 points with a -standard deviation of 30, and the correlation-aware detector wins on 22 of the 28 machines and loses on 1. -Average precision, by contrast, is a wash under this protocol: +0.006, and 14 wins to 14 losses. So choose -`"timeseries"` for telemetry because it surfaces more incidents, not because it ranks rows better. - - -Fitting SMD's own held-out training period instead — the conventional protocol, with no overlap between -training and scored rows — gives 57.5% against 65.3% at a 1% budget, with average precision 0.277 against -0.423. Both detectors do *worse* that way, because SMD's training period drifts from its test period, and a -model fitted on a sample of the rows it will score does not inherit that drift. - -That is a property of how DQX trains rather than of the benchmark, so the table above is the representative -one. It is not an artefact of scoring rows that were trained on: restricting the ranking metrics to the rows -the model never saw gives ROC-AUC 0.826 against 0.845 and average precision 0.380 against 0.382 — the same -numbers. Event coverage is not reported for that check, because dropping a quarter of the rows breaks -incidents into fragments and would count a different set of events. - -Both protocols come from one archived run. The comparison is between *estimators* on raw metric matrices: -DQX's own feature engineering, including `baseline_by` and `baseline_over_time`, is not part of it. - +Measured on the Server Machine Dataset: real machine telemetry, 28 machines, 38 metrics, 327 labelled +incidents. Trained on an earlier period and scored on a later one, which is what you do when you train once +and check new data: -That is telemetry, which is what `"timeseries"` is for. On ordinary tabular data the two detectors are -closer than this table might suggest, so choose on the *shape* of the anomaly you expect rather than on an -expected accuracy gap. Two things independent of accuracy do favour the default: `"tabular"` trains an -ensemble, so it can report `confidence_std`, and its contributions come from SHAP. Detection quality on -DQX's own synthetic fixtures is published in [Benchmarks](/docs/reference/benchmarks). +| `profile` | Incidents surfaced, 1% budget | Average precision | +|---|---|---| +| `"tabular"` | 57.5% | 0.277 | +| `"timeseries"` | **65.3%** | **0.423** | + +An incident counts as surfaced if the detector flags at least one of its rows, so the first column measures +whether you would have been paged. The comparison is between detectors on raw metric matrices; DQX's own +feature engineering is not part of it. + +That is telemetry, which is what `"timeseries"` is for. On ordinary tabular data the two are closer than +this table suggests, so **choose on the shape of the anomaly you expect, not on an expected accuracy gap.** +Two things independent of accuracy do favour the default: `"tabular"` trains an ensemble, so it can report +`confidence_std`, and its contributions come from SHAP. Detection quality on DQX's own synthetic fixtures is +published in [Benchmarks](/docs/reference/benchmarks). + +One capability difference is worth knowing because no accuracy number shows it. A **category that never +appeared in training** — a new payment type, an unrecognised status code — is caught by `"timeseries"` but +generally not by `"tabular"`. Both encode the row the same way, but only a detector that reads features +jointly can tell that the encoding is impossible; Isolation Forest splits one feature at a time and scores +such a row as ordinary. If unrecognised values are something you need to fail on, that is a membership +question rather than an anomaly one — use `is_in_list` or `foreign_key` on the column, under either profile. DQX does not detect which profile you need. Getting it right cannot be verified without labelled diff --git a/src/databricks/labs/dqx/anomaly/temporal.py b/src/databricks/labs/dqx/anomaly/temporal.py index f9da7fe4b..0eb0e1fc5 100644 --- a/src/databricks/labs/dqx/anomaly/temporal.py +++ b/src/databricks/labs/dqx/anomaly/temporal.py @@ -251,6 +251,19 @@ def _holdout_residual_scale(seconds: np.ndarray, values: np.ndarray, basis: Temp This is the statistic changepoint counts are chosen by. It has to be measured *after* the fit window because that is where over-flexible trends go wrong: an in-sample criterion is blind to it. + + Subtracting the residual median makes this a measure of *spread*, so it cannot see a constant forecast + bias -- a basis whose predictions are uniformly off by the same amount scores as well as one that is + right. That is deliberate, and the blind spot is exactly the case that cannot matter downstream: a + constant offset in a ``_rel_time`` feature cancels before it can reach a score. The correlation-aware + detector centres on the training mean, so the shift subtracts back out; IsolationForest draws split + thresholds from each feature's observed range, so shifting a column shifts its candidates with it and + the partition is unchanged. Measured against no offset, offsets of 5 and 500 move the correlation-aware + score by 1.7e-14 and 1.9e-12 relative -- floating-point residue from the centring, twelve orders of + magnitude below the quantile spacing that decides a severity percentile -- and leave IsolationForest + bit-identical. Non-constant error, a wrong slope or the wrong shape, inflates residual spread instead, + which this statistic does penalise. So do not "fix" this by scoring bias: it would trade a criterion + that tracks what matters for one that also tracks what cannot. """ order = np.argsort(seconds) t_sorted, v_sorted = seconds[order], values[order] diff --git a/tests/unit/test_anomaly_temporal_fit.py b/tests/unit/test_anomaly_temporal_fit.py index 760d07cbc..f915b59be 100644 --- a/tests/unit/test_anomaly_temporal_fit.py +++ b/tests/unit/test_anomaly_temporal_fit.py @@ -11,6 +11,7 @@ import numpy as np import pytest +from sklearn.ensemble import IsolationForest from sklearn.linear_model import Ridge from databricks.labs.dqx.anomaly.temporal import ( @@ -25,6 +26,7 @@ select_basis, trend_strength, ) +from databricks.labs.dqx.anomaly.timeseries_detector import MahalanobisDetector HOUR = 3600.0 DAY = 86400.0 @@ -460,3 +462,40 @@ def test_a_metric_with_more_than_half_identical_values_still_fits(): assert "metric" in fitted assert all(np.isfinite(fitted["metric"])) + + +def test_a_constant_offset_in_a_relative_feature_cannot_reach_a_score(): + """Why the selection criterion is allowed to be blind to constant forecast bias. + + ``_holdout_residual_scale`` subtracts the residual median, so it measures spread and scores a + uniformly-biased basis as well as an unbiased one. That looks like a gap in the objective until you + ask what a constant offset in a ``_rel_time`` column can actually do downstream: nothing. The + correlation-aware detector centres on the training mean, so the shift cancels exactly; IsolationForest + picks split thresholds from each feature's observed range, so shifting the whole column shifts the + thresholds with it and leaves the partition identical. + + The cancellation is algebraic, so what survives is floating-point residue from centring a shifted + column, not sensitivity to the offset. Measured relative movement in the correlation-aware score: + 1.7e-14 at an offset of 5 and 1.9e-12 at 500 -- twelve orders of magnitude below the quantile spacing + that decides a severity percentile, so no flag can turn on it. IsolationForest is bit-identical at + both, because shifting a column shifts its candidate split thresholds with it. + + Asserted at a tolerance rather than at bit equality, since bit equality is false and asserting it + would have made this test a statement about float arithmetic instead of about the criterion. + """ + rng = np.random.default_rng(0) + base = rng.normal(0.0, 1.0, (2000, 3)) + probe_base = rng.normal(0.0, 1.0, (50, 3)) + + def scores_with_offset(offset: float) -> tuple[np.ndarray, np.ndarray]: + shift = np.array([0.0, 0.0, offset]) # only the derived, time-relative column moves + detector = MahalanobisDetector().fit(base + shift) + forest = IsolationForest(n_estimators=50, random_state=0).fit(base + shift) + probe = probe_base + shift + return detector.score_samples(probe), forest.score_samples(probe) + + reference_maha, reference_forest = scores_with_offset(0.0) + for offset in (5.0, 500.0): + maha, forest = scores_with_offset(offset) + np.testing.assert_allclose(maha, reference_maha, rtol=1e-10) + assert np.array_equal(forest, reference_forest), f"IsolationForest scores moved at offset {offset}" From cba27d667decebbcdf3871629ae9942b55c8e865 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 22:03:02 +0100 Subject: [PATCH 099/107] Fix the broken doc anchor that was failing the Docusaurus build The schema table linked to #how-it-works, but the heading is "How it works under the hood", so the anchor is #how-it-works-under-the-hood. Docusaurus fails the build on broken anchors rather than warning, so this took the whole docs job down. Introduced two commits ago, in the same change that added the sentence. It also took the five codecov statuses with it: .github/codecov.yml sets require_ci_to_pass with wait_for_ci, so those report failure when any CI job fails, whatever the coverage actually is. Verified with `make docs-build` locally, which is what CI runs and what I should have run before pushing the change that broke it. A cross-check of every in-page anchor in that file against every heading found no others. Co-authored-by: Isaac --- docs/dqx/docs/guide/row_anomaly_detection/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 8b0d1fd90..57658f447 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -652,7 +652,7 @@ Each element is a **struct** with a shared “wide” schema. Currently, the onl | `is_anomaly` | boolean | `true` if `severity_percentile` ≥ threshold. | | `threshold` | double | Severity percentile threshold used (e.g. 95.0). | | `model` | string | Full model name (e.g. Unity Catalog name). | -| `contributions` | map<string, double> | Contribution percentages (0–100). Keyed by **source column** for `profile="timeseries"` and by engineered feature for `profile="tabular"` — see step 4 of [How it works](#how-it-works). On by default (`enable_contributions=True`); populated only for anomalous rows (`null` for non-anomalous rows or if you set it `False`). | +| `contributions` | map<string, double> | Contribution percentages (0–100). Keyed by **source column** for `profile="timeseries"` and by engineered feature for `profile="tabular"` — see step 4 of [How it works](#how-it-works-under-the-hood). On by default (`enable_contributions=True`); populated only for anomalous rows (`null` for non-anomalous rows or if you set it `False`). | | `confidence_std` | double | Ensemble score standard deviation. Present when `enable_confidence_std=True`; `null` otherwise. | | `is_new_baseline` | boolean | `true` when the row's group was absent from training, in which case `score` and `severity_percentile` are `null` and the row is **not** flagged. See [Groups that appear after training](#groups-that-appear-after-training). | | `new_baseline_key` | string | The unrecognised group key, for rows where `is_new_baseline` is `true`; `null` otherwise. | From c4ac1353716de2a8027ec65612a94167f1495b6c Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Mon, 7 Sep 2026 22:36:53 +0100 Subject: [PATCH 100/107] Explain the column that caused the anomaly, not a view of it, a normalising feature, or one member Three defects in feature attribution, all of which could name the wrong column, confidently, in business language, on a row the detector had flagged correctly. One fix is the prerequisite for the other two, so they land together. TreeSHAP's sign was being discarded rather than read. It explains the ensemble's average *path length*, not a score -- its base value is a path length around 12 -- and a row is anomalous when it is isolated in few splits, so a NEGATIVE value drives the anomaly and a positive one argues the row is ordinary. Using the magnitude added the second to the first and reported the sum as one number. The review that raised this had the sign backwards, and implementing it as described would have been far worse than the status quo. Measured on 200 rows each carrying exactly one deliberately anomalous feature, the largest value named the true culprit 199/200 under the orientation that ships, 199/200 using magnitude alone, and 0/200 using the positive side -- which reliably names the most ordinary feature instead. So magnitude was very nearly right; on rows anomalous enough to be shown, the evidence is 97.8% anomaly-driving at a severity of 95 and 99.5% at 99. Correcting it left the top driver unchanged on every row measured and reordered the ranks below it on roughly a quarter of rows. The map stays non-negative and sums to 100, so this is not a schema change. Orientation happens where the values are produced; the clip happens once, at the end, in format_shap_contributions. That ordering is load-bearing rather than tidy. Averaging across ensemble members and summing a source column's views both rely on SHAP's exact additivity, and a value arguing the row is normal has to be able to cancel one arguing it is not: two views at +3 and -1 net to 2 but clip first and they sum to 3; two members at +4 and -2 average to 1 but clip first and it is 2. An earlier draft of this change clipped at the point of production, which would have made both operations decompositions of nothing while every test still passed. A row with no anomaly-driving evidence was given 1/n for every key -- "every feature contributed equally", which is an explanation with no input behind it and the more misleading of the two failures because the numbers look unremarkable. Those rows now carry the all-null map that a row with a null feature already produced, and which every consumer already renders as unknown. Zero-valued entries are dropped from messages, prompts and the pattern key too, rather than being listed as "quantity (0%)": exact zeros went from rare to common when the normalising side stopped earning a share. An ensemble scored with every member and explained with one. The reported score is the mean across members and so is confidence_std, but contributions came from models[0], which is not a representative member -- they differ only by random seed, so it is whichever trained first. They disagree more than that framing suggests: on 200 rows with the default three members, the least-agreeing pair named a different top driver on 130 of them and the closest pair still differed on 92. Contributions are now the mean, which decomposes the mean path length exactly. The entry point takes the models behind the score rather than a model, so the mistake cannot be remade by omission, and misaligned members raise instead of being averaged into a confident, plausible, wrong answer. Cost is about 7% more at three members, because attribution runs only on rows above the threshold; there is deliberately no knob for it, since the honest lever is enable_contributions=False and it already exists. Both detectors now report one entry per source column. The correlation-aware one already did, from the previous commit's joint marginalisation; a tree model gets the plain sum within each block, which is exact because SHAP is additive and which the corrected sign is what makes possible. The failure it fixes is different in shape from the correlation-aware one -- the evidence is split rather than misdirected, 49.3% and 50.7% across two views of a column whose true share is 100% -- and it still changes the answer: with three views, a column whose true share was 74.5% left each view near 25%, handing the top spot to a derived view of itself while an unrelated single-view column sat tied alongside. The ensemble path never received blocks at all, and the default profile *is* the ensemble, so that plumbing is added. One knock-on worth stating: the shared integration contract asserted contribution keys were a subset of the *engineered* feature names. That was true only by accident on an all-numeric fixture, where a column's only feature is named after it, and false the moment a categorical appeared. It now asserts the actual contract, the caller's own columns. Both demos taught that a contribution reads "X vs its expected level at that time". It no longer does, and that is the trade this makes: naming the column right costs the signal of which comparison objected, which is now carried by baseline_over_time being set rather than by the key. Both prints say so. Also brought into line: compute_contributions_for_matrix carried both original defects in the same module, which is how one module came to hold two answers to what a negative SHAP value means; the correlation-aware detector's docstring claimed every consumer takes abs(); and the guide documented enable_contributions as defaulting to False when it is True. Co-authored-by: Isaac --- .../dqx_demo_anomaly_tabular_transactions.py | 7 +- demos/dqx_demo_anomaly_timeseries_fleet.py | 6 +- .../guide/row_anomaly_detection/index.mdx | 12 +- .../row_anomaly_detection/troubleshooting.mdx | 2 +- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 32 +- .../labs/dqx/anomaly/ensemble_scorer.py | 25 +- .../labs/dqx/anomaly/explainability.py | 358 ++++++++++++++---- .../labs/dqx/anomaly/single_model_scorer.py | 4 +- .../labs/dqx/anomaly/timeseries_detector.py | 11 +- .../test_anomaly_ensemble.py | 23 +- .../test_anomaly_timeseries_profile.py | 27 +- .../test_anomaly_attribution_orientation.py | 174 +++++++++ .../unit/test_anomaly_ensemble_attribution.py | 185 +++++++++ tests/unit/test_anomaly_llm_explainer.py | 7 +- tests/unit/test_anomaly_shap_gating.py | 16 +- .../test_anomaly_source_block_attribution.py | 136 +++++++ 16 files changed, 905 insertions(+), 120 deletions(-) create mode 100644 tests/unit/test_anomaly_attribution_orientation.py create mode 100644 tests/unit/test_anomaly_ensemble_attribution.py create mode 100644 tests/unit/test_anomaly_source_block_attribution.py diff --git a/demos/dqx_demo_anomaly_tabular_transactions.py b/demos/dqx_demo_anomaly_tabular_transactions.py index cc69b96e1..0167b6f6a 100644 --- a/demos/dqx_demo_anomaly_tabular_transactions.py +++ b/demos/dqx_demo_anomaly_tabular_transactions.py @@ -637,9 +637,10 @@ def generate_daily_volume(n_days: int, seed: int, stalled_week: bool = False): # COMMAND ---------- # DBTITLE 1,Read the contributions, which name the expected level -print("💡 The contributions say ' vs its expected level at that time', not 'unusual '.") -print(" That distinction is the whole point: every one of these counts sits inside the range the") -print(" history covers. Only their position against the trend is wrong.\n") +print("💡 The contributions name the metric, not the comparison that objected to it. Every one of") +print(" these counts sits inside the range the history covers, so what is wrong is their position") +print(" against the trend rather than their value -- but the map says 'this metric mattered', and") +print(" it is baseline_over_time being set that tells you the comparison it mattered against.\n") display( volume_result.filter(volume_anomaly.getField("is_anomaly")) diff --git a/demos/dqx_demo_anomaly_timeseries_fleet.py b/demos/dqx_demo_anomaly_timeseries_fleet.py index dd0498d57..519ccb576 100644 --- a/demos/dqx_demo_anomaly_timeseries_fleet.py +++ b/demos/dqx_demo_anomaly_timeseries_fleet.py @@ -668,8 +668,10 @@ def generate_wear_history(n_hours: int, seed: int, late_fault: bool = False): # COMMAND ---------- # DBTITLE 1,Read the contributions, which name the expected level -print("💡 Contributions read 'X vs its expected level at that time', not 'unusual X'.") -print(" The distinction is real: every one of these readings sits inside the history's own range.") +print("💡 Contributions name the metric, once, however many ways the model compared it.") +print(" Read them as 'this metric mattered', not 'this metric's value was extreme': every one of") +print(" these readings sits inside the history's own range, and it is baseline_over_time being set") +print(" that tells you the comparison they failed.") display( spark.table(wear_scored) .filter(anomaly.getField("is_anomaly")) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 57658f447..7fb463e56 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -558,9 +558,11 @@ feature engineering is not part of it. That is telemetry, which is what `"timeseries"` is for. On ordinary tabular data the two are closer than this table suggests, so **choose on the shape of the anomaly you expect, not on an expected accuracy gap.** -Two things independent of accuracy do favour the default: `"tabular"` trains an ensemble, so it can report -`confidence_std`, and its contributions come from SHAP. Detection quality on DQX's own synthetic fixtures is -published in [Benchmarks](/docs/reference/benchmarks). +One thing independent of accuracy favours the default: `"tabular"` trains an ensemble, so it can report +`confidence_std`. Its contributions come from SHAP rather than an exact decomposition, but both profiles +report them the same way — one entry per column you passed, and on an ensemble averaged across members, so +the score, `confidence_std` and the contributions all describe the same aggregate. Detection quality on +DQX's own synthetic fixtures is published in [Benchmarks](/docs/reference/benchmarks). One capability difference is worth knowing because no accuracy number shows it. A **category that never appeared in training** — a new payment type, an unrecognised status code — is caught by `"timeseries"` but @@ -621,7 +623,7 @@ For full parameter and schema details, see [Row Anomaly Detection in Quality Che 1. **Feature engineering** (automatic): DQX detects column types and creates features (numerical standardized, categorical one-hot or frequency-encoded, temporal expanded to hour/day/month/weekend). 2. **Smart sampling and training**: DQX samples your data (default 30%, capped at 1M rows), trains the detector your `profile` selects — an ensemble of Isolation Forest models by default, or a single correlation-aware model for `profile="timeseries"` — and captures baseline statistics for drift detection. 3. **Model registry**: Models and metadata live in MLflow and a Delta table; each `train()` call registers one model under its name and archives the previously active version. -4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. Feature contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. How they are computed depends on the detector — SHAP for Isolation Forest, an exact decomposition for the correlation-aware detector — and so does what the map's keys name: the correlation-aware detector reports one entry per **source column you passed**, while Isolation Forest reports one per engineered feature (so a single column can appear as several entries, for example `signup_hour_sin`). Either way the values are percentages of the same total. +4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. Feature contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. How they are computed depends on the detector — SHAP for Isolation Forest, an exact decomposition for the correlation-aware detector — but both report **one entry per column you passed**, not per engineered feature, so you read back your own column names. That matters because DQX derives up to three features from one numeric column; scoring each separately splits the column's evidence between them and understates it. Each value is that column's share of the evidence that made the row look unusual. A column that made the row look *more* normal shows `0`, and a flagged row for which nothing pointed towards an anomaly carries an all-null map rather than an invented even split. With an ensemble, the contributions are the mean across members, so the score, `confidence_std` and the contributions all describe the same aggregate. 5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping (categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group, for example region or product category). A discovered grouping is used as `baseline_by` (see [Group-aware anomaly detection](#group-aware-anomaly-detection)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. When you *do* pass `columns`, you have decided what to measure, so DQX leaves the comparison pooled rather than adding a grouping you did not ask for. If your data looks grouped it says so in a warning naming the grouping to pass. ### Which algorithm, and why @@ -652,7 +654,7 @@ Each element is a **struct** with a shared “wide” schema. Currently, the onl | `is_anomaly` | boolean | `true` if `severity_percentile` ≥ threshold. | | `threshold` | double | Severity percentile threshold used (e.g. 95.0). | | `model` | string | Full model name (e.g. Unity Catalog name). | -| `contributions` | map<string, double> | Contribution percentages (0–100). Keyed by **source column** for `profile="timeseries"` and by engineered feature for `profile="tabular"` — see step 4 of [How it works](#how-it-works-under-the-hood). On by default (`enable_contributions=True`); populated only for anomalous rows (`null` for non-anomalous rows or if you set it `False`). | +| `contributions` | map<string, double> | Each column's share (0–100) of the evidence that made the row look unusual, keyed by **the columns you passed** under either profile — see step 4 of [How it works](#how-it-works-under-the-hood). A column that argued the row was normal shows `0`. On by default (`enable_contributions=True`); populated only for anomalous rows (`null` for non-anomalous rows or if you set it `False`), and all-null on a flagged row where nothing pointed towards an anomaly. | | `confidence_std` | double | Ensemble score standard deviation. Present when `enable_confidence_std=True`; `null` otherwise. | | `is_new_baseline` | boolean | `true` when the row's group was absent from training, in which case `score` and `severity_percentile` are `null` and the row is **not** flagged. See [Groups that appear after training](#groups-that-appear-after-training). | | `new_baseline_key` | string | The unrecognised group key, for rows where `is_new_baseline` is `true`; `null` otherwise. | diff --git a/docs/dqx/docs/guide/row_anomaly_detection/troubleshooting.mdx b/docs/dqx/docs/guide/row_anomaly_detection/troubleshooting.mdx index fb3710c8e..d7277baca 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/troubleshooting.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/troubleshooting.mdx @@ -191,7 +191,7 @@ Sampling doesn't hurt model quality if the sample is representative! has_no_row_anomalies( model_name=model_name, registry_table=registry_table, - enable_contributions=False, # 10x faster, default is False + enable_contributions=False, # 10x faster; the default is True ) # Investigation: slower, with explanations diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 87049fc1a..32034d71b 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -65,8 +65,11 @@ ), ( "IsolationForest", - "individual feature values. A high contribution means this feature's own value was unusual for the " - "rows it was compared against.", + "how far each metric's own value sits from the values the model was trained on. A high contribution " + "means this metric was unusual for the rows it was compared against. Where a metric is compared " + "several ways at once -- against the whole table, against its own group, against its expected level " + "at that time -- the share covers all of those together, so it says the metric was involved, not " + "which comparison objected.", ), ) _DEFAULT_ATTRIBUTION_SEMANTICS = _ATTRIBUTION_SEMANTICS[-1][1] @@ -328,20 +331,24 @@ def redaction_set(redact_columns: tuple[str, ...], metadata: SparkFeatureMetadat def _pattern_spark_expr(contributions_col: str, redact_set: frozenset[str]) -> Column: """Pattern key as a pure-Spark-SQL expression (no Python UDFs shipped to executors). - Drops null and redacted entries, takes the top-2 features by |value| desc, + Drops null, zero and redacted entries, takes the top-2 features by value desc, sorts their names asc, and joins with '+'. Empty or null maps yield 'unknown'. - Ranking uses absolute value so signed SHAP contributions pick the same top-2 - as `format_contributions_map`. Implemented in SQL so Databricks Connect / - serverless workers don't need the dqx package installed. + Zero entries are dropped because a feature that earned no share did not contribute, + and pairing it into the key would group rows by a feature neither of them was flagged + for. Values are non-negative shares, so ordering by value and by |value| agree; the + abs() is kept only so the comparator matches `format_contributions_map`'s. + Implemented in SQL so Databricks Connect / serverless workers don't need the dqx + package installed. """ col = f"`{contributions_col}`" if redact_set: redact_arr = "array(" + ", ".join(f"'{_sql_string_literal(r)}'" for r in sorted(redact_set)) + ")" entries = ( - f"filter(map_entries({col}), e -> e.value is not null " f"and not array_contains({redact_arr}, e.key))" + f"filter(map_entries({col}), e -> e.value is not null and e.value > 0 " + f"and not array_contains({redact_arr}, e.key))" ) else: - entries = f"filter(map_entries({col}), e -> e.value is not null)" + entries = f"filter(map_entries({col}), e -> e.value is not null and e.value > 0)" sql = ( f"case when {col} is null or size({entries}) = 0 then 'unknown' " f"else concat_ws('+', array_sort(transform(slice(array_sort({entries}, " @@ -456,7 +463,12 @@ def _format_contributions_sql(top_n: int, labels: dict[str, str] | None = None) Mirrors *format_contributions_map* but stays inside Spark so per-group prompts can be assembled without a driver-side loop. Null/empty maps yield 'unknown'; entries are sorted by - absolute value descending and percentages are normalised against the L1 sum of |value|. + value descending and percentages are normalised against their sum. + + Null *and zero* entries are dropped, matching *format_contributions_map*: a feature that earned no + share contributed nothing, and listing it as 'quantity (0%)' hands the model a driver to explain that + the attribution says nothing about. Values are non-negative shares, so ordering by value and by + |value| agree; the abs() is kept only to keep the two implementations' comparators identical. *labels* maps an engineered feature name to its human label; when supplied, each key is rendered as its label ('amount_rel_baseline' -> 'amount vs its group baseline'), falling back @@ -465,7 +477,7 @@ def _format_contributions_sql(top_n: int, labels: dict[str, str] | None = None) dropped sensitive keys upstream in *_aggregate_groups_spark*, so labelling never re-exposes a redacted feature. """ - entries = "filter(map_entries(`mean_contributions`), e -> e.value is not null)" + entries = "filter(map_entries(`mean_contributions`), e -> e.value is not null and e.value > 0)" sorted_entries = ( f"array_sort({entries}, (a, b) -> " f"case when abs(b.value) > abs(a.value) then 1 " diff --git a/src/databricks/labs/dqx/anomaly/ensemble_scorer.py b/src/databricks/labs/dqx/anomaly/ensemble_scorer.py index 7a96adcc2..47a0aa8dc 100644 --- a/src/databricks/labs/dqx/anomaly/ensemble_scorer.py +++ b/src/databricks/labs/dqx/anomaly/ensemble_scorer.py @@ -23,6 +23,7 @@ from databricks.labs.dqx.anomaly.model_loader import load_and_validate_model from databricks.labs.dqx.anomaly.model_registry import AnomalyModelRecord from databricks.labs.dqx.anomaly.explainability import compute_gated_shap_contributions +from databricks.labs.dqx.anomaly.feature_naming import source_block_indices def serialize_ensemble_models( @@ -76,11 +77,16 @@ def create_ensemble_scoring_udf_with_contributions( schema: StructType, quantile_points: list[tuple[float, float]] | None = None, threshold: float | None = None, + blocks: dict[str, list[int]] | None = None, ): - """Create ensemble scoring UDF with SHAP contributions. + """Create ensemble scoring UDF with feature contributions. - When *quantile_points* and *threshold* are provided, SHAP runs only for rows whose + When *quantile_points* and *threshold* are provided, attribution runs only for rows whose mean-score severity reaches the threshold; other rows get a null contributions map. + + Contributions are the mean across every member, matching the score and *anomaly_score_std*, which are + also aggregates over all of them. Explaining one member while scoring with all of them made the + explanation depend on which member happened to be trained first. """ @pandas_udf(schema) # type: ignore[call-overload] @@ -97,12 +103,13 @@ def ensemble_scoring_udf(*cols: pd.Series) -> pd.DataFrame: "anomaly_score": mean_scores, "anomaly_score_std": std_scores, "anomaly_contributions": compute_gated_shap_contributions( - models[0], + models, feature_matrix, engineered_feature_cols, mean_scores, quantile_points, threshold, + blocks, ), } @@ -139,8 +146,15 @@ def score_ensemble_models( schema = prepare_ensemble_scoring_schema(enable_contributions) if enable_contributions: + # Blocks are a pure function of the persisted metadata, so they are built once on the driver and + # closed over rather than rebuilt per partition -- the same reasoning as the single-model scorer. ensemble_scoring_udf = create_ensemble_scoring_udf_with_contributions( - models_bytes, engineered_feature_cols, schema, quantile_points, threshold + models_bytes, + engineered_feature_cols, + schema, + quantile_points, + threshold, + source_block_indices(feature_metadata), ) else: ensemble_scoring_udf = create_ensemble_scoring_udf(models_bytes, engineered_feature_cols, schema) @@ -186,12 +200,13 @@ def score_ensemble_models_local( if enable_contributions: result["anomaly_contributions"] = compute_gated_shap_contributions( - models[0], + models, feature_matrix, engineered_feature_cols, mean_scores, quantile_points, threshold, + source_block_indices(feature_metadata), ) result_pdf = pd.DataFrame(result) diff --git a/src/databricks/labs/dqx/anomaly/explainability.py b/src/databricks/labs/dqx/anomaly/explainability.py index 08172ba80..0112d4dd1 100644 --- a/src/databricks/labs/dqx/anomaly/explainability.py +++ b/src/databricks/labs/dqx/anomaly/explainability.py @@ -1,11 +1,22 @@ -"""SHAP-based explainability for row anomaly detection. +"""Feature attribution for row anomaly detection: which columns made a row look unusual. + +Two detectors supply attribution by different means -- TreeSHAP for a tree model, an exact decomposition +for the correlation-aware one -- and one invariant lets everything downstream treat them alike: +**attribution is oriented so that a larger value means more responsible for the anomaly.** TreeSHAP does +not arrive that way, because it explains a path length that *shrinks* as a row becomes more isolated, so +:func:`_oriented_towards_anomaly` flips it. + +Attribution stays signed for as long as it is being combined. Averaging across ensemble members and +summing a source column's engineered views both depend on SHAP's exact additivity, and a value arguing +the row is normal has to be able to cancel one arguing it is not. :func:`format_shap_contributions` is +the single place the sign is dropped, producing the public map: non-negative shares of the +anomaly-driving evidence, totalling 100, keyed by the columns the caller passed. -Provides contribution formatting and computation for scoring pipelines, plus -TreeSHAP-based feature contribution analysis for reporting and messages. Requires the 'anomaly' extras: pip install databricks-labs-dqx[anomaly] """ import logging +from collections.abc import Sequence from typing import Any import mlflow.sklearn as mlflow_sklearn @@ -33,32 +44,99 @@ logger = logging.getLogger(__name__) +def _oriented_towards_anomaly(shap_values: np.ndarray) -> np.ndarray: + """Flip TreeSHAP's sign so that larger means *more* responsible for the anomaly. + + TreeSHAP on an isolation forest explains the ensemble's **average path length**, not a score: its + base value is a path length (around 12 on a few thousand rows), and a row is anomalous when it is + isolated in *few* splits. So a negative SHAP value shortens the path and drives the anomaly, while a + positive one lengthens it and argues the row is ordinary. Negating puts this branch on the same + footing as the correlation-aware detector's own attribution, whose values are already non-negative + with larger meaning more responsible -- one orientation for both, which is what lets everything + downstream average and group attributions without asking where they came from. + + The sign is easy to get backwards, and backwards is not a subtle error. Measured on 200 rows each + carrying exactly one deliberately anomalous feature, the feature with the largest value was the true + culprit 199 times out of 200 under this orientation, 199 out of 200 using the magnitude alone, and + **0 out of 200** using the un-negated positive part -- which reliably names the most *ordinary* + feature instead. Magnitude alone was the previous behaviour and was very nearly right, because on + rows anomalous enough to be shown at all the evidence is overwhelmingly anomaly-driving: 97.8% of it + at a severity of 95, 99.5% at 99. It was still wrong in kind, adding evidence of normality to + evidence of anomaly and reporting the sum as one number. + + **Deliberately does not clip.** Negative values -- features arguing the row is normal -- have to + survive until after any averaging across ensemble members or summing across a source column's + engineered views, because those operations rely on SHAP's exact additivity and the negatives are + what makes them cancel correctly. Clipping first would overstate: two views at +3 and -1 net to 2, + but clipping first sums to 3, and two ensemble members at +4 and -2 average to 1 rather than 2. + :func:`format_shap_contributions` is the one place the clip happens, once, at the very end. + + Args: + shap_values: Raw TreeSHAP values, shape ``(n_rows, n_features)``. + + Returns: + Signed array of the same shape, oriented so larger means more responsible for the anomaly. + """ + return -shap_values + + def format_shap_contributions( - shap_values: np.ndarray, + attribution: np.ndarray, valid_indices: np.ndarray, num_rows: int, - engineered_feature_cols: list[str], + keys: list[str], ) -> list[dict[str, float | None]]: - """Format SHAP values into contribution dictionaries.""" - num_features = len(engineered_feature_cols) - contributions: list[dict[str, float | None]] = [{c: None for c in engineered_feature_cols} for _ in range(num_rows)] + """Normalise an attribution matrix into per-row percentage maps. + + Takes attribution oriented so that larger means more responsible for the anomaly -- TreeSHAP via + :func:`_oriented_towards_anomaly`, or the correlation-aware detector's own decomposition, which is + already oriented that way and already non-negative. + + **This is the one place the clip happens, and it has to be here rather than earlier.** Negative + values are features arguing the row is *normal*, and a normalising feature is not a driver, so they + earn no share. But they must survive averaging across ensemble members and summing across a source + column's engineered views first, because both rely on SHAP's exact additivity and the negatives are + what makes them cancel: two views at +3 and -1 net to 2, while clipping first sums to 3. Clipping + last keeps every intermediate step an honest decomposition and still yields a non-negative public + map. The correlation-aware branch is unaffected, since its values are non-negative to begin with. + + A row whose attribution is entirely non-positive gets the all-``None`` map. It used to be given + ``1 / n`` for every key, which reads as "every feature contributed equally" when what is true is that + nothing is known -- invented rather than measured, and the more misleading of the two because the + numbers look ordinary. The all-``None`` shape is already what a row with a null feature produces, so + every consumer already handles it: *_pattern_spark_expr* and *_format_contributions_sql* both drop + null entries and fall back to ``'unknown'``. Rare in practice -- no such row among 200 measured at a + severity of 95 -- but "could not judge" has to stay distinguishable from "judged". + + Args: + attribution: Per-key attribution, shape ``(n_valid_rows, len(keys))``, oriented so larger means + more responsible. May contain negatives; they are dropped here. + valid_indices: Boolean mask over the original rows, marking which reached the attribution. + num_rows: Row count of the caller's frame, so the returned list aligns with it. + keys: Names for the columns of *attribution* -- source columns, or engineered feature names. + + Returns: + One map per row: percentages summing to 100 where there was anomaly-driving evidence, all-``None`` + otherwise. + """ + num_keys = len(keys) + contributions: list[dict[str, float | None]] = [{key: None for key in keys} for _ in range(num_rows)] - if shap_values.size == 0: + if attribution.size == 0 or num_keys == 0: return contributions - abs_shap = np.abs(shap_values) - totals = abs_shap.sum(axis=1, keepdims=True) - normalized = np.divide(abs_shap, totals, out=np.zeros_like(abs_shap), where=totals > 0) - if num_features > 0: - normalized[totals.squeeze(axis=1) == 0] = 1.0 / num_features + magnitudes = np.maximum(attribution, 0.0) + totals = magnitudes.sum(axis=1, keepdims=True) + normalized = np.divide(magnitudes, totals, out=np.zeros_like(magnitudes), where=totals > 0) + has_attribution = totals.squeeze(axis=1) > 0 valid_row_idx = 0 for i in range(num_rows): if valid_indices[i]: - contributions[i] = { - engineered_feature_cols[j]: round(float(normalized[valid_row_idx, j] * 100.0), 1) - for j in range(num_features) - } + if has_attribution[valid_row_idx]: + contributions[i] = { + keys[j]: round(float(normalized[valid_row_idx, j] * 100.0), 1) for j in range(num_keys) + } valid_row_idx += 1 return contributions @@ -84,16 +162,25 @@ def compute_row_attributions( Whatever the source, the values feed the same *format_shap_contributions*, so the emitted map has identical scaling and null handling either way. - *blocks* switches an estimator that supports it to **source-block** attribution, keyed by the source - column rather than by engineered feature. That is not cosmetic: explaining engineered features one at a - time is unsound when several of them share a source, because dropping one view leaves another copy of - the same information behind. Measured, adding one affine duplicate of a metric moved the reported cause - from 99.8% that metric to 99.9% an unrelated one, while the score did not move. Blocking restores - 99.7%/0.3%, matching the undisturbed model. - - Only the exact-attribution branch takes it. TreeSHAP's correct block aggregation is a **signed** sum, - and the tree path still discards sign (see *format_shap_contributions*), so blocking there would add - magnitudes that should have cancelled. That asymmetry is deliberate and tracked separately. + *blocks* switches to **source-block** attribution, keyed by the source column rather than by engineered + feature. That is not cosmetic: explaining engineered features one at a time is unsound when several of + them share a source, because one view's evidence says nothing about how much the *column* mattered. + Both detectors need it and both get it, by different arithmetic. + + For the correlation-aware detector the block value is an exact joint marginalisation, because dropping + one view leaves another copy of the same information behind and the per-view drops are therefore + almost nothing. Measured: adding one affine duplicate of a metric moved the reported cause from 99.8% + that metric to 99.9% an unrelated one, while the score did not move; blocking restores 99.7%/0.3%. + + For a tree model the block value is a plain **sum** of the oriented values within the block, which is + exact because SHAP is additive -- each value is that feature's share of the path length, so a source + column's share is the sum over its views, with any view that argued the row was normal correctly + cancelling part of the others. This is why the orientation must not clip before here. The failure it + fixes is different in shape from the correlation-aware one: the evidence is *split* rather than + misdirected, so a metric's true 100% shows up as 49.3% and 50.7% across two views. That still changes + the answer, because DQX gives a numeric column up to three views: measured on a column whose true share + was 74.5%, splitting it three ways left each view near 25% and handed the top spot to a *derived view* + of it, with an unrelated single-view column tied alongside. Returns: ``(attribution, valid_indices, keys)`` where *keys* names the columns of *attribution* -- source @@ -105,25 +192,141 @@ def compute_row_attributions( feature_values = scaler.transform(feature_matrix) if scaler else feature_matrix.values valid_indices = ~pd.isna(feature_values).any(axis=1) - blocked = bool(blocks) and hasattr(estimator, "block_contributions") and len(engineered_feature_cols) > 1 + blocked = bool(blocks) and len(engineered_feature_cols) > 1 keys = list(blocks) if blocked and blocks is not None else engineered_feature_cols attribution = np.array([]) if valid_indices.any(): - rows = feature_values[valid_indices] - if len(engineered_feature_cols) == 1: - attribution = np.ones((len(rows), 1)) - elif blocked and blocks is not None: - attribution = estimator.block_contributions(rows, [blocks[key] for key in keys]) - elif hasattr(estimator, "feature_contributions"): - attribution = estimator.feature_contributions(rows) - else: - explainer = SHAP.TreeExplainer(estimator) - attribution = explainer.shap_values(rows) + attribution = _attribute( + estimator, + feature_values[valid_indices], + len(engineered_feature_cols), + [blocks[key] for key in keys] if blocked and blocks is not None else None, + ) return attribution, valid_indices, keys +def _attribute( + estimator: Any, rows: np.ndarray, num_features: int, block_indices: list[list[int]] | None +) -> np.ndarray: + """Attribution from whichever estimator this is, already grouped into blocks if blocks were asked for. + + Dispatch order matters. An estimator offering ``block_contributions`` supplies an exact joint + marginalisation, which is the only correct grouping for a non-additive attribution; a tree model gets + the plain sum, which is correct because SHAP is additive. A model with a single feature has nothing to + decompose, so that feature takes the whole share. + + Args: + estimator: The bare estimator, already unwrapped from any pipeline. + rows: Feature values for the rows to attribute, scaled if the model carries a scaler. + num_features: Width of the engineered feature space. + block_indices: Feature positions per block, or ``None`` to attribute per feature. + + Returns: + Signed attribution, oriented so larger means more responsible, one column per block when blocked. + """ + if num_features == 1: + return np.ones((len(rows), 1)) + if block_indices is not None and hasattr(estimator, "block_contributions"): + return np.asarray(estimator.block_contributions(rows, block_indices)) + if hasattr(estimator, "feature_contributions"): + return np.asarray(estimator.feature_contributions(rows)) + + per_feature = _oriented_towards_anomaly(np.asarray(SHAP.TreeExplainer(estimator).shap_values(rows))) + return _sum_within_blocks(per_feature, block_indices) if block_indices is not None else per_feature + + +def _sum_within_blocks(attribution: np.ndarray, block_indices: list[list[int]]) -> np.ndarray: + """Collapse per-feature attribution into one column per block by summing within each. + + Correct only for an additive attribution, which TreeSHAP is: each value is that feature's signed share + of the explained quantity, so a group's share is the plain sum over its members. An estimator whose + attribution is *not* additive -- the correlation-aware detector's leave-one-out drops, where the views + of one column each measure almost nothing on their own -- must not come through here; it supplies + ``block_contributions`` instead. + + Args: + attribution: Oriented per-feature attribution, shape ``(n_rows, n_features)``. + block_indices: Feature positions per block, in the order the blocks' keys are reported. + + Returns: + Array of shape ``(n_rows, len(block_indices))``. Signed, like its input: the clip belongs to + :func:`format_shap_contributions`, after this. + """ + return np.column_stack([attribution[:, indices].sum(axis=1) for indices in block_indices]) + + +def mean_row_attributions( + models: Sequence[Any], + feature_matrix: pd.DataFrame, + engineered_feature_cols: list[str], + blocks: dict[str, list[int]] | None = None, +) -> tuple[np.ndarray, np.ndarray, list[str]]: + """Attribution for the *aggregate* an ensemble reports, rather than for one of its members. + + An ensemble's reported score is the mean over members, and so is its ``confidence_std``, but the + explanation used to come from ``models[0]`` alone. Members differ only by random seed, which makes + member zero an arbitrary choice, not a representative one -- and they disagree far more than that + framing suggests: measured on 200 rows with the default three members, the pair that agreed least + named a different top driver on 130 of them, and the closest pair still differed on 92. So the row + was flagged by a committee and explained by whichever member happened to be trained first. + + Averaging is the aggregate that matches what the score does. SHAP is additive per member, so the mean + of the per-feature values decomposes the mean predicted path length exactly -- which is why + :func:`_oriented_towards_anomaly` must not clip before this runs. + + One honest caveat. The reported score is ``mean(-score_samples)``, and ``score_samples`` is a strictly + monotone but *nonlinear* transform of path length, so this decomposes the mean path length rather than + the mean score. That is acceptable because nothing reads these numbers as quantities: they are + normalised to shares and consumed only as a ranking, and since the transform is strictly decreasing, a + feature that shortens the path in every member also raises the score in every member. Only the + weighting would differ. + + Averaging also does not make the explanation *stable*, only unbiased between members -- with three + members it remains noisy, and the fix for that is more members, not a different aggregate. + + Args: + models: The models behind the reported score, in any order. A single-element sequence returns the + single-model attribution untouched, with no averaging. + feature_matrix: Rows to attribute, already engineered. + engineered_feature_cols: Feature names, positionally matching *feature_matrix*. + blocks: Optional source-column grouping, forwarded to :func:`compute_row_attributions`. + + Returns: + ``(attribution, valid_indices, keys)``, matching :func:`compute_row_attributions`. + + Raises: + InvalidParameterError: If *models* is empty, or if members disagree on the attribution's shape or + keys. Averaging those would blend columns naming different features into a confident, + plausible-looking, wrong explanation -- the same failure this function exists to remove, so it + fails loudly instead of falling back to one member. + """ + if not models: + raise InvalidParameterError("At least one model is required to attribute a row.") + + attribution, valid_indices, keys = compute_row_attributions( + models[0], feature_matrix, engineered_feature_cols, blocks + ) + if len(models) == 1 or attribution.size == 0: + return attribution, valid_indices, keys + + members = [attribution] + for model in models[1:]: + member, member_valid, member_keys = compute_row_attributions( + model, feature_matrix, engineered_feature_cols, blocks + ) + if member_keys != keys or member.shape != attribution.shape or not np.array_equal(member_valid, valid_indices): + raise InvalidParameterError( + f"Ensemble members produced attributions that cannot be averaged: {len(keys)} keys with " + f"shape {attribution.shape} against {len(member_keys)} keys with shape {member.shape}. " + "Averaging them would mix different feature layouts." + ) + members.append(member) + + return np.mean(np.stack(members), axis=0), valid_indices, keys + + # Severity-gating margin for in-UDF SHAP computation. The UDF recomputes severity from raw # scores with numpy while the authoritative severity is a Spark expression over the same # quantile points; the epsilon makes the UDF-side gate slightly over-inclusive so floating-point @@ -170,7 +373,7 @@ def severity_from_scores(scores: np.ndarray, quantile_points: list[tuple[float, def compute_gated_shap_contributions( - model_local: Any, + models: Sequence[Any], feature_matrix: pd.DataFrame, engineered_feature_cols: list[str], scores: np.ndarray, @@ -178,18 +381,34 @@ def compute_gated_shap_contributions( threshold: float | None, blocks: dict[str, list[int]] | None = None, ) -> list[dict[str, float | None] | None]: - """Compute SHAP contributions only for rows whose severity reaches the anomaly threshold. + """Attribute only the rows whose severity reaches the anomaly threshold. TreeSHAP costs an order of magnitude more than scoring itself, and contributions are only surfaced for anomalous rows, so computing SHAP for the typically tiny anomalous subset instead of every row removes most of the contributions cost. Rows below the threshold get - ``None`` (a null map). When *quantile_points* or *threshold* is unavailable, SHAP is - computed for all rows (previous behaviour). + ``None`` (a null map). When *quantile_points* or *threshold* is unavailable, attribution runs + for all rows (previous behaviour). + + *models* is the set of models behind the reported score, not one model: for an ensemble that is every + member, and the attribution is their mean via :func:`mean_row_attributions`. The parameter is a + sequence rather than a single model precisely so the ensemble mistake this replaced -- scoring with a + committee and explaining with whichever member trained first -- cannot be made again by omission. A + single-element sequence behaves exactly as passing that model alone did. Note that an sklearn + ``Pipeline`` is itself indexable, so passing one bare where a sequence is expected is not a type error; + it fails a frame later inside the explainer rather than silently attributing a pipeline step. + + Cost of the ensemble aggregate is modest because the gate does the heavy lifting. Attribution runs on + the rows above the threshold, typically well under 1% of them, at roughly ten times scoring cost -- + about a tenth of one scoring pass. Scoring an N-member ensemble already costs N passes, so the total + moves from about ``N + 0.1`` to ``N + 0.1N``: near 7% more at the default three members. That is the + reason there is no setting for how many members to attribute. Such a knob would offer a choice between + a correct explanation and a few percent of runtime, and the honest lever for anyone who does not want + the cost is *enable_contributions=False*, which already exists. """ num_rows = len(feature_matrix) if not quantile_points or threshold is None: - attribution, valid_indices, keys = compute_row_attributions( - model_local, feature_matrix, engineered_feature_cols, blocks + attribution, valid_indices, keys = mean_row_attributions( + models, feature_matrix, engineered_feature_cols, blocks ) return list(format_shap_contributions(attribution, valid_indices, num_rows, keys)) @@ -198,9 +417,7 @@ def compute_gated_shap_contributions( contributions: list[dict[str, float | None] | None] = [None] * num_rows if anomalous_positions.size: subset = feature_matrix.iloc[anomalous_positions] - attribution, valid_indices, keys = compute_row_attributions( - model_local, subset, engineered_feature_cols, blocks - ) + attribution, valid_indices, keys = mean_row_attributions(models, subset, engineered_feature_cols, blocks) subset_contributions = format_shap_contributions(attribution, valid_indices, len(subset), keys) for position, contribution in zip(anomalous_positions.tolist(), subset_contributions): contributions[position] = contribution @@ -214,9 +431,13 @@ def format_contributions_map(contributions_map: dict[str, float | None] | None, contributions_map: Dictionary mapping feature names to contribution values (0-100 range) top_n: Number of top contributors to include + Features with no share are omitted rather than rendered as ``name (0%)``. Since attribution stopped + crediting features that argued the row was *normal*, an exact zero is now common -- and naming one as a + contributor states that it contributed, which is what the reader takes from seeing it in this list. + Returns: Formatted string like "amount (85%), quantity (10%), discount (5%)" - Empty string if contributions_map is None or empty + Empty string if contributions_map is None, empty, or holds nothing that contributed Example: >>> format_contributions_map(dict(amount=85.0, quantity=10.0), 2) @@ -225,17 +446,12 @@ def format_contributions_map(contributions_map: dict[str, float | None] | None, if not contributions_map: return "" - # Sort by absolute contribution value (descending) to rank by impact magnitude - sorted_contribs = sorted( - contributions_map.items(), key=lambda x: abs(x[1]) if x[1] is not None else 0.0, reverse=True - ) - - # Take top N - top_contribs = sorted_contribs[:top_n] + contributed = [(col, val) for col, val in contributions_map.items() if val is not None and val > 0.0] + # Rank by share, descending. Values are non-negative shares of the anomaly-driving evidence. + top_contribs = sorted(contributed, key=lambda item: item[1], reverse=True)[:top_n] # Format as string: "amount (85%), quantity (10%), discount (5%)" - parts = [f"{col} ({val:.0f}%)" for col, val in top_contribs if val is not None] - return ", ".join(parts) + return ", ".join(f"{col} ({val:.0f}%)" for col, val in top_contribs) def create_optimal_tree_explainer(tree_model: Any) -> Any: @@ -256,7 +472,17 @@ def create_optimal_tree_explainer(tree_model: Any) -> Any: def compute_contributions_for_matrix( model_local: Any, feature_matrix: np.ndarray, columns: list[str] ) -> list[dict[str, float | None]]: - """Compute normalized SHAP contributions for a feature matrix.""" + """Compute normalised contributions for a raw feature matrix, one row at a time. + + Shares the semantics of the scoring path rather than reimplementing them: values are oriented via + :func:`_oriented_towards_anomaly`, the side arguing the row is normal earns no share, and a row with + no anomaly-driving evidence gets all-``None`` instead of an invented uniform split. Keeping the two in + step matters more than the small duplication -- one module giving two different answers to "what does a + negative SHAP value mean" is how the original defect survived as long as it did. + + Unlike the scoring path, contributions here are fractions of 1 rather than percentages, which is the + existing contract of this function and its caller. + """ # If model is a Pipeline (due to feature scaling), extract components # SHAP's TreeExplainer only supports tree models, not pipelines # A Pipeline no longer necessarily contains a scaler: DQX fits the forest without one, since an @@ -287,15 +513,17 @@ def compute_contributions_for_matrix( contributions_list.append({col: None for col in columns}) continue - shap_values = explainer.shap_values(feature_matrix[i : i + 1])[0] - abs_shap = np.abs(shap_values) - total = abs_shap.sum() + driving = np.maximum( + _oriented_towards_anomaly(np.asarray(explainer.shap_values(feature_matrix[i : i + 1]))[0]), 0.0 + ) + total = driving.sum() + contributions: dict[str, float | None] if total > 0: - normalized = abs_shap / total - contributions: dict[str, float | None] = {col: float(normalized[j]) for j, col in enumerate(columns)} + normalized = driving / total + contributions = {col: float(normalized[j]) for j, col in enumerate(columns)} else: - contributions = {col: 1.0 / len(columns) for col in columns} + contributions = {col: None for col in columns} contributions_list.append(contributions) diff --git a/src/databricks/labs/dqx/anomaly/single_model_scorer.py b/src/databricks/labs/dqx/anomaly/single_model_scorer.py index fdecca0ad..ab162abe7 100644 --- a/src/databricks/labs/dqx/anomaly/single_model_scorer.py +++ b/src/databricks/labs/dqx/anomaly/single_model_scorer.py @@ -67,7 +67,7 @@ def predict_with_shap_udf(*cols: pd.Series) -> pd.DataFrame: scores = -model_local.score_samples(feature_matrix) contributions_list = compute_gated_shap_contributions( - model_local, + [model_local], feature_matrix, engineered_feature_cols, scores, @@ -162,7 +162,7 @@ def score_with_sklearn_model_local( if enable_contributions: result["anomaly_contributions"] = compute_gated_shap_contributions( - sklearn_model, + [sklearn_model], feature_matrix, engineered_feature_cols, scores, diff --git a/src/databricks/labs/dqx/anomaly/timeseries_detector.py b/src/databricks/labs/dqx/anomaly/timeseries_detector.py index b3aed4284..3738be7c8 100644 --- a/src/databricks/labs/dqx/anomaly/timeseries_detector.py +++ b/src/databricks/labs/dqx/anomaly/timeseries_detector.py @@ -220,11 +220,12 @@ def feature_contributions(self, X: np.ndarray) -> np.ndarray: The tempting alternative, the exactly-additive ``cᵢ = (x−μ)ᵢ·zᵢ`` with ``Σᵢ cᵢ = d²``, is **wrong for this pipeline**: its terms can be negative when features are correlated. With ``Σ = [[1, 0.9], [0.9, 1]]`` and ``x−μ = (1.0, 0.5)`` it gives ``(2.895, −1.053)`` — the second - feature *reduced* the distance. Every consumer downstream takes ``abs()`` and renormalises - (``explainability.format_shap_contributions``, ``_pattern_spark_expr``, - ``_format_contributions_sql``), so that term would be presented to an LLM as a 27% *driver* of - the anomaly and written into a narrative. Additivity buys nothing here, because nothing - downstream consumes it; non-negativity is what correctness requires. + feature *reduced* the distance. ``explainability.format_shap_contributions`` drops values at or + below zero and renormalises, so that term would not be reported as a 27% driver -- it would be + dropped entirely, and the feature that *did* reduce the distance would silently vanish from an + explanation whose remaining shares no longer describe the distance they came from. Additivity + buys nothing here, because nothing downstream consumes it; non-negativity is what correctness + requires, and it has to come from the formula rather than from a clip. Constant-in-training features are excluded from the distance and reported as ``0.0``, so the returned width always matches the trained feature count and therefore diff --git a/tests/integration_anomaly/test_anomaly_ensemble.py b/tests/integration_anomaly/test_anomaly_ensemble.py index 97aebf2fa..bf6d85bc0 100644 --- a/tests/integration_anomaly/test_anomaly_ensemble.py +++ b/tests/integration_anomaly/test_anomaly_ensemble.py @@ -125,7 +125,14 @@ def test_ensemble_with_feature_contributions( anomaly_scorer, anomaly_registry_prefix, ): - """Test that ensemble works with feature contributions.""" + """The ensemble's contributions describe the aggregate it scored with, and honour the public contract. + + Previously this asserted only that the map was not null, which passed while the map came from one + arbitrary member -- the member that happened to train first. The contract is asserted here instead: + keys are the columns the caller passed, values are non-negative, and they total 100. That has to hold + after the mean across members has survived cloudpickle, the pandas UDF and the Spark map round-trip, + which is the part no unit test can reach. + """ unique_id = make_random(8).lower() model_name = f"{anomaly_registry_prefix}.test_ensemble_contributions_{make_random(4).lower()}" registry_table = f"{anomaly_registry_prefix}.{unique_id}_registry" @@ -157,4 +164,16 @@ def test_ensemble_with_feature_contributions( rows_by_id = {row["transaction_id"]: row for row in result_df.collect()} row = rows_by_id[2] assert row["_dq_info"][0]["anomaly"]["confidence_std"] is not None - assert row["_dq_info"][0]["anomaly"]["contributions"] is not None + + contributions = row["_dq_info"][0]["anomaly"]["contributions"] + assert contributions is not None + + # Keyed by the caller's own columns, not by engineered feature names: one column reports once + # however many features were derived from it. + unknown = sorted(set(contributions) - {"amount", "quantity", "discount"}) + assert not unknown, f"contribution keys {unknown} are not columns the caller passed" + + values = [v for v in contributions.values() if v is not None] + assert values, "the flagged row's contributions map held only nulls" + assert min(values) >= 0.0, f"contributions must be non-negative, got {min(values)}" + assert abs(sum(values) - 100.0) < 0.5, f"contributions should be normalised to 100, summed to {sum(values)}" diff --git a/tests/integration_anomaly/test_anomaly_timeseries_profile.py b/tests/integration_anomaly/test_anomaly_timeseries_profile.py index 97cf77a56..3bc7ebc4a 100644 --- a/tests/integration_anomaly/test_anomaly_timeseries_profile.py +++ b/tests/integration_anomaly/test_anomaly_timeseries_profile.py @@ -119,20 +119,26 @@ def _train_both_profiles(spark, quick_model_factory, columns, train_rows): return models -def _assert_contribution_contract(contributions_series, engineered_names: set[str]) -> None: - """Every map is keyed by the persisted contract, non-negative, and normalised to 100. - - Non-negativity is the load-bearing one: the leave-one-out attribution is non-negative by - construction (the precision matrix is PSD), which is what lets it reuse the SHAP formatter - unchanged. A negative value here would mean the formula regressed, not the formatting. +def _assert_contribution_contract(contributions_series, source_columns: set[str]) -> None: + """Every map is keyed by the caller's own columns, non-negative, and normalised to 100. + + Keys are asserted against the **source columns**, not the engineered feature names, because that is + the contract: attribution is grouped so one column reports once however many features were derived + from it. Asserting engineered names would pass by accident on an all-numeric fixture, where each + column's only feature is named after it, and would fail the moment a categorical appeared. + + Non-negativity is the other load-bearing one. Both detectors produce it by construction rather than + by clipping -- the leave-one-out attribution because the precision matrix is PSD, and the tree path + because the formatter drops the side that argued the row was normal -- so a negative here means a + formula regressed. """ for contributions in contributions_series: assert contributions is not None, "a flagged row carried no contributions map" - unknown = sorted(set(contributions) - engineered_names) - assert not unknown, f"contribution keys {unknown} are not in the persisted engineered feature names" + unknown = sorted(set(contributions) - source_columns) + assert not unknown, f"contribution keys {unknown} are not columns the caller passed" values = [v for v in contributions.values() if v is not None] assert values, "a flagged row's contributions map held only nulls" - assert min(values) >= 0.0, f"leave-one-out contributions must be non-negative, got {min(values)}" + assert min(values) >= 0.0, f"contributions must be non-negative, got {min(values)}" assert abs(sum(values) - 100.0) < 0.5, f"contributions should be normalised to 100, summed to {sum(values)}" @@ -251,10 +257,11 @@ def test_timeseries_profile_end_to_end( # 3. Contributions honour the persisted feature contract on every flagged row. engineered_names = set(_engineered_feature_names(spark, timeseries_registry, timeseries_model)) + assert engineered_names, "the persisted feature contract should be readable back from the registry" flagged = timeseries[timeseries["flagged"] == 1.0] assert not flagged.empty, "no row was flagged, so the contributions assertions would pass vacuously" - _assert_contribution_contract(flagged["contributions"], engineered_names) + _assert_contribution_contract(flagged["contributions"], set(columns)) # Gating must actually have happened. Attribution costs an order of magnitude more than scoring, # so computing it for every row is a performance regression rather than a cosmetic one. Asserted as diff --git a/tests/unit/test_anomaly_attribution_orientation.py b/tests/unit/test_anomaly_attribution_orientation.py new file mode 100644 index 000000000..5c635a9e6 --- /dev/null +++ b/tests/unit/test_anomaly_attribution_orientation.py @@ -0,0 +1,174 @@ +"""What a contribution means, and the sign convention the whole map depends on (no Spark, no workspace). + +TreeSHAP on an isolation forest explains the ensemble's average *path length*, so a negative value is +what drove the anomaly and a positive one argues the row is ordinary. Getting that backwards is not a +small error, and it is not detectable from any test that only checks the map's shape -- so the +orientation is pinned here against ground truth, and the two alternatives are pinned as failures. + +The arithmetic tests build the attribution matrix by hand, which is the only way to state the semantics +without depending on whatever SHAP happens to produce for a fixture. +""" + +import numpy as np +import pandas as pd +import pytest +from sklearn.ensemble import IsolationForest + +from databricks.labs.dqx.anomaly.explainability import ( + compute_row_attributions, + format_contributions_map, + format_shap_contributions, +) + +_KEYS = ["amount", "quantity"] + + +def _single_row(values: list[float], keys: list[str] | None = None) -> dict[str, float | None]: + """Format one row's attribution, returning just that row's map.""" + names = keys if keys is not None else _KEYS + return format_shap_contributions(np.array([values]), np.array([True]), 1, names)[0] + + +# ── what earns a share ─────────────────────────────────────────────────────────────────────────────── + + +def test_a_feature_arguing_the_row_is_normal_earns_no_share(): + """A normalising feature is not a driver, and used to be reported as one. + + Attribution arrives oriented so larger means more responsible; a negative value is a feature that + made the row look *more* ordinary. Taking the magnitude added that to the evidence for the anomaly + and reported the sum as one number, so a feature that argued against the flag was rendered as having + caused a fifth of it. + """ + contributions = _single_row([8.0, -2.0]) + + assert contributions == {"amount": 100.0, "quantity": 0.0} + + +def test_the_emitted_shares_still_total_one_hundred(): + """The public contract, which the fix had to preserve rather than trade away. + + Two test suites and the documented schema of ``_dq_info[].anomaly.contributions`` depend on the map + being non-negative and summing to 100. Dropping the normalising side changes the denominator, so this + asserts the sum survived that change. + """ + contributions = _single_row([6.0, 2.0, -3.0], keys=["a", "b", "c"]) + + assert sum(v for v in contributions.values() if v is not None) == pytest.approx(100.0, abs=0.5) + assert min(v for v in contributions.values() if v is not None) >= 0.0 + + +# ── absence of evidence is not evidence ────────────────────────────────────────────────────────────── + + +def test_a_row_with_no_anomaly_driving_evidence_gets_no_explanation(): + """Every feature says the row looks normal, so there is nothing to report. + + This used to emit ``1 / n`` for every key -- "all features contributed equally" -- which is a claim + with no input behind it, and the more misleading failure of the two because the numbers look + unremarkable. The all-null map is what a row with a null feature already produces, and the consumers + already render it as ``unknown``. + """ + contributions = _single_row([-1.0, -2.0]) + + assert contributions == {"amount": None, "quantity": None} + + +def test_an_all_zero_attribution_row_gets_no_explanation(): + """The exact branch the invented uniform split used to occupy: a total of zero.""" + contributions = _single_row([0.0, 0.0]) + + assert contributions == {"amount": None, "quantity": None} + + +def test_a_row_without_evidence_does_not_shift_the_rows_after_it(): + """The off-by-one this restructure could have introduced, and the reason it is tested directly. + + Skipping a row while formatting must not skip the attribution cursor, or every row after the first + unexplained one is given another row's numbers -- plausible output, wrong row, invisible without an + assertion like this one. + """ + attribution = np.array([[9.0, 1.0], [-1.0, -1.0], [1.0, 3.0]]) + + contributions = format_shap_contributions(attribution, np.array([True, True, True]), 3, _KEYS) + + assert contributions[0] == {"amount": 90.0, "quantity": 10.0} + assert contributions[1] == {"amount": None, "quantity": None} + assert contributions[2] == {"amount": 25.0, "quantity": 75.0} + + +def test_a_null_feature_row_and_an_unexplained_row_can_both_appear_at_once(): + """The two independent reasons for a null map compose, and the cursor copes with both.""" + attribution = np.array([[-1.0, -1.0], [2.0, 2.0]]) + + contributions = format_shap_contributions(attribution, np.array([True, False, True]), 3, _KEYS) + + assert contributions[0] == {"amount": None, "quantity": None} + assert contributions[1] == {"amount": None, "quantity": None} + assert contributions[2] == {"amount": 50.0, "quantity": 50.0} + + +# ── ground truth: the orientation itself ───────────────────────────────────────────────────────────── + + +@pytest.fixture +def forest_and_features() -> tuple[IsolationForest, list[str]]: + """An isolation forest on three independent standard normals, so any anomaly is one we injected.""" + rng = np.random.default_rng(7) + columns = ["a", "b", "c"] + train = pd.DataFrame(rng.normal(0, 1, (600, 3)), columns=columns) + return IsolationForest(n_estimators=200, random_state=0).fit(train), columns + + +@pytest.mark.parametrize("culprit", [0, 1, 2]) +def test_the_deliberately_anomalous_feature_is_named_as_the_top_driver( + forest_and_features: tuple[IsolationForest, list[str]], culprit: int +): + """Ground truth, end to end through real TreeSHAP: one feature is anomalous, and it must be named.""" + forest, columns = forest_and_features + values = [0.1, 0.1, 0.1] + values[culprit] = 9.0 + probe = pd.DataFrame([values], columns=columns) + + attribution, valid_indices, keys = compute_row_attributions(forest, probe, columns) + contributions = format_shap_contributions(attribution, valid_indices, 1, keys)[0] + + named = max(contributions, key=lambda k: contributions[k] or 0.0) + assert named == columns[culprit], f"expected {columns[culprit]}, got {contributions}" + + +@pytest.mark.parametrize("culprit", [0, 1, 2]) +def test_keeping_the_un_negated_side_would_name_the_most_ordinary_feature_instead( + forest_and_features: tuple[IsolationForest, list[str]], culprit: int +): + """The alternative a reader might reach for, pinned as wrong so nobody reaches for it again. + + A review of this code proposed keeping the *positive* SHAP values, on the reading that positive means + "drove the anomaly". It is the opposite: because the explained quantity is a path length, the positive + side is the evidence that the row is ordinary. Measured over 200 single-culprit rows, that choice + named the true culprit 0 times; the orientation in use named it 199 times. + + Asserted by reconstructing the rejected alternative from the same attribution, so this test cannot + drift away from the code it is arguing about. + """ + forest, columns = forest_and_features + values = [0.1, 0.1, 0.1] + values[culprit] = 9.0 + probe = pd.DataFrame([values], columns=columns) + + attribution, _, _ = compute_row_attributions(forest, probe, columns) + # compute_row_attributions already negated, so negating again recovers the raw SHAP values. + rejected = np.maximum(-attribution, 0.0) + + assert rejected.argmax() != culprit, "the un-negated side should not name the true culprit" + + +def test_a_zero_share_feature_is_not_rendered_as_a_contributor(): + """Dropping the normalising side makes exact zeros common, and a zero is not a driver. + + Before this, nearly every feature had some magnitude and so a non-zero share. Now a feature that + argued the row was normal lands on exactly 0.0, and rendering it produces "quantity (0%)" in a + message or an LLM prompt -- naming something that contributed nothing, which is the same invention + the all-null map exists to avoid, one layer down. + """ + assert format_contributions_map({"amount": 100.0, "quantity": 0.0}, 3) == "amount (100%)" diff --git a/tests/unit/test_anomaly_ensemble_attribution.py b/tests/unit/test_anomaly_ensemble_attribution.py new file mode 100644 index 000000000..f80a789c9 --- /dev/null +++ b/tests/unit/test_anomaly_ensemble_attribution.py @@ -0,0 +1,185 @@ +"""An ensemble's explanation must describe the aggregate it scored with (no Spark, no workspace). + +The score an ensemble reports is the mean over its members, and so is *anomaly_score_std*. The +explanation used to come from ``models[0]``, which is not a representative member -- members differ only +by random seed, so member zero is just the one that trained first. They also disagree far more than that +framing suggests: measured on 200 rows with the default three members, the least-agreeing pair named a +different top driver on 130 of them and the closest pair still differed on 92. +""" + +import numpy as np +import pandas as pd +import pytest +from sklearn.ensemble import IsolationForest + +from databricks.labs.dqx.anomaly.explainability import ( + compute_gated_shap_contributions, + compute_row_attributions, + format_shap_contributions, + mean_row_attributions, +) +from databricks.labs.dqx.errors import InvalidParameterError + +_COLUMNS = ["amount", "quantity", "discount"] +_QUANTILE_POINTS = [(50.0, 0.4), (90.0, 0.5), (95.0, 0.55), (99.0, 0.6)] + + +@pytest.fixture +def members() -> list[IsolationForest]: + """Three members differing only by seed, mirroring how *ensemble_training* builds them.""" + rng = np.random.default_rng(11) + train = pd.DataFrame(rng.normal(0, 1, (500, 3)), columns=_COLUMNS) + return [IsolationForest(n_estimators=120, random_state=42 + i).fit(train) for i in range(3)] + + +@pytest.fixture +def probe() -> pd.DataFrame: + """Rows spanning ordinary to clearly anomalous, so the gate has something to include and exclude.""" + return pd.DataFrame( + [[0.2, -0.1, 0.3], [7.0, 0.2, -0.4], [-6.5, 5.0, 0.1]], + columns=_COLUMNS, + ) + + +def _map_for(attribution: np.ndarray, valid: np.ndarray, keys: list[str]) -> list[dict[str, float | None]]: + return list(format_shap_contributions(attribution, valid, len(attribution), keys)) + + +def test_the_attribution_is_the_mean_over_members_not_the_first_one( + members: list[IsolationForest], probe: pd.DataFrame +): + """The defect: scored by a committee, explained by whichever member trained first. + + Built by averaging the per-member attributions independently, so this asserts the aggregate rather + than re-deriving whatever the implementation happens to do. + """ + per_member = [compute_row_attributions(model, probe, _COLUMNS)[0] for model in members] + expected = np.mean(np.stack(per_member), axis=0) + + attribution, _, keys = mean_row_attributions(members, probe, _COLUMNS) + + assert keys == _COLUMNS + np.testing.assert_allclose(attribution, expected, rtol=1e-12) + # And it is genuinely not member zero's, or the test would pass without the fix. + assert not np.allclose(attribution, per_member[0]) + + +def test_averaging_happens_before_the_clip_so_a_disagreeing_member_can_cancel( + members: list[IsolationForest], probe: pd.DataFrame +): + """Why the orientation must not clip: clip-then-mean and mean-then-clip are different numbers. + + Averaging is only an exact decomposition of the mean path length while the negatives survive. If a + member decides a feature made the row look *normal*, that has to pull the mean down rather than be + read as zero -- two members at +4 and -2 average to 1, but clipping first averages to 2. + + Pinned by asserting the intermediate is still signed, which is the property clip-then-mean destroys. + """ + attribution, _, _ = mean_row_attributions(members, probe, _COLUMNS) + + assert (attribution < 0).any(), "the averaged attribution should still carry the normalising side" + + +def test_member_order_does_not_change_the_explanation(members: list[IsolationForest], probe: pd.DataFrame): + """Member order is an accident of training, so it must not reach the output. + + Compared with a tolerance, not exactly: floating-point addition is not associative, so summing three + members in a different order gives a bit-different answer. Asserting bit equality here would be a + statement about float arithmetic rather than about the aggregate, and it would fail. + """ + original, _, keys = mean_row_attributions(members, probe, _COLUMNS) + reordered, _, reordered_keys = mean_row_attributions([members[2], members[0], members[1]], probe, _COLUMNS) + + assert reordered_keys == keys + np.testing.assert_allclose(reordered, original, rtol=1e-9) + + +def test_every_member_influences_the_result(probe: pd.DataFrame): + """The behavioural statement of the fix, which survives a refactor that the arithmetic test would not. + + Two members are fitted so that each one attributes the same row predominantly to a *different* + feature. Explaining either alone names only that member's favourite; the aggregate has to reflect + both, because both took part in the score. + """ + rng = np.random.default_rng(3) + rows = 500 + tight_amount = pd.DataFrame( + np.column_stack([rng.normal(0, 0.05, rows), rng.normal(0, 3.0, rows), rng.normal(0, 1, rows)]), columns=_COLUMNS + ) + tight_quantity = pd.DataFrame( + np.column_stack([rng.normal(0, 3.0, rows), rng.normal(0, 0.05, rows), rng.normal(0, 1, rows)]), columns=_COLUMNS + ) + one = IsolationForest(n_estimators=200, random_state=0).fit(tight_amount) + two = IsolationForest(n_estimators=200, random_state=0).fit(tight_quantity) + row = pd.DataFrame([[1.0, 1.0, 0.0]], columns=_COLUMNS) + + top_one = _map_for(*compute_row_attributions(one, row, _COLUMNS)[:2], _COLUMNS)[0] + top_two = _map_for(*compute_row_attributions(two, row, _COLUMNS)[:2], _COLUMNS)[0] + combined = _map_for(*mean_row_attributions([one, two], row, _COLUMNS)[:2], _COLUMNS)[0] + + named_one = max(top_one, key=lambda k: top_one[k] or 0.0) + named_two = max(top_two, key=lambda k: top_two[k] or 0.0) + assert named_one != named_two, "fixture must produce members that disagree, or this proves nothing" + + # The aggregate credits both members' features, rather than only one of them. + assert (combined[named_one] or 0.0) > 5.0 + assert (combined[named_two] or 0.0) > 5.0 + + +def test_a_single_member_is_left_exactly_as_the_single_model_path(members: list[IsolationForest], probe: pd.DataFrame): + """The single-model path must be bit-identical: no averaging, no division by one, no stacking.""" + expected, expected_valid, expected_keys = compute_row_attributions(members[0], probe, _COLUMNS) + actual, actual_valid, actual_keys = mean_row_attributions([members[0]], probe, _COLUMNS) + + assert actual_keys == expected_keys + assert np.array_equal(actual_valid, expected_valid) + assert np.array_equal(actual, expected) + + +def test_the_gated_entry_point_takes_every_member(members: list[IsolationForest], probe: pd.DataFrame): + """The seam the scorers actually call, so the fix is wired and not merely available. + + Checks the plumbing and the gate, not the aggregate: whether the numbers are the mean is asserted + directly on the attribution above, which is the level where it cannot be masked by clipping, + normalising and rounding. On a row with one overwhelming driver every member agrees anyway, so a map + comparison here would prove nothing and would fail for the wrong reason. + """ + scores = np.array([0.40, 0.62, 0.61]) + + contributions = compute_gated_shap_contributions(members, probe, _COLUMNS, scores, _QUANTILE_POINTS, threshold=95.0) + + assert contributions[0] is None, "an ordinary row should not be attributed at all" + for anomalous in contributions[1:]: + assert anomalous is not None + assert sum(v for v in anomalous.values() if v is not None) == pytest.approx(100.0, abs=0.5) + assert min(v for v in anomalous.values() if v is not None) >= 0.0 + + +# ── failures that must be loud ─────────────────────────────────────────────────────────────────────── + + +class _WrongWidthEstimator: + """An estimator whose attribution has the wrong number of columns. + + A real second IsolationForest cannot produce this -- sklearn would raise inside ``score_samples`` + first -- so a stand-in is the only way to reach the guard. + """ + + def feature_contributions(self, rows: np.ndarray) -> np.ndarray: + return np.ones((len(rows), 1)) + + +def test_members_whose_attributions_cannot_be_aligned_are_rejected(members: list[IsolationForest], probe: pd.DataFrame): + """Averaging misaligned columns would blend different features into one confident, wrong answer. + + That is indistinguishable from correct output downstream -- exactly the failure mode this whole + function exists to remove -- so it raises rather than quietly falling back to one member. + """ + with pytest.raises(InvalidParameterError, match="cannot be averaged"): + mean_row_attributions([members[0], _WrongWidthEstimator()], probe, _COLUMNS) + + +def test_no_models_is_rejected(probe: pd.DataFrame): + """No silent empty map and no division by zero.""" + with pytest.raises(InvalidParameterError, match="At least one model"): + mean_row_attributions([], probe, _COLUMNS) diff --git a/tests/unit/test_anomaly_llm_explainer.py b/tests/unit/test_anomaly_llm_explainer.py index 7dc7ce668..dec6f74ab 100644 --- a/tests/unit/test_anomaly_llm_explainer.py +++ b/tests/unit/test_anomaly_llm_explainer.py @@ -297,12 +297,15 @@ def test_attribution_semantics_distinguishes_correlation_from_value_anomalies(): assert "do not assert either" in correlation values = llm_explainer.attribution_semantics("IsolationForest") - assert "feature's own value" in values + assert "metric's own value" in values assert correlation != values value_based = llm_explainer.attribution_semantics("IsolationForest") - assert "own value was unusual" in value_based + assert "was unusual for the rows it was compared against" in value_based assert "relationship" not in value_based + # A key covers a column with every comparison made of it, so the reading must not promise which + # comparison objected -- a share of 60% on a metric compared three ways says the metric was involved. + assert "not which comparison objected" in value_based assert correlation != value_based diff --git a/tests/unit/test_anomaly_shap_gating.py b/tests/unit/test_anomaly_shap_gating.py index 39bbd5ed4..9fed9515c 100644 --- a/tests/unit/test_anomaly_shap_gating.py +++ b/tests/unit/test_anomaly_shap_gating.py @@ -41,7 +41,7 @@ def test_gated_contributions_computed_only_for_anomalous_rows(fitted_model_and_f # Scores are passed in explicitly; only the last row's severity reaches the threshold. scores = np.array([1.0, 1.2, 1.1, 10.0]) contributions = compute_gated_shap_contributions( - model, features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=85.0 + [model], features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=85.0 ) assert contributions[:3] == [None, None, None] assert isinstance(contributions[3], dict) @@ -54,7 +54,7 @@ def test_gated_contributions_fall_back_to_all_rows_without_quantile_points(fitte model, features = fitted_model_and_features scores = np.array([1.0, 1.2, 1.1, 10.0]) contributions = compute_gated_shap_contributions( - model, features, ["amount", "quantity"], scores, quantile_points=None, threshold=85.0 + [model], features, ["amount", "quantity"], scores, quantile_points=None, threshold=85.0 ) assert all(isinstance(c, dict) for c in contributions) @@ -64,7 +64,7 @@ def test_gated_contributions_epsilon_includes_threshold_boundary(fitted_model_an # Severity of score 4.0 is exactly 90; with threshold 90 the boundary row must be included. scores = np.array([1.0, 1.0, 1.0, 4.0]) contributions = compute_gated_shap_contributions( - model, features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=90.0 + [model], features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=90.0 ) assert contributions[:3] == [None, None, None] assert isinstance(contributions[3], dict) @@ -96,7 +96,7 @@ def test_gating_behaves_identically_for_a_non_shap_estimator(fitted_mahalanobis_ scores = np.array([1.0, 1.2, 1.1, 10.0]) contributions = compute_gated_shap_contributions( - model, features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=85.0 + [model], features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=85.0 ) assert contributions[:3] == [None, None, None] @@ -111,7 +111,7 @@ def test_non_shap_contributions_are_map_compatible_with_the_shap_path(fitted_mah scores = np.array([1.0, 1.2, 1.1, 10.0]) contributions = compute_gated_shap_contributions( - model, features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=85.0 + [model], features, ["amount", "quantity"], scores, QUANTILE_POINTS, threshold=85.0 ) anomalous = contributions[3] @@ -125,7 +125,7 @@ def test_non_shap_contributions_are_never_negative(fitted_mahalanobis_and_featur model, features = fitted_mahalanobis_and_features contributions = compute_gated_shap_contributions( - model, features, ["amount", "quantity"], np.array([9.0, 9.0, 9.0, 10.0]), QUANTILE_POINTS, threshold=10.0 + [model], features, ["amount", "quantity"], np.array([9.0, 9.0, 9.0, 10.0]), QUANTILE_POINTS, threshold=10.0 ) for row in contributions: @@ -138,7 +138,7 @@ def test_the_deviating_feature_dominates_the_attribution(fitted_mahalanobis_and_ model, features = fitted_mahalanobis_and_features contributions = compute_gated_shap_contributions( - model, features, ["amount", "quantity"], np.array([1.0, 1.2, 1.1, 10.0]), QUANTILE_POINTS, threshold=85.0 + [model], features, ["amount", "quantity"], np.array([1.0, 1.2, 1.1, 10.0]), QUANTILE_POINTS, threshold=85.0 ) assert contributions[3]["amount"] > contributions[3]["quantity"] @@ -150,7 +150,7 @@ def test_rows_with_nulls_get_an_all_none_map(fitted_mahalanobis_and_features): features = pd.DataFrame({"amount": [9999.0, np.nan], "quantity": [1.0, 2.0]}) contributions = compute_gated_shap_contributions( - model, features, ["amount", "quantity"], np.array([10.0, 10.0]), QUANTILE_POINTS, threshold=10.0 + [model], features, ["amount", "quantity"], np.array([10.0, 10.0]), QUANTILE_POINTS, threshold=10.0 ) assert all(value is None for value in contributions[1].values()) diff --git a/tests/unit/test_anomaly_source_block_attribution.py b/tests/unit/test_anomaly_source_block_attribution.py new file mode 100644 index 000000000..66708a760 --- /dev/null +++ b/tests/unit/test_anomaly_source_block_attribution.py @@ -0,0 +1,136 @@ +"""Attribution keyed by the column a reader passed, not by the views feature engineering made of it. + +DQX gives one numeric column up to three engineered views: the metric, its deviation from its group's +baseline, and its deviation from its expected level at that time. Explaining views one at a time answers +a question nobody asked -- how much did *this view* matter -- and the answer does not add up to how much +the column mattered. + +Both detectors need blocking and both get it, by different arithmetic: an exact joint marginalisation for +the correlation-aware detector, whose per-view drops are each almost nothing, and a plain sum for a tree +model, which is exact because SHAP is additive. This file covers the tree path; the correlation-aware +path is covered in test_anomaly_mahalanobis_detector.py. +""" + +import numpy as np +import pandas as pd +import pytest +from sklearn.ensemble import IsolationForest + +from databricks.labs.dqx.anomaly.explainability import compute_row_attributions, format_shap_contributions + + +def _shares(model: IsolationForest, row: pd.DataFrame, columns: list[str], blocks=None) -> dict[str, float]: + attribution, valid, keys = compute_row_attributions(model, row, columns, blocks) + emitted = format_shap_contributions(attribution, valid, 1, keys)[0] + return {key: value for key, value in emitted.items() if value is not None} + + +def test_a_single_view_per_column_is_left_exactly_as_the_unblocked_map(): + """Blocking generalises the per-feature map; it must not redefine it. + + With one view per column every block is a singleton, so the sum over each block is the value itself. + Asserted rather than argued, because otherwise every model without derived features would silently + change its explanations. + """ + rng = np.random.default_rng(5) + columns = ["amount", "quantity"] + train = pd.DataFrame(rng.normal(0, 1, (500, 2)), columns=columns) + model = IsolationForest(n_estimators=150, random_state=0).fit(train) + row = pd.DataFrame([[6.0, 0.3]], columns=columns) + + unblocked = _shares(model, row, columns) + blocked = _shares(model, row, columns, {"amount": [0], "quantity": [1]}) + + assert blocked == unblocked + + +def test_the_views_of_one_column_are_summed_into_that_column(): + """The defect: one column's evidence split across its views, so the column understates itself. + + A constant temporal expectation makes ``amount_rel_time`` an affine duplicate of *amount*. The tree + model then splits the evidence roughly in half between them -- measured at 49.3% and 50.7% where the + same data with a single view attributes 100% to the column. Summing the block recovers it exactly. + """ + rng = np.random.default_rng(0) + rows = 3000 + amount = rng.normal(0, 1, rows) + quantity = rng.normal(0, 1, rows) + + single_view = pd.DataFrame({"amount": amount, "quantity": quantity}) + undisturbed = IsolationForest(n_estimators=200, random_state=0).fit(single_view) + truth = _shares(undisturbed, pd.DataFrame([[8.0, 0.5]], columns=["amount", "quantity"]), ["amount", "quantity"]) + + columns = ["amount", "amount_rel_time", "quantity"] + expanded = pd.DataFrame({"amount": amount, "amount_rel_time": amount - 3.0, "quantity": quantity}) + model = IsolationForest(n_estimators=200, random_state=0).fit(expanded) + row = pd.DataFrame([[8.0, 5.0, 0.5]], columns=columns) + + per_view = _shares(model, row, columns) + # Pinned so the fix's absence is not mistaken for the test being vacuous. + assert per_view["amount"] < 70.0, f"expected the per-view form to understate amount, got {per_view}" + + blocked = _shares(model, row, columns, {"amount": [0, 1], "quantity": [2]}) + + assert blocked["amount"] == pytest.approx(truth["amount"], abs=1.0) + assert set(blocked) == {"amount", "quantity"} + + +def test_splitting_a_columns_evidence_can_hand_the_top_spot_to_a_derived_view(): + """Why this is material rather than cosmetic, since a reader only ever reads the top entry. + + Three views divide a column's share three ways. Measured on a column whose true share was 74.5%, each + view landed near 25% -- so an unrelated single-view column sat level with them, and the name the map + put first was a *derived view* of the real driver rather than the column itself. Blocking names the + column. + """ + rng = np.random.default_rng(1) + rows = 3000 + driver = rng.normal(0, 1, rows) + other = rng.normal(0, 1, rows) + columns = ["driver", "driver_rel_baseline", "driver_rel_time", "other"] + train = pd.DataFrame( + { + "driver": driver, + "driver_rel_baseline": driver - 3.0, + "driver_rel_time": driver * 0.5 + 1.0, + "other": other, + } + ) + model = IsolationForest(n_estimators=300, random_state=0).fit(train) + row = pd.DataFrame([[5.0, 2.0, 3.5, 3.0]], columns=columns) + + per_view = _shares(model, row, columns) + named_per_view = max(per_view, key=lambda k: per_view[k]) + + blocks = {"driver": [0, 1, 2], "other": [3]} + blocked = _shares(model, row, columns, blocks) + named_blocked = max(blocked, key=lambda k: blocked[k]) + + assert named_per_view != "driver", f"fixture must reproduce the dilution, got {per_view}" + assert named_blocked == "driver", f"blocking should name the source column, got {blocked}" + + +def test_a_block_sums_signed_values_so_a_normalising_view_cancels(): + """The reason the orientation must not clip before blocking. + + A block's value is the net effect of its views on the path length. If one view drove the anomaly and + another argued the row was ordinary, the second has to reduce the first. Clipping the views before + summing would report the block as more responsible than the model found it -- two views at +3 and -1 + net to 2, but clipping first sums to 3. + """ + rng = np.random.default_rng(9) + rows = 2000 + columns = ["metric", "metric_rel_time", "other"] + train = pd.DataFrame(rng.normal(0, 1, (rows, 3)), columns=columns) + model = IsolationForest(n_estimators=200, random_state=0).fit(train) + row = pd.DataFrame([[4.0, -3.0, 0.2]], columns=columns) + + per_view, _, _ = compute_row_attributions(model, row, columns) + blocked, _, keys = compute_row_attributions(model, row, columns, {"metric": [0, 1], "other": [2]}) + + assert keys == ["metric", "other"] + # The block is the signed sum, not the sum of the clipped parts. + np.testing.assert_allclose(blocked[0, 0], per_view[0, 0] + per_view[0, 1], rtol=1e-12) + clipped_sum = float(np.maximum(per_view[0, :2], 0.0).sum()) + if per_view[0, :2].min() < 0: + assert blocked[0, 0] < clipped_sum From da4f6988caf076b99507e8eae74d489d1067d989 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 8 Sep 2026 10:29:52 +0100 Subject: [PATCH 101/107] Tell the explanation model what a contribution was measured against, not just that it mattered Keying contributions by source column closed a channel nobody had noticed was load-bearing. Before it, the map was keyed by engineered feature and the prompt applied a label per key, so "revenue_rel_time" reached the model as "revenue vs its expected level at that time". Now the key is "revenue", the label falls through to the column's own name, and the comparison the model objected to is not in the prompt at all. Grouping was never affected, because baseline_grouping has always been its own field. Temporal conditioning had no such field -- it only ever arrived through the feature label -- so the two axes were asymmetric long before this, and keying by source is what made it bite. The gap matters more than completeness. A metric judged against its expected level at a point in time can sit comfortably inside every range the table has ever held and still be wrong for when it arrived. A model given a large share and no comparison reaches for "unusually high", which is the one claim the evidence cannot support -- and this is the same failure the exemplar rewrite removed, arriving by a different route. So temporal_baseline becomes a first-class input alongside baseline_grouping, carrying the time column or 'none'. Both fields' descriptions now say what they license rather than only what they are, the instructions tie the claim to the comparison ("say the metric departed from whichever comparison those fields describe" and do not fall back on calling it unusual outright), and both exemplars carry the field in different states with the temporal one modelling the reading. Showing one state teaches a smaller model to treat it as the default and stop reading the field, which is how this would come back. Verified that the metadata reaches it rather than assuming: ExplanationContext.from_scoring_config is given the parsed feature metadata at scoring_run.py, and baseline_over_time lives on that metadata, so the field cannot silently read 'none' on a conditioned model. Eight tests, and each pins a property rather than a wording: the field exists and sits next to its sibling, the helper reports the column or 'none' and copes with absent metadata, the description carries the claim about a normal-looking value, both exemplar states appear, the temporal exemplar avoids directional language, and the instructions carry both reconciliation clauses. Snapshot regenerated. Co-authored-by: Isaac --- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 66 +++++++++++++-- tests/resources/ai_query_prompt_header.txt | 10 ++- tests/unit/test_anomaly_llm_explainer.py | 83 +++++++++++++++++++ 3 files changed, 147 insertions(+), 12 deletions(-) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 32034d71b..4b1a8aafe 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -43,6 +43,12 @@ "produce the identical number. So never say a value was high, low, above, below, elevated, " "inflated, dropped, spiked, or missing. Say it departed from its expected pattern, and leave " "which way unsaid. The same applies to drift magnitudes, which are also unsigned.\n" + "What a contribution is measured AGAINST is given to you, in baseline_grouping and " + "temporal_baseline, and it changes what you may claim. Say the metric departed from whichever " + "comparison those fields describe -- its group's normal, the level expected at that time, or the " + "table as a whole when both are 'none'. A metric judged against its group or its own history can be " + "entirely ordinary for the table and still be wrong, so do not fall back on calling it unusual " + "outright when a narrower comparison is what objected.\n" "Be direct and concrete: name the metrics, their shares and the group size without hedging " "phrases like 'The data shows', 'It appears that', or 'might indicate'. Being direct means " "stating plainly what the inputs contain — it does not license asserting a direction, a cause, " @@ -114,7 +120,17 @@ def attribution_semantics(algorithm: str | None) -> str: "baseline_grouping", "The columns whose values define each row's baseline group, e.g. 'region' or " "'region, product'. Anomalies are judged relative to the row's own group baseline; " - "'none' when the model is not grouped.", + "'none' when the model is not grouped. When set, a value can be ordinary for the table as a " + "whole and still be wrong for its own group, so say the metric departed from what its group " + "normally looks like rather than that it was unusual outright.", + ), + ( + "temporal_baseline", + "The time column each metric is judged along, e.g. 'event_ts', or 'none'. When set, each metric " + "is compared against the level expected of it AT THAT POINT IN TIME, not against its whole " + "history. So its value can sit well inside the range the data has always covered and still be " + "wrong for when it arrived: say it departed from the level expected at that time. Do not call it " + "unusual, high or low for the metric overall, because the comparison was never against that.", ), ("threshold", "The severity percentile threshold configured by the user (0–100)."), ( @@ -151,9 +167,14 @@ def attribution_semantics(algorithm: str | None) -> str: # and its mirror at (-8, -0.5) score identically (65.387) with identical contribution maps, so half of # those explanations were backwards, stated confidently, in business language. # -# The two also differ in *attribution_basis*, which is the field deciding how the contributions read. -# Showing only one reading would leave the other untaught; the values here are abbreviated forms of what -# *attribution_semantics* emits, since the exemplars exist to pin shape rather than to restate the header. +# The pair also differs in every field that changes how a contribution may be described, and differs +# deliberately: *attribution_basis*, which decides whether a share can be read as a feature's own value +# being unusual, and *temporal_baseline*, which decides whether "unusual" is against the metric's whole +# range or against the level expected of it at one moment. A field the header tells the model to follow +# has to appear in the demonstrations, and in both states, or the model learns to treat whichever state +# it saw as the default and stops reading the field. The *attribution_basis* values here are abbreviated +# forms of what *attribution_semantics* emits, since the exemplars pin shape rather than restate the +# header. _PROMPT_EXAMPLES = ( "Example (relationship basis, no drift):\n" "attribution_basis: each metric's position once the others are accounted for\n" @@ -162,25 +183,28 @@ def attribution_semantics(algorithm: str | None) -> str: "severity_range: mean 97.4, min 95.1, max 99.8\n" "confidence: high\n" "baseline_grouping: region\n" + "temporal_baseline: none\n" "threshold: 95.0\n" "drift_summary: none\n" 'Response: {"narrative":"Across 312 rows, amount departs most from what its own region implies ' '(61%), with quantity next (22%).","business_impact":"Amount values that do not match their ' "region's usual pattern distort revenue reporting if processed unchanged.\",\"action\":" '"Reconcile amount against source orders for the affected regions."}\n\n' - "Example (value basis, with drift):\n" + "Example (value basis, judged against time, with drift):\n" "attribution_basis: each feature's own value compared against the rows it was scored against\n" "feature_contributions: latency_ms (74%), retries (12%)\n" "group_size: 88 rows\n" "severity_range: mean 98.9, min 97.0, max 99.9\n" "confidence: mixed\n" "baseline_grouping: none\n" + "temporal_baseline: event_ts\n" "threshold: 95.0\n" "drift_summary: drift detected: latency_ms=4.12\n" - 'Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from ' - 'its training baseline; retries contribute modestly (12%).","business_impact":"Latency that no ' - 'longer matches its baseline risks SLA breaches for downstream consumers.","action":"Compare ' - 'latency_ms against the training baseline to find what changed."}' + 'Response: {"narrative":"88 rows are dominated by latency_ms (74%), which departs from the level ' + 'expected of it at that point in time and has also drifted from its training baseline; retries ' + 'contribute modestly (12%).","business_impact":"Latency that no longer tracks its expected level ' + 'risks SLA breaches for downstream consumers.","action":"Compare latency_ms against its expected ' + 'level for that period rather than against its overall range."}' ) if TYPE_CHECKING: @@ -371,6 +395,26 @@ def _baseline_grouping_str(metadata: SparkFeatureMetadata | None) -> str: return ", ".join(metadata.baseline_by) +def _temporal_baseline_str(metadata: SparkFeatureMetadata | None) -> str: + """The time column each metric is judged along, e.g. 'event_ts', or 'none'. + + The sibling of :func:`_baseline_grouping_str`, and it exists for the same reason: how a row was judged + is a per-run fact the model cannot infer from the contributions. Grouping has always been told to the + model; temporal conditioning never was. It used to leak through by accident, because contributions were + keyed by engineered feature and one of those keys rendered as "X vs its expected level at that time". + Attribution is now keyed by source column -- so one column reports once however many ways it was + compared -- and that accidental channel closed with it. + + This matters for more than completeness. When a metric is judged against its own history, its value can + sit comfortably inside the range the table has ever held and still be wrong for *when* it arrived. A + model told only that the metric mattered will reach for "unusually high", which is the one thing the + evidence does not say. Like the grouping columns, this is a column *name* and carries no row values. + """ + if metadata is None or not metadata.baseline_over_time: + return "none" + return metadata.baseline_over_time + + def _human_labels(metadata: SparkFeatureMetadata | None) -> dict[str, str]: """Engineered-name -> human-label map for the model's features, omitting identity labels. @@ -513,6 +557,7 @@ def _build_ai_query_prompt_column( instructions and field semantics. """ baseline_grouping = _baseline_grouping_str(ctx.feature_metadata) + temporal_baseline = _temporal_baseline_str(ctx.feature_metadata) confidence_expr = ( F.when((F.col("mean_std").isNull()) | F.lit(not is_ensemble), F.lit("n/a")) .when(F.col("mean_std") < F.lit(_CONFIDENCE_HIGH_BELOW), F.lit("high")) @@ -546,6 +591,9 @@ def _build_ai_query_prompt_column( F.lit("baseline_grouping: "), F.lit(baseline_grouping), F.lit("\n"), + F.lit("temporal_baseline: "), + F.lit(temporal_baseline), + F.lit("\n"), F.lit("threshold: "), F.lit(str(ctx.threshold)), F.lit("\n"), diff --git a/tests/resources/ai_query_prompt_header.txt b/tests/resources/ai_query_prompt_header.txt index d765a3eea..8f1be6071 100644 --- a/tests/resources/ai_query_prompt_header.txt +++ b/tests/resources/ai_query_prompt_header.txt @@ -1,5 +1,6 @@ You are a data quality analyst. Given aggregate metadata for a GROUP of anomalous rows sharing the same contribution pattern, explain in plain business language why the model flagged this group. Your explanation will be shown for every row in the group — describe the pattern, not a specific row. You are describing what the model measured, not diagnosing a root cause: the inputs cannot establish one. The inputs carry NO DIRECTION. A contribution says how much a metric mattered to the score, never whether it was high or low: a metric far above its norm and one equally far below produce the identical number. So never say a value was high, low, above, below, elevated, inflated, dropped, spiked, or missing. Say it departed from its expected pattern, and leave which way unsaid. The same applies to drift magnitudes, which are also unsigned. +What a contribution is measured AGAINST is given to you, in baseline_grouping and temporal_baseline, and it changes what you may claim. Say the metric departed from whichever comparison those fields describe -- its group's normal, the level expected at that time, or the table as a whole when both are 'none'. A metric judged against its group or its own history can be entirely ordinary for the table and still be wrong, so do not fall back on calling it unusual outright when a narrower comparison is what objected. Be direct and concrete: name the metrics, their shares and the group size without hedging phrases like 'The data shows', 'It appears that', or 'might indicate'. Being direct means stating plainly what the inputs contain — it does not license asserting a direction, a cause, or a value they do not contain. Do not restate the input field names back to the user, and do not invent feature names, values, or baseline groups that are not present in the input. Inputs: @@ -8,7 +9,8 @@ Inputs: - group_size: Number of rows in this group, e.g. '312 rows'. - severity_range: Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'. - confidence: How closely the ensemble's members agreed on the score: 'high' / 'mixed' / 'low', or 'n/a' when one model did the scoring. Members differ only by random seed on the same training data, so this measures the stability of the score, NOT how reliable the flag is or whether the data has since changed. Do not present it to the reader as confidence in the finding. -- baseline_grouping: The columns whose values define each row's baseline group, e.g. 'region' or 'region, product'. Anomalies are judged relative to the row's own group baseline; 'none' when the model is not grouped. +- baseline_grouping: The columns whose values define each row's baseline group, e.g. 'region' or 'region, product'. Anomalies are judged relative to the row's own group baseline; 'none' when the model is not grouped. When set, a value can be ordinary for the table as a whole and still be wrong for its own group, so say the metric departed from what its group normally looks like rather than that it was unusual outright. +- temporal_baseline: The time column each metric is judged along, e.g. 'event_ts', or 'none'. When set, each metric is compared against the level expected of it AT THAT POINT IN TIME, not against its whole history. So its value can sit well inside the range the data has always covered and still be wrong for when it arrived: say it departed from the level expected at that time. Do not call it unusual, high or low for the metric overall, because the comparison was never against that. - threshold: The severity percentile threshold configured by the user (0–100). - drift_summary: Baseline drift signal from the scoring run, e.g. 'drift detected: amount=4.12; quantity=3.55' or 'none'. If drift is present, explicitly frame the narrative vs baseline. @@ -24,17 +26,19 @@ group_size: 312 rows severity_range: mean 97.4, min 95.1, max 99.8 confidence: high baseline_grouping: region +temporal_baseline: none threshold: 95.0 drift_summary: none Response: {"narrative":"Across 312 rows, amount departs most from what its own region implies (61%), with quantity next (22%).","business_impact":"Amount values that do not match their region's usual pattern distort revenue reporting if processed unchanged.","action":"Reconcile amount against source orders for the affected regions."} -Example (value basis, with drift): +Example (value basis, judged against time, with drift): attribution_basis: each feature's own value compared against the rows it was scored against feature_contributions: latency_ms (74%), retries (12%) group_size: 88 rows severity_range: mean 98.9, min 97.0, max 99.9 confidence: mixed baseline_grouping: none +temporal_baseline: event_ts threshold: 95.0 drift_summary: drift detected: latency_ms=4.12 -Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from its training baseline; retries contribute modestly (12%).","business_impact":"Latency that no longer matches its baseline risks SLA breaches for downstream consumers.","action":"Compare latency_ms against the training baseline to find what changed."} +Response: {"narrative":"88 rows are dominated by latency_ms (74%), which departs from the level expected of it at that point in time and has also drifted from its training baseline; retries contribute modestly (12%).","business_impact":"Latency that no longer tracks its expected level risks SLA breaches for downstream consumers.","action":"Compare latency_ms against its expected level for that period rather than against its overall range."} diff --git a/tests/unit/test_anomaly_llm_explainer.py b/tests/unit/test_anomaly_llm_explainer.py index dec6f74ab..715f7d818 100644 --- a/tests/unit/test_anomaly_llm_explainer.py +++ b/tests/unit/test_anomaly_llm_explainer.py @@ -435,3 +435,86 @@ def test_ensemble_agreement_is_not_presented_as_confidence_in_the_finding(): assert "random seed" in description assert "NOT how reliable the flag is" in description + + +# ── how the row was judged is a per-run fact the contributions cannot carry ─────────────────────────── + + +def test_the_temporal_baseline_reaches_the_prompt_as_its_own_field(): + """The gap this closes: grouping was told to the model and temporal conditioning was not. + + Temporal conditioning used to leak through by accident, because contributions were keyed by + engineered feature and one of those keys rendered as "X vs its expected level at that time". Keying by + source column closed that channel -- correctly, since one column should report once however many ways + it was compared -- and left the model with no way to know the comparison was against time at all. + """ + input_names = [name for name, _ in llm_explainer._PROMPT_INPUT_FIELDS] + + assert "temporal_baseline" in input_names + # Sibling of the grouping field, and adjacent to it, because they answer the same question. + assert input_names.index("temporal_baseline") == input_names.index("baseline_grouping") + 1 + + +@pytest.mark.parametrize("baseline_over_time, expected", [("event_ts", "event_ts"), ("", "none")]) +def test_the_temporal_baseline_string_reports_the_time_column_or_none(baseline_over_time, expected): + metadata = SparkFeatureMetadata( + column_infos=[{"name": "revenue", "category": "numeric"}], + categorical_frequency_maps={}, + onehot_categories={}, + engineered_feature_names=["revenue"], + baseline_over_time=baseline_over_time, + ) + + assert llm_explainer._temporal_baseline_str(metadata) == expected + + +def test_the_temporal_baseline_string_is_none_without_metadata(): + """A caller who built the context directly gets the conservative answer rather than a crash.""" + assert llm_explainer._temporal_baseline_str(None) == "none" + + +def test_the_temporal_field_says_a_normal_looking_value_can_still_be_wrong(): + """The substantive capability, not just the field's presence. + + A metric compared against its expected level at a moment can sit inside every range the table has + ever held. Without being told that, a model reads a large share and reaches for "unusually high", + which is the one claim the evidence cannot support. + """ + description = dict(llm_explainer._PROMPT_INPUT_FIELDS)["temporal_baseline"] + + assert "AT THAT POINT IN TIME" in description + assert "still be wrong for when it arrived" in description + + +def test_both_comparison_states_are_demonstrated_in_the_exemplars(): + """A field the header tells the model to follow has to appear in the demonstrations, in both states. + + Showing only one state teaches the model to treat it as the default and stop reading the field, which + is how the same mistake would come back. + """ + values = [ + line.partition("temporal_baseline: ")[2] + for line in llm_explainer._PROMPT_EXAMPLES.splitlines() + if line.startswith("temporal_baseline: ") + ] + + assert len(values) == 2, "each exemplar must state whether a temporal baseline applied" + assert sorted(values) == ["event_ts", "none"] + + +def test_the_temporal_exemplar_describes_the_expected_level_rather_than_an_extreme_value(): + """The exemplar has to model the reading, since that is what a smaller model copies.""" + responses = [line for line in llm_explainer._PROMPT_EXAMPLES.splitlines() if line.startswith("Response: ")] + temporal_response = responses[1] + + assert "expected of it at that point in time" in temporal_response + for word in ("far above", "far below", "elevated", "inflated", "spiked"): + assert word not in temporal_response.lower() + + +def test_the_instructions_tie_the_claim_to_what_it_was_measured_against(): + """Otherwise "avoid hedging" plus a large share reads as licence to call the metric unusual outright.""" + header = llm_explainer._render_ai_query_prompt_header() + + assert "measured AGAINST is given to you" in header + assert "do not fall back on calling it unusual outright" in header From 5cafe9c5e188087d4e8940b410c432951d863789 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 8 Sep 2026 11:02:53 +0100 Subject: [PATCH 102/107] Unit-test the row-at-a-time attribution variant, which had no unit coverage at all codecov failed on the core component for the previous commit, and it was right to. Fixing compute_contributions_for_matrix -- a second implementation of the same semantics, carrying the same two defects, which is how one module came to hold two answers to what a negative SHAP value means -- touched lines that no unit test reaches. Its only caller, compute_feature_contributions, has no caller anywhere in src/, and both are exercised only from tests/integration_anomaly/, which the pull-request coverage job does not run. That is worth stating rather than working around: a pull request uploads unit-only coverage while the main baseline comes from nightly's full suite, deliberately, so any edit to integration-only code fails patch coverage. The answer is unit tests, not a threshold. Five, each pinning a property rather than a line: - ground-truth orientation, so this copy cannot drift from the scoring path's sign convention - fractions of 1 rather than percentages, because the two contracts genuinely differ and "aligning" them would rescale this function's caller by a hundred - no invented uniform split when nothing drove the anomaly, the defect this copy also had - a row carrying a null is attributed as nothing and does not take another row's numbers with it - the pipeline unwrap for both shapes, since models trained by older versions carry a RobustScaler and newer ones carry none, and the scaler has to be applied rather than merely tolerated Measured on the module: 71% to 86%, with the range covering this function no longer listed as missed. What remains uncovered is compute_feature_contributions, which takes a Spark DataFrame and so is out of reach from a unit test by construction. `make test`, which is what CI runs, reports 2767 passed. Two ad-hoc coverage invocations of my own reported 19 failures along the way; those were artefacts of the flags, do not reproduce under `make test` or plain pytest, and are not a repo issue. Also folds two ad-hoc `max(...)` expressions in the new file into one named helper, which removes a closure over a loop variable rather than silencing the warning about it. Co-authored-by: Isaac --- .../test_anomaly_attribution_orientation.py | 92 ++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_anomaly_attribution_orientation.py b/tests/unit/test_anomaly_attribution_orientation.py index 5c635a9e6..82ddd2cdd 100644 --- a/tests/unit/test_anomaly_attribution_orientation.py +++ b/tests/unit/test_anomaly_attribution_orientation.py @@ -13,8 +13,11 @@ import pandas as pd import pytest from sklearn.ensemble import IsolationForest +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import RobustScaler from databricks.labs.dqx.anomaly.explainability import ( + compute_contributions_for_matrix, compute_row_attributions, format_contributions_map, format_shap_contributions, @@ -29,6 +32,11 @@ def _single_row(values: list[float], keys: list[str] | None = None) -> dict[str, return format_shap_contributions(np.array([values]), np.array([True]), 1, names)[0] +def _top_key(contributions: dict[str, float | None]) -> str: + """The key a reader would take as the driver: the largest share, nulls treated as no share.""" + return max(contributions, key=lambda key: contributions[key] or 0.0) + + # ── what earns a share ─────────────────────────────────────────────────────────────────────────────── @@ -133,7 +141,7 @@ def test_the_deliberately_anomalous_feature_is_named_as_the_top_driver( attribution, valid_indices, keys = compute_row_attributions(forest, probe, columns) contributions = format_shap_contributions(attribution, valid_indices, 1, keys)[0] - named = max(contributions, key=lambda k: contributions[k] or 0.0) + named = _top_key(contributions) assert named == columns[culprit], f"expected {columns[culprit]}, got {contributions}" @@ -172,3 +180,85 @@ def test_a_zero_share_feature_is_not_rendered_as_a_contributor(): the all-null map exists to avoid, one layer down. """ assert format_contributions_map({"amount": 100.0, "quantity": 0.0}, 3) == "amount (100%)" + + +# ── the row-at-a-time variant, which has to agree with the scoring path ─────────────────────────────── + + +def _matrix_forest(columns: int = 3) -> IsolationForest: + rng = np.random.default_rng(4) + return IsolationForest(n_estimators=150, random_state=0).fit(rng.normal(0, 1, (400, columns))) + + +def test_the_matrix_variant_names_the_deliberately_anomalous_feature(): + """Same orientation as the scoring path, asserted independently. + + This function is a second implementation of the same semantics -- it existed with the same two defects + the scoring path had, which is how one module came to hold two answers to what a negative SHAP value + means. Pinning it separately is what keeps them from drifting apart again. + """ + forest = _matrix_forest() + probe = np.array([[9.0, 0.1, 0.1]]) + + contributions = compute_contributions_for_matrix(forest, probe, ["a", "b", "c"])[0] + + named = _top_key(contributions) + assert named == "a", f"expected the perturbed feature, got {contributions}" + + +def test_the_matrix_variant_reports_fractions_rather_than_percentages(): + """Its contract differs from the scoring path's on purpose, so the difference is pinned. + + The scoring path emits 0-100 because that is what ``_dq_info[].anomaly.contributions`` documents. This + one emits fractions of 1, which is what its own caller expects. Recording that here stops a later + reader "aligning" them and silently rescaling the other consumer by a hundred. + """ + forest = _matrix_forest() + + contributions = compute_contributions_for_matrix(forest, np.array([[8.0, 0.2, 0.2]]), ["a", "b", "c"])[0] + + values = [v for v in contributions.values() if v is not None] + assert sum(values) == pytest.approx(1.0) + assert max(values) <= 1.0 + + +def test_the_matrix_variant_invents_nothing_when_no_feature_drove_the_anomaly(): + """The uniform-split defect, in the copy that also carried it. + + Built from a hand-made attribution rather than hunting for a real row with no anomaly-driving + evidence, by using a model whose every feature is constant so nothing can be isolated. + """ + constant = IsolationForest(n_estimators=50, random_state=0).fit(np.zeros((200, 2))) + + contributions = compute_contributions_for_matrix(constant, np.array([[0.0, 0.0]]), ["a", "b"])[0] + + assert contributions == {"a": None, "b": None} + + +def test_the_matrix_variant_returns_nulls_for_a_row_it_cannot_score(): + """A row carrying a null cannot be attributed, and must not take another row's numbers with it.""" + forest = _matrix_forest(columns=2) + probe = np.array([[np.nan, 1.0], [7.0, 0.1]]) + + contributions = compute_contributions_for_matrix(forest, probe, ["a", "b"]) + + assert contributions[0] == {"a": None, "b": None} + assert contributions[1]["a"] is not None + + +def test_the_matrix_variant_unwraps_a_pipeline_and_its_scaler(): + """Models trained by older versions carry a RobustScaler in the pipeline; newer ones carry none. + + Both shapes have to work, and the scaler has to be *applied* rather than merely tolerated, or the + attribution is computed on unscaled values the estimator never saw. + """ + rng = np.random.default_rng(6) + train = rng.normal(0, 1, (400, 2)) + scaled = Pipeline([("scaler", RobustScaler()), ("model", IsolationForest(n_estimators=150, random_state=0))]) + scaled.fit(train) + bare = Pipeline([("model", IsolationForest(n_estimators=150, random_state=0))]).fit(train) + + probe = np.array([[8.0, 0.1]]) + for label, model in (("with a scaler", scaled), ("without one", bare)): + contributions = compute_contributions_for_matrix(model, probe, ["a", "b"])[0] + assert _top_key(contributions) == "a", f"pipeline {label} named {contributions}" From 3ea5d8836145972544bf3de76ceaddd6c5031a65 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 8 Sep 2026 13:48:04 +0100 Subject: [PATCH 103/107] Stop attribution and explanations claiming more than the evidence carries Round 4 of review. Six findings, five of them mine, and two of those correct claims I had asserted as measured. Reproduced all six before changing anything; the reviewer's own script is the regression check. An attribution matrix and the names it will be reported under could disagree, silently. keys become source columns whenever blocking is requested, but the dispatch fell through to per-engineered-feature attribution for an estimator without block_contributions, and the formatter normalises across the full width while writing only as many entries as there are keys. Reproduced with a synthetic estimator, and independently by the reviewer restoring a real by-value-pickled detector: three columns, two keys, a published map of {x: 0.0, y: 0.0}, and a score identical to a correctly explained row. Nothing downstream can detect that. Refused rather than degraded. Summing leave-one-out drops would reinstate the error blocking exists to remove, and emitting per-feature keys instead would make the map's key vocabulary vary per model, which the guide now promises is uniform -- a query would break on some models with no error. Two guards, because they answer different questions: the dispatch knows both that blocking was asked for and what the estimator can do, so it can name a remedy; the formatter is the funnel where a column acquires a name, so a structural check there covers any producer including a hand-built matrix. Reachable without archaeology, which is worth recording: the config hash covers only columns and the comparison bases, so it is not version-sensitive and an intermediate-commit model passes validation unchanged, and a third-party estimator arrives through a documented extension point. The prompt inferred a mechanism from a shape that does not carry one. Spread contributions were to be described as "the metrics no longer agreeing with each other"; two metrics with training correlation 6e-18, both marginally extreme, give 50%/50%. It now says each metric contributed and names both readings the shape is consistent with. It also inferred which comparison objected. baseline_grouping and temporal_baseline establish which comparisons were available, not which drove a share -- and a metric whose temporal fit was rejected gets a constant zero for its time-relative feature, so a time column does not even establish that the comparison ran for it. Both fields now read as context. The exemplar that taught this was worse than the instruction: its input line carried "amount vs its group baseline (61%)", a form the pipeline can no longer produce at all now that keys are source columns, and it demonstrated exactly the pinning the instruction was being corrected for. Fixing prose while leaving the demonstration would have changed nothing, because a smaller serving model weights the example more heavily. Both exemplars now state impact conditionally as well. The guide claimed the contributions and the score describe the same aggregate, in two places. Averaging oriented TreeSHAP decomposes the members' mean isolation depth; the score is the mean of a nonlinear function of that depth. Measured, 46 adjacent reversals among 600 probes. The shares are a ranking of columns, not a breakdown of the score, and mean_row_attributions' docstring already said so. The unseen-category claim was too broad. It holds for one-hot columns, at or below the cardinality threshold of 20, where the indicators sum to one and an unseen value sets none. Above it the column is one frequency number; with near-uniform training frequencies that coordinate is constant, the detector drops it as such, and a known and an unseen value both score 0.1837. Narrowed, with the membership-check steer kept. The temporal-bias justification proved the wrong invariance, and this one is a claim I got wrong rather than a wording slip. The docstring said a constant forecast bias "cannot matter" and warned against fixing it. The test shifted training and scoring data together and refitted, which measures translation invariance of refitting -- 9.95e-14, true and relevant, since selection compares candidates all refitted on the same data and a shared level error cancels from the ratio. It says nothing about a bias appearing only after the fit window. With the detector fixed, +5 on one held-out residual dimension leaves the residual MAD at 1.0026, invisible to the criterion, while rows above the training p99 go from 1.3% to 98.4%. Docstring narrowed to the invariance that holds, test renamed to what it proves, and the counterexample added so the limitation is executable. Drift detection and the staleness horizon are the mechanisms for it; this criterion is not. Inherited, and the only finding not mine: the ensemble path discarded the injected registry. A module-level helper built EnsembleTrainer() with no argument, so a registry handed to the strategy was honoured for a single model and dropped for an ensemble -- the default path, ensemble_size being 3 -- while the class's own docstring advertised the injection it defeated. The helper is deleted rather than given a parameter: it had one caller and existed only to flatten a typed result into a tuple, so removing it removes the trap. Its generated API page is gitignored, so that cost nothing. Tests. R1's refusal and the formatter guard, with the working path pinned beside them so the guard cannot be satisfied by refusing everything. R2 through the public attribution_semantics plus the rendered header. R3 on the public training path with create_autospec, asserting the injected registry was used and the default never consulted -- and honest in its docstring that reaching that assertion without Spark depends on the registry being consulted before feature engineering. R6 both ways: the invariance under its accurate name, and the counterexample. An existing ensemble test moved to the new message, because the width guard now fires where the attribution is produced rather than where it is averaged; the cross-member check stays, since per-member masks come from each member's own scaler and only it sees a divergence there. Deliberately not added: an assertion on the 46-of-600 reversal count. It depends on seed and library versions, and pinning it would repeat a mistake from earlier in this branch where a test measured float arithmetic instead of the property it claimed. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 36 +++++--- .../labs/dqx/anomaly/anomaly_llm_explainer.py | 68 ++++++++------ .../labs/dqx/anomaly/ensemble_training.py | 20 ---- .../labs/dqx/anomaly/explainability.py | 57 ++++++++++++ src/databricks/labs/dqx/anomaly/temporal.py | 30 ++++-- .../labs/dqx/anomaly/training_strategies.py | 16 +++- tests/resources/ai_query_prompt_header.txt | 12 +-- .../unit/test_anomaly_ensemble_attribution.py | 16 +++- ...est_anomaly_ensemble_registry_injection.py | 92 +++++++++++++++++++ tests/unit/test_anomaly_llm_explainer.py | 85 +++++++++++++---- .../test_anomaly_source_block_attribution.py | 81 ++++++++++++++++ tests/unit/test_anomaly_temporal_fit.py | 84 ++++++++++++++--- 12 files changed, 478 insertions(+), 119 deletions(-) create mode 100644 tests/unit/test_anomaly_ensemble_registry_injection.py diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 7fb463e56..7bbf7d969 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -560,16 +560,30 @@ That is telemetry, which is what `"timeseries"` is for. On ordinary tabular data this table suggests, so **choose on the shape of the anomaly you expect, not on an expected accuracy gap.** One thing independent of accuracy favours the default: `"tabular"` trains an ensemble, so it can report `confidence_std`. Its contributions come from SHAP rather than an exact decomposition, but both profiles -report them the same way — one entry per column you passed, and on an ensemble averaged across members, so -the score, `confidence_std` and the contributions all describe the same aggregate. Detection quality on -DQX's own synthetic fixtures is published in [Benchmarks](/docs/reference/benchmarks). - -One capability difference is worth knowing because no accuracy number shows it. A **category that never -appeared in training** — a new payment type, an unrecognised status code — is caught by `"timeseries"` but -generally not by `"tabular"`. Both encode the row the same way, but only a detector that reads features -jointly can tell that the encoding is impossible; Isolation Forest splits one feature at a time and scores -such a row as ordinary. If unrecognised values are something you need to fail on, that is a membership -question rather than an anomaly one — use `is_in_list` or `foreign_key` on the column, under either profile. +report them the same way — one entry per column you passed, and on an ensemble the mean across members: the +same members, in the same proportions, that produced the score. Read the result as a **ranking of columns** +rather than a breakdown of the score's magnitude. Averaging decomposes the members' mean isolation depth +exactly, while the score is the mean of a strictly monotone but *nonlinear* function of that depth, so the +two order rows almost identically without being the same quantity. Detection quality on DQX's own synthetic +fixtures is published in [Benchmarks](/docs/reference/benchmarks). + +One capability difference is worth knowing because no accuracy number shows it, and it is narrower than it +first looks. A **category that never appeared in training** — a new payment type, an unrecognised status +code — is caught by `"timeseries"` and generally not by `"tabular"`, **but only where the column is one-hot +encoded**, which is when its cardinality is at or below the categorical threshold (20 by default). There +every category keeps its own indicator, the indicators sum to one on every training row, and an unseen value +sets none of them: a detector reading features jointly sees a row off the surface all its training data lay +on, while Isolation Forest splits one feature at a time, so an all-zero row sits inside every indicator's own +range and scores as ordinary. + +Above that threshold the column is frequency-encoded into a single number and an unseen value becomes `0.0`, +which carries much less. If the training frequencies were near-uniform that coordinate barely varies, the +correlation-aware detector drops it from the distance as constant, and an unseen value does not move the +score at all — measured with 21 equally frequent categories, a known and an unseen value both score 0.1837. + +So neither profile is a substitute for a vocabulary check. If unrecognised values are something you need to +fail on, that is a membership question rather than an anomaly one — use `is_in_list` or `foreign_key` on the +column, under either profile. DQX does not detect which profile you need. Getting it right cannot be verified without labelled @@ -623,7 +637,7 @@ For full parameter and schema details, see [Row Anomaly Detection in Quality Che 1. **Feature engineering** (automatic): DQX detects column types and creates features (numerical standardized, categorical one-hot or frequency-encoded, temporal expanded to hour/day/month/weekend). 2. **Smart sampling and training**: DQX samples your data (default 30%, capped at 1M rows), trains the detector your `profile` selects — an ensemble of Isolation Forest models by default, or a single correlation-aware model for `profile="timeseries"` — and captures baseline statistics for drift detection. 3. **Model registry**: Models and metadata live in MLflow and a Delta table; each `train()` call registers one model under its name and archives the previously active version. -4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. Feature contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. How they are computed depends on the detector — SHAP for Isolation Forest, an exact decomposition for the correlation-aware detector — but both report **one entry per column you passed**, not per engineered feature, so you read back your own column names. That matters because DQX derives up to three features from one numeric column; scoring each separately splits the column's evidence between them and understates it. Each value is that column's share of the evidence that made the row look unusual. A column that made the row look *more* normal shows `0`, and a flagged row for which nothing pointed towards an anomaly carries an all-null map rather than an invented even split. With an ensemble, the contributions are the mean across members, so the score, `confidence_std` and the contributions all describe the same aggregate. +4. **Scoring and explanation**: Raw scores are normalized to a 0–100 severity percentile. Feature contributions (which features drove each score) are on by default and computed only for anomalous rows; set `enable_contributions=False` to skip them for the fastest scoring. How they are computed depends on the detector — SHAP for Isolation Forest, an exact decomposition for the correlation-aware detector — but both report **one entry per column you passed**, not per engineered feature, so you read back your own column names. That matters because DQX derives up to three features from one numeric column; scoring each separately splits the column's evidence between them and understates it. Each value is that column's share of the evidence that made the row look unusual. A column that made the row look *more* normal shows `0`, and a flagged row for which nothing pointed towards an anomaly carries an all-null map rather than an invented even split. With an ensemble the contributions are the mean across members — the aggregate the score comes from — but what averaging decomposes exactly is the members' mean isolation depth, not the mean score, which is a nonlinear function of it. Read the shares as a ranking of columns, not as a breakdown of the score. 5. **Auto-discovery of columns and grouping**: When you call `train()` without `columns`, DQX selects numeric columns with enough variance as features. It also looks for a grouping (categorical or low-cardinality columns with 2–50 distinct values, low null rate, and enough rows per group, for example region or product category). A discovered grouping is used as `baseline_by` (see [Group-aware anomaly detection](#group-aware-anomaly-detection)), so it costs one model regardless of how many groups it turns out to have. To compare against the whole table instead, pass `baseline_by=[]`. When you *do* pass `columns`, you have decided what to measure, so DQX leaves the comparison pooled rather than adding a grouping you did not ask for. If your data looks grouped it says so in a warning naming the grouping to pass. ### Which algorithm, and why diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 4b1a8aafe..8cc3d903f 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -43,12 +43,14 @@ "produce the identical number. So never say a value was high, low, above, below, elevated, " "inflated, dropped, spiked, or missing. Say it departed from its expected pattern, and leave " "which way unsaid. The same applies to drift magnitudes, which are also unsigned.\n" - "What a contribution is measured AGAINST is given to you, in baseline_grouping and " - "temporal_baseline, and it changes what you may claim. Say the metric departed from whichever " - "comparison those fields describe -- its group's normal, the level expected at that time, or the " - "table as a whole when both are 'none'. A metric judged against its group or its own history can be " - "entirely ordinary for the table and still be wrong, so do not fall back on calling it unusual " - "outright when a narrower comparison is what objected.\n" + "baseline_grouping and temporal_baseline tell you which comparisons were AVAILABLE to the model, not " + "which one objected. A metric may be compared against the table, against its own group and against " + "its expected level at that time all at once, and the contribution you are given is the total across " + "those, so it cannot say which comparison drove it. Treat these fields as context that widens what " + "the number is consistent with: when either is set, a metric can be entirely ordinary for the table " + "and still have departed from a narrower comparison, so do not call it unusual outright. State that " + "the metric contributed and name the comparisons that were in use; do not assign the departure to " + "one of them.\n" "Be direct and concrete: name the metrics, their shares and the group size without hedging " "phrases like 'The data shows', 'It appears that', or 'might indicate'. Being direct means " "stating plainly what the inputs contain — it does not license asserting a direction, a cause, " @@ -66,8 +68,10 @@ "metric does not fit the pattern the others imply, which can happen either because its own value " "moved a long way or because it stopped tracking the others while staying inside its normal range. " "The input does not distinguish those two cases, so do not assert either: say the metric does not " - "fit the pattern. When the contributions are spread across several metrics, describe it as the " - "metrics no longer agreeing with each other rather than as any one of them being abnormal.", + "fit the pattern. Contributions spread across several metrics mean each of them contributed, and " + "nothing more -- that happens when metrics stop agreeing with each other, and equally when several " + "unrelated metrics are each unusual at the same time. Name the metrics that contributed; do not " + "claim a relationship between them broke.", ), ( "IsolationForest", @@ -102,10 +106,10 @@ def attribution_semantics(algorithm: str | None) -> str: ), ( "feature_contributions", - "Mean contributions across the group, already named for a reader, e.g. 'amount vs its " - "group baseline (82%), quantity (11%), discount (5%)'. A phrase like 'X vs its group " - "baseline' means X was unusual relative to its own baseline group, not in absolute terms. " - "These are aggregated relative importances — not raw data values.", + "Mean share across the group of the evidence the model acted on, per column the caller named, " + "e.g. 'amount (82%), quantity (11%), discount (5%)'. One entry per column however many ways that " + "column was compared, so a share says the column was involved and not which comparison objected. " + "These are aggregated relative importances — not raw data values, and not percentages of the score.", ), ("group_size", "Number of rows in this group, e.g. '312 rows'."), ("severity_range", "Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'."), @@ -126,11 +130,14 @@ def attribution_semantics(algorithm: str | None) -> str: ), ( "temporal_baseline", - "The time column each metric is judged along, e.g. 'event_ts', or 'none'. When set, each metric " - "is compared against the level expected of it AT THAT POINT IN TIME, not against its whole " - "history. So its value can sit well inside the range the data has always covered and still be " - "wrong for when it arrived: say it departed from the level expected at that time. Do not call it " - "unusual, high or low for the metric overall, because the comparison was never against that.", + "The time column the model was allowed to judge metrics along, e.g. 'event_ts', or 'none'. When " + "set, a metric MAY have been compared against the level expected of it at that point in time as " + "well as against its overall range -- but not every metric is: where no expectation could be " + "fitted for one, that comparison contributes nothing for it. So this widens what a contribution " + "is consistent with rather than explaining it: a metric can sit well inside the range the data " + "has always covered and still have departed from what was expected when it arrived. Do not call a " + "metric unusual, high or low overall on the strength of this field, and do not state that it " + "departed from its expected level -- say the comparison was available.", ), ("threshold", "The severity percentile threshold configured by the user (0–100)."), ( @@ -178,7 +185,7 @@ def attribution_semantics(algorithm: str | None) -> str: _PROMPT_EXAMPLES = ( "Example (relationship basis, no drift):\n" "attribution_basis: each metric's position once the others are accounted for\n" - "feature_contributions: amount vs its group baseline (61%), quantity (22%)\n" + "feature_contributions: amount (61%), quantity (22%)\n" "group_size: 312 rows\n" "severity_range: mean 97.4, min 95.1, max 99.8\n" "confidence: high\n" @@ -186,10 +193,10 @@ def attribution_semantics(algorithm: str | None) -> str: "temporal_baseline: none\n" "threshold: 95.0\n" "drift_summary: none\n" - 'Response: {"narrative":"Across 312 rows, amount departs most from what its own region implies ' - '(61%), with quantity next (22%).","business_impact":"Amount values that do not match their ' - "region's usual pattern distort revenue reporting if processed unchanged.\",\"action\":" - '"Reconcile amount against source orders for the affected regions."}\n\n' + 'Response: {"narrative":"Across 312 rows, amount accounts for most of what the model measured (61%), ' + 'with quantity next (22%); these rows are judged against their own region.","business_impact":"If ' + "amount is wrong on these rows, revenue reporting for the affected regions would be affected too.\"," + '"action":"Reconcile amount against source orders for the affected regions."}\n\n' "Example (value basis, judged against time, with drift):\n" "attribution_basis: each feature's own value compared against the rows it was scored against\n" "feature_contributions: latency_ms (74%), retries (12%)\n" @@ -200,11 +207,11 @@ def attribution_semantics(algorithm: str | None) -> str: "temporal_baseline: event_ts\n" "threshold: 95.0\n" "drift_summary: drift detected: latency_ms=4.12\n" - 'Response: {"narrative":"88 rows are dominated by latency_ms (74%), which departs from the level ' - 'expected of it at that point in time and has also drifted from its training baseline; retries ' - 'contribute modestly (12%).","business_impact":"Latency that no longer tracks its expected level ' - 'risks SLA breaches for downstream consumers.","action":"Compare latency_ms against its expected ' - 'level for that period rather than against its overall range."}' + 'Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from its ' + 'training baseline; retries contribute modestly (12%). These rows are judged against expected levels ' + 'over time as well as overall.","business_impact":"If latency_ms is genuinely off on these rows, ' + 'downstream consumers with SLAs would be the first to notice.","action":"Compare latency_ms against ' + 'its expected level for that period as well as against its overall range."}' ) if TYPE_CHECKING: @@ -330,9 +337,10 @@ def from_scoring_config( def redaction_set(redact_columns: tuple[str, ...], metadata: SparkFeatureMetadata | None = None) -> frozenset[str]: """Columns to redact, plus every engineered feature derived from them. - Redaction matches contribution keys exactly, and which vocabulary those keys use depends on the - detector: source columns where attribution is blocked by source, engineered feature names on the - tree path. Covering both is why the source column *and* its descendants go into the set. So + Redaction matches contribution keys exactly, and both detectors now key by source column, which + :func:`compute_row_attributions` enforces rather than leaves to chance. The set still covers the + engineered names as well, deliberately: a redaction that silently under-covers is a privacy failure, so + it costs nothing to keep both vocabularies while attribution shape is a runtime property. So redacting ``amount`` must also stop ``amount_rel_baseline`` -- a signed log-ratio of the same column -- and redacting ``country`` must stop ``country_US``, ``country_DE``, ``country_freq`` and ``country_is_null``. A caller naming a column sensitive means every feature derived from it is diff --git a/src/databricks/labs/dqx/anomaly/ensemble_training.py b/src/databricks/labs/dqx/anomaly/ensemble_training.py index 190437936..28592262b 100644 --- a/src/databricks/labs/dqx/anomaly/ensemble_training.py +++ b/src/databricks/labs/dqx/anomaly/ensemble_training.py @@ -148,23 +148,3 @@ def _register_models( ) model_uris.append(model_uri) return model_uris - - -def train_ensemble( - train_df: DataFrame, - val_df: DataFrame, - columns: list[str], - params: AnomalyParams, - ensemble_size: int, - model_name: str, -) -> tuple[list[str], dict[str, Any], dict[str, float], dict[str, float], SparkFeatureMetadata]: - """Train ensemble of models with different random seeds.""" - trainer = EnsembleTrainer() - result = trainer.train(train_df, val_df, columns, params, ensemble_size, model_name) - return ( - result.model_uris, - result.hyperparams, - result.aggregated_metrics, - result.score_quantiles, - result.feature_metadata, - ) diff --git a/src/databricks/labs/dqx/anomaly/explainability.py b/src/databricks/labs/dqx/anomaly/explainability.py index 0112d4dd1..8e9bc0f31 100644 --- a/src/databricks/labs/dqx/anomaly/explainability.py +++ b/src/databricks/labs/dqx/anomaly/explainability.py @@ -125,6 +125,16 @@ def format_shap_contributions( if attribution.size == 0 or num_keys == 0: return contributions + # The funnel where column j acquires the name keys[j]. A mismatch here is invisible downstream: the + # normalisation runs over every column while only the first len(keys) are emitted, so the map looks + # ordinary and describes the wrong features. Guarded again here rather than trusting + # compute_row_attributions, because this function is reachable with a hand-built matrix. + if attribution.ndim != 2 or attribution.shape[1] != num_keys: + raise InvalidParameterError( + f"Attribution of shape {attribution.shape} cannot be reported under {num_keys} keys: the " + "emitted map would name only the leading columns while normalising across all of them." + ) + magnitudes = np.maximum(attribution, 0.0) totals = magnitudes.sum(axis=1, keepdims=True) normalized = np.divide(magnitudes, totals, out=np.zeros_like(magnitudes), where=totals > 0) @@ -203,10 +213,39 @@ def compute_row_attributions( len(engineered_feature_cols), [blocks[key] for key in keys] if blocked and blocks is not None else None, ) + _reject_malformed_attribution(attribution, keys, int(valid_indices.sum()), estimator) return attribution, valid_indices, keys +def _reject_malformed_attribution(attribution: np.ndarray, keys: list[str], expected_rows: int, estimator: Any) -> None: + """Refuse an attribution whose shape does not match the keys it will be reported under. + + :func:`format_shap_contributions` pairs column *j* with ``keys[j]`` and normalises across the full + width, so a mismatch does not raise anywhere -- it emits a plausible map built from the wrong columns. + Observed when an estimator supplied one value per engineered feature while the keys were source + columns: three columns, two keys, and a published map of ``{'x': 0.0, 'y': 0.0}`` on a row whose score + was identical to a correctly explained one. Nothing downstream can detect that, which is exactly why + the check belongs here rather than in a consumer. + + Args: + attribution: The matrix about to be returned. + keys: The names its columns will be reported under. + expected_rows: Number of rows that reached attribution. + estimator: The bare estimator, named in the error so the failing model is identifiable. + + Raises: + InvalidParameterError: If the width or row count disagrees with what was asked for. + """ + if attribution.ndim != 2 or attribution.shape[1] != len(keys) or attribution.shape[0] != expected_rows: + raise InvalidParameterError( + f"{type(estimator).__name__} produced attribution of shape " + f"{attribution.shape} for {expected_rows} rows and {len(keys)} keys " + f"({', '.join(keys[:4])}{'...' if len(keys) > 4 else ''}). Reporting it would pair values with " + "the wrong names. Retrain the model so its attribution matches the persisted feature contract." + ) + + def _attribute( estimator: Any, rows: np.ndarray, num_features: int, block_indices: list[list[int]] | None ) -> np.ndarray: @@ -217,6 +256,14 @@ def _attribute( the plain sum, which is correct because SHAP is additive. A model with a single feature has nothing to decompose, so that feature takes the whole share. + An estimator whose attribution is non-additive and which cannot group it is **refused** rather than + quietly attributed per feature. Summing leave-one-out drops would reinstate the error blocking exists to + remove -- each view of a shared source measures almost nothing on its own -- and returning per-feature + values under source-column keys silently pairs numbers with the wrong names. This is reachable because + :mod:`databricks.labs.dqx.anomaly.timeseries_detector` is registered with cloudpickle *by value*, so an + older class definition travels inside a persisted model and can be restored without the method. A + version string cannot substitute for the check: the capability is a property of the restored object. + Args: estimator: The bare estimator, already unwrapped from any pipeline. rows: Feature values for the rows to attribute, scaled if the model carries a scaler. @@ -225,12 +272,22 @@ def _attribute( Returns: Signed attribution, oriented so larger means more responsible, one column per block when blocked. + + Raises: + InvalidParameterError: If source blocks were asked for and this estimator's attribution can be + neither grouped by it nor summed soundly. """ if num_features == 1: return np.ones((len(rows), 1)) if block_indices is not None and hasattr(estimator, "block_contributions"): return np.asarray(estimator.block_contributions(rows, block_indices)) if hasattr(estimator, "feature_contributions"): + if block_indices is not None: + raise InvalidParameterError( + f"{type(estimator).__name__} supplies per-feature attribution but cannot group it by source " + "column, and its values are not additive, so they cannot be summed. This model predates " + "source-column attribution; retrain it to get explanations." + ) return np.asarray(estimator.feature_contributions(rows)) per_feature = _oriented_towards_anomaly(np.asarray(SHAP.TreeExplainer(estimator).shap_values(rows))) diff --git a/src/databricks/labs/dqx/anomaly/temporal.py b/src/databricks/labs/dqx/anomaly/temporal.py index 0eb0e1fc5..c4b826c72 100644 --- a/src/databricks/labs/dqx/anomaly/temporal.py +++ b/src/databricks/labs/dqx/anomaly/temporal.py @@ -254,16 +254,26 @@ def _holdout_residual_scale(seconds: np.ndarray, values: np.ndarray, basis: Temp Subtracting the residual median makes this a measure of *spread*, so it cannot see a constant forecast bias -- a basis whose predictions are uniformly off by the same amount scores as well as one that is - right. That is deliberate, and the blind spot is exactly the case that cannot matter downstream: a - constant offset in a ``_rel_time`` feature cancels before it can reach a score. The correlation-aware - detector centres on the training mean, so the shift subtracts back out; IsolationForest draws split - thresholds from each feature's observed range, so shifting a column shifts its candidates with it and - the partition is unchanged. Measured against no offset, offsets of 5 and 500 move the correlation-aware - score by 1.7e-14 and 1.9e-12 relative -- floating-point residue from the centring, twelve orders of - magnitude below the quantile spacing that decides a severity percentile -- and leave IsolationForest - bit-identical. Non-constant error, a wrong slope or the wrong shape, inflates residual spread instead, - which this statistic does penalise. So do not "fix" this by scoring bias: it would trade a criterion - that tracks what matters for one that also tracks what cannot. + right. The blind spot is **bounded rather than harmless**, and the bound is where the fit is. + + What it legitimately need not see: selection compares candidate bases that are all refitted on the same + data, and a level error shared by a candidate and its reference cancels from the ratio they are judged + on. Translating a metric and refitting moves the correlation-aware score by 1.7e-14 relative and leaves + IsolationForest bit-identical, because the detector centres on the training mean and the forest draws + split thresholds from each feature's observed range. So the *choice* is translation-invariant, and a + refit already cancels the part selection could have acted on. + + What it cannot cover, and what an earlier version of this docstring wrongly claimed it could: a bias + appearing only *after* the fit window, in the residuals a fixed model produces on future rows. The + detector is centred on training residuals that never saw the shift, so nothing cancels. Measured on a + fitted detector with +5 added to one held-out residual dimension, the residual MAD is unchanged at + 1.0026 -- this statistic is blind to it -- while the fraction of rows above the training p99 goes from + **1.3% to 98.4%**. Drift detection and the staleness horizon are what surface that; this criterion is + not one of them. + + Non-constant error, a wrong slope or the wrong shape, does inflate residual spread and is penalised. So + do not "fix" this by scoring in-window bias, and equally do not read it as a claim that forecast bias is + harmless. """ order = np.argsort(seconds) t_sorted, v_sorted = seconds[order], values[order] diff --git a/src/databricks/labs/dqx/anomaly/training_strategies.py b/src/databricks/labs/dqx/anomaly/training_strategies.py index 156ac9f60..8cc581f99 100644 --- a/src/databricks/labs/dqx/anomaly/training_strategies.py +++ b/src/databricks/labs/dqx/anomaly/training_strategies.py @@ -22,7 +22,7 @@ prepare_engineered_pandas, prepare_training_features, ) -from databricks.labs.dqx.anomaly.ensemble_training import train_ensemble +from databricks.labs.dqx.anomaly.ensemble_training import EnsembleTrainer from databricks.labs.dqx.anomaly.mlflow_registry import ModelRegistryBase, get_default_registry from databricks.labs.dqx.anomaly.timeseries_detector import fit_mahalanobis_model from databricks.labs.dqx.anomaly.types import TrainingResult @@ -120,10 +120,16 @@ def train( ensemble_size = params.ensemble_size if allow_ensemble and params.ensemble_size else 1 if ensemble_size > 1: - model_uris, hyperparams, validation_metrics, score_quantiles, feature_metadata = train_ensemble( - train_df, val_df, columns, params, ensemble_size, model_name - ) - model_uri = ",".join(model_uris) + # The injected registry, not the default one. A module-level helper used to construct + # EnsembleTrainer() with no argument here, so a registry passed to this strategy was honoured + # for a single model and silently ignored for an ensemble -- which is the default path, and the + # one where a test's fake registry would therefore have reached MLflow instead. + result = EnsembleTrainer(self._registry).train(train_df, val_df, columns, params, ensemble_size, model_name) + model_uri = ",".join(result.model_uris) + hyperparams = result.hyperparams + validation_metrics = result.aggregated_metrics + score_quantiles = result.score_quantiles + feature_metadata = result.feature_metadata run_id = "ensemble" else: model, hyperparams, feature_metadata = fit_isolation_forest(train_df, columns, params) diff --git a/tests/resources/ai_query_prompt_header.txt b/tests/resources/ai_query_prompt_header.txt index 8f1be6071..20ea61dad 100644 --- a/tests/resources/ai_query_prompt_header.txt +++ b/tests/resources/ai_query_prompt_header.txt @@ -1,16 +1,16 @@ You are a data quality analyst. Given aggregate metadata for a GROUP of anomalous rows sharing the same contribution pattern, explain in plain business language why the model flagged this group. Your explanation will be shown for every row in the group — describe the pattern, not a specific row. You are describing what the model measured, not diagnosing a root cause: the inputs cannot establish one. The inputs carry NO DIRECTION. A contribution says how much a metric mattered to the score, never whether it was high or low: a metric far above its norm and one equally far below produce the identical number. So never say a value was high, low, above, below, elevated, inflated, dropped, spiked, or missing. Say it departed from its expected pattern, and leave which way unsaid. The same applies to drift magnitudes, which are also unsigned. -What a contribution is measured AGAINST is given to you, in baseline_grouping and temporal_baseline, and it changes what you may claim. Say the metric departed from whichever comparison those fields describe -- its group's normal, the level expected at that time, or the table as a whole when both are 'none'. A metric judged against its group or its own history can be entirely ordinary for the table and still be wrong, so do not fall back on calling it unusual outright when a narrower comparison is what objected. +baseline_grouping and temporal_baseline tell you which comparisons were AVAILABLE to the model, not which one objected. A metric may be compared against the table, against its own group and against its expected level at that time all at once, and the contribution you are given is the total across those, so it cannot say which comparison drove it. Treat these fields as context that widens what the number is consistent with: when either is set, a metric can be entirely ordinary for the table and still have departed from a narrower comparison, so do not call it unusual outright. State that the metric contributed and name the comparisons that were in use; do not assign the departure to one of them. Be direct and concrete: name the metrics, their shares and the group size without hedging phrases like 'The data shows', 'It appears that', or 'might indicate'. Being direct means stating plainly what the inputs contain — it does not license asserting a direction, a cause, or a value they do not contain. Do not restate the input field names back to the user, and do not invent feature names, values, or baseline groups that are not present in the input. Inputs: - attribution_basis: What the feature_contributions below are measuring, which differs by detector and decides how you may describe the pattern. Follow this field rather than assuming a reading: one basis supports saying a feature's own value was unusual, the other does not. -- feature_contributions: Mean contributions across the group, already named for a reader, e.g. 'amount vs its group baseline (82%), quantity (11%), discount (5%)'. A phrase like 'X vs its group baseline' means X was unusual relative to its own baseline group, not in absolute terms. These are aggregated relative importances — not raw data values. +- feature_contributions: Mean share across the group of the evidence the model acted on, per column the caller named, e.g. 'amount (82%), quantity (11%), discount (5%)'. One entry per column however many ways that column was compared, so a share says the column was involved and not which comparison objected. These are aggregated relative importances — not raw data values, and not percentages of the score. - group_size: Number of rows in this group, e.g. '312 rows'. - severity_range: Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'. - confidence: How closely the ensemble's members agreed on the score: 'high' / 'mixed' / 'low', or 'n/a' when one model did the scoring. Members differ only by random seed on the same training data, so this measures the stability of the score, NOT how reliable the flag is or whether the data has since changed. Do not present it to the reader as confidence in the finding. - baseline_grouping: The columns whose values define each row's baseline group, e.g. 'region' or 'region, product'. Anomalies are judged relative to the row's own group baseline; 'none' when the model is not grouped. When set, a value can be ordinary for the table as a whole and still be wrong for its own group, so say the metric departed from what its group normally looks like rather than that it was unusual outright. -- temporal_baseline: The time column each metric is judged along, e.g. 'event_ts', or 'none'. When set, each metric is compared against the level expected of it AT THAT POINT IN TIME, not against its whole history. So its value can sit well inside the range the data has always covered and still be wrong for when it arrived: say it departed from the level expected at that time. Do not call it unusual, high or low for the metric overall, because the comparison was never against that. +- temporal_baseline: The time column the model was allowed to judge metrics along, e.g. 'event_ts', or 'none'. When set, a metric MAY have been compared against the level expected of it at that point in time as well as against its overall range -- but not every metric is: where no expectation could be fitted for one, that comparison contributes nothing for it. So this widens what a contribution is consistent with rather than explaining it: a metric can sit well inside the range the data has always covered and still have departed from what was expected when it arrived. Do not call a metric unusual, high or low overall on the strength of this field, and do not state that it departed from its expected level -- say the comparison was available. - threshold: The severity percentile threshold configured by the user (0–100). - drift_summary: Baseline drift signal from the scoring run, e.g. 'drift detected: amount=4.12; quantity=3.55' or 'none'. If drift is present, explicitly frame the narrative vs baseline. @@ -21,7 +21,7 @@ Respond with ONLY a JSON object. Field rules: Example (relationship basis, no drift): attribution_basis: each metric's position once the others are accounted for -feature_contributions: amount vs its group baseline (61%), quantity (22%) +feature_contributions: amount (61%), quantity (22%) group_size: 312 rows severity_range: mean 97.4, min 95.1, max 99.8 confidence: high @@ -29,7 +29,7 @@ baseline_grouping: region temporal_baseline: none threshold: 95.0 drift_summary: none -Response: {"narrative":"Across 312 rows, amount departs most from what its own region implies (61%), with quantity next (22%).","business_impact":"Amount values that do not match their region's usual pattern distort revenue reporting if processed unchanged.","action":"Reconcile amount against source orders for the affected regions."} +Response: {"narrative":"Across 312 rows, amount accounts for most of what the model measured (61%), with quantity next (22%); these rows are judged against their own region.","business_impact":"If amount is wrong on these rows, revenue reporting for the affected regions would be affected too.","action":"Reconcile amount against source orders for the affected regions."} Example (value basis, judged against time, with drift): attribution_basis: each feature's own value compared against the rows it was scored against @@ -41,4 +41,4 @@ baseline_grouping: none temporal_baseline: event_ts threshold: 95.0 drift_summary: drift detected: latency_ms=4.12 -Response: {"narrative":"88 rows are dominated by latency_ms (74%), which departs from the level expected of it at that point in time and has also drifted from its training baseline; retries contribute modestly (12%).","business_impact":"Latency that no longer tracks its expected level risks SLA breaches for downstream consumers.","action":"Compare latency_ms against its expected level for that period rather than against its overall range."} +Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from its training baseline; retries contribute modestly (12%). These rows are judged against expected levels over time as well as overall.","business_impact":"If latency_ms is genuinely off on these rows, downstream consumers with SLAs would be the first to notice.","action":"Compare latency_ms against its expected level for that period as well as against its overall range."} diff --git a/tests/unit/test_anomaly_ensemble_attribution.py b/tests/unit/test_anomaly_ensemble_attribution.py index f80a789c9..390cebfd7 100644 --- a/tests/unit/test_anomaly_ensemble_attribution.py +++ b/tests/unit/test_anomaly_ensemble_attribution.py @@ -169,13 +169,21 @@ def feature_contributions(self, rows: np.ndarray) -> np.ndarray: return np.ones((len(rows), 1)) -def test_members_whose_attributions_cannot_be_aligned_are_rejected(members: list[IsolationForest], probe: pd.DataFrame): +def test_a_member_whose_attribution_is_malformed_is_rejected(members: list[IsolationForest], probe: pd.DataFrame): """Averaging misaligned columns would blend different features into one confident, wrong answer. - That is indistinguishable from correct output downstream -- exactly the failure mode this whole - function exists to remove -- so it raises rather than quietly falling back to one member. + Two guards stand between that and the output, and this exercises the earlier one. A member whose + attribution does not match the keys it would be reported under is refused where it is *produced*, by + ``compute_row_attributions``, so the failure names the offending estimator rather than surfacing as an + averaging problem one frame later. That guard also covers the single-model path, which averaging never + touches. + + ``mean_row_attributions`` keeps its own cross-member check. Width can no longer differ between members + -- every member is validated against the same key list -- but the per-row valid mask is derived from + each member's own scaler, so two members can still disagree about which rows were attributable, and only + the cross-member check sees that. """ - with pytest.raises(InvalidParameterError, match="cannot be averaged"): + with pytest.raises(InvalidParameterError, match="Reporting it would pair values with the wrong names"): mean_row_attributions([members[0], _WrongWidthEstimator()], probe, _COLUMNS) diff --git a/tests/unit/test_anomaly_ensemble_registry_injection.py b/tests/unit/test_anomaly_ensemble_registry_injection.py new file mode 100644 index 000000000..555095333 --- /dev/null +++ b/tests/unit/test_anomaly_ensemble_registry_injection.py @@ -0,0 +1,92 @@ +"""A registry handed to a training strategy must be the one it uses (no Spark, no workspace). + +The strategy accepts a ``ModelRegistryBase`` so a caller can substitute a backend and a test can +substitute a fake. That held for a single model and silently did not for an ensemble: a module-level +helper constructed ``EnsembleTrainer()`` with no argument, so the injected registry was dropped on the +path taken by default -- ``ensemble_size`` is 3 -- and a test's fake registry would have reached MLflow +instead of recording the call. + +Reaching the assertion without Spark depends on ``EnsembleTrainer.train`` consulting the registry before +it engineers any features. That ordering is load-bearing for the test and not for the product, so it is +stated here rather than left implicit: if the two are ever swapped, this test fails for the wrong reason +and its docstring is where to look. +""" + +from unittest.mock import create_autospec + +import pytest +from pyspark.sql import DataFrame + +from databricks.labs.dqx.anomaly.mlflow_registry import ( + ModelRegistryBase, + get_default_registry, + set_default_registry, +) +from databricks.labs.dqx.anomaly.training_strategies import IsolationForestTrainingStrategy +from databricks.labs.dqx.config import AnomalyParams + + +class _RegistryReached(Exception): + """Raised by whichever registry is consulted first, to identify it without doing any real work.""" + + +@pytest.fixture +def default_registry_spy(): + """Install a fake as the process default and restore the real one afterwards. + + Installed through the public *set_default_registry*, which the module documents as being for exactly + this. It matters for more than tidiness: without it the real MLflow registry would run against the + repository's local tracking store on the failing path and leave state behind. + """ + original = get_default_registry() + spy = create_autospec(ModelRegistryBase, instance=True) + set_default_registry(spy) + try: + yield spy + finally: + set_default_registry(original) + + +def _train(strategy: IsolationForestTrainingStrategy, ensemble_size: int) -> None: + train_df = create_autospec(DataFrame, instance=True) + val_df = create_autospec(DataFrame, instance=True) + strategy.train( + train_df, + val_df, + ["amount", "quantity"], + AnomalyParams(ensemble_size=ensemble_size), + "cat.sch.model", + allow_ensemble=True, + ) + + +def test_the_ensemble_path_uses_the_injected_registry(default_registry_spy): + """The defect: an ensemble ignored the injected registry and used the default. + + Both halves are asserted. That the injected registry was consulted is the fix; that the default was + not touched at all is what makes this a statement about injection rather than about which of two + registries happens to be reached first. + """ + injected = create_autospec(ModelRegistryBase, instance=True) + injected.ensure_registry_configured.side_effect = _RegistryReached + + with pytest.raises(_RegistryReached): + _train(IsolationForestTrainingStrategy(registry=injected), ensemble_size=3) + + injected.ensure_registry_configured.assert_called_once() + assert not default_registry_spy.method_calls, "the default registry should never have been consulted" + + +def test_the_single_model_path_also_uses_the_injected_registry(default_registry_spy): + """The path that already worked, pinned so the fix cannot be mistaken for the whole story. + + Reaches the registry later than the ensemble path -- after fitting -- so any Spark stand-in fails + first; what matters here is only that the default registry is still never the one consulted. + """ + injected = create_autospec(ModelRegistryBase, instance=True) + injected.ensure_registry_configured.side_effect = _RegistryReached + + with pytest.raises(Exception): # pylint: disable=broad-exception-caught + _train(IsolationForestTrainingStrategy(registry=injected), ensemble_size=1) + + assert not default_registry_spy.method_calls, "the default registry should never have been consulted" diff --git a/tests/unit/test_anomaly_llm_explainer.py b/tests/unit/test_anomaly_llm_explainer.py index 715f7d818..bff026794 100644 --- a/tests/unit/test_anomaly_llm_explainer.py +++ b/tests/unit/test_anomaly_llm_explainer.py @@ -473,17 +473,26 @@ def test_the_temporal_baseline_string_is_none_without_metadata(): assert llm_explainer._temporal_baseline_str(None) == "none" -def test_the_temporal_field_says_a_normal_looking_value_can_still_be_wrong(): - """The substantive capability, not just the field's presence. - - A metric compared against its expected level at a moment can sit inside every range the table has - ever held. Without being told that, a model reads a large share and reaches for "unusually high", - which is the one claim the evidence cannot support. +def test_the_temporal_field_describes_an_available_comparison_not_one_that_happened(): + """The field widens what a contribution is consistent with; it does not explain it. + + An earlier version of this test pinned the opposite, and was wrong. Setting a time column does not mean + every metric was judged against time: where no expectation could be fitted for a metric, feature + engineering emits a constant zero for its time-relative feature, so that comparison contributes nothing + for it. Nor is the contribution attributable to one comparison even when several ran, because the share + is the total across them. + + What the field legitimately buys is the *absence* of a wrong conclusion: with it set, a metric can be + ordinary for the table and still have departed from a narrower comparison, so "unusually high" stops + being the obvious reading of a large share. """ description = dict(llm_explainer._PROMPT_INPUT_FIELDS)["temporal_baseline"] - assert "AT THAT POINT IN TIME" in description - assert "still be wrong for when it arrived" in description + assert "MAY have been compared" in description + assert "not every metric is" in description + assert "say the comparison was available" in description + # The overclaim this replaced: asserting the departure rather than the availability. + assert "departed from the level expected" not in description def test_both_comparison_states_are_demonstrated_in_the_exemplars(): @@ -502,19 +511,59 @@ def test_both_comparison_states_are_demonstrated_in_the_exemplars(): assert sorted(values) == ["event_ts", "none"] -def test_the_temporal_exemplar_describes_the_expected_level_rather_than_an_extreme_value(): - """The exemplar has to model the reading, since that is what a smaller model copies.""" +def test_no_exemplar_attributes_the_departure_to_a_particular_comparison(): + """An exemplar is an instruction, so it must not model a claim the inputs cannot support. + + The previous version of this test required the temporal exemplar to say the metric "departs from the + level expected of it at that point in time" -- which is precisely the attribution that is unavailable. + Both exemplars now name the comparisons in use and stop there. + """ + responses = [line for line in llm_explainer._PROMPT_EXAMPLES.splitlines() if line.startswith("Response: ")] + assert len(responses) == 2 + + for response in responses: + lowered = response.lower() + for phrase in ("departs from the level expected", "no longer tracks its expected level"): + assert phrase not in lowered, f"exemplar attributes the departure to one comparison: {phrase}" + for word in ("far above", "far below", "elevated", "inflated", "spiked"): + assert word not in lowered + + +def test_the_exemplars_state_business_impact_conditionally(): + """Whether a contribution means real damage depends on facts the model does not have. + + An unusual amount need not distort revenue and a latency change need not breach an SLA -- the row may + be legitimate. Both exemplars previously asserted the consequence outright, which is what a smaller + model copies into every explanation it writes. + """ responses = [line for line in llm_explainer._PROMPT_EXAMPLES.splitlines() if line.startswith("Response: ")] - temporal_response = responses[1] - assert "expected of it at that point in time" in temporal_response - for word in ("far above", "far below", "elevated", "inflated", "spiked"): - assert word not in temporal_response.lower() + for response in responses: + impact = response.partition('"business_impact":"')[2].partition('","')[0] + assert impact.lower().startswith("if "), f"impact should be conditional, got {impact!r}" -def test_the_instructions_tie_the_claim_to_what_it_was_measured_against(): - """Otherwise "avoid hedging" plus a large share reads as licence to call the metric unusual outright.""" +def test_the_instructions_present_the_comparisons_as_context_not_cause(): + """Otherwise "avoid hedging" plus a large share reads as licence to name a responsible comparison.""" header = llm_explainer._render_ai_query_prompt_header() - assert "measured AGAINST is given to you" in header - assert "do not fall back on calling it unusual outright" in header + assert "which comparisons were AVAILABLE to the model, not which one objected" in header + assert "do not assign the departure to one of them" in header + assert "do not call it unusual outright" in header + + +def test_spread_contributions_do_not_license_a_broken_relationship_claim(): + """Spread means several metrics contributed. It does not identify a mechanism. + + A review probe settled this: two metrics with training correlation 6e-18, both marginally extreme at + once, produce contributions of 50% and 50%. Nothing about that shape distinguishes metrics that stopped + agreeing with each other from unrelated metrics that happened to be unusual together, so the earlier + instruction to describe spread as "the metrics no longer agreeing" was asserting a mechanism from + evidence that does not carry one. + """ + correlation = llm_explainer.attribution_semantics("Mahalanobis") + + assert "each of them contributed, and nothing more" in correlation + assert "do not claim a relationship between them broke" in correlation.lower() + # The instruction this replaced. + assert "describe it as the metrics no longer agreeing with each other" not in correlation diff --git a/tests/unit/test_anomaly_source_block_attribution.py b/tests/unit/test_anomaly_source_block_attribution.py index 66708a760..c109347f0 100644 --- a/tests/unit/test_anomaly_source_block_attribution.py +++ b/tests/unit/test_anomaly_source_block_attribution.py @@ -17,6 +17,8 @@ from sklearn.ensemble import IsolationForest from databricks.labs.dqx.anomaly.explainability import compute_row_attributions, format_shap_contributions +from databricks.labs.dqx.anomaly.timeseries_detector import MahalanobisDetector +from databricks.labs.dqx.errors import InvalidParameterError def _shares(model: IsolationForest, row: pd.DataFrame, columns: list[str], blocks=None) -> dict[str, float]: @@ -134,3 +136,82 @@ def test_a_block_sums_signed_values_so_a_normalising_view_cancels(): clipped_sum = float(np.maximum(per_view[0, :2], 0.0).sum()) if per_view[0, :2].min() < 0: assert blocked[0, 0] < clipped_sum + + +# ── the attribution contract: shape must match the names it will be reported under ──────────────────── + + +class _PerFeatureOnly: + """An estimator that can attribute per feature but cannot group by source column. + + Reachable for two reasons, neither hypothetical. ``timeseries_detector`` is registered with cloudpickle + *by value*, so an older class definition travels inside a persisted model and is restored without + methods added since; and the config hash covers only columns and the comparison bases, so such a model + passes scoring-time validation unchanged. A third-party estimator supplied through the documented + training-strategy extension point is the other. + """ + + def feature_contributions(self, rows: np.ndarray) -> np.ndarray: + return np.tile(np.array([1e-5, 1e-5, 0.13]), (len(rows), 1)) + + +_BLOCKED_COLUMNS = ["x", "x_rel_time", "y"] +_BLOCKS = {"x": [0, 1], "y": [2]} + + +def test_an_estimator_that_cannot_group_by_source_is_refused_rather_than_misreported(): + """The defect: per-feature width reported under source-column names, silently. + + Left alone this published ``{'x': 0.0, 'y': 0.0}`` -- nothing identified as a driver -- on a row whose + score was identical to a correctly explained one. Both halves of that are bad: the substantial value + landed in the normalising denominator and was then never emitted, and the two names it did emit belonged + to different columns than the numbers behind them. + + Summing the drops instead is not an option: they are not additive, which is the whole reason source + blocking exists for this detector. + """ + frame = pd.DataFrame([[8.0, 5.0, 0.5]], columns=_BLOCKED_COLUMNS) + + with pytest.raises(InvalidParameterError, match="cannot group it by source column"): + compute_row_attributions(_PerFeatureOnly(), frame, _BLOCKED_COLUMNS, _BLOCKS) + + +def test_the_same_estimator_is_still_attributed_when_no_blocks_are_asked_for(): + """The refusal is about the combination, not about the estimator. + + Without this the guard could be satisfied by rejecting the estimator outright, which would break a + caller who never wanted source blocking in the first place. + """ + frame = pd.DataFrame([[8.0, 5.0, 0.5]], columns=_BLOCKED_COLUMNS) + + attribution, valid, keys = compute_row_attributions(_PerFeatureOnly(), frame, _BLOCKED_COLUMNS) + + assert keys == _BLOCKED_COLUMNS + assert attribution.shape == (1, 3) + assert valid.all() + + +def test_an_estimator_that_can_group_by_source_is_unaffected(): + """The guard must not be satisfiable by refusing everything, so the working path is pinned beside it.""" + rng = np.random.default_rng(0) + metric = rng.normal(0, 1, 2000) + train = np.column_stack([metric, metric - 3.0, rng.normal(0, 1, 2000)]) + detector = MahalanobisDetector().fit(train) + frame = pd.DataFrame([[8.0, 5.0, 0.5]], columns=_BLOCKED_COLUMNS) + + attribution, _, keys = compute_row_attributions(detector, frame, _BLOCKED_COLUMNS, _BLOCKS) + + assert keys == ["x", "y"] + assert attribution.shape == (1, 2) + + +def test_the_formatter_refuses_a_matrix_that_does_not_match_its_keys(): + """Guarded twice on purpose, because this function is reachable with a hand-built matrix. + + ``compute_row_attributions`` catches the case it can diagnose, with a remedy. This catches any producer + -- a future branch, a caller assembling values itself -- at the point where a column acquires a name. + """ + three_wide = np.array([[1e-5, 1e-5, 0.13]]) + + with pytest.raises(InvalidParameterError, match="cannot be reported under 2 keys"): + format_shap_contributions(three_wide, np.array([True]), 1, ["x", "y"]) diff --git a/tests/unit/test_anomaly_temporal_fit.py b/tests/unit/test_anomaly_temporal_fit.py index f915b59be..500718843 100644 --- a/tests/unit/test_anomaly_temporal_fit.py +++ b/tests/unit/test_anomaly_temporal_fit.py @@ -16,6 +16,7 @@ from databricks.labs.dqx.anomaly.temporal import ( CANDIDATE_PERIODS_SECONDS, + MAD_TO_SIGMA, MIN_SEASONAL_CYCLES, SEASONAL_HARMONICS, TemporalBasis, @@ -464,24 +465,25 @@ def test_a_metric_with_more_than_half_identical_values_still_fits(): assert all(np.isfinite(fitted["metric"])) -def test_a_constant_offset_in_a_relative_feature_cannot_reach_a_score(): - """Why the selection criterion is allowed to be blind to constant forecast bias. +def test_translating_a_metric_and_refitting_leaves_the_scores_unchanged(): + """Translation invariance *of refitting* -- which is the property selection actually relies on. - ``_holdout_residual_scale`` subtracts the residual median, so it measures spread and scores a - uniformly-biased basis as well as an unbiased one. That looks like a gap in the objective until you - ask what a constant offset in a ``_rel_time`` column can actually do downstream: nothing. The - correlation-aware detector centres on the training mean, so the shift cancels exactly; IsolationForest - picks split thresholds from each feature's observed range, so shifting the whole column shifts the - thresholds with it and leaves the partition identical. + Named for what it measures, after an earlier version of this test was used to support a stronger claim + it does not establish. It shifts the training data *and* the scored data and refits, so it shows that a + level error shared by a candidate basis and its reference cancels from the ratio they are judged on. + That is why ``_holdout_residual_scale`` may measure spread alone: the *choice* between bases is + unaffected by a shared translation. - The cancellation is algebraic, so what survives is floating-point residue from centring a shifted - column, not sensitivity to the offset. Measured relative movement in the correlation-aware score: - 1.7e-14 at an offset of 5 and 1.9e-12 at 500 -- twelve orders of magnitude below the quantile spacing - that decides a severity percentile, so no flag can turn on it. IsolationForest is bit-identical at - both, because shifting a column shifts its candidate split thresholds with it. + It is **not** evidence that forecast bias is harmless. A bias present only in future residuals, with + the detector already fitted, does reach the score -- see + :func:`test_a_bias_appearing_only_after_the_fit_window_does_reach_the_score`. - Asserted at a tolerance rather than at bit equality, since bit equality is false and asserting it - would have made this test a statement about float arithmetic instead of about the criterion. + The cancellation is algebraic, so what survives is floating-point residue from centring a shifted + column: 1.7e-14 relative at an offset of 5 and 1.9e-12 at 500, twelve orders of magnitude below the + quantile spacing that decides a severity percentile. IsolationForest is bit-identical, because shifting + a column shifts its candidate split thresholds with it. Asserted at a tolerance, since bit equality is + false for the correlation-aware detector and asserting it would make this a statement about float + arithmetic. """ rng = np.random.default_rng(0) base = rng.normal(0.0, 1.0, (2000, 3)) @@ -499,3 +501,55 @@ def scores_with_offset(offset: float) -> tuple[np.ndarray, np.ndarray]: maha, forest = scores_with_offset(offset) np.testing.assert_allclose(maha, reference_maha, rtol=1e-10) assert np.array_equal(forest, reference_forest), f"IsolationForest scores moved at offset {offset}" + + +def test_the_basis_choice_is_unchanged_by_translating_the_metric(): + """The criterion-level statement, which is what the docstring is really about. + + The test above measures the *detector*; this one measures the selector. ``select_basis`` compares + candidates all refitted on the same data, and standardisation centres by median, so adding a constant to + a metric must not change which basis wins. Asserted through the public function rather than through + ``_holdout_residual_scale``, so it survives a change of internal statistic. + """ + seconds = np.arange(0.0, 60 * DAY, HOUR) + values = 100.0 + 0.002 * seconds + np.sin(seconds * 2 * np.pi / DAY) * 5.0 + + chosen, _ = select_basis(seconds, {"m": values}) + shifted, _ = select_basis(seconds, {"m": values + 5.0}) + + assert chosen.to_dict() == shifted.to_dict() + + +def test_a_bias_appearing_only_after_the_fit_window_does_reach_the_score(): + """The counterexample, recorded so the limitation is executable rather than merely described. + + An earlier docstring here claimed a constant forecast bias "cannot matter" because centring cancels it. + That holds only when the bias is present while the model is fitted. A bias that appears *after* the fit + window meets a detector centred on training residuals that never saw it, so nothing cancels. + + Both halves are asserted, because together they are the finding: the selection criterion is blind to it + (the residual spread is unchanged, so no basis choice would differ) *and* it dominates the score. That + is why drift detection and the staleness horizon exist and why this criterion is not the mechanism for + it -- not an argument for making the criterion bias-sensitive, which a refit already handles. + """ + rng = np.random.default_rng(0) + train_residuals = rng.normal(0.0, 1.0, (4000, 2)) + detector = MahalanobisDetector().fit(train_residuals) + alert_threshold = np.percentile(-detector.score_samples(train_residuals), 99) + + holdout = rng.normal(0.0, 1.0, (2000, 2)) + shifted = holdout + np.array([0.0, 5.0]) + + def robust_spread(residuals: np.ndarray) -> float: + column = residuals[:, 1] + return float(np.median(np.abs(column - np.median(column))) * MAD_TO_SIGMA) + + def alert_fraction(residuals: np.ndarray) -> float: + return float((-detector.score_samples(residuals) >= alert_threshold).mean()) + + # The criterion cannot see it: a median-centred spread is translation-invariant by construction. + assert robust_spread(shifted) == pytest.approx(robust_spread(holdout), rel=1e-12) + + # The score can: measured 1.3% against 98.4% on this fixture. + assert alert_fraction(holdout) < 0.05 + assert alert_fraction(shifted) > 0.90 From 4b97463666abd3eeb4e2c2e822399df6031e5032 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Tue, 8 Sep 2026 13:54:48 +0100 Subject: [PATCH 104/107] Drop a lint suppression, and the near-vacuous test that needed it The previous commit added `# pylint: disable=broad-exception-caught` to a registry-injection test, which AGENTS.md forbids outright and the no-cheat workflow rejects by design. Mine, and correctly caught. Deleted rather than reworded, because the suppression was propping up a test that asserted almost nothing: that the default registry is not consulted on the single-model path, which fails before reaching any registry at all. Its `pytest.raises(Exception)` existed only to swallow whichever error the Spark stand-ins produced first, so the assertion could never have distinguished the fix from its absence. What covers that path is stated in the module docstring instead: the single-model path already forwarded the injected registry before this change, and it fits before consulting it, so reaching the call without a real Spark session is not possible -- the integration suite is what exercises it. The ensemble test, which is the one the defect was about, is unchanged. Verified with the workflow's own command rather than by inspection: the diff against main now yields zero bytes of `# pylint: disable`. Co-authored-by: Isaac --- ...est_anomaly_ensemble_registry_injection.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/tests/unit/test_anomaly_ensemble_registry_injection.py b/tests/unit/test_anomaly_ensemble_registry_injection.py index 555095333..c83217be1 100644 --- a/tests/unit/test_anomaly_ensemble_registry_injection.py +++ b/tests/unit/test_anomaly_ensemble_registry_injection.py @@ -10,6 +10,10 @@ it engineers any features. That ordering is load-bearing for the test and not for the product, so it is stated here rather than left implicit: if the two are ever swapped, this test fails for the wrong reason and its docstring is where to look. + +Only the ensemble path is covered. The single-model path already forwarded the injected registry, and it +fits before consulting it, so reaching that call without a real Spark session is not possible -- the +existing integration tests are what cover it. """ from unittest.mock import create_autospec @@ -75,18 +79,3 @@ def test_the_ensemble_path_uses_the_injected_registry(default_registry_spy): injected.ensure_registry_configured.assert_called_once() assert not default_registry_spy.method_calls, "the default registry should never have been consulted" - - -def test_the_single_model_path_also_uses_the_injected_registry(default_registry_spy): - """The path that already worked, pinned so the fix cannot be mistaken for the whole story. - - Reaches the registry later than the ensemble path -- after fitting -- so any Spark stand-in fails - first; what matters here is only that the default registry is still never the one consulted. - """ - injected = create_autospec(ModelRegistryBase, instance=True) - injected.ensure_registry_configured.side_effect = _RegistryReached - - with pytest.raises(Exception): # pylint: disable=broad-exception-caught - _train(IsolationForestTrainingStrategy(registry=injected), ensemble_size=1) - - assert not default_registry_spy.method_calls, "the default registry should never have been consulted" From 7c4fee9f3ddb03fa00301850dd19c9bbaf79e5a4 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 9 Sep 2026 10:44:08 +0100 Subject: [PATCH 105/107] Publish a severity a reader can compare against the threshold and get the flag's answer A black-box evaluation of this branch found displayed severity disagreeing with threshold interpretation in 477 of 912 scoring cells, every audited disagreement sitting exactly at displayed threshold equality. The decision was never wrong. The flag, the Spark contributions gate, the numpy gate inside the UDF and both AI-explanation gates all read the full-precision severity, and is_anomaly matched _errors on every measured row. Only the published number was lossy: it was rounded to one decimal, so a row at 94.96 was shown as 95.0 and not flagged at a threshold of 95. Nothing was broken except the reader's ability to interpret what they were shown -- which the guide actively invited, telling them to use that value for thresholds, and which the check's own docstring promised outright: "Records with severity_percentile >= threshold are flagged". That promise was false before this commit. Floored instead of rounded, and the decision left untouched. Flooring gives displayed <= actual, so displayed >= threshold implies actual >= threshold; taking the floor at the threshold's own precision gives the converse. The two verdicts are then exactly equivalent, and not one flag changes -- the decision expression is byte-identical. The precision is derived from the threshold rather than fixed at one decimal, which removes a limitation instead of documenting one. Measured over dense sweeps around each threshold plus the float neighbours of every gridpoint, since the implementation multiplies before flooring and that is where such an implementation drifts: rounding disagrees on 560 sampled values at each of 90, 95, 99, 99.5 and 99.9; flooring at one decimal disagrees on none of those but on 560 at a threshold of 99.95; flooring at the threshold's precision disagrees on none at any of them, up to 99.995. Rounding the *decision* to match the display would also have made the two agree, and is the wrong trade: it would begin flagging every row in [threshold - 0.05, threshold), changing the detector to fix a presentation defect. The advisory that raised this says so explicitly, and the arithmetic agrees. Two follow-on corrections. The error message rounded the already-published value a second time, which would have undone the fix and could show a value above the threshold on a row that was not flagged; it now quotes the published number as it stands. And it said "exceeded threshold" for an inclusive comparison, so it now says "reached". Documented where a reader meets it: the check's output description, the guide's schema table -- which is the sentence that made this bite -- and the helper itself, which carries the measurements so a later contributor restoring round() sees the numbers that argue against them. A test pins that too, because round() is shorter, looks tidier, and passes everything else. One honest limit on the evidence. The 0-disagreement result comes from a Python mirror of the Spark expression, not from Spark: the property is arithmetic, so that is a fair test of the property, but "Spark agrees with this arithmetic" and "no flag changed on real data" both need the workspace suite and are recorded as not measured rather than passed. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 4 +- .../labs/dqx/anomaly/check_funcs.py | 14 ++- .../labs/dqx/anomaly/scoring_utils.py | 59 ++++++++- .../test_anomaly_ai_explanation.py | 12 +- tests/unit/test_anomaly_severity_display.py | 116 ++++++++++++++++++ 5 files changed, 194 insertions(+), 11 deletions(-) create mode 100644 tests/unit/test_anomaly_severity_display.py diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 7bbf7d969..a6cdfc974 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -664,8 +664,8 @@ Each element is a **struct** with a shared “wide” schema. Currently, the onl |--------|------|-------------| | `check_name` | string | Always `"has_no_row_anomalies"` for this check. | | `score` | double | Raw model score (0–1). Use for diagnostics only. | -| `severity_percentile` | double | Normalized score 0–100. **Use this for thresholds and ordering.** | -| `is_anomaly` | boolean | `true` if `severity_percentile` ≥ threshold. | +| `severity_percentile` | double | Normalized score 0–100. **Use this for thresholds and ordering.** Published so that `severity_percentile >= threshold` gives the same answer as `is_anomaly`: it is floored to the threshold's own precision rather than rounded, so it never reads higher than the value the flag was decided on. | +| `is_anomaly` | boolean | `true` if `severity_percentile` ≥ threshold, and the authoritative decision. | | `threshold` | double | Severity percentile threshold used (e.g. 95.0). | | `model` | string | Full model name (e.g. Unity Catalog name). | | `contributions` | map<string, double> | Each column's share (0–100) of the evidence that made the row look unusual, keyed by **the columns you passed** under either profile — see step 4 of [How it works](#how-it-works-under-the-hood). A column that argued the row was normal shows `0`. On by default (`enable_contributions=True`); populated only for anomalous rows (`null` for non-anomalous rows or if you set it `False`), and all-null on a flagged row where nothing pointed towards an anomaly. | diff --git a/src/databricks/labs/dqx/anomaly/check_funcs.py b/src/databricks/labs/dqx/anomaly/check_funcs.py index 7680c54e1..a2d55edb8 100644 --- a/src/databricks/labs/dqx/anomaly/check_funcs.py +++ b/src/databricks/labs/dqx/anomaly/check_funcs.py @@ -140,8 +140,11 @@ def has_no_row_anomalies( Output columns: - _dq_info: Array of structs (one element per dataset-level check). For example: - _dq_info[0].anomaly.score: Raw anomaly score (model-relative) - - _dq_info[0].anomaly.severity_percentile: Severity percentile (0–100) - - _dq_info[0].anomaly.is_anomaly: Boolean flag + - _dq_info[0].anomaly.severity_percentile: Severity percentile (0–100), published so that + comparing it against *threshold* gives the same verdict as *is_anomaly*. It is floored to the + threshold's own precision rather than rounded, so it never reads higher than the value the flag + was decided on + - _dq_info[0].anomaly.is_anomaly: Boolean flag, and the authoritative decision - _dq_info[0].anomaly.threshold: Severity percentile threshold used (0–100) - _dq_info[0].anomaly.model: Model name - _dq_info[0].anomaly.contributions: feature contributions as percentages (0–100); populated @@ -288,11 +291,14 @@ def apply(df: DataFrame) -> DataFrame: result = run_anomaly_scoring(df_to_score, config, registry_table, model_name) return result.drop(row_id_col) + # The published severity is already floored to the precision this threshold needs, so it is quoted as + # it stands: rounding it again here would undo that and could show a value above the threshold on a row + # that was not flagged. "Reached" rather than "exceeded", because the comparison is inclusive. message = F.concat_ws( "", F.lit("Anomaly severity "), - F.round(F.col(output_columns.info).anomaly.severity_percentile, 1).cast("string"), - F.lit(f" exceeded threshold {threshold}"), + F.col(output_columns.info).anomaly.severity_percentile.cast("string"), + F.lit(f" reached threshold {threshold}"), ) condition_expr = F.col(output_columns.info).anomaly.is_anomaly return make_condition(condition_expr, message, "has_row_anomalies"), apply, output_columns.info diff --git a/src/databricks/labs/dqx/anomaly/scoring_utils.py b/src/databricks/labs/dqx/anomaly/scoring_utils.py index 3a6cb4df8..9928541d0 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_utils.py +++ b/src/databricks/labs/dqx/anomaly/scoring_utils.py @@ -31,6 +31,63 @@ # Register anomaly field for the wide _dq_info struct (so merge gets a consistent schema) register_dq_info_field("anomaly", anomaly_info_struct_schema) +# Floor to at least one decimal, so a severity stays readable when the threshold is a whole number. +_MIN_DISPLAYED_SEVERITY_DECIMALS = 1 + + +def displayed_severity_decimals(threshold: float) -> int: + """How many decimals the displayed severity needs so it can be compared against *threshold*. + + The threshold's own precision, and never fewer than one. See + :func:`displayed_severity_expr` for why the two have to match. + + Args: + threshold: The severity percentile a row must reach to be flagged. + + Returns: + Decimal places to keep in the published severity. + """ + text = f"{float(threshold):.10f}".rstrip("0") + decimals = len(text.partition(".")[2]) + return max(_MIN_DISPLAYED_SEVERITY_DECIMALS, decimals) + + +def displayed_severity_expr(severity: Column, threshold: float) -> Column: + """Publish a severity a reader can compare against the threshold and reach the same verdict. + + The flag is decided on the full-precision severity, and so is every other consumer -- the + contributions gate on both the Spark and numpy sides, and the AI-explanation gates. Only the published + number was lossy: it was *rounded* to one decimal, so a row at 94.96 was shown as 95.0 and not flagged + at a threshold of 95. Nothing was wrong with the decision, but a reader comparing the two numbers + disagreed with it, which a black-box evaluation found in 477 of 912 scoring cells -- every audited + disagreement sitting exactly at displayed threshold equality. + + Flooring instead of rounding is what closes it, and it changes no flag at all. Flooring gives + ``displayed <= actual``, so ``displayed >= threshold`` implies ``actual >= threshold``; and because the + floor is taken at the threshold's own precision, ``actual >= threshold`` implies + ``displayed >= threshold`` too. The two verdicts therefore agree exactly. + + The precision is derived rather than fixed at one decimal, because a fixed decimal is correct only for + thresholds that happen to be that precise. Measured over dense sweeps around each threshold plus the + float neighbours of every gridpoint: rounding disagrees on 560 sampled values at each of 90, 95, 99, + 99.5 and 99.9; flooring at one decimal disagrees on none of those but on 560 at a threshold of 99.95; + flooring at the threshold's precision disagrees on none at any of them, up to 99.995. + + Rounding the *decision* to match the display would also make the two agree, and is the wrong trade: it + would begin flagging every row in ``[threshold - 0.05, threshold)``, changing the detector to fix a + presentation defect. + + Args: + severity: The full-precision severity column. Null passes through null. + threshold: The severity percentile a row must reach to be flagged. + + Returns: + The severity to publish: never above the true value, and never below it by enough to change how it + compares against *threshold*. + """ + scale = float(10 ** displayed_severity_decimals(threshold)) + return F.floor(severity * F.lit(scale)) / F.lit(scale) + def create_null_scored_dataframe( df: DataFrame, @@ -152,7 +209,7 @@ def add_info_column( anomaly_info_fields = { "check_name": F.lit("has_no_row_anomalies"), "score": F.round(F.col(score_col), 3), - "severity_percentile": F.round(F.col(severity_col), 1), + "severity_percentile": displayed_severity_expr(F.col(severity_col), threshold), "is_anomaly": is_anomaly, "threshold": F.lit(threshold), "model": F.lit(model_name), diff --git a/tests/integration_anomaly/test_anomaly_ai_explanation.py b/tests/integration_anomaly/test_anomaly_ai_explanation.py index 7378f7fd5..68e555376 100644 --- a/tests/integration_anomaly/test_anomaly_ai_explanation.py +++ b/tests/integration_anomaly/test_anomaly_ai_explanation.py @@ -85,10 +85,14 @@ def test_ai_query_explanation_populated_for_anomalous_row( for feat in explanation["top_features"].split("+"): assert feat in explanation["top_drivers"] assert explanation["group_size"] == 1 - # Single-row group: group_avg_severity is the row's severity. The struct's - # severity_percentile is rounded to 1 decimal while group_avg_severity is full precision, - # so compare with a tolerance that absorbs that rounding rather than exact equality. - assert explanation["group_avg_severity"] == pytest.approx(anomaly_info["severity_percentile"], abs=0.1) + # Single-row group, so group_avg_severity is this row's own severity at full precision. The struct's + # severity_percentile is *floored* to the threshold's precision so that comparing it against the + # threshold agrees with is_anomaly, which means it can sit up to one whole display step below the true + # value -- a wider gap than the half-step the previous rounding allowed. The tolerance is one step plus + # a margin, rather than exactly one step, so a value floored by almost the full step does not fail on a + # boundary. Direction is asserted separately: the published value never reads high. + assert explanation["group_avg_severity"] == pytest.approx(anomaly_info["severity_percentile"], abs=0.15) + assert anomaly_info["severity_percentile"] <= explanation["group_avg_severity"] + 1e-9 def test_ai_query_explanation_null_for_non_anomalous_row( diff --git a/tests/unit/test_anomaly_severity_display.py b/tests/unit/test_anomaly_severity_display.py new file mode 100644 index 000000000..da05c622f --- /dev/null +++ b/tests/unit/test_anomaly_severity_display.py @@ -0,0 +1,116 @@ +"""The published severity must lead a reader to the same verdict as the flag (no Spark, no workspace). + +The flag is decided on the full-precision severity. The published number used to be *rounded* to one +decimal, so a row at 94.96 was shown as 95.0 and not flagged at a threshold of 95: nothing wrong with the +decision, but a reader comparing the two numbers disagreed with it. A black-box evaluation found that in 477 +of 912 scoring cells, every audited disagreement sitting exactly at displayed threshold equality. + +Flooring at the threshold's own precision closes it without touching a single flag. These tests pin the +equivalence itself rather than the formula, so a different implementation of the same guarantee passes. +""" + +import math + +import pytest + +from databricks.labs.dqx.anomaly.scoring_utils import displayed_severity_decimals + +# The documented thresholds, plus finer ones to show the guarantee does not stop at one decimal. +_THRESHOLDS = [90.0, 95.0, 99.0, 99.5, 99.9, 99.95, 99.995] + + +def _displayed(severity: float, threshold: float) -> float: + """The published value, in Python, mirroring *displayed_severity_expr*'s arithmetic. + + Duplicated deliberately: the Spark expression cannot be evaluated without a session, and the property + under test is arithmetic. The integration suite is what checks that Spark agrees with this. + """ + scale = float(10 ** displayed_severity_decimals(threshold)) + return math.floor(severity * scale) / scale + + +def _candidates(threshold: float) -> list[float]: + """Values around a threshold, including the float neighbours of each gridpoint. + + The dense sweep covers ordinary cases; the neighbours are where a scaling-then-flooring implementation + would break, because ``10 * x`` need not be exact. + """ + values = [threshold + step * 1e-5 for step in range(-500, 501)] + for base in (threshold - 0.1, threshold, threshold + 0.1): + for direction in (math.inf, -math.inf): + value = base + for _ in range(50): + value = math.nextafter(value, direction) + values.append(value) + return values + + +@pytest.mark.parametrize("threshold", _THRESHOLDS) +def test_a_reader_comparing_the_published_severity_reaches_the_flags_verdict(threshold: float): + """The contract, stated as the equivalence it is. + + Swept either side of the threshold and across the float neighbours of every gridpoint, because the + implementation multiplies before flooring and that is exactly where such an implementation would drift. + """ + disagreements = [ + value for value in _candidates(threshold) if (_displayed(value, threshold) >= threshold) != (value >= threshold) + ] + + assert not disagreements, f"{len(disagreements)} values disagree at {threshold}, e.g. {disagreements[:3]}" + + +@pytest.mark.parametrize("threshold", [90.0, 95.0, 99.0, 99.5, 99.9]) +def test_rounding_to_one_decimal_would_break_that_equivalence(threshold: float): + """The defect, pinned so the fix cannot be reverted quietly. + + Without this a later contributor could restore ``round(severity, 1)`` -- which is shorter, looks tidier, + and passes every other test in the suite -- and reintroduce the disagreement. + + Parametrised over the one-decimal thresholds only, which is where the defect was observed and where + rounding is guaranteed to break: a value in ``[t - 0.05, t)`` rounds up onto the threshold while the + flag, decided on the true value, says no. At a finer threshold the relationship between the two + precisions changes and rounding need not disagree on any sampled value, so claiming it there would be + asserting more than the arithmetic supports. + """ + disagreements = [ + value for value in _candidates(threshold) if (round(value, 1) >= threshold) != (value >= threshold) + ] + + assert disagreements, f"expected rounding to disagree somewhere near {threshold}" + + +def test_the_published_severity_never_reads_higher_than_the_decided_one(): + """The one-sided half of the guarantee, and the reason flooring was chosen over rounding. + + Overstating is the harmful direction: it invites a reader to believe a row crossed the threshold when + the flag says it did not. Understating cannot mislead that way, and the equivalence above bounds how + far it may understate. + """ + for threshold in _THRESHOLDS: + for value in _candidates(threshold): + assert _displayed(value, threshold) <= value + + +@pytest.mark.parametrize( + "threshold, expected", + [(95.0, 1), (99.0, 1), (99.5, 1), (99.9, 1), (99.95, 2), (99.995, 3), (95, 1)], +) +def test_the_display_precision_follows_the_threshold(threshold: float, expected: int): + """Derived rather than fixed, because a fixed precision is right only for thresholds that match it. + + Measured: flooring at one decimal disagrees on none of the one-decimal thresholds but on 560 sampled + values at a threshold of 99.95. Deriving the precision disagrees on none at any of them. + """ + assert displayed_severity_decimals(threshold) == expected + + +def test_a_whole_number_threshold_still_shows_a_decimal(): + """Readability: a severity of 95 should not print as an integer just because the threshold is one.""" + assert displayed_severity_decimals(95.0) == 1 + assert _displayed(95.04, 95.0) == 95.0 + + +def test_a_saturated_severity_is_published_unchanged(): + """The tail expression reaches 100 exactly; flooring must not shave it to 99.9.""" + for threshold in _THRESHOLDS: + assert _displayed(100.0, threshold) == 100.0 From 21f68a57b220b82954e9019e30426a8be01ee358 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 9 Sep 2026 11:13:06 +0100 Subject: [PATCH 106/107] Floor the displayed severity in decimal space, not by scaling The previous commit's arithmetic was wrong in the direction that matters. `floor(severity * 10**n) / 10**n` reads as equivalent to flooring at n decimals and is not: the multiply is a double operation that can round up across an integer, so a row one ulp below the threshold displays exactly at it -- the display says the row is at the threshold and the flag says it is not, which is the defect this work set out to remove, in its harmful direction. Enumerated over every legal threshold at each precision, the scaling form disagrees at 98 of 1000 one-decimal thresholds, 140 of 1001 two-decimal and 30 of 1001 three-decimal. The decimal form disagrees at none. My original test swept values densely but only seven thresholds, all of them clean, which is why it passed on defective arithmetic -- so the tests now sweep thresholds as well as values. `floor(expr, scale)` goes through `call_function` because the two-argument Python wrapper is PySpark 4 only, while the supported databricks-connect floor of 15.4 ships a 3.5 client. The SQL function has taken a scale since Spark 3.3. Also documents the precision caveat where a user meets it: severity carries the precision the *scoring* threshold needed, so counting alerts at a finer cutoff than that undercounts rather than misleads quietly. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 9 +++ .../labs/dqx/anomaly/scoring_utils.py | 22 ++++-- tests/unit/test_anomaly_severity_display.py | 72 +++++++++++++++++-- 3 files changed, 94 insertions(+), 9 deletions(-) diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index a6cdfc974..93b91bfa0 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -360,6 +360,15 @@ for cutoff in (90, 95, 98, 99, 99.5): print(f"threshold {cutoff}: {alerts} alerts ({alerts / total:.2%} of rows)") ``` +One precision caveat, because it decides which cutoffs this loop can answer. The published +`severity_percentile` carries exactly the precision the *scoring* threshold needed — one decimal for a +threshold of 95, two for 99.95 — which is what makes `severity >= threshold` agree with `is_anomaly`. So +counts are exact at any cutoff at or coarser than that precision, and **undercount** at a finer one: scored +at `threshold=95`, every severity in `[99.95, 100)` was published as `99.9`, so a cutoff of `99.95` misses +those rows. Stay at or below the scoring threshold's precision, or rescore at the finer threshold. The +`threshold` field sits next to `severity_percentile` in the same struct, so a scored table always says which +precision it carries. + Pick the cutoff whose alert count matches how many rows someone can actually investigate. Both anomaly demos under `demos/` print this table, and one of them chooses its threshold from it rather than inheriting the default. diff --git a/src/databricks/labs/dqx/anomaly/scoring_utils.py b/src/databricks/labs/dqx/anomaly/scoring_utils.py index 9928541d0..e350814b9 100644 --- a/src/databricks/labs/dqx/anomaly/scoring_utils.py +++ b/src/databricks/labs/dqx/anomaly/scoring_utils.py @@ -70,8 +70,23 @@ def displayed_severity_expr(severity: Column, threshold: float) -> Column: The precision is derived rather than fixed at one decimal, because a fixed decimal is correct only for thresholds that happen to be that precise. Measured over dense sweeps around each threshold plus the float neighbours of every gridpoint: rounding disagrees on 560 sampled values at each of 90, 95, 99, - 99.5 and 99.9; flooring at one decimal disagrees on none of those but on 560 at a threshold of 99.95; - flooring at the threshold's precision disagrees on none at any of them, up to 99.995. + 99.5 and 99.9, while flooring at one decimal disagrees on none of those but on 560 at a threshold of + 99.95. + + **The floor is taken in decimal space, not by scaling and flooring**, and that distinction is the whole + correctness argument rather than a stylistic one. ``floor(severity * 10**n) / 10**n`` looks equivalent + and is not: the multiply is a double operation that can round *up* across an integer, so the floor lands + exactly on the threshold for a row whose true severity is one ulp below it -- reintroducing this defect + in its harmful direction, where the display reads at the threshold and the flag says no. Enumerated over + every legal threshold at each precision, the scaling form disagrees at 98 of 1000 one-decimal thresholds + (0.9, 1.8, 3.6 and so on), 140 of 1001 two-decimal, and 30 of 1001 three-decimal; the decimal form + disagrees at none. The failing thresholds are all outside the commonly used range, which is exactly why + a sweep of 90, 95, 99, 99.5 and 99.9 misses them -- so the tests sweep thresholds too, not only values. + + ``floor(expr, scale)`` is called through :func:`pyspark.sql.functions.call_function` rather than + ``F.floor(col, scale)`` because the two-argument Python wrapper only exists in PySpark 4, while this + package supports a ``databricks-connect`` floor of 15.4 whose client is PySpark 3.5. The SQL function + itself has taken a scale since Spark 3.3, so every supported runtime provides it. Rounding the *decision* to match the display would also make the two agree, and is the wrong trade: it would begin flagging every row in ``[threshold - 0.05, threshold)``, changing the detector to fix a @@ -85,8 +100,7 @@ def displayed_severity_expr(severity: Column, threshold: float) -> Column: The severity to publish: never above the true value, and never below it by enough to change how it compares against *threshold*. """ - scale = float(10 ** displayed_severity_decimals(threshold)) - return F.floor(severity * F.lit(scale)) / F.lit(scale) + return F.call_function("floor", severity, F.lit(displayed_severity_decimals(threshold))) def create_null_scored_dataframe( diff --git a/tests/unit/test_anomaly_severity_display.py b/tests/unit/test_anomaly_severity_display.py index da05c622f..dfbbc7be7 100644 --- a/tests/unit/test_anomaly_severity_display.py +++ b/tests/unit/test_anomaly_severity_display.py @@ -10,21 +10,36 @@ """ import math +from decimal import ROUND_FLOOR, Decimal import pytest from databricks.labs.dqx.anomaly.scoring_utils import displayed_severity_decimals -# The documented thresholds, plus finer ones to show the guarantee does not stop at one decimal. -_THRESHOLDS = [90.0, 95.0, 99.0, 99.5, 99.9, 99.95, 99.995] +# The commonly used thresholds, plus finer ones, plus three that a first version of this fix got WRONG. +# 0.9, 90.01 and 99.01 are here because scaling-then-flooring disagrees with the flag on the double one ulp +# below each of them, and none of the ordinary thresholds show it. An earlier version of this file swept +# values densely at seven clean thresholds and passed against a defective implementation. +_THRESHOLDS = [90.0, 95.0, 99.0, 99.5, 99.9, 99.95, 99.995, 0.9, 90.01, 99.01, 99.058] def _displayed(severity: float, threshold: float) -> float: - """The published value, in Python, mirroring *displayed_severity_expr*'s arithmetic. + """The published value, in Python, mirroring *displayed_severity_expr*. - Duplicated deliberately: the Spark expression cannot be evaluated without a session, and the property - under test is arithmetic. The integration suite is what checks that Spark agrees with this. + Flooring in decimal space on the shortest representation, which is what Spark's ``floor(expr, scale)`` + does for a double: it routes through ``BigDecimal.valueOf``, i.e. ``Double.toString``. Deliberately not + ``floor(severity * scale) / scale`` -- that is the form this fix replaced, and + :func:`test_scaling_then_flooring_would_reintroduce_the_defect` pins why. + + A mirror, so it proves a property of the arithmetic and not of the product. The integration suite is + what checks Spark agrees with it. """ + quantum = Decimal(1).scaleb(-displayed_severity_decimals(threshold)) + return float(Decimal(repr(severity)).quantize(quantum, rounding=ROUND_FLOOR)) + + +def _scaled_then_floored(severity: float, threshold: float) -> float: + """The rejected form, kept so its failure is executable rather than described.""" scale = float(10 ** displayed_severity_decimals(threshold)) return math.floor(severity * scale) / scale @@ -114,3 +129,50 @@ def test_a_saturated_severity_is_published_unchanged(): """The tail expression reaches 100 exactly; flooring must not shave it to 99.9.""" for threshold in _THRESHOLDS: assert _displayed(100.0, threshold) == 100.0 + + +def _threshold_grid(start: float, stop: float, step: float, places: int) -> list[float]: + """Every legal threshold at one precision, so the sweep covers thresholds and not only values.""" + count = int(round((stop - start) / step)) + return [round(start + step * k, places) for k in range(count + 1)] + + +@pytest.mark.parametrize( + "grid", + [ + pytest.param(_threshold_grid(0.1, 100.0, 0.1, 1), id="every_one_decimal_threshold"), + pytest.param(_threshold_grid(90.0, 100.0, 0.01, 2), id="every_two_decimal_threshold_from_90"), + pytest.param(_threshold_grid(99.0, 100.0, 0.001, 3), id="every_three_decimal_threshold_from_99"), + ], +) +def test_the_equivalence_holds_at_every_legal_threshold_not_just_the_common_ones(grid: list[float]): + """Swept over thresholds, which is the axis the first version of this fix left untested. + + Its predecessor swept values densely at seven thresholds that all happen to be clean, so it passed + against an implementation that disagreed with the flag at 268 other legal thresholds. The failure was + always one ulp below the threshold and always in the harmful direction: the display read at the + threshold while the row was not flagged. + """ + failures = [] + for threshold in grid: + probe = threshold + for _ in range(4): + probe = math.nextafter(probe, -math.inf) + if (_displayed(probe, threshold) >= threshold) != (probe >= threshold): + failures.append((threshold, probe)) + break + + assert not failures, f"{len(failures)} thresholds disagree, e.g. {failures[:3]}" + + +@pytest.mark.parametrize("threshold, probe", [(0.9, 0.8999999999999999), (90.01, 90.00999999999999)]) +def test_scaling_then_flooring_would_reintroduce_the_defect(threshold: float, probe: float): + """The rejected implementation, pinned at two of the thresholds where it fails. + + ``severity * 10**n`` is a double multiply and can round up across an integer, after which the floor + lands on the threshold for a row below it. Without this test the shorter form looks equivalent, reads + more simply, and passes every threshold anyone would think to try. + """ + assert probe < threshold + assert _scaled_then_floored(probe, threshold) >= threshold, "expected the rejected form to overstate" + assert _displayed(probe, threshold) < threshold, "the shipped form must not" From 7f6bf4ddb1d9fb9221f70c84fc72a01faaec0423 Mon Sep 17 00:00:00 2001 From: Varun Bhandary Date: Wed, 9 Sep 2026 11:13:28 +0100 Subject: [PATCH 107/107] Say when redaction withheld evidence, instead of implying the rest is everything Redaction did its literal job: a black-box audit of ten groups found no redacted column name in any prompt or output. What it also did was drop the redacted column's share before the contributions map was built, so the survivors were renormalised to sum to 100 with nothing recording that anything had gone. A visible column holding about 7% of one group's evidence was published at 100%, and the narrative said it accounted for "all of what the model measured". All ten groups failed evidence-scope clarity while passing redaction, which is why the two are asserted separately here: this is a faithfulness defect inside a privacy feature, not a leak. The aggregation now carries the withheld and total evidence per group and derives a coarse state -- complete, partial, limited, cut at half. Coarse deliberately: a share would disclose by proportion what a redacted name discloses by identity, so an observer learns which side of one boundary a group sits on and nothing finer. The state is published as `ai_explanation.evidence_scope` so a consumer can filter or escalate without parsing prose, told to the model in a field it is instructed to read *before* the shares it qualifies, and appended to the narrative as a deterministic clause -- because an instruction alone is not enforcement. A model that claims completeness anyway cannot be repaired by appending a caveat: the result contradicts itself and a reader believes the first half. Those narratives are replaced with the group's own facts instead. The replacement states no limit of its own, since the clause appended straight after it says that once, with the band's precision. The limited-disclosure exemplar had the same defect the guard exists to catch -- it declared most evidence withheld and then called the rows dominated by the one metric it could see, the exact redirect the limited band forbids. A worked example outweighs an instruction, so it is rewritten, and a test now asserts no exemplar response would trip our own guard. `group_avg_severity` is floored at the projection to match the severity published beside it, and not upstream, where the unfloored mean also ranks groups against max_groups. The disclosure strings and the rendered prompt become public rather than reaching past an underscore from the tests: the three states are already a documented field value, and the header is what the endpoint receives. Co-authored-by: Isaac --- .../guide/row_anomaly_detection/index.mdx | 6 +- .../labs/dqx/anomaly/anomaly_info_schema.py | 8 + .../labs/dqx/anomaly/anomaly_llm_explainer.py | 228 ++++++++++++++-- .../labs/dqx/anomaly/check_funcs.py | 18 +- tests/resources/ai_query_prompt_header.txt | 7 +- .../unit/test_anomaly_evidence_disclosure.py | 258 ++++++++++++++++++ 6 files changed, 497 insertions(+), 28 deletions(-) create mode 100644 tests/unit/test_anomaly_evidence_disclosure.py diff --git a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx index 93b91bfa0..63b02d3b9 100644 --- a/docs/dqx/docs/guide/row_anomaly_detection/index.mdx +++ b/docs/dqx/docs/guide/row_anomaly_detection/index.mdx @@ -692,7 +692,9 @@ The nested `ai_explanation` struct (populated when AI explanations are on, which | `action` | string | What an analyst should investigate. | | `top_features` | string | Deterministic top-2 contributing features (e.g. `amount+quantity`), the group's pattern key. | | `group_size` | long | Number of anomalous rows in this anomaly group. | -| `group_avg_severity` | double | Mean `severity_percentile` across the group. | +| `top_drivers` | string | The same drivers rendered for display, as human labels with their shares (e.g. `amount vs its group baseline (74%), quantity (12%)`). | +| `group_avg_severity` | double | Mean `severity_percentile` across the group, floored the same way as `severity_percentile` above. | +| `evidence_scope` | string | How much of the contributing evidence the explanation was allowed to show: `complete` when nothing was withheld, `partial` when redacted columns held a minority of the group's evidence, `limited` when they held the majority. Deliberately coarse — a share would disclose by proportion what a redacted name discloses by identity. Filter or escalate on this rather than parsing the narrative. Only `redact_columns` produces anything other than `complete`. | **Access in PySpark:** use `F.element_at(F.col("_dq_info"), 1)` for the first element (1-based), then `.getField("anomaly").getField("severity_percentile")` etc. Alternatively `F.col("_dq_info").getItem(0)` for 0-based index (see [Troubleshooting](/docs/guide/row_anomaly_detection/troubleshooting) for Spark Connect friendly patterns). @@ -728,7 +730,7 @@ checks = [ * Explanations are **on by default** and call a Model Serving endpoint, so they add per-run LLM cost. `max_groups` (default 500) caps how many anomaly groups the model is called for per run. Set `enable_ai_explanation=False` to turn explanations off. * No serving endpoint? If the configured endpoint isn't reachable (e.g. Foundation Model APIs aren't enabled in the workspace), explanations are skipped with a warning and scoring still completes, so nothing breaks. -* `redact_columns` keeps the listed feature names out of the prompt (their contribution keys are shown as ``). +* `redact_columns` keeps the listed feature names out of the prompt. Their contribution entries are **dropped** from what the model is shown, not relabelled, and they are excluded from the group's pattern key too. The remaining shares are renormalised across what is left, so the explanation says how much of the *disclosed* evidence each column carried and states plainly when evidence was withheld — a column shown at 100% can be a small part of what the model actually measured. Note the scope: this governs what is sent to the serving endpoint. Your own `_dq_info[0].anomaly.contributions` map is unchanged and still lists every column, so anyone who can read the scored table can see the full picture; `redact_columns` is not an access control. ## Practical examples (non-technical) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py index df8347569..57f60ef1b 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_info_schema.py @@ -25,6 +25,14 @@ StructField("action", StringType(), True), StructField("group_size", LongType(), True), StructField("group_avg_severity", DoubleType(), True), + # Appended last, and it must stay last: add_info_column casts this struct positionally, so schema + # order and construction order have to match. _dq_info therefore grows wider, and appending to a + # Delta table that already holds the narrower shape needs mergeSchema. + # + # How much of the contributing evidence the explanation was allowed to show, as a coarse state + # rather than a share -- a share would disclose by proportion what a redacted name discloses by + # identity. Machine-readable so a consumer can filter or escalate on it instead of parsing prose. + StructField("evidence_scope", StringType(), True), ] ) diff --git a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py index 8cc3d903f..7b4ff510e 100644 --- a/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py +++ b/src/databricks/labs/dqx/anomaly/anomaly_llm_explainer.py @@ -24,6 +24,7 @@ from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema from databricks.labs.dqx.anomaly.feature_naming import engineered_from, human_label +from databricks.labs.dqx.anomaly.scoring_utils import displayed_severity_expr from databricks.labs.dqx.anomaly.transformers import BASELINE_RELATIVE_SUFFIX, SparkFeatureMetadata from databricks.labs.dqx.config import LLMModelConfig from databricks.labs.dqx.errors import InvalidParameterError @@ -104,12 +105,19 @@ def attribution_semantics(algorithm: str | None) -> str: "you may describe the pattern. Follow this field rather than assuming a reading: one basis " "supports saying a feature's own value was unusual, the other does not.", ), + ( + "evidence_disclosure", + "Whether you are being shown all of the evidence. Read this BEFORE feature_contributions, because " + "it decides what the shares below mean and what you may say about them.", + ), ( "feature_contributions", "Mean share across the group of the evidence the model acted on, per column the caller named, " "e.g. 'amount (82%), quantity (11%), discount (5%)'. One entry per column however many ways that " "column was compared, so a share says the column was involved and not which comparison objected. " - "These are aggregated relative importances — not raw data values, and not percentages of the score.", + "These are aggregated relative importances — not raw data values, and not percentages of the score. " + "The shares are normalised across the entries listed here only, so when evidence_disclosure says " + "any was withheld they are shares of what is disclosed and not of the whole decision.", ), ("group_size", "Number of rows in this group, e.g. '312 rows'."), ("severity_range", "Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'."), @@ -185,6 +193,8 @@ def attribution_semantics(algorithm: str | None) -> str: _PROMPT_EXAMPLES = ( "Example (relationship basis, no drift):\n" "attribution_basis: each metric's position once the others are accounted for\n" + "evidence_disclosure: every contributing metric is shown to you, so the shares below cover all of " + "the evidence the model used.\n" "feature_contributions: amount (61%), quantity (22%)\n" "group_size: 312 rows\n" "severity_range: mean 97.4, min 95.1, max 99.8\n" @@ -199,6 +209,8 @@ def attribution_semantics(algorithm: str | None) -> str: '"action":"Reconcile amount against source orders for the affected regions."}\n\n' "Example (value basis, judged against time, with drift):\n" "attribution_basis: each feature's own value compared against the rows it was scored against\n" + "evidence_disclosure: MOST of the contributing evidence cannot be disclosed and is absent from the " + "shares below, so what remains is a minority of what the model used.\n" "feature_contributions: latency_ms (74%), retries (12%)\n" "group_size: 88 rows\n" "severity_range: mean 98.9, min 97.0, max 99.9\n" @@ -207,11 +219,19 @@ def attribution_semantics(algorithm: str | None) -> str: "temporal_baseline: event_ts\n" "threshold: 95.0\n" "drift_summary: drift detected: latency_ms=4.12\n" - 'Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from its ' - 'training baseline; retries contribute modestly (12%). These rows are judged against expected levels ' - 'over time as well as overall.","business_impact":"If latency_ms is genuinely off on these rows, ' - 'downstream consumers with SLAs would be the first to notice.","action":"Compare latency_ms against ' - 'its expected level for that period as well as against its overall range."}' + # This exemplar declares that most of the contributing evidence is withheld, so its response has to + # demonstrate the limited-band rule rather than contradict it. The earlier wording called the rows + # "dominated by latency_ms" and sent the reader straight there -- the precise redirect the limited + # instruction forbids, and a worked example outweighs an instruction. It still names both shares; what + # it drops is the claim that they settle the matter. It also does not restate the disclosure limit: the + # deterministic clause is appended to every withheld narrative, and an exemplar that stated it too would + # model saying it twice. + 'Response: {"narrative":"Of what can be shown across these 88 rows, latency_ms carries most (74%) ' + 'and retries a small part (12%); latency_ms has also moved from its training baseline. These rows are ' + 'judged against expected levels over time as well as overall.","business_impact":"If these rows are ' + 'wrong, downstream consumers with SLAs would be the first to notice.","action":"Start with the rows ' + 'themselves rather than one field: compare latency_ms against its expected level for that period, and ' + 'take account of the evidence this explanation could not include."}' ) if TYPE_CHECKING: @@ -403,6 +423,111 @@ def _baseline_grouping_str(metadata: SparkFeatureMetadata | None) -> str: return ", ".join(metadata.baseline_by) +# Coarse disclosure states. Deliberately three, and deliberately unlabelled by any number: a reader who +# knows the boundaries must not be able to read a proportion back out of the state. "Most" versus "some" +# is the distinction that changes what a narrative may claim, and finer bands would buy nothing while +# leaking more about a column the caller asked to keep out of the prompt. +DISCLOSURE_COMPLETE = "complete" +DISCLOSURE_PARTIAL = "partial" +DISCLOSURE_LIMITED = "limited" + +# Above this share of the evidence being withheld, what remains cannot carry the explanation on its own. +_LIMITED_DISCLOSURE_SHARE = 0.5 + +DISCLOSURE_PROMPT_TEXT: dict[str, str] = { + DISCLOSURE_COMPLETE: "every contributing metric is shown to you, so the shares below cover all of the " + "evidence the model used.", + DISCLOSURE_PARTIAL: "SOME contributing evidence cannot be disclosed and is absent from the shares below, " + "which therefore cover only what is shown. Do not present a share as a portion of the whole decision, " + "and do not describe the metrics you can see as the reason the row was flagged.", + DISCLOSURE_LIMITED: "MOST of the contributing evidence cannot be disclosed and is absent from the shares " + "below, so what remains is a minority of what the model used. Name what you can see, say plainly that " + "the explanation is limited by evidence that cannot be shown, and do not direct the reader to " + "investigate a metric on the strength of a share this small. Never guess what the withheld evidence was.", +} + +# Appended verbatim to the narrative, after sanitisation, whenever evidence was withheld. Deterministic +# because the advisory that raised this is explicit that a prompt instruction is not a control: the +# qualification has to survive a model that ignores it. +DISCLOSURE_NARRATIVE_CLAUSE: dict[str, str] = { + DISCLOSURE_PARTIAL: " Some contributing evidence could not be disclosed, so these shares cover only " + "what is shown here.", + DISCLOSURE_LIMITED: " Most of the contributing evidence could not be disclosed, so this explanation " + "covers a minority of what the model used.", +} + +# Phrasings that claim the listed metrics are the whole of what the model measured. Appending a +# "some evidence was withheld" clause to one of these produces a narrative that contradicts itself, so the +# claim is replaced rather than qualified. Deliberately narrow: these are the shapes actually observed, and a +# broad pattern would swallow legitimate prose. It cannot catch a narrative that merely *implies* +# completeness -- what limits that is the model no longer being shown a bare "100%". +TOTALISING_CLAIM_PATTERN = ( + r"(?i)(all of what the model measured" + r"|accounts? for all|accounted for all|account for all" # every conjugation seen; "accounts"/"account" + r"|the whole (decision|picture)" + r"|100% of" + r"|fully explains?)" +) + +# Used when a totalising claim has to be replaced. Built from the group's own facts, so it is useful rather +# than merely safe -- the advisory is explicit that a fallback must not be a bare refusal, and equally must +# not be a raw unreviewed model response. +# Replaces a narrative that claimed completeness. Deliberately flat: it states the group size and the +# disclosed shares and nothing else -- no direction, no cause, no adjective the inputs cannot support. It +# asserts no limit of its own either, because the clause appended straight after it says that once, with +# the band's precision. +DISCLOSURE_FALLBACK_TEMPLATE = "Across %s rows, the shares that can be shown are %s." + + +def _disclosure_state_expr(withheld_evidence: Column, total_evidence: Column) -> Column: + """Bucket how much of a group's evidence was withheld into a coarse state. + + Redaction keeps a column out of the prompt, which it does correctly -- no name has ever leaked. What it + also did was remove that column's share from the map the narrative is built from, after which the + remaining shares were renormalised to sum to 100 with nothing saying so. A column holding 7% of a + group's evidence was consequently described as accounting for "all of what the model measured (100%)". + + A state rather than the share itself, because the share would disclose by proportion roughly what the + name discloses by identity. Three states are enough to change what a narrative may claim and coarse + enough that the boundaries cannot be inverted into a measurement. + + Args: + withheld_evidence: Summed absolute contribution of the redacted keys in the group. + total_evidence: Summed absolute contribution of every key in the group. + + Returns: + One of *DISCLOSURE_COMPLETE*, *DISCLOSURE_PARTIAL* or *DISCLOSURE_LIMITED*. A group with no + measurable evidence at all counts as complete: there is nothing withheld to qualify. + """ + return ( + F.when((total_evidence.isNull()) | (total_evidence <= F.lit(0.0)), F.lit(DISCLOSURE_COMPLETE)) + .when(withheld_evidence <= F.lit(0.0), F.lit(DISCLOSURE_COMPLETE)) + .when(withheld_evidence / total_evidence > F.lit(_LIMITED_DISCLOSURE_SHARE), F.lit(DISCLOSURE_LIMITED)) + .otherwise(F.lit(DISCLOSURE_PARTIAL)) + ) + + +def _disclosure_prompt_expr(state: Column) -> Column: + """The sentence the model reads for a group's disclosure state.""" + expr = F.lit(DISCLOSURE_PROMPT_TEXT[DISCLOSURE_COMPLETE]) + for name in (DISCLOSURE_PARTIAL, DISCLOSURE_LIMITED): + expr = F.when(state == F.lit(name), F.lit(DISCLOSURE_PROMPT_TEXT[name])).otherwise(expr) + return expr + + +def _disclosure_clause_expr(state: Column) -> Column: + """The qualification appended to a narrative, or an empty string when nothing was withheld. + + Appended after sanitisation so it cannot be dropped, reworded or truncated away by the model. The + prompt asks for the same qualification in the model's own words; this is what makes it a guarantee + rather than a request, which is what the advisory that raised this defect requires. + """ + expr = F.lit("") + for name, clause in DISCLOSURE_NARRATIVE_CLAUSE.items(): + expr = F.when(state == F.lit(name), F.lit(clause)).otherwise(expr) + return expr + + def _temporal_baseline_str(metadata: SparkFeatureMetadata | None) -> str: """The time column each metric is judged along, e.g. 'event_ts', or 'none'. @@ -468,7 +593,7 @@ def _render_ai_query_prompt_header() -> str: return "\n".join(lines) -_AI_QUERY_PROMPT_HEADER = _render_ai_query_prompt_header() +AI_QUERY_PROMPT_HEADER = _render_ai_query_prompt_header() # Databricks Model Serving endpoint name rules: 1–63 chars, must start with a letter, then any # of [letter, digit, hyphen, underscore]. This is the *platform's own* naming constraint — any @@ -513,9 +638,16 @@ def _resolve_ai_query_endpoint(model_name: str) -> str: def _format_contributions_sql(top_n: int, labels: dict[str, str] | None = None) -> Column: """Spark expression producing 'feat_a (82%), feat_b (11%)' from a ``mean_contributions`` map. - Mirrors *format_contributions_map* but stays inside Spark so per-group prompts can be - assembled without a driver-side loop. Null/empty maps yield 'unknown'; entries are sorted by - value descending and percentages are normalised against their sum. + Stays inside Spark so per-group prompts can be assembled without a driver-side loop. Null/empty maps + yield 'unknown'; entries are sorted by value descending and percentages are normalised against their sum. + + It is **not** a mirror of *format_contributions_map*, and the difference is the point: that function + prints the stored 0-100 values as they are, so a map missing entries renders shares that sum to less + than 100 and understates. This one renormalises across what it is given, so a map missing entries + renders shares that still total 100 and therefore overstates. The two agree only on a complete map. + Renormalising is deliberate here -- publishing the true shares of a redacted map would disclose the + withheld proportion exactly -- and it is why the caller appends a scope qualifier when anything was + withheld, rather than leaving the numbers to speak for themselves. Null *and zero* entries are dropped, matching *format_contributions_map*: a feature that earned no share contributed nothing, and listing it as 'quantity (0%)' hands the model a driver to explain that @@ -561,7 +693,7 @@ def _build_ai_query_prompt_column( """Assemble the per-row prompt string sent to ``ai_query``. Per-group fields come from columns added by *_aggregate_groups_spark*; per-run fields are - constants for the whole call. The shared header (*_AI_QUERY_PROMPT_HEADER*) holds the + constants for the whole call. The shared header (*AI_QUERY_PROMPT_HEADER*) holds the instructions and field semantics. """ baseline_grouping = _baseline_grouping_str(ctx.feature_metadata) @@ -580,10 +712,13 @@ def _build_ai_query_prompt_column( ) group_size_expr = F.concat(F.col("group_size").cast(StringType()), F.lit(" rows")) return F.concat( - F.lit(_AI_QUERY_PROMPT_HEADER), + F.lit(AI_QUERY_PROMPT_HEADER), F.lit("attribution_basis: "), F.lit(attribution_semantics(ctx.algorithm)), F.lit("\n"), + F.lit("evidence_disclosure: "), + _disclosure_prompt_expr(F.col("__disclosure")), + F.lit("\n"), F.lit("feature_contributions: "), F.col("feature_contributions"), F.lit("\n"), @@ -633,12 +768,25 @@ def _aggregate_groups_spark( F.max(severity_col).alias("severity_max"), F.avg(score_std_col).alias("mean_std"), ) + # Withheld keys are aggregated alongside the disclosed ones rather than filtered away first, because + # how much was withheld is itself needed downstream. Filtering here is what made the explanation + # misleading: the shares were renormalised over whatever survived, with nothing recording that anything + # had gone. Measured on one group, a visible column holding 7% of the evidence was rendered as 100%. exploded = anomalous.select(F.col(pattern_col), F.explode(F.col(contributions_col)).alias("__k", "__v")) - if redact_set: - exploded = exploded.filter(~F.col("__k").isin(list(redact_set))) per_key_mean = exploded.groupBy(pattern_col, "__k").agg(F.avg("__v").alias("__mean")) - per_pattern_contrib = per_key_mean.groupBy(pattern_col).agg( - F.map_from_entries(F.collect_list(F.struct(F.col("__k"), F.col("__mean")))).alias("mean_contributions") + withheld = F.col("__k").isin(list(redact_set)) if redact_set else F.lit(False) + per_pattern_contrib = ( + per_key_mean.groupBy(pattern_col) + .agg( + # collect_list drops nulls, so the `when` keeps only disclosed keys out of the emitted map. The + # redacted names never reach it, which is the property redaction exists for and is unchanged. + F.map_from_entries(F.collect_list(F.when(~withheld, F.struct(F.col("__k"), F.col("__mean"))))).alias( + "mean_contributions" + ), + F.sum(F.when(withheld, F.abs(F.col("__mean"))).otherwise(F.lit(0.0))).alias("__withheld_evidence"), + F.sum(F.abs(F.col("__mean"))).alias("__total_evidence"), + ) + .withColumn("__disclosure", _disclosure_state_expr(F.col("__withheld_evidence"), F.col("__total_evidence"))) ) # Single-action total/kept accounting: window aggregates over ``primary`` carry run-level @@ -707,7 +855,16 @@ def _call_llm_for_groups_ai_query( # prompt and the struct's top_drivers, so a reader sees the same human phrasing the LLM did. enriched = kept_groups_sdf.withColumn( "feature_contributions", - _format_contributions_sql(_TOP_N, _human_labels(ctx.feature_metadata)), + # Labelled at the point of rendering, not left to the prompt field alone. This string is what the + # model reads *and* what the struct publishes as top_drivers, so the qualification travels with the + # numbers instead of depending on a separate field being honoured. + F.concat( + _format_contributions_sql(_TOP_N, _human_labels(ctx.feature_metadata)), + F.when( + F.col("__disclosure") != F.lit(DISCLOSURE_COMPLETE), + F.lit(" — shares of disclosed evidence only"), + ).otherwise(F.lit("")), + ), ).withColumn( "__prompt", _build_ai_query_prompt_column(ctx, is_ensemble, drift_summary), @@ -760,16 +917,46 @@ def _sanitize(col_name: str) -> Column: f"then regexp_replace({capped}, '\\\\s\\\\S*$', '') else {cleaned} end" ) + # The disclosure clause is appended *after* sanitisation, so the length cap applies to the model's own + # words and the qualification cannot be truncated off the end of a long narrative. A narrative the model + # failed to produce stays null: qualifying nothing would publish a clause with no explanation attached. + disclosure_clause = _disclosure_clause_expr(F.col("__disclosure")) + withheld = F.col("__disclosure") != F.lit(DISCLOSURE_COMPLETE) + # A model that asserts completeness cannot be corrected by appending a caveat -- the result contradicts + # itself, and a reader believes the first half. Replaced with the group's own facts instead. The clause is + # then appended either way, so the qualification is present whichever branch produced the text. + model_narrative = _sanitize("narrative") + safe_narrative = F.when( + withheld & model_narrative.rlike(TOTALISING_CLAIM_PATTERN), + F.format_string( + DISCLOSURE_FALLBACK_TEMPLATE, + F.col("group_size").cast(StringType()), + # The source column, not the top_drivers alias: that alias is created in this same select. + F.col("feature_contributions"), + ), + ).otherwise(model_narrative) return parsed.select( F.col(pattern_col), - _sanitize("narrative").alias("narrative"), + F.when(model_narrative.isNotNull(), F.concat(safe_narrative, disclosure_clause)) + .otherwise(F.lit(None).cast(StringType())) + .alias("narrative"), _sanitize("business_impact").alias("business_impact"), _sanitize("action").alias("action"), # Human-labelled drivers carried through for the struct's top_drivers. Built by us from the # contributions map, not the LLM, so it needs no sanitisation. F.col("feature_contributions").alias("top_drivers"), F.col("group_size").cast(LongType()).alias("group_size"), - F.col("group_avg_severity").cast(DoubleType()).alias("group_avg_severity"), + # Floored here, at the boundary where it becomes visible, and deliberately not in + # _aggregate_groups_spark: the unfloored mean also drives __rank_score there, so flooring it + # upstream would change which groups survive max_groups -- a behaviour change for a presentation + # fix, which is the same trap avoided by never flooring the value the flag reads. Published beside + # the floored severity_percentile in one struct, so the two must be on the same grid. + displayed_severity_expr(F.col("group_avg_severity"), ctx.threshold) + .cast(DoubleType()) + .alias("group_avg_severity"), + # Carried through so the struct can publish it as evidence_scope: a consumer should be able to + # filter or escalate on limited-evidence explanations without parsing the narrative for a phrase. + F.col("__disclosure"), ) @@ -796,6 +983,8 @@ def _attach_explanation_struct( F.col("action").alias("action"), F.col("group_size").alias("group_size"), F.col("group_avg_severity").alias("group_avg_severity"), + # Last, matching ai_explanation_struct_schema: add_info_column casts positionally. + F.col("__disclosure").alias("evidence_scope"), ), ).otherwise(_build_empty_explanation_column()), ).drop( @@ -806,6 +995,7 @@ def _attach_explanation_struct( "action", "group_size", "group_avg_severity", + "__disclosure", ) diff --git a/src/databricks/labs/dqx/anomaly/check_funcs.py b/src/databricks/labs/dqx/anomaly/check_funcs.py index a2d55edb8..19800368b 100644 --- a/src/databricks/labs/dqx/anomaly/check_funcs.py +++ b/src/databricks/labs/dqx/anomaly/check_funcs.py @@ -216,10 +216,13 @@ def has_no_row_anomalies( LLMModelConfig instance is accepted. The simplest dict form sets only *model_name* to a Databricks Model Serving endpoint. See the AI Explanations section of the Row Anomaly Detection reference docs for a full example. - redact_columns: Column names to exclude from the LLM prompt. Filters the contribution - map keys, the top-2 pattern key, and — when the scored model is segmented — any - matching segment key (emitted as ``key=`` so sensitive segmentation values - never reach the prompt). + redact_columns: Column names to exclude from the LLM prompt. Their contribution entries are + dropped from what the model is shown, and they are excluded from the top-2 pattern key. + The remaining shares are renormalised across what is left, so the explanation reports + shares of the *disclosed* evidence and says when evidence was withheld -- a column shown + at 100% can be a small part of what the model measured. This governs what reaches the + serving endpoint: *_dq_info[].anomaly.contributions* is unchanged and still lists every + column, so this is not an access control over the scored table. max_groups: Maximum number of distinct (segment, pattern) groups the LLM is called for per scoring run (default 500). Groups beyond this cap — ranked by group_size * group_avg_severity — get a null ai_explanation; a warning is logged. @@ -294,10 +297,15 @@ def apply(df: DataFrame) -> DataFrame: # The published severity is already floored to the precision this threshold needs, so it is quoted as # it stands: rounding it again here would undo that and could show a value above the threshold on a row # that was not flagged. "Reached" rather than "exceeded", because the comparison is inclusive. + # + # Coalesced because a row can be flagged with no severity at all: an unseen baseline group has a null + # severity, and a caller may choose to treat "cannot judge this row" as a violation. concat_ws drops + # nulls silently, so without this the message read "Anomaly severity reached threshold 95.0" -- a + # sentence asserting a number that does not exist. message = F.concat_ws( "", F.lit("Anomaly severity "), - F.col(output_columns.info).anomaly.severity_percentile.cast("string"), + F.coalesce(F.col(output_columns.info).anomaly.severity_percentile.cast("string"), F.lit("unavailable")), F.lit(f" reached threshold {threshold}"), ) condition_expr = F.col(output_columns.info).anomaly.is_anomaly diff --git a/tests/resources/ai_query_prompt_header.txt b/tests/resources/ai_query_prompt_header.txt index 20ea61dad..fea78d757 100644 --- a/tests/resources/ai_query_prompt_header.txt +++ b/tests/resources/ai_query_prompt_header.txt @@ -5,7 +5,8 @@ Be direct and concrete: name the metrics, their shares and the group size withou Inputs: - attribution_basis: What the feature_contributions below are measuring, which differs by detector and decides how you may describe the pattern. Follow this field rather than assuming a reading: one basis supports saying a feature's own value was unusual, the other does not. -- feature_contributions: Mean share across the group of the evidence the model acted on, per column the caller named, e.g. 'amount (82%), quantity (11%), discount (5%)'. One entry per column however many ways that column was compared, so a share says the column was involved and not which comparison objected. These are aggregated relative importances — not raw data values, and not percentages of the score. +- evidence_disclosure: Whether you are being shown all of the evidence. Read this BEFORE feature_contributions, because it decides what the shares below mean and what you may say about them. +- feature_contributions: Mean share across the group of the evidence the model acted on, per column the caller named, e.g. 'amount (82%), quantity (11%), discount (5%)'. One entry per column however many ways that column was compared, so a share says the column was involved and not which comparison objected. These are aggregated relative importances — not raw data values, and not percentages of the score. The shares are normalised across the entries listed here only, so when evidence_disclosure says any was withheld they are shares of what is disclosed and not of the whole decision. - group_size: Number of rows in this group, e.g. '312 rows'. - severity_range: Severity percentile range across the group, e.g. 'mean 97.4, min 95.1, max 99.8'. - confidence: How closely the ensemble's members agreed on the score: 'high' / 'mixed' / 'low', or 'n/a' when one model did the scoring. Members differ only by random seed on the same training data, so this measures the stability of the score, NOT how reliable the flag is or whether the data has since changed. Do not present it to the reader as confidence in the finding. @@ -21,6 +22,7 @@ Respond with ONLY a JSON object. Field rules: Example (relationship basis, no drift): attribution_basis: each metric's position once the others are accounted for +evidence_disclosure: every contributing metric is shown to you, so the shares below cover all of the evidence the model used. feature_contributions: amount (61%), quantity (22%) group_size: 312 rows severity_range: mean 97.4, min 95.1, max 99.8 @@ -33,6 +35,7 @@ Response: {"narrative":"Across 312 rows, amount accounts for most of what the mo Example (value basis, judged against time, with drift): attribution_basis: each feature's own value compared against the rows it was scored against +evidence_disclosure: MOST of the contributing evidence cannot be disclosed and is absent from the shares below, so what remains is a minority of what the model used. feature_contributions: latency_ms (74%), retries (12%) group_size: 88 rows severity_range: mean 98.9, min 97.0, max 99.9 @@ -41,4 +44,4 @@ baseline_grouping: none temporal_baseline: event_ts threshold: 95.0 drift_summary: drift detected: latency_ms=4.12 -Response: {"narrative":"88 rows are dominated by latency_ms (74%), which has also drifted from its training baseline; retries contribute modestly (12%). These rows are judged against expected levels over time as well as overall.","business_impact":"If latency_ms is genuinely off on these rows, downstream consumers with SLAs would be the first to notice.","action":"Compare latency_ms against its expected level for that period as well as against its overall range."} +Response: {"narrative":"Of what can be shown across these 88 rows, latency_ms carries most (74%) and retries a small part (12%); latency_ms has also moved from its training baseline. These rows are judged against expected levels over time as well as overall.","business_impact":"If these rows are wrong, downstream consumers with SLAs would be the first to notice.","action":"Start with the rows themselves rather than one field: compare latency_ms against its expected level for that period, and take account of the evidence this explanation could not include."} diff --git a/tests/unit/test_anomaly_evidence_disclosure.py b/tests/unit/test_anomaly_evidence_disclosure.py new file mode 100644 index 000000000..f1179fe37 --- /dev/null +++ b/tests/unit/test_anomaly_evidence_disclosure.py @@ -0,0 +1,258 @@ +"""When evidence is withheld, the explanation must say so (no Spark, no workspace). + +Redaction keeps a column out of the prompt, and it did that correctly -- a black-box audit found no name +leaking in any of ten tested groups. What it also did was drop that column's share from the map the +narrative is built from, after which the remaining shares were renormalised to sum to 100 with nothing +recording that anything had gone. A column holding about 7% of one group's evidence was consequently +described as accounting for "all of what the model measured (100%)". All ten groups failed evidence-scope +clarity while passing redaction, which is why those two properties are asserted separately here. + +The band boundaries and the wording are the product contract, so they are pinned directly. The Spark +plumbing that carries them is exercised by the integration suite. +""" + +import re + +import pytest + +from databricks.labs.dqx.anomaly.anomaly_info_schema import ai_explanation_struct_schema +from databricks.labs.dqx.anomaly.anomaly_llm_explainer import ( + AI_QUERY_PROMPT_HEADER, + DISCLOSURE_COMPLETE, + DISCLOSURE_FALLBACK_TEMPLATE, + DISCLOSURE_LIMITED, + DISCLOSURE_NARRATIVE_CLAUSE, + DISCLOSURE_PARTIAL, + DISCLOSURE_PROMPT_TEXT, + TOTALISING_CLAIM_PATTERN, +) + + +def _state(withheld: float, total: float) -> str: + """The band, mirroring *_disclosure_state_expr*'s arithmetic without a Spark session. + + Duplicated deliberately: the boundaries are the contract and are worth pinning in a fast test. The + integration suite is what checks the Spark expression agrees. + """ + if total is None or total <= 0.0: + return DISCLOSURE_COMPLETE + if withheld <= 0.0: + return DISCLOSURE_COMPLETE + if withheld / total > 0.5: + return DISCLOSURE_LIMITED + return DISCLOSURE_PARTIAL + + +@pytest.mark.parametrize( + "withheld, total, expected", + [ + pytest.param(0.0, 100.0, DISCLOSURE_COMPLETE, id="nothing_redacted"), + pytest.param(7.0, 100.0, DISCLOSURE_PARTIAL, id="minor_contributor_redacted"), + pytest.param(50.0, 100.0, DISCLOSURE_PARTIAL, id="exactly_half_is_not_yet_limited"), + pytest.param(93.0, 100.0, DISCLOSURE_LIMITED, id="the_measured_case_dominant_redacted"), + pytest.param(100.0, 100.0, DISCLOSURE_LIMITED, id="every_contributor_redacted"), + pytest.param(0.0, 0.0, DISCLOSURE_COMPLETE, id="no_evidence_at_all_has_nothing_to_qualify"), + ], +) +def test_the_disclosure_band_reflects_how_much_was_withheld(withheld: float, total: float, expected: str): + """The bands, including the case the defect was measured on: 93% withheld reads as limited.""" + assert _state(withheld, total) == expected + + +def test_a_dominant_redacted_contributor_is_not_reported_as_partial(): + """The distinction that changes what a narrative may claim. + + Under the old behaviour this group's visible 7% was rendered as 100%. Calling it merely *partial* would + let a narrative keep treating the remainder as the explanation; *limited* is what tells it not to. + """ + assert _state(93.0, 100.0) == DISCLOSURE_LIMITED + assert _state(93.0, 100.0) != DISCLOSURE_PARTIAL + + +def test_the_band_cannot_be_read_back_as_a_proportion(): + """Coarse on purpose: the state must not disclose by proportion what the name discloses by identity. + + Every share above the boundary maps to one string and every share below it to another, so an observer + learns which side of one boundary a group sits on and nothing finer. + """ + limited = {_state(share, 100.0) for share in (50.01, 60.0, 75.0, 93.0, 99.9, 100.0)} + partial = {_state(share, 100.0) for share in (0.1, 5.0, 7.0, 25.0, 49.9, 50.0)} + + assert limited == {DISCLOSURE_LIMITED} + assert partial == {DISCLOSURE_PARTIAL} + + +# ── what the model is told, and what it cannot omit ────────────────────────────────────────────────── + + +def test_every_band_has_prompt_wording_and_only_the_withheld_ones_carry_a_clause(): + """A complete group must not be qualified: saying evidence was withheld when none was is its own defect.""" + assert set(DISCLOSURE_PROMPT_TEXT) == { + DISCLOSURE_COMPLETE, + DISCLOSURE_PARTIAL, + DISCLOSURE_LIMITED, + } + assert set(DISCLOSURE_NARRATIVE_CLAUSE) == {DISCLOSURE_PARTIAL, DISCLOSURE_LIMITED} + + +@pytest.mark.parametrize("band", [DISCLOSURE_PARTIAL, DISCLOSURE_LIMITED]) +def test_the_prompt_forbids_treating_the_remainder_as_the_whole_decision(band: str): + """The specific false claim that was measured, forbidden in the field the model reads first.""" + text = DISCLOSURE_PROMPT_TEXT[band] + + assert "cannot be disclosed" in text + assert "only what is shown" in text or "minority of what the model used" in text + + +def test_the_limited_band_forbids_redirecting_the_reader_to_a_weak_contributor(): + """The other half of the failure: a confident action built on a share that is nearly all that is left.""" + text = DISCLOSURE_PROMPT_TEXT[DISCLOSURE_LIMITED] + + assert "do not direct the reader to investigate a metric on the strength of a share this small" in text + assert "Never guess what the withheld evidence was" in text + + +@pytest.mark.parametrize("band", [DISCLOSURE_PARTIAL, DISCLOSURE_LIMITED]) +def test_the_appended_clause_states_the_limitation_without_naming_anything(band: str): + """Deterministic, so a model that ignores the instruction cannot produce an unqualified narrative. + + It must also disclose nothing beyond the fact of withholding -- no name, no value, no share. + """ + clause = DISCLOSURE_NARRATIVE_CLAUSE[band] + + assert "could not be disclosed" in clause + for leak in ("%", "latency", "column", "field named"): + assert leak not in clause + + +def test_no_band_names_a_share_or_a_column(): + """Applies to every string this feature can publish, prompt wording included.""" + for text in (*DISCLOSURE_PROMPT_TEXT.values(), *DISCLOSURE_NARRATIVE_CLAUSE.values()): + assert "%" not in text + + +def test_both_exemplars_show_a_disclosure_state_and_they_differ(): + """A field the header tells the model to read first has to be demonstrated, and in both states. + + One state teaches the model to treat it as the default and stop reading the field -- the lesson this + module's own comments already record for the conditioning fields. + """ + values = [ + line.partition("evidence_disclosure: ")[2] + for line in AI_QUERY_PROMPT_HEADER.splitlines() + if line.startswith("evidence_disclosure: ") + ] + + assert len(values) == 2 + assert values[0] != values[1] + + +def test_the_disclosure_field_is_read_before_the_contributions_it_qualifies(): + """Order matters in a prompt: the qualification has to arrive before the numbers it applies to.""" + assert AI_QUERY_PROMPT_HEADER.index("- evidence_disclosure:") < AI_QUERY_PROMPT_HEADER.index( + "- feature_contributions:" + ) + + +def test_the_contributions_field_says_the_shares_may_cover_only_what_is_disclosed(): + """Because the shares themselves are renormalised, the field describing them has to say so.""" + description = AI_QUERY_PROMPT_HEADER.partition("- feature_contributions:")[2].partition("\n")[0] + + assert "normalised across the entries listed here only" in description + assert "not of the whole decision" in description + + +# ── a model that claims completeness cannot be fixed by appending a caveat ──────────────────────────── + + +@pytest.mark.parametrize( + "narrative", + [ + "event_ts accounts for all of what the model measured (100%).", + "These two metrics account for all of the decision.", + "latency_ms fully explains the flag across 88 rows.", + "amount is 100% of the evidence here.", + "This is the whole picture for these rows.", + ], +) +def test_a_totalising_claim_is_detected(narrative: str): + """The observed failure, and the shapes near it. + + The first entry is close to the sentence the evaluation actually recorded. Appending "some evidence was + withheld" to any of these produces a narrative that contradicts itself, and a reader believes the first + half -- so the claim has to be replaced rather than qualified. + """ + assert re.search(TOTALISING_CLAIM_PATTERN, narrative) + + +@pytest.mark.parametrize( + "narrative", + [ + "Across 312 rows, amount carries most of the evidence shown (61%), with quantity next (22%).", + "Of what can be shown across these 88 rows, latency_ms carries most (74%) and retries a small part (12%).", + "Of what can be shown, amount (61%) and quantity (22%) contributed.", + ], +) +def test_ordinary_prose_is_not_mistaken_for_a_totalising_claim(narrative: str): + """The guard must not fire on the house style, or it would replace good narratives with the fallback. + + The second entry is an exemplar response verbatim: if the guard flagged our own demonstration, every + partial group would get the fallback and the model's work would be discarded. + """ + assert not re.search(TOTALISING_CLAIM_PATTERN, narrative) + + +def test_the_fallback_carries_the_groups_facts_and_invents_nothing(): + """A fallback has to be useful, not a refusal -- and must not smuggle in a direction or a cause.""" + rendered = DISCLOSURE_FALLBACK_TEMPLATE % ("88", "latency_ms (74%), retries (12%)") + + assert "88 rows" in rendered + assert "latency_ms" in rendered + for invented in ("far above", "elevated", "because", "caused"): + assert invented not in rendered.lower() + + +@pytest.mark.parametrize("band", [DISCLOSURE_PARTIAL, DISCLOSURE_LIMITED]) +def test_the_fallback_states_the_limit_exactly_once_once_the_clause_is_appended(band: str): + """The fallback fires only when evidence was withheld, and the clause is appended on that same + condition -- so if the fallback asserted the limit too, every replaced narrative would say it twice, + the second time more precisely than the first. The clause is the one place that says it. + """ + rendered = DISCLOSURE_FALLBACK_TEMPLATE % ("88", "latency_ms (74%)") + DISCLOSURE_NARRATIVE_CLAUSE[band] + + assert rendered.lower().count("could not be disclosed") == 1 + assert "cannot fully disclose" not in rendered.lower() + + +def test_the_scope_is_published_as_a_field_not_only_as_prose(): + """A consumer should be able to filter limited-evidence explanations without parsing a sentence. + + Position is load-bearing: the struct is cast positionally, so evidence_scope must be last in the schema + and last in the construction. This pins the schema half. + """ + names = [field.name for field in ai_explanation_struct_schema.fields] + + assert names[-1] == "evidence_scope" + assert ( + dict(zip(names, (f.dataType.simpleString() for f in ai_explanation_struct_schema.fields)))["evidence_scope"] + == "string" + ) + + +def test_no_exemplar_response_would_be_replaced_by_our_own_guard(): + """A few-shot example outweighs an instruction, so an exemplar that trips the guard is worse than none. + + This is the defect the limited-disclosure exemplar had when first written: it declared that most of the + evidence was withheld and then described the rows as dominated by the one metric it could see, which is + the redirect the limited band explicitly forbids. Extracted from the rendered header rather than the + source table so it tracks what the endpoint is actually sent. + """ + narratives = [ + line.partition('"narrative":"')[2].partition('","')[0] + for line in AI_QUERY_PROMPT_HEADER.splitlines() + if line.startswith("Response: {") + ] + + assert len(narratives) == 2, "both exemplars should carry a narrative" + for narrative in narratives: + assert not re.search(TOTALISING_CLAIM_PATTERN, narrative), narrative