Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
[![CI](https://github.com/finite-sample/streamcal/actions/workflows/ci.yml/badge.svg)](https://github.com/finite-sample/streamcal/actions/workflows/ci.yml)
[![Docs](https://github.com/finite-sample/streamcal/actions/workflows/docs.yml/badge.svg)](https://finite-sample.github.io/streamcal/)
[![PyPI](https://img.shields.io/pypi/v/streamcal)](https://pypi.org/project/streamcal/)
[![PyPI Downloads](https://static.pepy.tech/badge/streamcal)](https://pepy.tech/projects/streamcal)
[![Python](https://img.shields.io/pypi/pyversions/streamcal)](https://pypi.org/project/streamcal/)
[![License: MIT](https://img.shields.io/pypi/l/streamcal)](https://github.com/finite-sample/streamcal/blob/main/LICENSE)

Expand Down
307 changes: 205 additions & 102 deletions benchmarks/render.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,75 @@
from __future__ import annotations

import json
import math
import re
from pathlib import Path
from statistics import mean, median

FAMILIES = {
"raw": "Uncalibrated",
"online-logistic": "Online logistic regression",
}
METHODS = {"isotonic": "isotonic", "sigmoid": "sigmoid", "temperature": "temperature"}
POLICIES = {
"accumulating": "refit on all history",
"frozen": "fit once, never updated",
"rolling": "refit on recent window",
}


def label(name):
"""Turn a benchmark configuration name into a readable method name.

Args:
name: Configuration name such as ``stream/b=20/d=0.5/p=1.0``.

Returns:
str: Name a reader can interpret without the benchmark source.
"""
if name.startswith("stream/"):
bins, decay, prior = re.findall(r"=([\d.]+)", name)
return (
f"**streamcal** ({bins} bins, decay {float(decay):g}, "
f"prior {float(prior):g})"
)
if name.startswith("online-logistic"):
return FAMILIES["online-logistic"]
if name in FAMILIES:
return FAMILIES[name]
policy, method = name.split("/")[0].split("-")
text = f"{METHODS[method].capitalize()}, {POLICIES[policy]}"
if policy == "rolling":
window = re.search(r"w=(\d+)", name).group(1)
text += f" (last {int(window):,})"
return text


def size(n_bytes):
"""Format a byte count in the smallest readable unit."""
if n_bytes < 1024:
return f"{n_bytes:.0f}\N{NO-BREAK SPACE}B"
if n_bytes < 1024**2:
return (
f"{n_bytes / 1024:.0f}\N{NO-BREAK SPACE}KB"
if n_bytes >= 10 * 1024
else (f"{n_bytes / 1024:.1f}\N{NO-BREAK SPACE}KB")
)
return f"{n_bytes / 1024**2:.0f}\N{NO-BREAK SPACE}MB"


def millis(value):
"""Format a latency in milliseconds to the precision that matters."""
if value < 0.1:
return "<0.1"
return f"{value:.1f}" if value < 10 else f"{value:.0f}"


def percent(value):
"""Format a signed percentage, keeping one decimal only for small values."""
text = f"{value:+.1f}" if abs(value) < 10 else f"{value:+.0f}"
return text.replace("-", "\N{MINUS SIGN}") + "%"


def row(*values):
"""Render one Markdown table row."""
Expand All @@ -17,172 +83,209 @@ def main():
quality = json.loads(Path("benchmarks/results/quality.json").read_text())
resources = json.loads(Path("benchmarks/results/resources.json").read_text())
electricity = quality["electricity"]
test = {r["name"]: r for r in electricity["test"]}
raw_brier = test["raw"]["brier"]
streamcal = electricity["selected"]["stream"]

lines = [
"<!-- Generated by python -m benchmarks.render -->",
"",
"## Chronological Elec2 results",
"## Results on the electricity stream",
"",
row(
"Method",
"Brier",
"Array bytes",
"Serialized bytes",
"p95 predict ms",
"p95 update ms",
),
"|---|---:|---:|---:|---:|---:|",
"Brier score is the mean squared gap between forecast and outcome;",
"lower is better. Change is relative to the uncalibrated model.",
"State is the calibrator's saved (serialized) size. Update time is the",
"95th-percentile time to learn from one batch of up to 336 labels.",
"",
row("Method", "Brier", "Change vs. uncalibrated", "State", "Update ms"),
"|---|---:|---:|---:|---:|",
]
lines.extend(
row(
r["name"],
f"{r['brier']:.4f}",
f"{r['state_bytes']:,}",
f"{r['serialized_bytes']:,}",
f"{r['p95_predict_ms']:.3f}",
f"{r['p95_update_ms']:.3f}",
label(r["name"]),
f"{r['brier']:.3f}",
"—" if r["name"] == "raw" else percent(100 * (r["brier"] / raw_brier - 1)),
size(r["serialized_bytes"]),
millis(r["p95_update_ms"]),
)
for r in electricity["test"]
for r in sorted(electricity["test"], key=lambda r: r["brier"])
)
lines += [
"",
"Times measure calls on batches of up to 336 observations.",
"Array bytes include upstream fitted models. Python scalar state",
"is reflected in serialized size. Serialization is not process memory.",
"## Streamcal against each alternative",
"",
f"Using {label(streamcal)} instead of each alternative. A negative",
"change means streamcal's Brier score is lower (better), as a",
"percentage of the alternative's score. The range is a 95% interval",
"from a block bootstrap that respects the order of the stream.",
"",
row("Reference", "Brier difference", "Paired 95% interval"),
"|---|---:|---:|",
row("Instead of", "Brier change", "95% interval", "Verdict"),
"|---|---:|---:|---|",
]
lines.extend(
row(name, f"{delta:.4f}", f"[{lower:.4f}, {upper:.4f}]")
for name, (delta, lower, upper) in electricity["paired_brier_intervals"].items()
)
for name, (delta, lower, upper) in sorted(
electricity["paired_brier_intervals"].items(),
key=lambda item: item[1][0] / test[item[0]]["brier"],
):
base = test[name]["brier"]
if upper < 0:
verdict = "streamcal better"
elif lower > 0:
verdict = "streamcal worse"
else:
verdict = "no clear difference"
lines.append(
row(
label(name),
percent(100 * delta / base),
f"{percent(100 * lower / base)} to {percent(100 * upper / base)}",
verdict,
)
)
lines += [
"",
"Differences are streamcal minus reference. Intervals are marginal,",
"not simultaneous across methods. Crossing zero establishes neither",
"superiority nor equivalence.",
"Each interval is for one comparison on its own; they are not",
"adjusted for making eleven at once. “No clear difference” means the",
"data cannot separate the two, not that they are equivalent.",
"",
"## Constrained validation choices",
"## Choosing under a budget",
"",
"Every row requires p95 prediction calls <=100 ms. Update budgets also",
"apply to p95 calls. Choices use validation data. The final column",
"checks the budgets again on test.",
"`TradeoffReport.select` picks the most accurate method that fits a",
"latency and memory budget, using only validation data. The last",
"column re-checks the budget on the test period, where timings can",
"differ from validation.",
"",
row(
"Objective",
"Update budget ms",
"State budget bytes",
"Choice",
"Test objective",
"Test budget met",
"Optimizing",
"Update budget",
"Memory budget",
"Chosen",
"Test score",
"Budget held on test?",
),
"|---|---:|---:|---|---:|---|",
]
objectives = {"brier": "Brier", "binned_calibration_error": "Calibration error"}
for point in electricity["operating_points"]:
chosen = point["selected"]
score = (
"N/A"
if point["test"] is None
else (f"{point['test'][point['objective']]:.4f}")
"—" if point["test"] is None else f"{point['test'][point['objective']]:.3f}"
)
if point["test_resource_feasible"]:
held = "yes"
else:
held = f"no ({point['test']['p95_update_ms']:.2f} ms)"

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Handle operating points with no feasible selection

When the validation budget admits no method, electricity() in benchmarks/suite.py deliberately records selected, test, and test_resource_feasible as None. This falsey branch then indexes point["test"], causing python -m benchmarks.render to fail with a TypeError instead of emitting the intended “nothing fits” row. Handle the None case before formatting the failed-budget detail.

Useful? React with 👍 / 👎.

lines.append(
row(
point["objective"],
point["max_update_ms"],
f"{point['max_serialized_bytes']:,}",
point["selected"] or "No feasible choice",
objectives[point["objective"]],
f"{point['max_update_ms']:g} ms",
size(point["max_serialized_bytes"]),
label(chosen) if chosen else "nothing fits",
score,
point["test_resource_feasible"],
held,
)
)
lines += [
"",
"## Controlled expected-result experiments",
"## Simulations with a known answer",
"",
"Mean squared distance to the known conditional probability over three",
"seeds. Lower is better. Settings are fixed across scenarios. Sparse",
"cases reveal one in ten labels; delayed cases reveal them 0-399 events",
"later. No settings are chosen from these displayed results.",
"In simulated streams the true probability is known, so each method's",
"error can be measured directly: the typical distance between its",
"forecast and the true probability, in percentage points (root mean",
"squared, averaged over three seeds). Lower is better. The same",
"settings are used in every scenario.",
"",
row("Scenario", "Raw", "Streamcal", "Rolling isotonic", "Online logistic"),
row(
"Scenario",
"Uncalibrated",
"streamcal",
"Rolling isotonic",
"Online logistic",
),
"|---|---:|---:|---:|---:|",
]
for scenario in (
"calibrated",
"stationary",
"abrupt",
"gradual",
"sparse",
"delayed",
):
scenarios = {
"calibrated": "Already calibrated",
"stationary": "Miscalibrated, stable",
"abrupt": "Sudden shift",
"gradual": "Gradual drift",
"sparse": "Only 1 in 10 labels seen",
"delayed": "Labels arrive late",
}
for scenario, description in scenarios.items():
values = [
mean(
r["known_truth_mse"]
for r in quality["controlled"]
if r["scenario"] == scenario and r["name"] == name
math.sqrt(
mean(
r["known_truth_mse"]
for r in quality["controlled"]
if r["scenario"] == scenario and r["name"] == name
)
)
for name in ("raw", "stream", "rolling-isotonic", "online-logistic")
]
lines.append(row(scenario, *(f"{v:.5f}" for v in values)))
lines.append(row(description, *(f"{100 * v:.1f}" for v in values)))
lines += [
"",
"## Isolated resource measurements",
"Late labels arrive between 0 and 399 events after their forecast.",
"",
"## Resource use in isolation",
"",
"Medians of two fresh processes: 10,000 observations, batches of 100.",
"Rolling and accumulating references refit every 16 batches here.",
"Each method run alone in a fresh process on 10,000 observations in",
"batches of 100 (median of two runs). Rolling and accumulating methods",
"refit every 16 batches here.",
"",
row(
"Method",
"Arrays bytes",
"Serialized bytes",
"Peak process MiB",
"p95 predict ms",
"p95 update ms",
"Array state",
"Saved size",
"Whole process",
"Predict ms",
"Update ms",
),
"|---|---:|---:|---:|---:|---:|---:|",
"|---|---:|---:|---:|---:|---:|",
]
for method in ("stream", "rolling", "accumulating", "online-logistic"):
names = {
"stream": "streamcal",
"rolling": "Rolling isotonic",
"accumulating": "Isotonic, all history",
"online-logistic": "Online logistic regression",
}
for method, name in names.items():
rows = [
r
for r in resources["measurements"]
if r["method"] == method
and r["observations"] == 10000
and r["batch_size"] == 100
]
values = {
key: median(r[key] for r in rows)
for key in (
"numeric_array_bytes",
"serialized_bytes",
"peak_process_rss_bytes",
"p95_predict_call_ms",
"p95_update_call_ms",
)
}

def mid(key, rows=rows):
return median(r[key] for r in rows)

lines.append(
row(
method,
f"{values['numeric_array_bytes']:.0f}",
f"{values['serialized_bytes']:.0f}",
f"{values['peak_process_rss_bytes'] / 2**20:.1f}",
f"{values['p95_predict_call_ms']:.3f}",
f"{values['p95_update_call_ms']:.3f}",
name,
size(mid("numeric_array_bytes")),
size(mid("serialized_bytes")),
size(mid("peak_process_rss_bytes")),
millis(mid("p95_predict_call_ms")),
millis(mid("p95_update_call_ms")),
)
)
environment = quality["environment"]
lines += [
"",
"Peak RSS includes interpreter, common imports, model and temporaries.",
"The JSON also records baseline RSS, single-event calls, batches of",
"1,000, stream lengths 1,000 and 10,000, and 100,000-observation runs",
"with batches of 1,000.",
"Online logistic regression keeps its state in Python dictionaries,",
"which the array count does not see; its saved size does. The",
"whole-process figure is dominated by Python and its libraries,",
"which is why it barely differs between methods. Times are 95th",
"percentiles per call.",
"",
"## Provenance",
"",
f"Python {quality['environment']['python']}.",
quality["environment"]["platform"],
"",
f"Quality source SHA-256: `{quality['environment']['source_sha256']}`.",
"",
f"Resource source SHA-256: `{resources['environment']['source_sha256']}`.",
f"Measured with streamcal {environment['streamcal']} on Python "
f"{environment['python']} ({environment['platform']}). Source "
f"fingerprint `{environment['source_sha256'][:12]}`; the full SHA-256",
"is in the JSON files.",
"",
]
Path("docs/evidence-results.md").write_text("\n".join(lines))
Expand Down
Loading
Loading