33from __future__ import annotations
44
55import json
6+ import math
7+ import re
68from pathlib import Path
79from statistics import mean , median
810
11+ FAMILIES = {
12+ "raw" : "Uncalibrated" ,
13+ "online-logistic" : "Online logistic regression" ,
14+ }
15+ METHODS = {"isotonic" : "isotonic" , "sigmoid" : "sigmoid" , "temperature" : "temperature" }
16+ POLICIES = {
17+ "accumulating" : "refit on all history" ,
18+ "frozen" : "fit once, never updated" ,
19+ "rolling" : "refit on recent window" ,
20+ }
21+
22+
23+ def label (name ):
24+ """Turn a benchmark configuration name into a readable method name.
25+
26+ Args:
27+ name: Configuration name such as ``stream/b=20/d=0.5/p=1.0``.
28+
29+ Returns:
30+ str: Name a reader can interpret without the benchmark source.
31+ """
32+ if name .startswith ("stream/" ):
33+ bins , decay , prior = re .findall (r"=([\d.]+)" , name )
34+ return (
35+ f"**streamcal** ({ bins } bins, decay { float (decay ):g} , "
36+ f"prior { float (prior ):g} )"
37+ )
38+ if name .startswith ("online-logistic" ):
39+ return FAMILIES ["online-logistic" ]
40+ if name in FAMILIES :
41+ return FAMILIES [name ]
42+ policy , method = name .split ("/" )[0 ].split ("-" )
43+ text = f"{ METHODS [method ].capitalize ()} , { POLICIES [policy ]} "
44+ if policy == "rolling" :
45+ window = re .search (r"w=(\d+)" , name ).group (1 )
46+ text += f" (last { int (window ):,} )"
47+ return text
48+
49+
50+ def size (n_bytes ):
51+ """Format a byte count in the smallest readable unit."""
52+ if n_bytes < 1024 :
53+ return f"{ n_bytes :.0f} \N{NO-BREAK SPACE} B"
54+ if n_bytes < 1024 ** 2 :
55+ return (
56+ f"{ n_bytes / 1024 :.0f} \N{NO-BREAK SPACE} KB"
57+ if n_bytes >= 10 * 1024
58+ else (f"{ n_bytes / 1024 :.1f} \N{NO-BREAK SPACE} KB" )
59+ )
60+ return f"{ n_bytes / 1024 ** 2 :.0f} \N{NO-BREAK SPACE} MB"
61+
62+
63+ def millis (value ):
64+ """Format a latency in milliseconds to the precision that matters."""
65+ if value < 0.1 :
66+ return "<0.1"
67+ return f"{ value :.1f} " if value < 10 else f"{ value :.0f} "
68+
69+
70+ def percent (value ):
71+ """Format a signed percentage, keeping one decimal only for small values."""
72+ text = f"{ value :+.1f} " if abs (value ) < 10 else f"{ value :+.0f} "
73+ return text .replace ("-" , "\N{MINUS SIGN} " ) + "%"
74+
975
1076def row (* values ):
1177 """Render one Markdown table row."""
@@ -17,172 +83,209 @@ def main():
1783 quality = json .loads (Path ("benchmarks/results/quality.json" ).read_text ())
1884 resources = json .loads (Path ("benchmarks/results/resources.json" ).read_text ())
1985 electricity = quality ["electricity" ]
86+ test = {r ["name" ]: r for r in electricity ["test" ]}
87+ raw_brier = test ["raw" ]["brier" ]
88+ streamcal = electricity ["selected" ]["stream" ]
89+
2090 lines = [
2191 "<!-- Generated by python -m benchmarks.render -->" ,
2292 "" ,
23- "## Chronological Elec2 results " ,
93+ "## Results on the electricity stream " ,
2494 "" ,
25- row (
26- "Method" ,
27- "Brier" ,
28- "Array bytes" ,
29- "Serialized bytes" ,
30- "p95 predict ms" ,
31- "p95 update ms" ,
32- ),
33- "|---|---:|---:|---:|---:|---:|" ,
95+ "Brier score is the mean squared gap between forecast and outcome;" ,
96+ "lower is better. Change is relative to the uncalibrated model." ,
97+ "State is the calibrator's saved (serialized) size. Update time is the" ,
98+ "95th-percentile time to learn from one batch of up to 336 labels." ,
99+ "" ,
100+ row ("Method" , "Brier" , "Change vs. uncalibrated" , "State" , "Update ms" ),
101+ "|---|---:|---:|---:|---:|" ,
34102 ]
35103 lines .extend (
36104 row (
37- r ["name" ],
38- f"{ r ['brier' ]:.4f} " ,
39- f"{ r ['state_bytes' ]:,} " ,
40- f"{ r ['serialized_bytes' ]:,} " ,
41- f"{ r ['p95_predict_ms' ]:.3f} " ,
42- f"{ r ['p95_update_ms' ]:.3f} " ,
105+ label (r ["name" ]),
106+ f"{ r ['brier' ]:.3f} " ,
107+ "—" if r ["name" ] == "raw" else percent (100 * (r ["brier" ] / raw_brier - 1 )),
108+ size (r ["serialized_bytes" ]),
109+ millis (r ["p95_update_ms" ]),
43110 )
44- for r in electricity ["test" ]
111+ for r in sorted ( electricity ["test" ], key = lambda r : r [ "brier" ])
45112 )
46113 lines += [
47114 "" ,
48- "Times measure calls on batches of up to 336 observations." ,
49- "Array bytes include upstream fitted models. Python scalar state" ,
50- "is reflected in serialized size. Serialization is not process memory." ,
115+ "## Streamcal against each alternative" ,
116+ "" ,
117+ f"Using { label (streamcal )} instead of each alternative. A negative" ,
118+ "change means streamcal's Brier score is lower (better), as a" ,
119+ "percentage of the alternative's score. The range is a 95% interval" ,
120+ "from a block bootstrap that respects the order of the stream." ,
51121 "" ,
52- row ("Reference " , "Brier difference " , "Paired 95% interval" ),
53- "|---|---:|---:|" ,
122+ row ("Instead of " , "Brier change " , "95% interval" , "Verdict " ),
123+ "|---|---:|---:|---| " ,
54124 ]
55- lines .extend (
56- row (name , f"{ delta :.4f} " , f"[{ lower :.4f} , { upper :.4f} ]" )
57- for name , (delta , lower , upper ) in electricity ["paired_brier_intervals" ].items ()
58- )
125+ for name , (delta , lower , upper ) in sorted (
126+ electricity ["paired_brier_intervals" ].items (),
127+ key = lambda item : item [1 ][0 ] / test [item [0 ]]["brier" ],
128+ ):
129+ base = test [name ]["brier" ]
130+ if upper < 0 :
131+ verdict = "streamcal better"
132+ elif lower > 0 :
133+ verdict = "streamcal worse"
134+ else :
135+ verdict = "no clear difference"
136+ lines .append (
137+ row (
138+ label (name ),
139+ percent (100 * delta / base ),
140+ f"{ percent (100 * lower / base )} to { percent (100 * upper / base )} " ,
141+ verdict ,
142+ )
143+ )
59144 lines += [
60145 "" ,
61- "Differences are streamcal minus reference. Intervals are marginal, " ,
62- "not simultaneous across methods. Crossing zero establishes neither " ,
63- "superiority nor equivalence ." ,
146+ "Each interval is for one comparison on its own; they are not " ,
147+ "adjusted for making eleven at once. “No clear difference” means the " ,
148+ "data cannot separate the two, not that they are equivalent ." ,
64149 "" ,
65- "## Constrained validation choices " ,
150+ "## Choosing under a budget " ,
66151 "" ,
67- "Every row requires p95 prediction calls <=100 ms. Update budgets also" ,
68- "apply to p95 calls. Choices use validation data. The final column" ,
69- "checks the budgets again on test." ,
152+ "`TradeoffReport.select` picks the most accurate method that fits a" ,
153+ "latency and memory budget, using only validation data. The last" ,
154+ "column re-checks the budget on the test period, where timings can" ,
155+ "differ from validation." ,
70156 "" ,
71157 row (
72- "Objective " ,
73- "Update budget ms " ,
74- "State budget bytes " ,
75- "Choice " ,
76- "Test objective " ,
77- "Test budget met " ,
158+ "Optimizing " ,
159+ "Update budget" ,
160+ "Memory budget" ,
161+ "Chosen " ,
162+ "Test score " ,
163+ "Budget held on test? " ,
78164 ),
79165 "|---|---:|---:|---|---:|---|" ,
80166 ]
167+ objectives = {"brier" : "Brier" , "binned_calibration_error" : "Calibration error" }
81168 for point in electricity ["operating_points" ]:
169+ chosen = point ["selected" ]
82170 score = (
83- "N/A"
84- if point ["test" ] is None
85- else (f"{ point ['test' ][point ['objective' ]]:.4f} " )
171+ "—" if point ["test" ] is None else f"{ point ['test' ][point ['objective' ]]:.3f} "
86172 )
173+ if point ["test_resource_feasible" ]:
174+ held = "yes"
175+ else :
176+ held = f"no ({ point ['test' ]['p95_update_ms' ]:.2f} ms)"
87177 lines .append (
88178 row (
89- point ["objective" ],
90- point [" max_update_ms" ] ,
91- f" { point [' max_serialized_bytes' ]:, } " ,
92- point [ "selected" ] or "No feasible choice " ,
179+ objectives [ point ["objective" ] ],
180+ f" { point [' max_update_ms' ]:g } ms" ,
181+ size ( point [" max_serialized_bytes" ]) ,
182+ label ( chosen ) if chosen else "nothing fits " ,
93183 score ,
94- point [ "test_resource_feasible" ] ,
184+ held ,
95185 )
96186 )
97187 lines += [
98188 "" ,
99- "## Controlled expected-result experiments " ,
189+ "## Simulations with a known answer " ,
100190 "" ,
101- "Mean squared distance to the known conditional probability over three" ,
102- "seeds. Lower is better. Settings are fixed across scenarios. Sparse" ,
103- "cases reveal one in ten labels; delayed cases reveal them 0-399 events" ,
104- "later. No settings are chosen from these displayed results." ,
191+ "In simulated streams the true probability is known, so each method's" ,
192+ "error can be measured directly: the typical distance between its" ,
193+ "forecast and the true probability, in percentage points (root mean" ,
194+ "squared, averaged over three seeds). Lower is better. The same" ,
195+ "settings are used in every scenario." ,
105196 "" ,
106- row ("Scenario" , "Raw" , "Streamcal" , "Rolling isotonic" , "Online logistic" ),
197+ row (
198+ "Scenario" ,
199+ "Uncalibrated" ,
200+ "streamcal" ,
201+ "Rolling isotonic" ,
202+ "Online logistic" ,
203+ ),
107204 "|---|---:|---:|---:|---:|" ,
108205 ]
109- for scenario in (
110- "calibrated" ,
111- "stationary" ,
112- "abrupt" ,
113- "gradual" ,
114- "sparse" ,
115- "delayed" ,
116- ):
206+ scenarios = {
207+ "calibrated" : "Already calibrated" ,
208+ "stationary" : "Miscalibrated, stable" ,
209+ "abrupt" : "Sudden shift" ,
210+ "gradual" : "Gradual drift" ,
211+ "sparse" : "Only 1 in 10 labels seen" ,
212+ "delayed" : "Labels arrive late" ,
213+ }
214+ for scenario , description in scenarios .items ():
117215 values = [
118- mean (
119- r ["known_truth_mse" ]
120- for r in quality ["controlled" ]
121- if r ["scenario" ] == scenario and r ["name" ] == name
216+ math .sqrt (
217+ mean (
218+ r ["known_truth_mse" ]
219+ for r in quality ["controlled" ]
220+ if r ["scenario" ] == scenario and r ["name" ] == name
221+ )
122222 )
123223 for name in ("raw" , "stream" , "rolling-isotonic" , "online-logistic" )
124224 ]
125- lines .append (row (scenario , * (f"{ v :.5f } " for v in values )))
225+ lines .append (row (description , * (f"{ 100 * v :.1f } " for v in values )))
126226 lines += [
127227 "" ,
128- "## Isolated resource measurements" ,
228+ "Late labels arrive between 0 and 399 events after their forecast." ,
229+ "" ,
230+ "## Resource use in isolation" ,
129231 "" ,
130- "Medians of two fresh processes: 10,000 observations, batches of 100." ,
131- "Rolling and accumulating references refit every 16 batches here." ,
232+ "Each method run alone in a fresh process on 10,000 observations in" ,
233+ "batches of 100 (median of two runs). Rolling and accumulating methods" ,
234+ "refit every 16 batches here." ,
132235 "" ,
133236 row (
134237 "Method" ,
135- "Arrays bytes " ,
136- "Serialized bytes " ,
137- "Peak process MiB " ,
138- "p95 predict ms" ,
139- "p95 update ms" ,
238+ "Array state " ,
239+ "Saved size " ,
240+ "Whole process" ,
241+ "Predict ms" ,
242+ "Update ms" ,
140243 ),
141- "|---|---:|---:|---:|---:|---:|---:| " ,
244+ "|---|---:|---:|---:|---:|---:|" ,
142245 ]
143- for method in ("stream" , "rolling" , "accumulating" , "online-logistic" ):
246+ names = {
247+ "stream" : "streamcal" ,
248+ "rolling" : "Rolling isotonic" ,
249+ "accumulating" : "Isotonic, all history" ,
250+ "online-logistic" : "Online logistic regression" ,
251+ }
252+ for method , name in names .items ():
144253 rows = [
145254 r
146255 for r in resources ["measurements" ]
147256 if r ["method" ] == method
148257 and r ["observations" ] == 10000
149258 and r ["batch_size" ] == 100
150259 ]
151- values = {
152- key : median (r [key ] for r in rows )
153- for key in (
154- "numeric_array_bytes" ,
155- "serialized_bytes" ,
156- "peak_process_rss_bytes" ,
157- "p95_predict_call_ms" ,
158- "p95_update_call_ms" ,
159- )
160- }
260+
261+ def mid (key , rows = rows ):
262+ return median (r [key ] for r in rows )
263+
161264 lines .append (
162265 row (
163- method ,
164- f" { values [ ' numeric_array_bytes' ]:.0f } " ,
165- f" { values [ ' serialized_bytes' ]:.0f } " ,
166- f" { values [ ' peak_process_rss_bytes' ] / 2 ** 20 :.1f } " ,
167- f" { values [ ' p95_predict_call_ms' ]:.3f } " ,
168- f" { values [ ' p95_update_call_ms' ]:.3f } " ,
266+ name ,
267+ size ( mid ( " numeric_array_bytes" )) ,
268+ size ( mid ( " serialized_bytes" )) ,
269+ size ( mid ( " peak_process_rss_bytes" )) ,
270+ millis ( mid ( " p95_predict_call_ms" )) ,
271+ millis ( mid ( " p95_update_call_ms" )) ,
169272 )
170273 )
274+ environment = quality ["environment" ]
171275 lines += [
172276 "" ,
173- "Peak RSS includes interpreter, common imports, model and temporaries." ,
174- "The JSON also records baseline RSS, single-event calls, batches of" ,
175- "1,000, stream lengths 1,000 and 10,000, and 100,000-observation runs" ,
176- "with batches of 1,000." ,
277+ "Online logistic regression keeps its state in Python dictionaries," ,
278+ "which the array count does not see; its saved size does. The" ,
279+ "whole-process figure is dominated by Python and its libraries," ,
280+ "which is why it barely differs between methods. Times are 95th" ,
281+ "percentiles per call." ,
177282 "" ,
178283 "## Provenance" ,
179284 "" ,
180- f"Python { quality ['environment' ]['python' ]} ." ,
181- quality ["environment" ]["platform" ],
182- "" ,
183- f"Quality source SHA-256: `{ quality ['environment' ]['source_sha256' ]} `." ,
184- "" ,
185- f"Resource source SHA-256: `{ resources ['environment' ]['source_sha256' ]} `." ,
285+ f"Measured with streamcal { environment ['streamcal' ]} on Python "
286+ f"{ environment ['python' ]} ({ environment ['platform' ]} ). Source "
287+ f"fingerprint `{ environment ['source_sha256' ][:12 ]} `; the full SHA-256" ,
288+ "is in the JSON files." ,
186289 "" ,
187290 ]
188291 Path ("docs/evidence-results.md" ).write_text ("\n " .join (lines ))
0 commit comments