Repository navigation
Expand file tree
/
Copy pathraw.jsonl
More file actions
36 lines (36 loc) · 109 KB
/
Copy pathraw.jsonl
File metadata and controls
36 lines (36 loc) · 109 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "short_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 256, "engine_cache_reuse_calls": 31277, "engine_peak_bytes": 1791405568, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 48, "kv_cache_byte_utilization": 0.328972868, "loaded_tensors": 290, "measured_d2d_bytes": 542220288, "measured_d2d_calls": 4848, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 67104, "measured_h2d_calls": 57, "measured_ms": 549.011575, "measured_tokens": 240, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 66748416, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 0, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 1, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [109.844642, 110.174802, 110.587566, 110.77185, 110.379023, 110.90103, 182.43935, 182.736549], "request_completion_p50_ms": 110.679708, "request_completion_p95_ms": 182.63252935, "request_count": 8, "request_ttft_ms": [47.653534, 47.643903, 47.640074, 47.635893, 47.63386, 47.622371, 47.618706, 47.610473], "request_ttft_p50_ms": 47.6348765, "request_ttft_p95_ms": 47.65016315, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15541918989117524000, "tokens_per_second": 437.149253183, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5574.207175}
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "short_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 256, "engine_cache_reuse_calls": 31277, "engine_peak_bytes": 1791405568, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 48, "kv_cache_byte_utilization": 0.328972868, "loaded_tensors": 290, "measured_d2d_bytes": 542220288, "measured_d2d_calls": 4848, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 67104, "measured_h2d_calls": 57, "measured_ms": 545.58112, "measured_tokens": 240, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 66748416, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 2, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [108.995331, 109.21003, 109.578585, 109.732505, 109.387634, 109.857087, 181.005875, 181.188125], "request_completion_p50_ms": 109.655545, "request_completion_p95_ms": 181.1243375, "request_count": 8, "request_ttft_ms": [47.267636, 47.261795, 47.256806, 47.254159, 47.252552, 47.249724, 47.245256, 47.241337], "request_ttft_p50_ms": 47.2533555, "request_ttft_p95_ms": 47.26559165, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15541918989117524000, "tokens_per_second": 439.897920221, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5607.127735}
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "short_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 256, "engine_cache_reuse_calls": 31277, "engine_peak_bytes": 1791405568, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 48, "kv_cache_byte_utilization": 0.328972868, "loaded_tensors": 290, "measured_d2d_bytes": 542220288, "measured_d2d_calls": 4848, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 67104, "measured_h2d_calls": 57, "measured_ms": 548.571065, "measured_tokens": 240, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 66748416, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 3, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [109.693549, 109.992129, 110.376751, 110.54145, 110.169392, 110.658908, 182.006782, 182.304441], "request_completion_p50_ms": 110.4591005, "request_completion_p95_ms": 182.20026035, "request_count": 8, "request_ttft_ms": [47.536676, 47.534327, 47.529405, 47.516794, 47.514982, 47.504686, 47.500994, 47.495969], "request_ttft_p50_ms": 47.515888, "request_ttft_p95_ms": 47.53585385, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15541918989117524000, "tokens_per_second": 437.500289958, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5664.012977}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 29, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "short_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 400, "engine_cache_reuse_calls": 54044, "engine_peak_bytes": 1715514880, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.425116099, "loaded_tensors": 290, "measured_d2d_bytes": 542220288, "measured_d2d_calls": 4848, "measured_d2h_bytes": 1536, "measured_d2h_calls": 96, "measured_h2d_bytes": 67104, "measured_h2d_calls": 102, "measured_ms": 752.12681, "measured_tokens": 240, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 53993472, "positions_aware_batch_decode_calls": 29, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 1, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 0, 0, 1, 1], "request_completion_ms": [137.370974, 137.677276, 138.064225, 249.725588, 137.863883, 249.884288, 250.306259, 250.452332], "request_completion_p50_ms": 193.8949065, "request_completion_p95_ms": 250.40120645, "request_count": 8, "request_ttft_ms": [14.38958, 14.384076, 14.380463, 155.735979, 14.372993, 155.729167, 47.480908, 47.473214], "request_ttft_p50_ms": 30.931397, "request_ttft_p95_ms": 155.7335948, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15541918989117524000, "tokens_per_second": 319.095127057, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5739.527735}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 29, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "short_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 400, "engine_cache_reuse_calls": 54044, "engine_peak_bytes": 1715514880, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.425116099, "loaded_tensors": 290, "measured_d2d_bytes": 542220288, "measured_d2d_calls": 4848, "measured_d2h_bytes": 1536, "measured_d2h_calls": 96, "measured_h2d_bytes": 67104, "measured_h2d_calls": 102, "measured_ms": 755.735391, "measured_tokens": 240, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 53993472, "positions_aware_batch_decode_calls": 29, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 2, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 0, 0, 1, 1], "request_completion_ms": [137.721854, 138.033849, 138.372816, 250.091907, 138.215419, 250.236018, 250.620281, 250.730781], "request_completion_p50_ms": 194.2323615, "request_completion_p95_ms": 250.692106, "request_count": 8, "request_ttft_ms": [14.311593, 14.308751, 14.304855, 156.030211, 14.293363, 156.013977, 47.506101, 47.493668], "request_ttft_p50_ms": 30.9026305, "request_ttft_p95_ms": 156.0245291, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15541918989117524000, "tokens_per_second": 317.571471256, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5792.31178}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 29, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "short_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 400, "engine_cache_reuse_calls": 54044, "engine_peak_bytes": 1715514880, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.425116099, "loaded_tensors": 290, "measured_d2d_bytes": 542220288, "measured_d2d_calls": 4848, "measured_d2h_bytes": 1536, "measured_d2h_calls": 96, "measured_h2d_bytes": 67104, "measured_h2d_calls": 102, "measured_ms": 752.341947, "measured_tokens": 240, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 53993472, "positions_aware_batch_decode_calls": 29, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 3, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 0, 0, 1, 1], "request_completion_ms": [137.546064, 137.867339, 138.24695, 249.827758, 138.049246, 249.950272, 250.340514, 250.464323], "request_completion_p50_ms": 194.037354, "request_completion_p95_ms": 250.42098985, "request_count": 8, "request_ttft_ms": [14.294947, 14.2921, 14.288672, 155.90222, 14.281971, 155.896374, 47.483529, 47.476397], "request_ttft_p50_ms": 30.885672, "request_ttft_p95_ms": 155.9001739, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15541918989117524000, "tokens_per_second": 319.003879761, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5857.328425}
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "long_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 256, "engine_cache_reuse_calls": 31277, "engine_peak_bytes": 2437737984, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 16, "kv_cache_byte_utilization": 0.608042636, "loaded_tensors": 290, "measured_d2d_bytes": 940302336, "measured_d2d_calls": 4944, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 123552, "measured_h2d_calls": 57, "measured_ms": 658.335277, "measured_tokens": 336, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 123371520, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 1, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [144.385449, 144.56619, 216.551001, 216.7372, 217.026094, 217.160714, 216.867991, 217.270216], "request_completion_p50_ms": 216.8025955, "request_completion_p95_ms": 217.2318903, "request_count": 8, "request_ttft_ms": [81.991338, 81.986719, 81.979373, 81.973759, 81.96429, 81.958926, 81.952766, 81.947475], "request_ttft_p50_ms": 81.9690245, "request_ttft_p95_ms": 81.98972135, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.875, "status": "pass", "steps": 3, "token_checksum": 4065496277705958240, "tokens_per_second": 510.378239992, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5693.664155}
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "long_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 256, "engine_cache_reuse_calls": 31277, "engine_peak_bytes": 2437737984, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 16, "kv_cache_byte_utilization": 0.608042636, "loaded_tensors": 290, "measured_d2d_bytes": 940302336, "measured_d2d_calls": 4944, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 123552, "measured_h2d_calls": 57, "measured_ms": 654.213897, "measured_tokens": 336, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 123371520, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 2, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [143.713796, 143.902103, 215.85745, 216.045758, 216.320707, 216.448953, 216.167842, 216.570955], "request_completion_p50_ms": 216.1068, "request_completion_p95_ms": 216.5282543, "request_count": 8, "request_ttft_ms": [81.366111, 81.359009, 81.355723, 81.35276, 81.348312, 81.341068, 81.335613, 81.329711], "request_ttft_p50_ms": 81.350536, "request_ttft_p95_ms": 81.3636253, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.875, "status": "pass", "steps": 3, "token_checksum": 4065496277705958240, "tokens_per_second": 513.593492191, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5488.711195}
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "long_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 256, "engine_cache_reuse_calls": 31277, "engine_peak_bytes": 2437737984, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 16, "kv_cache_byte_utilization": 0.608042636, "loaded_tensors": 290, "measured_d2d_bytes": 940302336, "measured_d2d_calls": 4944, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 123552, "measured_h2d_calls": 57, "measured_ms": 662.645875, "measured_tokens": 336, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 123371520, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 3, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [146.595073, 146.891958, 219.232312, 219.516789, 219.88206, 219.968455, 219.656586, 220.095145], "request_completion_p50_ms": 219.5866875, "request_completion_p95_ms": 220.0508035, "request_count": 8, "request_ttft_ms": [83.850409, 83.839641, 83.835172, 83.827336, 83.822947, 83.769754, 83.766328, 83.761023], "request_ttft_p50_ms": 83.8251415, "request_ttft_p95_ms": 83.8466402, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.875, "status": "pass", "steps": 3, "token_checksum": 4065496277705958240, "tokens_per_second": 507.058162854, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5488.697712}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 37, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "long_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 416, "engine_cache_reuse_calls": 65668, "engine_peak_bytes": 2038681088, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.57372291, "loaded_tensors": 290, "measured_d2d_bytes": 940302336, "measured_d2d_calls": 4944, "measured_d2h_bytes": 1920, "measured_d2h_calls": 120, "measured_h2d_bytes": 123552, "measured_h2d_calls": 126, "measured_ms": 1154.879754, "measured_tokens": 336, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 72867840, "positions_aware_batch_decode_calls": 37, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 1, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 1, 1, 1, 1, 1, 1], "request_completion_ms": [144.418638, 144.598442, 215.408133, 215.586763, 215.919997, 382.603102, 215.771276, 382.787543], "request_completion_p50_ms": 215.6790195, "request_completion_p95_ms": 382.72298865, "request_count": 8, "request_ttft_ms": [11.165538, 11.159658, 55.263806, 55.258611, 55.248805, 250.024221, 55.232314, 250.007399], "request_ttft_p50_ms": 55.253708, "request_ttft_p95_ms": 250.0183333, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 32, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.4375, "status": "pass", "steps": 3, "token_checksum": 4065496277705958240, "tokens_per_second": 290.939380343, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5974.461063}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 37, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "long_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 416, "engine_cache_reuse_calls": 65668, "engine_peak_bytes": 2038681088, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.57372291, "loaded_tensors": 290, "measured_d2d_bytes": 940302336, "measured_d2d_calls": 4944, "measured_d2h_bytes": 1920, "measured_d2h_calls": 120, "measured_h2d_bytes": 123552, "measured_h2d_calls": 126, "measured_ms": 1161.013803, "measured_tokens": 336, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 72867840, "positions_aware_batch_decode_calls": 37, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 2, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 1, 1, 1, 1, 1, 1], "request_completion_ms": [146.89042, 147.222966, 218.416857, 218.738223, 219.117417, 386.886777, 218.866427, 386.960796], "request_completion_p50_ms": 218.802325, "request_completion_p95_ms": 386.93488935, "request_count": 8, "request_ttft_ms": [12.264834, 12.26101, 56.882085, 56.877615, 56.86545, 253.690462, 56.80875, 253.625239], "request_ttft_p50_ms": 56.8715325, "request_ttft_p95_ms": 253.66763395, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 32, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.4375, "status": "pass", "steps": 3, "token_checksum": 4065496277705958240, "tokens_per_second": 289.402244083, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5935.690175}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 37, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "long_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 416, "engine_cache_reuse_calls": 65668, "engine_peak_bytes": 2038681088, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.57372291, "loaded_tensors": 290, "measured_d2d_bytes": 940302336, "measured_d2d_calls": 4944, "measured_d2h_bytes": 1920, "measured_d2h_calls": 120, "measured_h2d_bytes": 123552, "measured_h2d_calls": 126, "measured_ms": 1155.809093, "measured_tokens": 336, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 72867840, "positions_aware_batch_decode_calls": 37, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 3, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 1, 1, 1, 1, 1, 1], "request_completion_ms": [145.005367, 145.232057, 216.372647, 216.579927, 217.017452, 384.174632, 216.839706, 384.355317], "request_completion_p50_ms": 216.7098165, "request_completion_p95_ms": 384.29207725, "request_count": 8, "request_ttft_ms": [11.331138, 11.326227, 55.470432, 55.466061, 55.458812, 251.303881, 55.447215, 251.291752], "request_ttft_p50_ms": 55.4624365, "request_ttft_p95_ms": 251.29963585, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 32, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.4375, "status": "pass", "steps": 3, "token_checksum": 4065496277705958240, "tokens_per_second": 290.705447842, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5739.92962}
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 19, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": false, "case": "delayed_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 200, "engine_cache_reuse_calls": 37159, "engine_peak_bytes": 2114571776, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 64, "kv_cache_byte_utilization": 0.46753876, "loaded_tensors": 290, "measured_d2d_bytes": 741261312, "measured_d2d_calls": 4896, "measured_d2h_bytes": 1920, "measured_d2h_calls": 60, "measured_h2d_bytes": 95328, "measured_h2d_calls": 69, "measured_ms": 656.924919, "measured_tokens": 288, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 94863360, "positions_aware_batch_decode_calls": 19, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 1, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [110.33302, 110.61053, 110.803328, 110.931958, 184.346123, 184.60537, 184.741828, 184.874445], "request_completion_p50_ms": 147.6390405, "request_completion_p95_ms": 184.82802905, "request_count": 8, "request_ttft_ms": [16.406144, 16.39946, 16.395532, 16.387524, 50.240779, 50.23309, 50.226215, 50.219789], "request_ttft_p50_ms": 33.3129665, "request_ttft_p95_ms": 50.23808785, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 2426372307028996544, "tokens_per_second": 438.406264811, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5677.541558}
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 19, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": false, "case": "delayed_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 200, "engine_cache_reuse_calls": 37159, "engine_peak_bytes": 2114571776, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 64, "kv_cache_byte_utilization": 0.46753876, "loaded_tensors": 290, "measured_d2d_bytes": 741261312, "measured_d2d_calls": 4896, "measured_d2h_bytes": 1920, "measured_d2h_calls": 60, "measured_h2d_bytes": 95328, "measured_h2d_calls": 69, "measured_ms": 652.500076, "measured_tokens": 288, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 94863360, "positions_aware_batch_decode_calls": 19, "post_run_gpu_state": {"gpu_use_percent": 5, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 2, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [108.344117, 108.560394, 108.741044, 108.875779, 184.500181, 184.672183, 184.81404, 184.946785], "request_completion_p50_ms": 146.68798, "request_completion_p95_ms": 184.90032425, "request_count": 8, "request_ttft_ms": [14.758194, 14.754721, 14.74909, 14.746457, 50.876341, 50.871478, 50.866799, 50.86211], "request_ttft_p50_ms": 32.810152, "request_ttft_p95_ms": 50.87463895, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 2426372307028996544, "tokens_per_second": 441.379258935, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5547.257379}
{"allocated_cache_bytes": 202899456, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 19, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": false, "case": "delayed_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 200, "engine_cache_reuse_calls": 37159, "engine_peak_bytes": 2114571776, "expected_cache_bytes": 202899456, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 64, "kv_cache_byte_utilization": 0.46753876, "loaded_tensors": 290, "measured_d2d_bytes": 741261312, "measured_d2d_calls": 4896, "measured_d2h_bytes": 1920, "measured_d2h_calls": 60, "measured_h2d_bytes": 95328, "measured_h2d_calls": 69, "measured_ms": 655.674977, "measured_tokens": 288, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 94863360, "positions_aware_batch_decode_calls": 19, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 2, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 3, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [108.751719, 109.015979, 109.173785, 109.325253, 185.030721, 185.268008, 185.4201, 185.564803], "request_completion_p50_ms": 147.177987, "request_completion_p95_ms": 185.51415695, "request_count": 8, "request_ttft_ms": [15.023274, 15.018463, 15.011531, 15.007334, 50.625907, 50.619547, 50.610944, 50.605889], "request_ttft_p50_ms": 32.8145815, "request_ttft_p95_ms": 50.623681, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 2426372307028996544, "tokens_per_second": 439.242017924, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5786.683639}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 22, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": true, "case": "delayed_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 284, "engine_cache_reuse_calls": 41590, "engine_peak_bytes": 2038681088, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 0, "kv_cache_byte_utilization": 0.746904025, "loaded_tensors": 290, "measured_d2d_bytes": 741261312, "measured_d2d_calls": 4896, "measured_d2h_bytes": 1152, "measured_d2h_calls": 72, "measured_h2d_bytes": 95328, "measured_h2d_calls": 78, "measured_ms": 701.206194, "measured_tokens": 288, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 94863360, "positions_aware_batch_decode_calls": 22, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 1, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 1, 1, 1, 1], "request_completion_ms": [123.319477, 123.504143, 123.68376, 123.8613, 203.006572, 203.188362, 203.360092, 203.534247], "request_completion_p50_ms": 163.433936, "request_completion_p95_ms": 203.47329275, "request_count": 8, "request_ttft_ms": [15.396263, 15.393517, 15.388255, 15.385092, 55.373751, 55.364616, 55.351382, 55.339974], "request_ttft_p50_ms": 35.3681185, "request_ttft_p95_ms": 55.37055375, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 2426372307028996544, "tokens_per_second": 410.72084426, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5729.907511}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 22, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": true, "case": "delayed_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 284, "engine_cache_reuse_calls": 41590, "engine_peak_bytes": 2038681088, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 0, "kv_cache_byte_utilization": 0.746904025, "loaded_tensors": 290, "measured_d2d_bytes": 741261312, "measured_d2d_calls": 4896, "measured_d2h_bytes": 1152, "measured_d2h_calls": 72, "measured_h2d_bytes": 95328, "measured_h2d_calls": 78, "measured_ms": 695.69226, "measured_tokens": 288, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 94863360, "positions_aware_batch_decode_calls": 22, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 2, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 1, 1, 1, 1], "request_completion_ms": [123.219795, 123.413971, 123.582404, 123.692237, 198.118574, 198.283076, 198.435308, 198.564075], "request_completion_p50_ms": 160.9054055, "request_completion_p95_ms": 198.51900655, "request_count": 8, "request_ttft_ms": [15.302808, 15.300165, 15.291577, 15.266653, 55.151354, 55.129674, 55.110725, 55.101656], "request_ttft_p50_ms": 35.202232, "request_ttft_p95_ms": 55.143766, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 2426372307028996544, "tokens_per_second": 413.976145142, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5724.751788}
{"allocated_cache_bytes": 127008768, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 22, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": true, "case": "delayed_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 284, "engine_cache_reuse_calls": 41590, "engine_peak_bytes": 2038681088, "expected_cache_bytes": 127008768, "generated_tokens": [[9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707], [9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879], [1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707, 1879, 9707]], "inactive_rows_skipped": 0, "kv_cache_byte_utilization": 0.746904025, "loaded_tensors": 290, "measured_d2d_bytes": 741261312, "measured_d2d_calls": 4896, "measured_d2h_bytes": 1152, "measured_d2h_calls": 72, "measured_h2d_bytes": 95328, "measured_h2d_calls": 78, "measured_ms": 696.531167, "measured_tokens": 288, "model": "qwen2.5-0.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 494032768, "peak_active_cache_bytes": 94863360, "positions_aware_batch_decode_calls": 22, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 0}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 3, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 1, 1, 1, 1], "request_completion_ms": [123.34111, 123.528066, 123.698647, 123.839334, 198.262838, 198.439265, 198.60959, 198.777744], "request_completion_p50_ms": 161.051086, "request_completion_p95_ms": 198.7188901, "request_count": 8, "request_ttft_ms": [15.430043, 15.422388, 15.413363, 15.373951, 55.178994, 55.168642, 55.155265, 55.142919], "request_ttft_p50_ms": 35.286481, "request_ttft_p95_ms": 55.1753708, "resident_weight_bytes": 1260477952, "revision": "060db6499f32faf8b98477b0a26969ef7d8b9987", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 2426372307028996544, "tokens_per_second": 413.477549383, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5739.624776}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "short_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 278, "engine_cache_reuse_calls": 36463, "engine_peak_bytes": 5335820288, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 9533, 14, 9217, 14, 17, 10, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 48, "kv_cache_byte_utilization": 0.328972868, "loaded_tensors": 339, "measured_d2d_bytes": 1070702592, "measured_d2d_calls": 5616, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 67104, "measured_h2d_calls": 57, "measured_ms": 931.845904, "measured_tokens": 240, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 155746304, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 1, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [189.523255, 189.708983, 189.999303, 190.136192, 189.849331, 190.270837, 309.411871, 309.621739], "request_completion_p50_ms": 190.0677475, "request_completion_p95_ms": 309.5482852, "request_count": 8, "request_ttft_ms": [84.654809, 84.652431, 84.649538, 84.646449, 84.644422, 84.64105, 84.637952, 84.633571], "request_ttft_p50_ms": 84.6454355, "request_ttft_p95_ms": 84.6539767, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15677142324983508573, "tokens_per_second": 257.553313235, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5917.505798}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "short_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 278, "engine_cache_reuse_calls": 36463, "engine_peak_bytes": 5335820288, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 9533, 14, 9217, 14, 17, 10, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 48, "kv_cache_byte_utilization": 0.328972868, "loaded_tensors": 339, "measured_d2d_bytes": 1070702592, "measured_d2d_calls": 5616, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 67104, "measured_h2d_calls": 57, "measured_ms": 936.086311, "measured_tokens": 240, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 155746304, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 2, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [190.039528, 190.377287, 190.772783, 190.938581, 190.57309, 191.098027, 310.839141, 311.153397], "request_completion_p50_ms": 190.855682, "request_completion_p95_ms": 311.0434074, "request_count": 8, "request_ttft_ms": [85.038833, 85.025008, 85.020798, 85.015788, 85.01351, 85.00867, 85.005299, 85.000181], "request_ttft_p50_ms": 85.014649, "request_ttft_p95_ms": 85.03399425, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15677142324983508573, "tokens_per_second": 256.386614332, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5948.210526}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "short_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 278, "engine_cache_reuse_calls": 36463, "engine_peak_bytes": 5335820288, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 9533, 14, 9217, 14, 17, 10, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 48, "kv_cache_byte_utilization": 0.328972868, "loaded_tensors": 339, "measured_d2d_bytes": 1070702592, "measured_d2d_calls": 5616, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 67104, "measured_h2d_calls": 57, "measured_ms": 936.065538, "measured_tokens": 240, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 155746304, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 3, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [189.536839, 189.868102, 190.270036, 190.426788, 190.090338, 190.584646, 310.073397, 310.388842], "request_completion_p50_ms": 190.348412, "request_completion_p95_ms": 310.27843625, "request_count": 8, "request_ttft_ms": [84.909696, 84.906682, 84.901298, 84.896943, 84.895202, 84.891525, 84.888968, 84.88506], "request_ttft_p50_ms": 84.8960725, "request_ttft_p95_ms": 84.9086411, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15677142324983508573, "tokens_per_second": 256.392304018, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5810.82736}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 29, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "short_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 446, "engine_cache_reuse_calls": 62986, "engine_peak_bytes": 5158742016, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 9533, 14, 9217, 14, 17, 10, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.425116099, "loaded_tensors": 339, "measured_d2d_bytes": 1070702592, "measured_d2d_calls": 5616, "measured_d2h_bytes": 1536, "measured_d2h_calls": 96, "measured_h2d_bytes": 67104, "measured_h2d_calls": 102, "measured_ms": 1276.946623, "measured_tokens": 240, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 125984768, "positions_aware_batch_decode_calls": 29, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 1, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 0, 0, 1, 1], "request_completion_ms": [232.957388, 233.095862, 233.385736, 424.157678, 233.224154, 424.367768, 424.853826, 425.024841], "request_completion_p50_ms": 328.771707, "request_completion_p95_ms": 424.96498575, "request_count": 8, "request_ttft_ms": [24.625524, 24.623042, 24.617817, 265.299503, 24.609579, 265.291229, 82.481723, 82.473014], "request_ttft_p50_ms": 53.549269, "request_ttft_p95_ms": 265.2966071, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15677142324983508573, "tokens_per_second": 187.948341518, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 6180.130417}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 29, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "short_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 446, "engine_cache_reuse_calls": 62986, "engine_peak_bytes": 5158742016, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 9533, 14, 9217, 14, 17, 10, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.425116099, "loaded_tensors": 339, "measured_d2d_bytes": 1070702592, "measured_d2d_calls": 5616, "measured_d2h_bytes": 1536, "measured_d2h_calls": 96, "measured_h2d_bytes": 67104, "measured_h2d_calls": 102, "measured_ms": 1284.916554, "measured_tokens": 240, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 125984768, "positions_aware_batch_decode_calls": 29, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 2, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 0, 0, 1, 1], "request_completion_ms": [233.758687, 233.98239, 234.578053, 425.361711, 234.18744, 425.59815, 426.301357, 426.605794], "request_completion_p50_ms": 329.969882, "request_completion_p95_ms": 426.49924105, "request_count": 8, "request_ttft_ms": [24.833868, 24.827697, 24.823856, 266.813604, 24.797525, 266.801342, 82.621808, 82.613233], "request_ttft_p50_ms": 53.7235505, "request_ttft_p95_ms": 266.8093123, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15677142324983508573, "tokens_per_second": 186.782557399, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5965.388829}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 29, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "short_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 446, "engine_cache_reuse_calls": 62986, "engine_peak_bytes": 5158742016, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 9533, 14, 9217, 14, 17, 10, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.425116099, "loaded_tensors": 339, "measured_d2d_bytes": 1070702592, "measured_d2d_calls": 5616, "measured_d2h_bytes": 1536, "measured_d2h_calls": 96, "measured_h2d_bytes": 67104, "measured_h2d_calls": 102, "measured_ms": 1285.850898, "measured_tokens": 240, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 8, 8, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 125984768, "positions_aware_batch_decode_calls": 29, "post_run_gpu_state": {"gpu_use_percent": 2, "physical_gpu_index": 2, "vram_percent": 1}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 3, "prompt_lengths": [256, 256, 512, 512, 256, 512, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 0, 0, 1, 1], "request_completion_ms": [234.475325, 234.656384, 235.179377, 426.984849, 234.816858, 427.311489, 428.001394, 428.302423], "request_completion_p50_ms": 331.082113, "request_completion_p95_ms": 428.19706285, "request_count": 8, "request_ttft_ms": [24.763365, 24.760286, 24.754584, 267.536569, 24.745607, 267.523431, 83.075972, 83.064769], "request_ttft_p50_ms": 53.914067, "request_ttft_p95_ms": 267.5319707, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.625, "status": "pass", "steps": 3, "token_checksum": 15677142324983508573, "tokens_per_second": 186.646834694, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 6077.502577}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "long_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 272, "engine_cache_reuse_calls": 36469, "engine_peak_bytes": 6074951680, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [151643, 151646, 271, 151649, 271, 17, 488, 220], [10, 17, 30, 151643, 151646, 271, 785, 2629, 315, 220, 17, 323, 220, 17, 374, 220], [17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 16819, 17767, 19], [73594, 151643, 151646, 271, 17, 10, 17, 16819, 220, 19, 382, 73594, 151643, 151646, 271, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271, 2, 151643, 151646, 271, 2, 4710, 2, 4710], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 16, "kv_cache_byte_utilization": 0.608042636, "loaded_tensors": 339, "measured_d2d_bytes": 1921769472, "measured_d2d_calls": 5712, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 123552, "measured_h2d_calls": 57, "measured_ms": 1127.503702, "measured_tokens": 336, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 287866880, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 2, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 1, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [250.458223, 250.763427, 372.962916, 373.244414, 373.617918, 373.777281, 373.427118, 373.946695], "request_completion_p50_ms": 373.335766, "request_completion_p95_ms": 373.8874001, "request_count": 8, "request_ttft_ms": [143.760236, 143.757385, 143.753398, 143.749328, 143.74526, 143.739804, 143.737115, 143.733305], "request_ttft_p50_ms": 143.747294, "request_ttft_p95_ms": 143.75923815, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.875, "status": "pass", "steps": 3, "token_checksum": 11073511144039895589, "tokens_per_second": 298.003456134, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5961.060427}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "long_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 272, "engine_cache_reuse_calls": 36469, "engine_peak_bytes": 6074951680, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [151643, 151646, 271, 151649, 271, 17, 488, 220], [10, 17, 30, 151643, 151646, 271, 785, 2629, 315, 220, 17, 323, 220, 17, 374, 220], [17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 16819, 17767, 19], [73594, 151643, 151646, 271, 17, 10, 17, 16819, 220, 19, 382, 73594, 151643, 151646, 271, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271, 2, 151643, 151646, 271, 2, 4710, 2, 4710], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 16, "kv_cache_byte_utilization": 0.608042636, "loaded_tensors": 339, "measured_d2d_bytes": 1921769472, "measured_d2d_calls": 5712, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 123552, "measured_h2d_calls": 57, "measured_ms": 1131.799249, "measured_tokens": 336, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 287866880, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 2, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [251.863304, 252.160502, 374.372165, 374.66654, 375.091335, 375.244066, 374.88525, 375.401729], "request_completion_p50_ms": 374.775895, "request_completion_p95_ms": 375.34654695, "request_count": 8, "request_ttft_ms": [145.29097, 145.286526, 145.28293, 145.278532, 145.274575, 145.268517, 145.265957, 145.262116], "request_ttft_p50_ms": 145.2765535, "request_ttft_p95_ms": 145.2894146, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.875, "status": "pass", "steps": 3, "token_checksum": 11073511144039895589, "tokens_per_second": 296.8724359, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5893.464368}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 15, "batched_prefill_calls": 2, "batched_prefill_rows": 6, "bucketed_cache": false, "case": "long_heavy_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 272, "engine_cache_reuse_calls": 36469, "engine_peak_bytes": 6074951680, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [151643, 151646, 271, 151649, 271, 17, 488, 220], [10, 17, 30, 151643, 151646, 271, 785, 2629, 315, 220, 17, 323, 220, 17, 374, 220], [17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 16819, 17767, 19], [73594, 151643, 151646, 271, 17, 10, 17, 16819, 220, 19, 382, 73594, 151643, 151646, 271, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271, 2, 151643, 151646, 271, 2, 4710, 2, 4710], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 16, "kv_cache_byte_utilization": 0.608042636, "loaded_tensors": 339, "measured_d2d_bytes": 1921769472, "measured_d2d_calls": 5712, "measured_d2h_bytes": 1536, "measured_d2h_calls": 48, "measured_h2d_bytes": 123552, "measured_h2d_calls": 57, "measured_ms": 1123.357909, "measured_tokens": 336, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 287866880, "positions_aware_batch_decode_calls": 15, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 3, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [251.450173, 251.674069, 373.492803, 373.728078, 374.054418, 374.213756, 373.884728, 374.367447], "request_completion_p50_ms": 373.806403, "request_completion_p95_ms": 374.31365515, "request_count": 8, "request_ttft_ms": [145.267721, 145.265079, 145.261959, 145.258784, 145.254714, 145.249993, 145.247499, 145.2437], "request_ttft_p50_ms": 145.256749, "request_ttft_p95_ms": 145.2667963, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 16, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.875, "status": "pass", "steps": 3, "token_checksum": 11073511144039895589, "tokens_per_second": 299.103248669, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5720.954964}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 37, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "long_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 458, "engine_cache_reuse_calls": 76534, "engine_peak_bytes": 5528307712, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [151643, 151646, 271, 151649, 271, 17, 488, 220], [10, 17, 30, 151643, 151646, 271, 785, 2629, 315, 220, 17, 323, 220, 17, 374, 220], [17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 16819, 17767, 19], [73594, 151643, 151646, 271, 17, 10, 17, 16819, 220, 19, 382, 73594, 151643, 151646, 271, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271, 2, 151643, 151646, 271, 2, 4710, 2, 4710], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.57372291, "loaded_tensors": 339, "measured_d2d_bytes": 1921769472, "measured_d2d_calls": 5712, "measured_d2h_bytes": 1920, "measured_d2h_calls": 120, "measured_h2d_bytes": 123552, "measured_h2d_calls": 126, "measured_ms": 1977.095178, "measured_tokens": 336, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 170024960, "positions_aware_batch_decode_calls": 37, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 1}, "prefill_batch_calls": 5, "process_run": 1, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 1, 1, 1, 1, 1, 1], "request_completion_ms": [250.187226, 250.393974, 370.143765, 370.355559, 370.768829, 655.767866, 370.542759, 655.975467], "request_completion_p50_ms": 370.449159, "request_completion_p95_ms": 655.90280665, "request_count": 8, "request_ttft_ms": [19.892251, 19.883442, 99.211537, 99.206586, 99.196364, 431.531755, 99.181971, 431.517584], "request_ttft_p50_ms": 99.201475, "request_ttft_p95_ms": 431.52679515, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 32, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.4375, "status": "pass", "steps": 3, "token_checksum": 11073511144039895589, "tokens_per_second": 169.946294816, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 6162.937795}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 37, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "long_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 458, "engine_cache_reuse_calls": 76534, "engine_peak_bytes": 5528307712, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [151643, 151646, 271, 151649, 271, 17, 488, 220], [10, 17, 30, 151643, 151646, 271, 785, 2629, 315, 220, 17, 323, 220, 17, 374, 220], [17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 16819, 17767, 19], [73594, 151643, 151646, 271, 17, 10, 17, 16819, 220, 19, 382, 73594, 151643, 151646, 271, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271, 2, 151643, 151646, 271, 2, 4710, 2, 4710], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.57372291, "loaded_tensors": 339, "measured_d2d_bytes": 1921769472, "measured_d2d_calls": 5712, "measured_d2h_bytes": 1920, "measured_d2h_calls": 120, "measured_h2d_bytes": 123552, "measured_h2d_calls": 126, "measured_ms": 1972.748382, "measured_tokens": 336, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 170024960, "positions_aware_batch_decode_calls": 37, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 2, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 1, 1, 1, 1, 1, 1], "request_completion_ms": [250.533275, 250.756682, 370.681306, 370.893792, 371.317595, 655.212093, 371.087452, 655.404211], "request_completion_p50_ms": 370.990622, "request_completion_p95_ms": 655.3369697, "request_count": 8, "request_ttft_ms": [19.468328, 19.464596, 99.727492, 99.72322, 99.715506, 432.017714, 99.69626, 431.998365], "request_ttft_p50_ms": 99.719363, "request_ttft_p95_ms": 432.01094185, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 32, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.4375, "status": "pass", "steps": 3, "token_checksum": 11073511144039895589, "tokens_per_second": 170.320758119, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 6279.36111}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 0, 0, 0, 0], "batch_decode_calls": 37, "batched_prefill_calls": 2, "batched_prefill_rows": 5, "bucketed_cache": true, "case": "long_heavy_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 458, "engine_cache_reuse_calls": 76534, "engine_peak_bytes": 5528307712, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [151643, 151646, 271, 151649, 271, 17, 488, 220], [10, 17, 30, 151643, 151646, 271, 785, 2629, 315, 220, 17, 323, 220, 17, 374, 220], [17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 16819, 17767, 19], [73594, 151643, 151646, 271, 17, 10, 17, 16819, 220, 19, 382, 73594, 151643, 151646, 271, 17], [38858, 79075, 90, 19, 92, 151643, 151646, 271, 2, 151643, 151646, 271, 2, 4710, 2, 4710], [151648, 198, 151649, 271, 334, 3838, 374, 220, 17, 10, 17, 30, 56177, 1249, 11625, 279], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 44, "kv_cache_byte_utilization": 0.57372291, "loaded_tensors": 339, "measured_d2d_bytes": 1921769472, "measured_d2d_calls": 5712, "measured_d2h_bytes": 1920, "measured_d2h_calls": 120, "measured_h2d_bytes": 123552, "measured_h2d_calls": 126, "measured_ms": 1970.561647, "measured_tokens": 336, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 16, 16, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 170024960, "positions_aware_batch_decode_calls": 37, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 5, "process_run": 3, "prompt_lengths": [256, 512, 1024, 1024, 2048, 2048, 1024, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 1, 1, 1, 1, 1, 1], "request_completion_ms": [249.143009, 249.303096, 368.549553, 368.761792, 369.184069, 653.076837, 368.955151, 653.26879], "request_completion_p50_ms": 368.8584715, "request_completion_p95_ms": 653.20160645, "request_count": 8, "request_ttft_ms": [19.35255, 19.348621, 98.790734, 98.785855, 98.777768, 430.040144, 98.76458, 430.026195], "request_ttft_p50_ms": 98.7818115, "request_ttft_p95_ms": 430.03526185, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 32, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 2, "slot_utilization": 0.4375, "status": "pass", "steps": 3, "token_checksum": 11073511144039895589, "tokens_per_second": 170.509763301, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 6474.470108}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 19, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": false, "case": "delayed_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 215, "engine_cache_reuse_calls": 43312, "engine_peak_bytes": 5705385984, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 151643, 151646, 271, 785, 23606, 220, 17, 488, 220, 17, 16819, 220, 19, 374, 264], [151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 374, 30339, 323, 3059, 304], [3838, 374, 220, 17, 10, 17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 64, "kv_cache_byte_utilization": 0.46753876, "loaded_tensors": 339, "measured_d2d_bytes": 1496236032, "measured_d2d_calls": 5664, "measured_d2h_bytes": 1920, "measured_d2h_calls": 60, "measured_h2d_bytes": 95328, "measured_h2d_calls": 69, "measured_ms": 1128.391962, "measured_tokens": 288, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 221347840, "positions_aware_batch_decode_calls": 19, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 1, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [193.893722, 194.070816, 194.223699, 194.370746, 320.736911, 320.952014, 321.095838, 321.238917], "request_completion_p50_ms": 257.5538285, "request_completion_p95_ms": 321.18883935, "request_count": 8, "request_ttft_ms": [27.512919, 27.510903, 27.50361, 27.493818, 95.387136, 95.382426, 95.37761, 95.372748], "request_ttft_p50_ms": 61.4428335, "request_ttft_p95_ms": 95.3854875, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 1177940916830374468, "tokens_per_second": 255.230460424, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 6005.396478}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 19, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": false, "case": "delayed_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 215, "engine_cache_reuse_calls": 43312, "engine_peak_bytes": 5705385984, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 151643, 151646, 271, 785, 23606, 220, 17, 488, 220, 17, 16819, 220, 19, 374, 264], [151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 374, 30339, 323, 3059, 304], [3838, 374, 220, 17, 10, 17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 64, "kv_cache_byte_utilization": 0.46753876, "loaded_tensors": 339, "measured_d2d_bytes": 1496236032, "measured_d2d_calls": 5664, "measured_d2h_bytes": 1920, "measured_d2h_calls": 60, "measured_h2d_bytes": 95328, "measured_h2d_calls": 69, "measured_ms": 1129.167446, "measured_tokens": 288, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 221347840, "positions_aware_batch_decode_calls": 19, "post_run_gpu_state": {"gpu_use_percent": 5, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 2, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [193.459483, 193.684555, 193.88836, 194.09408, 320.568259, 320.807062, 321.022731, 321.229142], "request_completion_p50_ms": 257.3311695, "request_completion_p95_ms": 321.15689815, "request_count": 8, "request_ttft_ms": [27.506556, 27.504567, 27.49658, 27.494041, 95.103541, 95.098681, 95.093708, 95.089091], "request_ttft_p50_ms": 61.2978235, "request_ttft_p95_ms": 95.10184, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 1177940916830374468, "tokens_per_second": 255.055174518, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5862.038794}
{"allocated_cache_bytes": 473432064, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 19, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": false, "case": "delayed_uniform", "continuous_cache_buckets": [], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 215, "engine_cache_reuse_calls": 43312, "engine_peak_bytes": 5705385984, "expected_cache_bytes": 473432064, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 151643, 151646, 271, 785, 23606, 220, 17, 488, 220, 17, 16819, 220, 19, 374, 264], [151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 374, 30339, 323, 3059, 304], [3838, 374, 220, 17, 10, 17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 64, "kv_cache_byte_utilization": 0.46753876, "loaded_tensors": 339, "measured_d2d_bytes": 1496236032, "measured_d2d_calls": 5664, "measured_d2h_bytes": 1920, "measured_d2h_calls": 60, "measured_h2d_bytes": 95328, "measured_h2d_calls": 69, "measured_ms": 1132.490598, "measured_tokens": 288, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 221347840, "positions_aware_batch_decode_calls": 19, "post_run_gpu_state": {"gpu_use_percent": 4, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 3, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [], "request_completion_ms": [194.737863, 195.06315, 195.245086, 195.406369, 321.788134, 322.09854, 322.262942, 322.417384], "request_completion_p50_ms": 258.5972515, "request_completion_p95_ms": 322.3633293, "request_count": 8, "request_ttft_ms": [27.583098, 27.580842, 27.576875, 27.572805, 95.846516, 95.83957, 95.833367, 95.827401], "request_ttft_p50_ms": 61.7052495, "request_ttft_p95_ms": 95.8440849, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 1177940916830374468, "tokens_per_second": 254.306747013, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5845.875058}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 22, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": true, "case": "delayed_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 315, "engine_cache_reuse_calls": 48471, "engine_peak_bytes": 5528307712, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 151643, 151646, 271, 785, 23606, 220, 17, 488, 220, 17, 16819, 220, 19, 374, 264], [151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 374, 30339, 323, 3059, 304], [3838, 374, 220, 17, 10, 17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 0, "kv_cache_byte_utilization": 0.746904025, "loaded_tensors": 339, "measured_d2d_bytes": 1496236032, "measured_d2d_calls": 5664, "measured_d2h_bytes": 1152, "measured_d2h_calls": 72, "measured_h2d_bytes": 95328, "measured_h2d_calls": 78, "measured_ms": 1200.045549, "measured_tokens": 288, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 221347840, "positions_aware_batch_decode_calls": 22, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 1, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 1, 1, 1, 1], "request_completion_ms": [216.734045, 217.045387, 217.206709, 217.321071, 343.136438, 343.41255, 343.578625, 343.742002], "request_completion_p50_ms": 280.2287545, "request_completion_p95_ms": 343.68482005, "request_count": 8, "request_ttft_ms": [27.717315, 27.704399, 27.699278, 27.652184, 102.571374, 102.56615, 102.556314, 102.547209], "request_ttft_p50_ms": 65.132262, "request_ttft_p95_ms": 102.5695456, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 1177940916830374468, "tokens_per_second": 239.990890546, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 6023.727047}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 22, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": true, "case": "delayed_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 315, "engine_cache_reuse_calls": 48471, "engine_peak_bytes": 5528307712, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 151643, 151646, 271, 785, 23606, 220, 17, 488, 220, 17, 16819, 220, 19, 374, 264], [151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 374, 30339, 323, 3059, 304], [3838, 374, 220, 17, 10, 17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 0, "kv_cache_byte_utilization": 0.746904025, "loaded_tensors": 339, "measured_d2d_bytes": 1496236032, "measured_d2d_calls": 5664, "measured_d2h_bytes": 1152, "measured_d2h_calls": 72, "measured_h2d_bytes": 95328, "measured_h2d_calls": 78, "measured_ms": 1196.42826, "measured_tokens": 288, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 221347840, "positions_aware_batch_decode_calls": 22, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 2, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 1, 1, 1, 1], "request_completion_ms": [215.935429, 216.115035, 216.246104, 216.382079, 342.444587, 342.628902, 342.76638, 342.899385], "request_completion_p50_ms": 279.413333, "request_completion_p95_ms": 342.85283325, "request_count": 8, "request_ttft_ms": [27.393561, 27.389547, 27.38197, 27.379201, 102.544425, 102.539275, 102.531212, 102.52377], "request_ttft_p50_ms": 64.9586655, "request_ttft_p95_ms": 102.5426225, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 1177940916830374468, "tokens_per_second": 240.716480569, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5937.288436}
{"allocated_cache_bytes": 296353792, "architecture": "gfx942:sramecc+:xnack-", "arrival_steps": [0, 0, 0, 0, 4, 4, 4, 4], "batch_decode_calls": 22, "batched_prefill_calls": 4, "batched_prefill_rows": 8, "bucketed_cache": true, "case": "delayed_bucket2", "continuous_cache_buckets": [{"max_sequence_length": 520, "max_slots": 4}, {"max_sequence_length": 2064, "max_slots": 4}], "continuous_diagnostics": false, "continuous_prefill_batch": true, "continuous_slots": 8, "deterministic_across_steps": true, "device": "hip:0", "device_name": "AMD Instinct MI300X VF", "device_total_bytes": 205822885888, "engine_backend_allocation_calls": 315, "engine_cache_reuse_calls": 48471, "engine_peak_bytes": 5528307712, "expected_cache_bytes": 296353792, "generated_tokens": [[220, 17, 10, 17, 30, 151643, 151646, 271], [17, 10, 17, 30, 151643, 151646, 151646, 271], [151648, 198, 151649, 271, 17, 488, 220, 17], [198, 151649, 271, 17, 10, 17, 10236, 255], [30, 151643, 151646, 271, 785, 23606, 220, 17, 488, 220, 17, 16819, 220, 19, 374, 264], [151643, 151646, 271, 785, 23606, 17767, 17, 488, 220, 17, 57758, 374, 30339, 323, 3059, 304], [3838, 374, 220, 17, 10, 17, 30, 151643, 151646, 271, 785, 23606, 17767, 17, 488, 220], [374, 220, 17, 10, 17, 30, 151643, 151646, 271, 151649, 271, 17, 488, 220, 17, 284]], "inactive_rows_skipped": 0, "kv_cache_byte_utilization": 0.746904025, "loaded_tensors": 339, "measured_d2d_bytes": 1496236032, "measured_d2d_calls": 5664, "measured_d2h_bytes": 1152, "measured_d2h_calls": 72, "measured_h2d_bytes": 95328, "measured_h2d_calls": 78, "measured_ms": 1201.065372, "measured_tokens": 288, "model": "deepseek-r1-distill-qwen-1.5b", "new_token_lengths": [8, 8, 8, 8, 16, 16, 16, 16], "parameter_count": 1777088000, "peak_active_cache_bytes": 221347840, "positions_aware_batch_decode_calls": 22, "post_run_gpu_state": {"gpu_use_percent": 3, "physical_gpu_index": 2, "vram_percent": 2}, "pre_run_gpu_state": {"gpu_use_percent": 1, "physical_gpu_index": 2, "vram_percent": 0}, "prefill_batch_calls": 4, "process_run": 3, "prompt_lengths": [256, 256, 512, 512, 1024, 1024, 2048, 2048], "prompt_offsets": [0, 1, 2, 3, 4, 5, 6, 7], "record_type": "official_continuous_serving_measurement", "request_bucket_indices": [0, 0, 0, 0, 1, 1, 1, 1], "request_completion_ms": [216.811162, 217.158376, 217.345699, 217.545756, 343.240587, 343.580084, 343.827924, 344.005668], "request_completion_p50_ms": 280.3931715, "request_completion_p95_ms": 343.9434576, "request_count": 8, "request_ttft_ms": [27.713602, 27.710423, 27.701708, 27.695615, 102.637204, 102.63187, 102.622796, 102.613247], "request_ttft_p50_ms": 65.1634245, "request_ttft_p95_ms": 102.6353371, "resident_weight_bytes": 4487960576, "revision": "ad9f0ae0864d7fbcd1cd905e3c6c5b069cc8b562", "row_prefill_calls": 8, "scheduler_steps": 20, "schema_version": 1, "selection_diagnostic_count": 0, "selection_diagnostics": [], "slot_admissions": 8, "slot_refills": 0, "slot_utilization": 0.6, "status": "pass", "steps": 3, "token_checksum": 1177940916830374468, "tokens_per_second": 239.787114602, "uniform_batch_decode_calls": 0, "warmup": 1, "warmup_ms": 5826.707116}