{ "schema_version": 1, "generated": "2026-09-07", "hardware": { "gpu": "NVIDIA GeForce RTX 5090", "arch": "Blackwell sm120", "vram_total_mib": 32607, "vram_usable_gib": 31.4, "vram_baseline_mib": 950, "vram_baseline_note": "desktop compositor (kwin_wayland + Xwayland)", "driver": "610.57.04", "host_ram_gib": 30, "cpu_threads": 16, "os": "Arch Linux", "runtime": "podman 6.1.0, CDI nvidia.com/gpu=all" }, "software": { "image": "docker.io/vllm/vllm-openai:v0.28.0", "vllm": "0.28.0", "torch": "2.13.0+cu130", "transformers": "5.15.1" }, "method": { "tools": "six fixed requests through the OpenAI-compatible endpoint with two tools declared; a check passes when the server returns what an agent framework needs (structured tool_calls, a used tool result, two calls in one turn, no false call, a streamed call, an exact path argument)", "loop": "a growing agent transcript replayed turn by turn; warm lets the server reuse its prefix cache, cold salts the system prompt so every turn is prefilled from scratch; output length is forced with ignore_eos", "shapes": "synthetic = a 600-word system prompt plus sixteen ~1.2K-token tool results, 128 tokens out per turn; typical and heavy = real Claude Code sessions from usage.json replayed at their recorded context and output sizes, turns that do not fit the server's window skipped and counted", "warmup": "one short turn before each configuration's runs, so compilation is not charged to turn 1" }, "toolcalls": [ { "label": "qwen3.6-35b-a3b", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-06", "vllm_version": "0.28.0", "passed": 6, "total": 6, "reasoning_field_seen": true, "checks": [ { "name": "single_call", "pass": true, "detail": "get_weather({\"city\": \"Helsinki\"})", "finish": "tool_calls", "output_tokens": 104, "latency_s": 0.427, "reasoning_tokens_seen": true }, { "name": "tool_result_roundtrip", "pass": true, "detail": "answer uses the result (78 chars)", "finish": "stop", "output_tokens": 24, "latency_s": 0.146, "reasoning_tokens_seen": false }, { "name": "parallel_calls", "pass": true, "detail": "2 calls in one turn: ['helsinki', 'oslo']", "finish": "tool_calls", "output_tokens": 147, "latency_s": 0.577, "reasoning_tokens_seen": true }, { "name": "no_false_call", "pass": true, "detail": "plain answer 'hello'", "finish": "stop", "output_tokens": 82, "latency_s": 0.346, "reasoning_tokens_seen": true }, { "name": "streaming_call", "pass": true, "detail": "get_weather({\"city\": \"Helsinki\"})", "finish": "tool_calls", "output_tokens": 104, "latency_s": 0.423, "reasoning_tokens_seen": true }, { "name": "path_argument", "pass": true, "detail": "read_file(path) exact", "finish": "tool_calls", "output_tokens": 96, "latency_s": 0.396, "reasoning_tokens_seen": true } ], "dimensions": { "model": "Qwen3.6-35B-A3B", "seqs": "32", "graphs": "on" } }, { "label": "nemotron-3.5-lightning", "model": "Nemotron-3.5-Lightning-30B-A3B", "generated": "2026-09-06", "vllm_version": "0.28.0", "passed": 5, "total": 6, "reasoning_field_seen": true, "checks": [ { "name": "single_call", "pass": true, "detail": "get_weather({\"city\": \"Helsinki\", \"unit\": \"celsius\"})", "finish": "tool_calls", "output_tokens": 75, "latency_s": 0.657, "reasoning_tokens_seen": true }, { "name": "tool_result_roundtrip", "pass": true, "detail": "answer uses the result (102 chars)", "finish": "stop", "output_tokens": 56, "latency_s": 0.215, "reasoning_tokens_seen": true }, { "name": "parallel_calls", "pass": false, "detail": "1 call(s): ['helsinki'] (finish=tool_calls)", "finish": "tool_calls", "output_tokens": 68, "latency_s": 0.245, "reasoning_tokens_seen": true }, { "name": "no_false_call", "pass": true, "detail": "plain answer 'hello'", "finish": "stop", "output_tokens": 43, "latency_s": 0.18, "reasoning_tokens_seen": true }, { "name": "streaming_call", "pass": true, "detail": "get_weather({\"city\": \"Helsinki\", \"unit\": \"celsius\"})", "finish": "tool_calls", "output_tokens": 75, "latency_s": 0.246, "reasoning_tokens_seen": true }, { "name": "path_argument", "pass": true, "detail": "read_file(path) exact", "finish": "tool_calls", "output_tokens": 51, "latency_s": 0.201, "reasoning_tokens_seen": true } ], "dimensions": { "model": "Nemotron-3.5-Lightning-30B-A3B", "seqs": "32", "graphs": "on" } }, { "label": "qwen3.8-27b text-only, CUDA graphs", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "passed": 6, "total": 6, "reasoning_field_seen": true, "checks": [ { "name": "single_call", "pass": true, "detail": "get_weather({\"city\": \"Helsinki\"})", "finish": "tool_calls", "output_tokens": 73, "latency_s": 1.321, "reasoning_tokens_seen": true }, { "name": "tool_result_roundtrip", "pass": true, "detail": "answer uses the result (246 chars)", "finish": "stop", "output_tokens": 143, "latency_s": 2.352, "reasoning_tokens_seen": true }, { "name": "parallel_calls", "pass": true, "detail": "2 calls in one turn: ['helsinki', 'oslo']", "finish": "tool_calls", "output_tokens": 130, "latency_s": 2.149, "reasoning_tokens_seen": true }, { "name": "no_false_call", "pass": true, "detail": "plain answer 'hello'", "finish": "stop", "output_tokens": 42, "latency_s": 0.738, "reasoning_tokens_seen": true }, { "name": "streaming_call", "pass": true, "detail": "get_weather({\"city\": \"Helsinki\"})", "finish": "tool_calls", "output_tokens": 73, "latency_s": 1.246, "reasoning_tokens_seen": true }, { "name": "path_argument", "pass": true, "detail": "read_file(path) exact", "finish": "tool_calls", "output_tokens": 63, "latency_s": 1.078, "reasoning_tokens_seen": true } ], "dimensions": { "model": "Qwen3.8-27B", "seqs": "32", "vision": "off", "graphs": "on" } }, { "label": "qwen3.8-27b default", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "passed": 6, "total": 6, "reasoning_field_seen": true, "checks": [ { "name": "single_call", "pass": true, "detail": "get_weather({\"city\": \"Helsinki\", \"unit\": \"celsius\"})", "finish": "tool_calls", "output_tokens": 84, "latency_s": 7.163, "reasoning_tokens_seen": true }, { "name": "tool_result_roundtrip", "pass": true, "detail": "answer uses the result (222 chars)", "finish": "stop", "output_tokens": 139, "latency_s": 5.671, "reasoning_tokens_seen": true }, { "name": "parallel_calls", "pass": true, "detail": "2 calls in one turn: ['helsinki', 'oslo']", "finish": "tool_calls", "output_tokens": 125, "latency_s": 5.083, "reasoning_tokens_seen": true }, { "name": "no_false_call", "pass": true, "detail": "plain answer 'hello'", "finish": "stop", "output_tokens": 41, "latency_s": 1.71, "reasoning_tokens_seen": true }, { "name": "streaming_call", "pass": true, "detail": "get_weather({\"city\": \"Helsinki\", \"unit\": \"celsius\"})", "finish": "tool_calls", "output_tokens": 84, "latency_s": 3.458, "reasoning_tokens_seen": true }, { "name": "path_argument", "pass": true, "detail": "read_file(path) exact", "finish": "tool_calls", "output_tokens": 61, "latency_s": 2.52, "reasoning_tokens_seen": true } ], "dimensions": { "model": "Qwen3.8-27B", "seqs": "8", "vision": "on", "graphs": "off" } } ], "loops": [ { "label": "qwen3.6-35b-a3b", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "turns": 16, "tool_words": 400, "system_words": 600, "output_tokens": 128, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 16, "turns_ok": 16, "status": "ok", "wall_s": 9.93, "ttft_first_turn_s": 0.072, "ttft_last_turn_s": 0.248, "ttft_mean_s": 0.179, "ttft_p95_s": 0.233, "context_tokens_last": 20644, "prompt_tokens_total": 183208, "turns_per_minute": 96.6, "per_turn": [ { "turn": 1, "context_tokens": 2306, "ttft_p50_s": 0.072, "ttft_max_s": 0.072, "decode_tok_s_per_stream": 289.8 }, { "turn": 2, "context_tokens": 3522, "ttft_p50_s": 0.135, "ttft_max_s": 0.135, "decode_tok_s_per_stream": 289.7 }, { "turn": 3, "context_tokens": 4738, "ttft_p50_s": 0.136, "ttft_max_s": 0.136, "decode_tok_s_per_stream": 289.8 }, { "turn": 4, "context_tokens": 5954, "ttft_p50_s": 0.143, "ttft_max_s": 0.143, "decode_tok_s_per_stream": 289.7 }, { "turn": 5, "context_tokens": 7170, "ttft_p50_s": 0.201, "ttft_max_s": 0.201, "decode_tok_s_per_stream": 289.1 }, { "turn": 6, "context_tokens": 8386, "ttft_p50_s": 0.156, "ttft_max_s": 0.156, "decode_tok_s_per_stream": 288.7 }, { "turn": 7, "context_tokens": 9602, "ttft_p50_s": 0.2, "ttft_max_s": 0.2, "decode_tok_s_per_stream": 288.4 }, { "turn": 8, "context_tokens": 10818, "ttft_p50_s": 0.173, "ttft_max_s": 0.173, "decode_tok_s_per_stream": 288.2 }, { "turn": 9, "context_tokens": 12034, "ttft_p50_s": 0.212, "ttft_max_s": 0.212, "decode_tok_s_per_stream": 287.5 }, { "turn": 10, "context_tokens": 13264, "ttft_p50_s": 0.206, "ttft_max_s": 0.206, "decode_tok_s_per_stream": 287.3 }, { "turn": 11, "context_tokens": 14494, "ttft_p50_s": 0.168, "ttft_max_s": 0.168, "decode_tok_s_per_stream": 287.9 }, { "turn": 12, "context_tokens": 15724, "ttft_p50_s": 0.216, "ttft_max_s": 0.216, "decode_tok_s_per_stream": 287.0 }, { "turn": 13, "context_tokens": 16954, "ttft_p50_s": 0.179, "ttft_max_s": 0.179, "decode_tok_s_per_stream": 287.5 }, { "turn": 14, "context_tokens": 18184, "ttft_p50_s": 0.228, "ttft_max_s": 0.228, "decode_tok_s_per_stream": 285.7 }, { "turn": 15, "context_tokens": 19414, "ttft_p50_s": 0.19, "ttft_max_s": 0.19, "decode_tok_s_per_stream": 285.7 }, { "turn": 16, "context_tokens": 20644, "ttft_p50_s": 0.248, "ttft_max_s": 0.248, "decode_tok_s_per_stream": 285.2 } ], "prefix_cache_hit_rate": 0.8077, "prefix_cache_queried_tokens": 183208 }, { "agents": 1, "mode": "cold", "turns_requested": 16, "turns_ok": 16, "status": "ok", "wall_s": 14.56, "ttft_first_turn_s": 0.132, "ttft_last_turn_s": 0.843, "ttft_mean_s": 0.468, "ttft_p95_s": 0.797, "context_tokens_last": 20651, "prompt_tokens_total": 183311, "turns_per_minute": 65.9, "per_turn": [ { "turn": 1, "context_tokens": 2312, "ttft_p50_s": 0.132, "ttft_max_s": 0.132, "decode_tok_s_per_stream": 289.1 }, { "turn": 2, "context_tokens": 3528, "ttft_p50_s": 0.194, "ttft_max_s": 0.194, "decode_tok_s_per_stream": 289.5 }, { "turn": 3, "context_tokens": 4744, "ttft_p50_s": 0.248, "ttft_max_s": 0.248, "decode_tok_s_per_stream": 289.3 }, { "turn": 4, "context_tokens": 5960, "ttft_p50_s": 0.258, "ttft_max_s": 0.258, "decode_tok_s_per_stream": 289.2 }, { "turn": 5, "context_tokens": 7176, "ttft_p50_s": 0.292, "ttft_max_s": 0.292, "decode_tok_s_per_stream": 288.3 }, { "turn": 6, "context_tokens": 8392, "ttft_p50_s": 0.32, "ttft_max_s": 0.32, "decode_tok_s_per_stream": 288.3 }, { "turn": 7, "context_tokens": 9608, "ttft_p50_s": 0.38, "ttft_max_s": 0.38, "decode_tok_s_per_stream": 288.2 }, { "turn": 8, "context_tokens": 10824, "ttft_p50_s": 0.419, "ttft_max_s": 0.419, "decode_tok_s_per_stream": 287.2 }, { "turn": 9, "context_tokens": 12040, "ttft_p50_s": 0.46, "ttft_max_s": 0.46, "decode_tok_s_per_stream": 287.6 }, { "turn": 10, "context_tokens": 13271, "ttft_p50_s": 0.536, "ttft_max_s": 0.536, "decode_tok_s_per_stream": 287.5 }, { "turn": 11, "context_tokens": 14501, "ttft_p50_s": 0.564, "ttft_max_s": 0.564, "decode_tok_s_per_stream": 287.4 }, { "turn": 12, "context_tokens": 15731, "ttft_p50_s": 0.641, "ttft_max_s": 0.641, "decode_tok_s_per_stream": 287.0 }, { "turn": 13, "context_tokens": 16961, "ttft_p50_s": 0.681, "ttft_max_s": 0.681, "decode_tok_s_per_stream": 287.0 }, { "turn": 14, "context_tokens": 18191, "ttft_p50_s": 0.738, "ttft_max_s": 0.738, "decode_tok_s_per_stream": 286.1 }, { "turn": 15, "context_tokens": 19421, "ttft_p50_s": 0.781, "ttft_max_s": 0.781, "decode_tok_s_per_stream": 285.8 }, { "turn": 16, "context_tokens": 20651, "ttft_p50_s": 0.843, "ttft_max_s": 0.843, "decode_tok_s_per_stream": 284.7 } ], "prefix_cache_hit_rate": 0.0119, "prefix_cache_queried_tokens": 183311 }, { "agents": 4, "mode": "warm", "turns_requested": 64, "turns_ok": 64, "status": "ok", "wall_s": 16.82, "ttft_first_turn_s": 0.374, "ttft_last_turn_s": 0.712, "ttft_mean_s": 0.426, "ttft_p95_s": 0.632, "context_tokens_last": 20644, "prompt_tokens_total": 732832, "turns_per_minute": 228.3, "per_turn": [ { "turn": 1, "context_tokens": 2306, "ttft_p50_s": 0.374, "ttft_max_s": 0.374, "decode_tok_s_per_stream": 210.2 }, { "turn": 2, "context_tokens": 3522, "ttft_p50_s": 0.236, "ttft_max_s": 0.237, "decode_tok_s_per_stream": 213.3 }, { "turn": 3, "context_tokens": 4738, "ttft_p50_s": 0.423, "ttft_max_s": 0.426, "decode_tok_s_per_stream": 212.2 }, { "turn": 4, "context_tokens": 5954, "ttft_p50_s": 0.278, "ttft_max_s": 0.279, "decode_tok_s_per_stream": 209.5 }, { "turn": 5, "context_tokens": 7170, "ttft_p50_s": 0.472, "ttft_max_s": 0.473, "decode_tok_s_per_stream": 212.3 }, { "turn": 6, "context_tokens": 8386, "ttft_p50_s": 0.312, "ttft_max_s": 0.314, "decode_tok_s_per_stream": 213.6 }, { "turn": 7, "context_tokens": 9602, "ttft_p50_s": 0.525, "ttft_max_s": 0.526, "decode_tok_s_per_stream": 213.4 }, { "turn": 8, "context_tokens": 10818, "ttft_p50_s": 0.373, "ttft_max_s": 0.374, "decode_tok_s_per_stream": 216.8 }, { "turn": 9, "context_tokens": 12034, "ttft_p50_s": 0.592, "ttft_max_s": 0.594, "decode_tok_s_per_stream": 215.5 }, { "turn": 10, "context_tokens": 13264, "ttft_p50_s": 0.493, "ttft_max_s": 0.494, "decode_tok_s_per_stream": 197.0 }, { "turn": 11, "context_tokens": 14494, "ttft_p50_s": 0.318, "ttft_max_s": 0.32, "decode_tok_s_per_stream": 211.3 }, { "turn": 12, "context_tokens": 15724, "ttft_p50_s": 0.562, "ttft_max_s": 0.564, "decode_tok_s_per_stream": 207.3 }, { "turn": 13, "context_tokens": 16954, "ttft_p50_s": 0.371, "ttft_max_s": 0.373, "decode_tok_s_per_stream": 203.8 }, { "turn": 14, "context_tokens": 18184, "ttft_p50_s": 0.63, "ttft_max_s": 0.632, "decode_tok_s_per_stream": 203.5 }, { "turn": 15, "context_tokens": 19414, "ttft_p50_s": 0.432, "ttft_max_s": 0.434, "decode_tok_s_per_stream": 207.9 }, { "turn": 16, "context_tokens": 20644, "ttft_p50_s": 0.712, "ttft_max_s": 0.714, "decode_tok_s_per_stream": 206.2 } ], "prefix_cache_hit_rate": 0.7958, "prefix_cache_queried_tokens": 732832 }, { "agents": 4, "mode": "cold", "turns_requested": 64, "turns_ok": 64, "status": "ok", "wall_s": 37.2, "ttft_first_turn_s": 0.377, "ttft_last_turn_s": 2.155, "ttft_mean_s": 1.229, "ttft_p95_s": 2.38, "context_tokens_last": 20651, "prompt_tokens_total": 733244, "turns_per_minute": 103.2, "per_turn": [ { "turn": 1, "context_tokens": 2312, "ttft_p50_s": 0.377, "ttft_max_s": 0.378, "decode_tok_s_per_stream": 214.2 }, { "turn": 2, "context_tokens": 3528, "ttft_p50_s": 0.487, "ttft_max_s": 0.487, "decode_tok_s_per_stream": 206.7 }, { "turn": 3, "context_tokens": 4744, "ttft_p50_s": 0.676, "ttft_max_s": 0.676, "decode_tok_s_per_stream": 203.0 }, { "turn": 4, "context_tokens": 5960, "ttft_p50_s": 0.767, "ttft_max_s": 0.794, "decode_tok_s_per_stream": 195.4 }, { "turn": 5, "context_tokens": 7176, "ttft_p50_s": 0.907, "ttft_max_s": 1.012, "decode_tok_s_per_stream": 182.2 }, { "turn": 6, "context_tokens": 8392, "ttft_p50_s": 1.072, "ttft_max_s": 1.195, "decode_tok_s_per_stream": 169.9 }, { "turn": 7, "context_tokens": 9608, "ttft_p50_s": 1.155, "ttft_max_s": 1.319, "decode_tok_s_per_stream": 153.7 }, { "turn": 8, "context_tokens": 10824, "ttft_p50_s": 1.197, "ttft_max_s": 1.563, "decode_tok_s_per_stream": 133.3 }, { "turn": 9, "context_tokens": 12040, "ttft_p50_s": 1.442, "ttft_max_s": 1.675, "decode_tok_s_per_stream": 140.5 }, { "turn": 10, "context_tokens": 13271, "ttft_p50_s": 1.5, "ttft_max_s": 1.907, "decode_tok_s_per_stream": 118.4 }, { "turn": 11, "context_tokens": 14501, "ttft_p50_s": 1.594, "ttft_max_s": 1.86, "decode_tok_s_per_stream": 116.2 }, { "turn": 12, "context_tokens": 15731, "ttft_p50_s": 1.786, "ttft_max_s": 2.086, "decode_tok_s_per_stream": 99.8 }, { "turn": 13, "context_tokens": 16961, "ttft_p50_s": 1.78, "ttft_max_s": 2.2, "decode_tok_s_per_stream": 81.8 }, { "turn": 14, "context_tokens": 18191, "ttft_p50_s": 1.904, "ttft_max_s": 2.282, "decode_tok_s_per_stream": 96.7 }, { "turn": 15, "context_tokens": 19421, "ttft_p50_s": 2.145, "ttft_max_s": 2.538, "decode_tok_s_per_stream": 77.8 }, { "turn": 16, "context_tokens": 20651, "ttft_p50_s": 2.155, "ttft_max_s": 2.623, "decode_tok_s_per_stream": 86.8 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 733244 } ], "dimensions": { "model": "Qwen3.6-35B-A3B", "seqs": "32", "graphs": "on" } }, { "label": "nemotron-3.5-lightning", "model": "Nemotron-3.5-Lightning-30B-A3B", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "turns": 16, "tool_words": 400, "system_words": 600, "output_tokens": 128, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 16, "turns_ok": 16, "status": "ok", "wall_s": 7.62, "ttft_first_turn_s": 0.144, "ttft_last_turn_s": 0.143, "ttft_mean_s": 0.156, "ttft_p95_s": 0.203, "context_tokens_last": 19174, "prompt_tokens_total": 170848, "turns_per_minute": 125.9, "per_turn": [ { "turn": 1, "context_tokens": 2231, "ttft_p50_s": 0.144, "ttft_max_s": 0.144, "decode_tok_s_per_stream": 417.2 }, { "turn": 2, "context_tokens": 3354, "ttft_p50_s": 0.122, "ttft_max_s": 0.122, "decode_tok_s_per_stream": 416.5 }, { "turn": 3, "context_tokens": 4477, "ttft_p50_s": 0.184, "ttft_max_s": 0.184, "decode_tok_s_per_stream": 416.5 }, { "turn": 4, "context_tokens": 5600, "ttft_p50_s": 0.112, "ttft_max_s": 0.112, "decode_tok_s_per_stream": 393.0 }, { "turn": 5, "context_tokens": 6723, "ttft_p50_s": 0.172, "ttft_max_s": 0.172, "decode_tok_s_per_stream": 392.0 }, { "turn": 6, "context_tokens": 7846, "ttft_p50_s": 0.121, "ttft_max_s": 0.121, "decode_tok_s_per_stream": 393.0 }, { "turn": 7, "context_tokens": 8969, "ttft_p50_s": 0.172, "ttft_max_s": 0.172, "decode_tok_s_per_stream": 392.5 }, { "turn": 8, "context_tokens": 10092, "ttft_p50_s": 0.126, "ttft_max_s": 0.126, "decode_tok_s_per_stream": 393.7 }, { "turn": 9, "context_tokens": 11215, "ttft_p50_s": 0.174, "ttft_max_s": 0.174, "decode_tok_s_per_stream": 392.2 }, { "turn": 10, "context_tokens": 12352, "ttft_p50_s": 0.131, "ttft_max_s": 0.131, "decode_tok_s_per_stream": 393.3 }, { "turn": 11, "context_tokens": 13489, "ttft_p50_s": 0.188, "ttft_max_s": 0.188, "decode_tok_s_per_stream": 393.3 }, { "turn": 12, "context_tokens": 14626, "ttft_p50_s": 0.139, "ttft_max_s": 0.139, "decode_tok_s_per_stream": 392.6 }, { "turn": 13, "context_tokens": 15763, "ttft_p50_s": 0.198, "ttft_max_s": 0.198, "decode_tok_s_per_stream": 391.4 }, { "turn": 14, "context_tokens": 16900, "ttft_p50_s": 0.15, "ttft_max_s": 0.15, "decode_tok_s_per_stream": 393.2 }, { "turn": 15, "context_tokens": 18037, "ttft_p50_s": 0.215, "ttft_max_s": 0.215, "decode_tok_s_per_stream": 392.1 }, { "turn": 16, "context_tokens": 19174, "ttft_p50_s": 0.143, "ttft_max_s": 0.143, "decode_tok_s_per_stream": 393.8 } ], "prefix_cache_hit_rate": 0.7972, "prefix_cache_queried_tokens": 170848 }, { "agents": 1, "mode": "cold", "turns_requested": 16, "turns_ok": 16, "status": "ok", "wall_s": 13.33, "ttft_first_turn_s": 0.179, "ttft_last_turn_s": 0.905, "ttft_mean_s": 0.514, "ttft_p95_s": 0.877, "context_tokens_last": 19181, "prompt_tokens_total": 170951, "turns_per_minute": 72.0, "per_turn": [ { "turn": 1, "context_tokens": 2237, "ttft_p50_s": 0.179, "ttft_max_s": 0.179, "decode_tok_s_per_stream": 416.9 }, { "turn": 2, "context_tokens": 3360, "ttft_p50_s": 0.189, "ttft_max_s": 0.189, "decode_tok_s_per_stream": 412.6 }, { "turn": 3, "context_tokens": 4483, "ttft_p50_s": 0.257, "ttft_max_s": 0.257, "decode_tok_s_per_stream": 415.4 }, { "turn": 4, "context_tokens": 5606, "ttft_p50_s": 0.283, "ttft_max_s": 0.283, "decode_tok_s_per_stream": 412.5 }, { "turn": 5, "context_tokens": 6729, "ttft_p50_s": 0.333, "ttft_max_s": 0.333, "decode_tok_s_per_stream": 392.5 }, { "turn": 6, "context_tokens": 7852, "ttft_p50_s": 0.365, "ttft_max_s": 0.365, "decode_tok_s_per_stream": 393.5 }, { "turn": 7, "context_tokens": 8975, "ttft_p50_s": 0.428, "ttft_max_s": 0.428, "decode_tok_s_per_stream": 393.1 }, { "turn": 8, "context_tokens": 10098, "ttft_p50_s": 0.467, "ttft_max_s": 0.467, "decode_tok_s_per_stream": 393.7 }, { "turn": 9, "context_tokens": 11221, "ttft_p50_s": 0.524, "ttft_max_s": 0.524, "decode_tok_s_per_stream": 392.4 }, { "turn": 10, "context_tokens": 12359, "ttft_p50_s": 0.57, "ttft_max_s": 0.57, "decode_tok_s_per_stream": 393.4 }, { "turn": 11, "context_tokens": 13496, "ttft_p50_s": 0.637, "ttft_max_s": 0.637, "decode_tok_s_per_stream": 393.0 }, { "turn": 12, "context_tokens": 14633, "ttft_p50_s": 0.678, "ttft_max_s": 0.678, "decode_tok_s_per_stream": 393.3 }, { "turn": 13, "context_tokens": 15770, "ttft_p50_s": 0.747, "ttft_max_s": 0.747, "decode_tok_s_per_stream": 392.9 }, { "turn": 14, "context_tokens": 16907, "ttft_p50_s": 0.79, "ttft_max_s": 0.79, "decode_tok_s_per_stream": 392.4 }, { "turn": 15, "context_tokens": 18044, "ttft_p50_s": 0.867, "ttft_max_s": 0.867, "decode_tok_s_per_stream": 393.1 }, { "turn": 16, "context_tokens": 19181, "ttft_p50_s": 0.905, "ttft_max_s": 0.905, "decode_tok_s_per_stream": 393.8 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 170951 }, { "agents": 4, "mode": "warm", "turns_requested": 64, "turns_ok": 64, "status": "ok", "wall_s": 14.51, "ttft_first_turn_s": 0.542, "ttft_last_turn_s": 0.379, "ttft_mean_s": 0.309, "ttft_p95_s": 0.538, "context_tokens_last": 19174, "prompt_tokens_total": 683392, "turns_per_minute": 264.7, "per_turn": [ { "turn": 1, "context_tokens": 2231, "ttft_p50_s": 0.542, "ttft_max_s": 0.547, "decode_tok_s_per_stream": 255.5 }, { "turn": 2, "context_tokens": 3354, "ttft_p50_s": 0.272, "ttft_max_s": 0.315, "decode_tok_s_per_stream": 271.8 }, { "turn": 3, "context_tokens": 4477, "ttft_p50_s": 0.418, "ttft_max_s": 0.421, "decode_tok_s_per_stream": 270.9 }, { "turn": 4, "context_tokens": 5600, "ttft_p50_s": 0.201, "ttft_max_s": 0.26, "decode_tok_s_per_stream": 232.1 }, { "turn": 5, "context_tokens": 6723, "ttft_p50_s": 0.408, "ttft_max_s": 0.434, "decode_tok_s_per_stream": 218.4 }, { "turn": 6, "context_tokens": 7846, "ttft_p50_s": 0.223, "ttft_max_s": 0.231, "decode_tok_s_per_stream": 220.7 }, { "turn": 7, "context_tokens": 8969, "ttft_p50_s": 0.382, "ttft_max_s": 0.419, "decode_tok_s_per_stream": 234.6 }, { "turn": 8, "context_tokens": 10092, "ttft_p50_s": 0.232, "ttft_max_s": 0.3, "decode_tok_s_per_stream": 230.2 }, { "turn": 9, "context_tokens": 11215, "ttft_p50_s": 0.402, "ttft_max_s": 0.444, "decode_tok_s_per_stream": 219.5 }, { "turn": 10, "context_tokens": 12352, "ttft_p50_s": 0.229, "ttft_max_s": 0.306, "decode_tok_s_per_stream": 219.2 }, { "turn": 11, "context_tokens": 13489, "ttft_p50_s": 0.423, "ttft_max_s": 0.476, "decode_tok_s_per_stream": 225.0 }, { "turn": 12, "context_tokens": 14626, "ttft_p50_s": 0.253, "ttft_max_s": 0.36, "decode_tok_s_per_stream": 201.8 }, { "turn": 13, "context_tokens": 15763, "ttft_p50_s": 0.443, "ttft_max_s": 0.505, "decode_tok_s_per_stream": 240.5 }, { "turn": 14, "context_tokens": 16900, "ttft_p50_s": 0.266, "ttft_max_s": 0.386, "decode_tok_s_per_stream": 196.8 }, { "turn": 15, "context_tokens": 18037, "ttft_p50_s": 0.437, "ttft_max_s": 0.601, "decode_tok_s_per_stream": 187.3 }, { "turn": 16, "context_tokens": 19174, "ttft_p50_s": 0.379, "ttft_max_s": 0.386, "decode_tok_s_per_stream": 242.5 } ], "prefix_cache_hit_rate": 0.8252, "prefix_cache_queried_tokens": 683392 }, { "agents": 4, "mode": "cold", "turns_requested": 64, "turns_ok": 64, "status": "ok", "wall_s": 35.36, "ttft_first_turn_s": 0.381, "ttft_last_turn_s": 1.522, "ttft_mean_s": 0.831, "ttft_p95_s": 1.589, "context_tokens_last": 19181, "prompt_tokens_total": 683804, "turns_per_minute": 108.6, "per_turn": [ { "turn": 1, "context_tokens": 2237, "ttft_p50_s": 0.381, "ttft_max_s": 0.387, "decode_tok_s_per_stream": 267.4 }, { "turn": 2, "context_tokens": 3360, "ttft_p50_s": 0.483, "ttft_max_s": 0.487, "decode_tok_s_per_stream": 265.3 }, { "turn": 3, "context_tokens": 4483, "ttft_p50_s": 0.592, "ttft_max_s": 0.775, "decode_tok_s_per_stream": 196.0 }, { "turn": 4, "context_tokens": 5606, "ttft_p50_s": 0.597, "ttft_max_s": 0.697, "decode_tok_s_per_stream": 200.3 }, { "turn": 5, "context_tokens": 6729, "ttft_p50_s": 0.656, "ttft_max_s": 0.856, "decode_tok_s_per_stream": 157.1 }, { "turn": 6, "context_tokens": 7852, "ttft_p50_s": 0.67, "ttft_max_s": 0.851, "decode_tok_s_per_stream": 134.4 }, { "turn": 7, "context_tokens": 8975, "ttft_p50_s": 0.776, "ttft_max_s": 0.965, "decode_tok_s_per_stream": 124.1 }, { "turn": 8, "context_tokens": 10098, "ttft_p50_s": 0.786, "ttft_max_s": 1.003, "decode_tok_s_per_stream": 112.9 }, { "turn": 9, "context_tokens": 11221, "ttft_p50_s": 0.973, "ttft_max_s": 1.15, "decode_tok_s_per_stream": 112.9 }, { "turn": 10, "context_tokens": 12359, "ttft_p50_s": 0.953, "ttft_max_s": 1.169, "decode_tok_s_per_stream": 73.8 }, { "turn": 11, "context_tokens": 13496, "ttft_p50_s": 1.053, "ttft_max_s": 1.317, "decode_tok_s_per_stream": 69.5 }, { "turn": 12, "context_tokens": 14633, "ttft_p50_s": 1.059, "ttft_max_s": 1.321, "decode_tok_s_per_stream": 61.8 }, { "turn": 13, "context_tokens": 15770, "ttft_p50_s": 1.161, "ttft_max_s": 1.469, "decode_tok_s_per_stream": 60.6 }, { "turn": 14, "context_tokens": 16907, "ttft_p50_s": 1.259, "ttft_max_s": 1.593, "decode_tok_s_per_stream": 55.7 }, { "turn": 15, "context_tokens": 18044, "ttft_p50_s": 1.327, "ttft_max_s": 1.695, "decode_tok_s_per_stream": 53.6 }, { "turn": 16, "context_tokens": 19181, "ttft_p50_s": 1.522, "ttft_max_s": 1.864, "decode_tok_s_per_stream": 100.6 } ], "prefix_cache_hit_rate": 0.0934, "prefix_cache_queried_tokens": 683804 } ], "dimensions": { "model": "Nemotron-3.5-Lightning-30B-A3B", "seqs": "32", "graphs": "on" } }, { "label": "qwen3.8-27b text-only, CUDA graphs", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "turns": 16, "tool_words": 400, "system_words": 600, "output_tokens": 128, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 16, "turns_ok": 16, "status": "ok", "wall_s": 36.95, "ttft_first_turn_s": 0.341, "ttft_last_turn_s": 0.31, "ttft_mean_s": 0.269, "ttft_p95_s": 0.362, "context_tokens_last": 20644, "prompt_tokens_total": 183208, "turns_per_minute": 26.0, "per_turn": [ { "turn": 1, "context_tokens": 2306, "ttft_p50_s": 0.341, "ttft_max_s": 0.341, "decode_tok_s_per_stream": 66.2 }, { "turn": 2, "context_tokens": 3522, "ttft_p50_s": 0.212, "ttft_max_s": 0.212, "decode_tok_s_per_stream": 62.5 }, { "turn": 3, "context_tokens": 4738, "ttft_p50_s": 0.192, "ttft_max_s": 0.192, "decode_tok_s_per_stream": 62.5 }, { "turn": 4, "context_tokens": 5954, "ttft_p50_s": 0.145, "ttft_max_s": 0.145, "decode_tok_s_per_stream": 62.5 }, { "turn": 5, "context_tokens": 7170, "ttft_p50_s": 0.284, "ttft_max_s": 0.284, "decode_tok_s_per_stream": 62.4 }, { "turn": 6, "context_tokens": 8386, "ttft_p50_s": 0.265, "ttft_max_s": 0.265, "decode_tok_s_per_stream": 62.4 }, { "turn": 7, "context_tokens": 9602, "ttft_p50_s": 0.233, "ttft_max_s": 0.233, "decode_tok_s_per_stream": 62.2 }, { "turn": 8, "context_tokens": 10818, "ttft_p50_s": 0.192, "ttft_max_s": 0.192, "decode_tok_s_per_stream": 62.0 }, { "turn": 9, "context_tokens": 12034, "ttft_p50_s": 0.345, "ttft_max_s": 0.345, "decode_tok_s_per_stream": 61.9 }, { "turn": 10, "context_tokens": 13264, "ttft_p50_s": 0.313, "ttft_max_s": 0.313, "decode_tok_s_per_stream": 61.8 }, { "turn": 11, "context_tokens": 14494, "ttft_p50_s": 0.282, "ttft_max_s": 0.282, "decode_tok_s_per_stream": 61.8 }, { "turn": 12, "context_tokens": 15724, "ttft_p50_s": 0.253, "ttft_max_s": 0.253, "decode_tok_s_per_stream": 61.8 }, { "turn": 13, "context_tokens": 16954, "ttft_p50_s": 0.195, "ttft_max_s": 0.195, "decode_tok_s_per_stream": 61.8 }, { "turn": 14, "context_tokens": 18184, "ttft_p50_s": 0.387, "ttft_max_s": 0.387, "decode_tok_s_per_stream": 61.7 }, { "turn": 15, "context_tokens": 19414, "ttft_p50_s": 0.353, "ttft_max_s": 0.353, "decode_tok_s_per_stream": 61.6 }, { "turn": 16, "context_tokens": 20644, "ttft_p50_s": 0.31, "ttft_max_s": 0.31, "decode_tok_s_per_stream": 61.5 } ], "prefix_cache_hit_rate": 0.8302, "prefix_cache_queried_tokens": 183208 }, { "agents": 1, "mode": "cold", "turns_requested": 16, "turns_ok": 16, "status": "ok", "wall_s": 54.23, "ttft_first_turn_s": 0.235, "ttft_last_turn_s": 2.601, "ttft_mean_s": 1.34, "ttft_p95_s": 2.461, "context_tokens_last": 20651, "prompt_tokens_total": 183311, "turns_per_minute": 17.7, "per_turn": [ { "turn": 1, "context_tokens": 2312, "ttft_p50_s": 0.235, "ttft_max_s": 0.235, "decode_tok_s_per_stream": 62.5 }, { "turn": 2, "context_tokens": 3528, "ttft_p50_s": 0.367, "ttft_max_s": 0.367, "decode_tok_s_per_stream": 62.5 }, { "turn": 3, "context_tokens": 4744, "ttft_p50_s": 0.503, "ttft_max_s": 0.503, "decode_tok_s_per_stream": 62.5 }, { "turn": 4, "context_tokens": 5960, "ttft_p50_s": 0.62, "ttft_max_s": 0.62, "decode_tok_s_per_stream": 62.4 }, { "turn": 5, "context_tokens": 7176, "ttft_p50_s": 0.766, "ttft_max_s": 0.766, "decode_tok_s_per_stream": 62.4 }, { "turn": 6, "context_tokens": 8392, "ttft_p50_s": 0.92, "ttft_max_s": 0.92, "decode_tok_s_per_stream": 62.3 }, { "turn": 7, "context_tokens": 9608, "ttft_p50_s": 1.066, "ttft_max_s": 1.066, "decode_tok_s_per_stream": 62.1 }, { "turn": 8, "context_tokens": 10824, "ttft_p50_s": 1.209, "ttft_max_s": 1.209, "decode_tok_s_per_stream": 61.9 }, { "turn": 9, "context_tokens": 12040, "ttft_p50_s": 1.368, "ttft_max_s": 1.368, "decode_tok_s_per_stream": 61.8 }, { "turn": 10, "context_tokens": 13271, "ttft_p50_s": 1.53, "ttft_max_s": 1.53, "decode_tok_s_per_stream": 61.8 }, { "turn": 11, "context_tokens": 14501, "ttft_p50_s": 1.703, "ttft_max_s": 1.703, "decode_tok_s_per_stream": 61.8 }, { "turn": 12, "context_tokens": 15731, "ttft_p50_s": 1.875, "ttft_max_s": 1.875, "decode_tok_s_per_stream": 61.8 }, { "turn": 13, "context_tokens": 16961, "ttft_p50_s": 2.037, "ttft_max_s": 2.037, "decode_tok_s_per_stream": 61.7 }, { "turn": 14, "context_tokens": 18191, "ttft_p50_s": 2.229, "ttft_max_s": 2.229, "decode_tok_s_per_stream": 61.6 }, { "turn": 15, "context_tokens": 19421, "ttft_p50_s": 2.415, "ttft_max_s": 2.415, "decode_tok_s_per_stream": 61.6 }, { "turn": 16, "context_tokens": 20651, "ttft_p50_s": 2.601, "ttft_max_s": 2.601, "decode_tok_s_per_stream": 61.5 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 183311 }, { "agents": 4, "mode": "warm", "turns_requested": 64, "turns_ok": 64, "status": "ok", "wall_s": 132.85, "ttft_first_turn_s": 0.87, "ttft_last_turn_s": 9.896, "ttft_mean_s": 3.698, "ttft_p95_s": 9.604, "context_tokens_last": 20644, "prompt_tokens_total": 732832, "turns_per_minute": 28.9, "per_turn": [ { "turn": 1, "context_tokens": 2306, "ttft_p50_s": 0.87, "ttft_max_s": 0.964, "decode_tok_s_per_stream": 46.3 }, { "turn": 2, "context_tokens": 3522, "ttft_p50_s": 0.387, "ttft_max_s": 0.432, "decode_tok_s_per_stream": 45.0 }, { "turn": 3, "context_tokens": 4738, "ttft_p50_s": 0.879, "ttft_max_s": 1.005, "decode_tok_s_per_stream": 41.2 }, { "turn": 4, "context_tokens": 5954, "ttft_p50_s": 1.457, "ttft_max_s": 1.649, "decode_tok_s_per_stream": 33.4 }, { "turn": 5, "context_tokens": 7170, "ttft_p50_s": 1.311, "ttft_max_s": 1.471, "decode_tok_s_per_stream": 30.0 }, { "turn": 6, "context_tokens": 8386, "ttft_p50_s": 1.448, "ttft_max_s": 1.623, "decode_tok_s_per_stream": 27.2 }, { "turn": 7, "context_tokens": 9602, "ttft_p50_s": 1.57, "ttft_max_s": 1.81, "decode_tok_s_per_stream": 25.0 }, { "turn": 8, "context_tokens": 10818, "ttft_p50_s": 1.728, "ttft_max_s": 1.921, "decode_tok_s_per_stream": 24.7 }, { "turn": 9, "context_tokens": 12034, "ttft_p50_s": 2.254, "ttft_max_s": 4.483, "decode_tok_s_per_stream": 25.8 }, { "turn": 10, "context_tokens": 13264, "ttft_p50_s": 3.152, "ttft_max_s": 5.121, "decode_tok_s_per_stream": 24.3 }, { "turn": 11, "context_tokens": 14494, "ttft_p50_s": 3.563, "ttft_max_s": 5.547, "decode_tok_s_per_stream": 23.4 }, { "turn": 12, "context_tokens": 15724, "ttft_p50_s": 5.829, "ttft_max_s": 6.51, "decode_tok_s_per_stream": 25.5 }, { "turn": 13, "context_tokens": 16954, "ttft_p50_s": 7.091, "ttft_max_s": 7.174, "decode_tok_s_per_stream": 24.7 }, { "turn": 14, "context_tokens": 18184, "ttft_p50_s": 7.755, "ttft_max_s": 8.901, "decode_tok_s_per_stream": 29.0 }, { "turn": 15, "context_tokens": 19414, "ttft_p50_s": 9.351, "ttft_max_s": 9.449, "decode_tok_s_per_stream": 27.9 }, { "turn": 16, "context_tokens": 20644, "ttft_p50_s": 9.896, "ttft_max_s": 9.994, "decode_tok_s_per_stream": 26.9 } ], "prefix_cache_hit_rate": 0.0128, "prefix_cache_queried_tokens": 732832 }, { "agents": 4, "mode": "cold", "turns_requested": 64, "turns_ok": 64, "status": "ok", "wall_s": 133.95, "ttft_first_turn_s": 0.682, "ttft_last_turn_s": 9.905, "ttft_mean_s": 3.721, "ttft_p95_s": 9.616, "context_tokens_last": 20651, "prompt_tokens_total": 733244, "turns_per_minute": 28.7, "per_turn": [ { "turn": 1, "context_tokens": 2312, "ttft_p50_s": 0.682, "ttft_max_s": 0.956, "decode_tok_s_per_stream": 44.3 }, { "turn": 2, "context_tokens": 3528, "ttft_p50_s": 0.775, "ttft_max_s": 0.904, "decode_tok_s_per_stream": 43.1 }, { "turn": 3, "context_tokens": 4744, "ttft_p50_s": 1.218, "ttft_max_s": 1.34, "decode_tok_s_per_stream": 38.0 }, { "turn": 4, "context_tokens": 5960, "ttft_p50_s": 1.187, "ttft_max_s": 1.334, "decode_tok_s_per_stream": 33.2 }, { "turn": 5, "context_tokens": 7176, "ttft_p50_s": 1.32, "ttft_max_s": 1.482, "decode_tok_s_per_stream": 30.0 }, { "turn": 6, "context_tokens": 8392, "ttft_p50_s": 1.456, "ttft_max_s": 1.632, "decode_tok_s_per_stream": 27.2 }, { "turn": 7, "context_tokens": 9608, "ttft_p50_s": 1.577, "ttft_max_s": 1.817, "decode_tok_s_per_stream": 25.0 }, { "turn": 8, "context_tokens": 10824, "ttft_p50_s": 1.733, "ttft_max_s": 1.926, "decode_tok_s_per_stream": 24.7 }, { "turn": 9, "context_tokens": 12040, "ttft_p50_s": 2.259, "ttft_max_s": 4.487, "decode_tok_s_per_stream": 25.7 }, { "turn": 10, "context_tokens": 13271, "ttft_p50_s": 3.155, "ttft_max_s": 5.125, "decode_tok_s_per_stream": 24.2 }, { "turn": 11, "context_tokens": 14501, "ttft_p50_s": 3.573, "ttft_max_s": 5.556, "decode_tok_s_per_stream": 23.4 }, { "turn": 12, "context_tokens": 15731, "ttft_p50_s": 5.831, "ttft_max_s": 6.508, "decode_tok_s_per_stream": 25.5 }, { "turn": 13, "context_tokens": 16961, "ttft_p50_s": 7.111, "ttft_max_s": 7.199, "decode_tok_s_per_stream": 24.7 }, { "turn": 14, "context_tokens": 18191, "ttft_p50_s": 7.767, "ttft_max_s": 8.91, "decode_tok_s_per_stream": 29.0 }, { "turn": 15, "context_tokens": 19421, "ttft_p50_s": 9.362, "ttft_max_s": 9.461, "decode_tok_s_per_stream": 27.9 }, { "turn": 16, "context_tokens": 20651, "ttft_p50_s": 9.905, "ttft_max_s": 10.001, "decode_tok_s_per_stream": 26.8 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 733244 } ], "dimensions": { "model": "Qwen3.8-27B", "seqs": "32", "vision": "off", "graphs": "on" } }, { "label": "qwen3.8-27b default", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "turns": 16, "tool_words": 400, "system_words": 600, "output_tokens": 128, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 16, "turns_ok": 16, "status": "ok", "wall_s": 85.84, "ttft_first_turn_s": 0.288, "ttft_last_turn_s": 0.358, "ttft_mean_s": 0.317, "ttft_p95_s": 0.413, "context_tokens_last": 20644, "prompt_tokens_total": 183208, "turns_per_minute": 11.2, "per_turn": [ { "turn": 1, "context_tokens": 2306, "ttft_p50_s": 0.288, "ttft_max_s": 0.288, "decode_tok_s_per_stream": 25.1 }, { "turn": 2, "context_tokens": 3522, "ttft_p50_s": 0.269, "ttft_max_s": 0.269, "decode_tok_s_per_stream": 25.2 }, { "turn": 3, "context_tokens": 4738, "ttft_p50_s": 0.274, "ttft_max_s": 0.274, "decode_tok_s_per_stream": 25.0 }, { "turn": 4, "context_tokens": 5954, "ttft_p50_s": 0.184, "ttft_max_s": 0.184, "decode_tok_s_per_stream": 25.2 }, { "turn": 5, "context_tokens": 7170, "ttft_p50_s": 0.343, "ttft_max_s": 0.343, "decode_tok_s_per_stream": 25.4 }, { "turn": 6, "context_tokens": 8386, "ttft_p50_s": 0.32, "ttft_max_s": 0.32, "decode_tok_s_per_stream": 25.3 }, { "turn": 7, "context_tokens": 9602, "ttft_p50_s": 0.3, "ttft_max_s": 0.3, "decode_tok_s_per_stream": 24.9 }, { "turn": 8, "context_tokens": 10818, "ttft_p50_s": 0.232, "ttft_max_s": 0.232, "decode_tok_s_per_stream": 25.3 }, { "turn": 9, "context_tokens": 12034, "ttft_p50_s": 0.402, "ttft_max_s": 0.402, "decode_tok_s_per_stream": 25.4 }, { "turn": 10, "context_tokens": 13264, "ttft_p50_s": 0.367, "ttft_max_s": 0.367, "decode_tok_s_per_stream": 25.2 }, { "turn": 11, "context_tokens": 14494, "ttft_p50_s": 0.332, "ttft_max_s": 0.332, "decode_tok_s_per_stream": 25.2 }, { "turn": 12, "context_tokens": 15724, "ttft_p50_s": 0.332, "ttft_max_s": 0.332, "decode_tok_s_per_stream": 24.9 }, { "turn": 13, "context_tokens": 16954, "ttft_p50_s": 0.229, "ttft_max_s": 0.229, "decode_tok_s_per_stream": 25.0 }, { "turn": 14, "context_tokens": 18184, "ttft_p50_s": 0.439, "ttft_max_s": 0.439, "decode_tok_s_per_stream": 25.2 }, { "turn": 15, "context_tokens": 19414, "ttft_p50_s": 0.404, "ttft_max_s": 0.404, "decode_tok_s_per_stream": 25.2 }, { "turn": 16, "context_tokens": 20644, "ttft_p50_s": 0.358, "ttft_max_s": 0.358, "decode_tok_s_per_stream": 25.1 } ], "prefix_cache_hit_rate": 0.8302, "prefix_cache_queried_tokens": 183208 }, { "agents": 1, "mode": "cold", "turns_requested": 16, "turns_ok": 16, "status": "ok", "wall_s": 104.42, "ttft_first_turn_s": 0.287, "ttft_last_turn_s": 2.849, "ttft_mean_s": 1.498, "ttft_p95_s": 2.699, "context_tokens_last": 20651, "prompt_tokens_total": 183311, "turns_per_minute": 9.2, "per_turn": [ { "turn": 1, "context_tokens": 2312, "ttft_p50_s": 0.287, "ttft_max_s": 0.287, "decode_tok_s_per_stream": 25.1 }, { "turn": 2, "context_tokens": 3528, "ttft_p50_s": 0.436, "ttft_max_s": 0.436, "decode_tok_s_per_stream": 25.3 }, { "turn": 3, "context_tokens": 4744, "ttft_p50_s": 0.621, "ttft_max_s": 0.621, "decode_tok_s_per_stream": 25.3 }, { "turn": 4, "context_tokens": 5960, "ttft_p50_s": 0.714, "ttft_max_s": 0.714, "decode_tok_s_per_stream": 25.3 }, { "turn": 5, "context_tokens": 7176, "ttft_p50_s": 0.874, "ttft_max_s": 0.874, "decode_tok_s_per_stream": 25.2 }, { "turn": 6, "context_tokens": 8392, "ttft_p50_s": 1.044, "ttft_max_s": 1.044, "decode_tok_s_per_stream": 25.3 }, { "turn": 7, "context_tokens": 9608, "ttft_p50_s": 1.22, "ttft_max_s": 1.22, "decode_tok_s_per_stream": 25.3 }, { "turn": 8, "context_tokens": 10824, "ttft_p50_s": 1.356, "ttft_max_s": 1.356, "decode_tok_s_per_stream": 25.3 }, { "turn": 9, "context_tokens": 12040, "ttft_p50_s": 1.529, "ttft_max_s": 1.529, "decode_tok_s_per_stream": 25.3 }, { "turn": 10, "context_tokens": 13271, "ttft_p50_s": 1.703, "ttft_max_s": 1.703, "decode_tok_s_per_stream": 25.4 }, { "turn": 11, "context_tokens": 14501, "ttft_p50_s": 1.888, "ttft_max_s": 1.888, "decode_tok_s_per_stream": 25.4 }, { "turn": 12, "context_tokens": 15731, "ttft_p50_s": 2.107, "ttft_max_s": 2.107, "decode_tok_s_per_stream": 25.3 }, { "turn": 13, "context_tokens": 16961, "ttft_p50_s": 2.244, "ttft_max_s": 2.244, "decode_tok_s_per_stream": 25.2 }, { "turn": 14, "context_tokens": 18191, "ttft_p50_s": 2.449, "ttft_max_s": 2.449, "decode_tok_s_per_stream": 25.1 }, { "turn": 15, "context_tokens": 19421, "ttft_p50_s": 2.649, "ttft_max_s": 2.649, "decode_tok_s_per_stream": 25.2 }, { "turn": 16, "context_tokens": 20651, "ttft_p50_s": 2.849, "ttft_max_s": 2.849, "decode_tok_s_per_stream": 25.3 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 183311 }, { "agents": 4, "mode": "warm", "turns_requested": 64, "turns_ok": 64, "status": "ok", "wall_s": 181.79, "ttft_first_turn_s": 0.786, "ttft_last_turn_s": 11.629, "ttft_mean_s": 3.512, "ttft_p95_s": 10.973, "context_tokens_last": 20644, "prompt_tokens_total": 732832, "turns_per_minute": 21.1, "per_turn": [ { "turn": 1, "context_tokens": 2306, "ttft_p50_s": 0.786, "ttft_max_s": 1.076, "decode_tok_s_per_stream": 22.7 }, { "turn": 2, "context_tokens": 3522, "ttft_p50_s": 0.447, "ttft_max_s": 0.48, "decode_tok_s_per_stream": 22.8 }, { "turn": 3, "context_tokens": 4738, "ttft_p50_s": 0.442, "ttft_max_s": 0.467, "decode_tok_s_per_stream": 23.0 }, { "turn": 4, "context_tokens": 5954, "ttft_p50_s": 0.239, "ttft_max_s": 0.24, "decode_tok_s_per_stream": 22.5 }, { "turn": 5, "context_tokens": 7170, "ttft_p50_s": 0.563, "ttft_max_s": 0.622, "decode_tok_s_per_stream": 21.9 }, { "turn": 6, "context_tokens": 8386, "ttft_p50_s": 2.363, "ttft_max_s": 3.23, "decode_tok_s_per_stream": 16.9 }, { "turn": 7, "context_tokens": 9602, "ttft_p50_s": 1.638, "ttft_max_s": 1.759, "decode_tok_s_per_stream": 16.6 }, { "turn": 8, "context_tokens": 10818, "ttft_p50_s": 2.024, "ttft_max_s": 2.217, "decode_tok_s_per_stream": 15.6 }, { "turn": 9, "context_tokens": 12034, "ttft_p50_s": 2.175, "ttft_max_s": 2.409, "decode_tok_s_per_stream": 14.9 }, { "turn": 10, "context_tokens": 13264, "ttft_p50_s": 2.233, "ttft_max_s": 6.861, "decode_tok_s_per_stream": 15.5 }, { "turn": 11, "context_tokens": 14494, "ttft_p50_s": 3.615, "ttft_max_s": 8.1, "decode_tok_s_per_stream": 15.3 }, { "turn": 12, "context_tokens": 15724, "ttft_p50_s": 4.306, "ttft_max_s": 8.623, "decode_tok_s_per_stream": 14.7 }, { "turn": 13, "context_tokens": 16954, "ttft_p50_s": 6.731, "ttft_max_s": 8.9, "decode_tok_s_per_stream": 14.3 }, { "turn": 14, "context_tokens": 18184, "ttft_p50_s": 5.121, "ttft_max_s": 9.299, "decode_tok_s_per_stream": 14.0 }, { "turn": 15, "context_tokens": 19414, "ttft_p50_s": 7.727, "ttft_max_s": 10.607, "decode_tok_s_per_stream": 14.7 }, { "turn": 16, "context_tokens": 20644, "ttft_p50_s": 11.629, "ttft_max_s": 13.318, "decode_tok_s_per_stream": 16.9 } ], "prefix_cache_hit_rate": 0.077, "prefix_cache_queried_tokens": 732832 }, { "agents": 4, "mode": "cold", "turns_requested": 64, "turns_ok": 64, "status": "ok", "wall_s": 187.81, "ttft_first_turn_s": 0.797, "ttft_last_turn_s": 11.648, "ttft_mean_s": 3.669, "ttft_p95_s": 10.955, "context_tokens_last": 20651, "prompt_tokens_total": 733244, "turns_per_minute": 20.4, "per_turn": [ { "turn": 1, "context_tokens": 2312, "ttft_p50_s": 0.797, "ttft_max_s": 1.091, "decode_tok_s_per_stream": 22.2 }, { "turn": 2, "context_tokens": 3528, "ttft_p50_s": 0.863, "ttft_max_s": 0.994, "decode_tok_s_per_stream": 21.8 }, { "turn": 3, "context_tokens": 4744, "ttft_p50_s": 1.361, "ttft_max_s": 1.479, "decode_tok_s_per_stream": 20.4 }, { "turn": 4, "context_tokens": 5960, "ttft_p50_s": 1.309, "ttft_max_s": 1.467, "decode_tok_s_per_stream": 19.1 }, { "turn": 5, "context_tokens": 7176, "ttft_p50_s": 1.451, "ttft_max_s": 1.622, "decode_tok_s_per_stream": 18.1 }, { "turn": 6, "context_tokens": 8392, "ttft_p50_s": 1.599, "ttft_max_s": 1.784, "decode_tok_s_per_stream": 16.8 }, { "turn": 7, "context_tokens": 9608, "ttft_p50_s": 1.653, "ttft_max_s": 1.776, "decode_tok_s_per_stream": 16.5 }, { "turn": 8, "context_tokens": 10824, "ttft_p50_s": 2.04, "ttft_max_s": 2.236, "decode_tok_s_per_stream": 15.5 }, { "turn": 9, "context_tokens": 12040, "ttft_p50_s": 2.193, "ttft_max_s": 2.426, "decode_tok_s_per_stream": 14.7 }, { "turn": 10, "context_tokens": 13271, "ttft_p50_s": 2.248, "ttft_max_s": 6.883, "decode_tok_s_per_stream": 15.4 }, { "turn": 11, "context_tokens": 14501, "ttft_p50_s": 3.644, "ttft_max_s": 8.131, "decode_tok_s_per_stream": 15.3 }, { "turn": 12, "context_tokens": 15731, "ttft_p50_s": 4.327, "ttft_max_s": 8.612, "decode_tok_s_per_stream": 14.8 }, { "turn": 13, "context_tokens": 16961, "ttft_p50_s": 6.732, "ttft_max_s": 8.879, "decode_tok_s_per_stream": 14.3 }, { "turn": 14, "context_tokens": 18191, "ttft_p50_s": 5.142, "ttft_max_s": 9.324, "decode_tok_s_per_stream": 14.0 }, { "turn": 15, "context_tokens": 19421, "ttft_p50_s": 7.722, "ttft_max_s": 10.588, "decode_tok_s_per_stream": 14.7 }, { "turn": 16, "context_tokens": 20651, "ttft_p50_s": 11.648, "ttft_max_s": 13.357, "decode_tok_s_per_stream": 16.8 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 733244 } ], "dimensions": { "model": "Qwen3.8-27B", "seqs": "8", "vision": "on", "graphs": "off" } }, { "label": "qwen3.6-35b-a3b · typical session", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "profile": "docs/usage.json", "session": "typical", "turns": 35, "turns_in_window": 33, "context_min": 919, "context_max": 65699, "output_tokens_p50": 252.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 32, "turns_ok": 32, "status": "ok", "wall_s": 92.96, "turns_over_window": 3, "ttft_first_turn_s": 2.854, "ttft_last_turn_s": 0.283, "ttft_mean_s": 0.774, "ttft_p95_s": 2.924, "context_tokens_last": 56033, "prompt_tokens_total": 1724942, "turns_per_minute": 20.7, "per_turn": [ { "turn": 1, "context_tokens": 50703, "ttft_p50_s": 2.854, "ttft_max_s": 2.854, "decode_tok_s_per_stream": 268.0, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58769, "ttft_p50_s": 0.836, "ttft_max_s": 0.836, "decode_tok_s_per_stream": 267.1, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59627, "ttft_p50_s": 0.237, "ttft_max_s": 0.237, "decode_tok_s_per_stream": 268.2, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 59995, "ttft_p50_s": 0.274, "ttft_max_s": 0.274, "decode_tok_s_per_stream": 265.7, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60346, "ttft_p50_s": 0.315, "ttft_max_s": 0.315, "decode_tok_s_per_stream": 265.1, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 60966, "ttft_p50_s": 0.316, "ttft_max_s": 0.316, "decode_tok_s_per_stream": 263.3, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62317, "ttft_p50_s": 0.318, "ttft_max_s": 0.318, "decode_tok_s_per_stream": 266.9, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63614, "ttft_p50_s": 0.416, "ttft_max_s": 0.416, "decode_tok_s_per_stream": 262.8, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1272, "ttft_p50_s": 0.135, "ttft_max_s": 0.135, "decode_tok_s_per_stream": 294.6, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 44897, "ttft_p50_s": 2.367, "ttft_max_s": 2.367, "decode_tok_s_per_stream": 271.4, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 45190, "ttft_p50_s": 0.247, "ttft_max_s": 0.247, "decode_tok_s_per_stream": 271.9, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 45452, "ttft_p50_s": 0.269, "ttft_max_s": 0.269, "decode_tok_s_per_stream": 270.9, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49422, "ttft_p50_s": 0.578, "ttft_max_s": 0.578, "decode_tok_s_per_stream": 267.5, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50484, "ttft_p50_s": 0.348, "ttft_max_s": 0.348, "decode_tok_s_per_stream": 267.5, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54341, "ttft_p50_s": 0.511, "ttft_max_s": 0.511, "decode_tok_s_per_stream": 264.8, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1276, "ttft_p50_s": 0.136, "ttft_max_s": 0.136, "decode_tok_s_per_stream": 295.0, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 44786, "ttft_p50_s": 2.338, "ttft_max_s": 2.338, "decode_tok_s_per_stream": 270.4, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60481, "ttft_p50_s": 1.519, "ttft_max_s": 1.519, "decode_tok_s_per_stream": 264.4, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60842, "ttft_p50_s": 0.364, "ttft_max_s": 0.364, "decode_tok_s_per_stream": 265.0, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61367, "ttft_p50_s": 0.199, "ttft_max_s": 0.199, "decode_tok_s_per_stream": 266.2, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61622, "ttft_p50_s": 0.25, "ttft_max_s": 0.25, "decode_tok_s_per_stream": 262.6, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62224, "ttft_p50_s": 0.274, "ttft_max_s": 0.274, "decode_tok_s_per_stream": 262.8, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 63972, "ttft_p50_s": 0.467, "ttft_max_s": 0.467, "decode_tok_s_per_stream": 263.6, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64736, "ttft_p50_s": 0.324, "ttft_max_s": 0.324, "decode_tok_s_per_stream": 265.0, "target_context": 64770, "output_tokens": 172 }, { "turn": 26, "context_tokens": 53213, "ttft_p50_s": 3.009, "ttft_max_s": 3.009, "decode_tok_s_per_stream": 269.6, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 54845, "ttft_p50_s": 0.348, "ttft_max_s": 0.348, "decode_tok_s_per_stream": 265.6, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 62436, "ttft_p50_s": 0.849, "ttft_max_s": 0.849, "decode_tok_s_per_stream": 267.1, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 64177, "ttft_p50_s": 0.457, "ttft_max_s": 0.457, "decode_tok_s_per_stream": 262.7, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 64542, "ttft_p50_s": 0.323, "ttft_max_s": 0.323, "decode_tok_s_per_stream": 265.6, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 65254, "ttft_p50_s": 0.368, "ttft_max_s": 0.368, "decode_tok_s_per_stream": 267.3, "target_context": 65290, "output_tokens": 106 }, { "turn": 34, "context_tokens": 55741, "ttft_p50_s": 3.235, "ttft_max_s": 3.235, "decode_tok_s_per_stream": 267.4, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 56033, "ttft_p50_s": 0.283, "ttft_max_s": 0.283, "decode_tok_s_per_stream": 273.3, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.8074, "prefix_cache_queried_tokens": 1724942 }, { "agents": 1, "mode": "cold", "turns_requested": 32, "turns_ok": 32, "status": "ok", "wall_s": 171.17, "turns_over_window": 3, "ttft_first_turn_s": 2.805, "ttft_last_turn_s": 3.306, "ttft_mean_s": 3.211, "ttft_p95_s": 4.084, "context_tokens_last": 55925, "prompt_tokens_total": 1724024, "turns_per_minute": 11.2, "per_turn": [ { "turn": 1, "context_tokens": 50709, "ttft_p50_s": 2.805, "ttft_max_s": 2.805, "decode_tok_s_per_stream": 267.0, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58775, "ttft_p50_s": 3.447, "ttft_max_s": 3.447, "decode_tok_s_per_stream": 265.7, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59633, "ttft_p50_s": 3.58, "ttft_max_s": 3.58, "decode_tok_s_per_stream": 267.3, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 60001, "ttft_p50_s": 3.586, "ttft_max_s": 3.586, "decode_tok_s_per_stream": 264.5, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60352, "ttft_p50_s": 3.648, "ttft_max_s": 3.648, "decode_tok_s_per_stream": 264.0, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 60972, "ttft_p50_s": 3.685, "ttft_max_s": 3.685, "decode_tok_s_per_stream": 262.1, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62323, "ttft_p50_s": 3.878, "ttft_max_s": 3.878, "decode_tok_s_per_stream": 264.9, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63620, "ttft_p50_s": 3.946, "ttft_max_s": 3.946, "decode_tok_s_per_stream": 260.5, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.134, "ttft_max_s": 0.134, "decode_tok_s_per_stream": 296.1, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 44675, "ttft_p50_s": 2.347, "ttft_max_s": 2.347, "decode_tok_s_per_stream": 268.9, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 44968, "ttft_p50_s": 2.39, "ttft_max_s": 2.39, "decode_tok_s_per_stream": 270.2, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 45230, "ttft_p50_s": 2.391, "ttft_max_s": 2.391, "decode_tok_s_per_stream": 269.5, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49431, "ttft_p50_s": 2.746, "ttft_max_s": 2.746, "decode_tok_s_per_stream": 266.5, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50493, "ttft_p50_s": 2.765, "ttft_max_s": 2.765, "decode_tok_s_per_stream": 265.8, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54350, "ttft_p50_s": 3.125, "ttft_max_s": 3.125, "decode_tok_s_per_stream": 264.0, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1283, "ttft_p50_s": 0.135, "ttft_max_s": 0.135, "decode_tok_s_per_stream": 296.2, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 44564, "ttft_p50_s": 2.345, "ttft_max_s": 2.345, "decode_tok_s_per_stream": 269.3, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60489, "ttft_p50_s": 3.644, "ttft_max_s": 3.644, "decode_tok_s_per_stream": 264.6, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60850, "ttft_p50_s": 3.703, "ttft_max_s": 3.703, "decode_tok_s_per_stream": 264.7, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61375, "ttft_p50_s": 3.762, "ttft_max_s": 3.762, "decode_tok_s_per_stream": 266.0, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61630, "ttft_p50_s": 3.818, "ttft_max_s": 3.818, "decode_tok_s_per_stream": 261.8, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62232, "ttft_p50_s": 3.847, "ttft_max_s": 3.847, "decode_tok_s_per_stream": 262.2, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 63980, "ttft_p50_s": 4.018, "ttft_max_s": 4.018, "decode_tok_s_per_stream": 262.2, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64744, "ttft_p50_s": 4.082, "ttft_max_s": 4.082, "decode_tok_s_per_stream": 264.4, "target_context": 64770, "output_tokens": 172 }, { "turn": 26, "context_tokens": 53220, "ttft_p50_s": 3.028, "ttft_max_s": 3.028, "decode_tok_s_per_stream": 268.5, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 54852, "ttft_p50_s": 3.192, "ttft_max_s": 3.192, "decode_tok_s_per_stream": 265.4, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 62443, "ttft_p50_s": 3.897, "ttft_max_s": 3.897, "decode_tok_s_per_stream": 266.0, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 64184, "ttft_p50_s": 4.019, "ttft_max_s": 4.019, "decode_tok_s_per_stream": 261.8, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 64549, "ttft_p50_s": 4.086, "ttft_max_s": 4.086, "decode_tok_s_per_stream": 264.3, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 65261, "ttft_p50_s": 4.134, "ttft_max_s": 4.134, "decode_tok_s_per_stream": 265.7, "target_context": 65290, "output_tokens": 106 }, { "turn": 34, "context_tokens": 55633, "ttft_p50_s": 3.256, "ttft_max_s": 3.256, "decode_tok_s_per_stream": 267.0, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 55925, "ttft_p50_s": 3.306, "ttft_max_s": 3.306, "decode_tok_s_per_stream": 271.9, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 1724024 }, { "agents": 4, "mode": "warm", "turns_requested": 128, "turns_ok": 128, "status": "ok", "wall_s": 203.99, "turns_over_window": 12, "ttft_first_turn_s": 7.378, "ttft_last_turn_s": 1.157, "ttft_mean_s": 1.864, "ttft_p95_s": 9.022, "context_tokens_last": 56033, "prompt_tokens_total": 6899768, "turns_per_minute": 37.6, "per_turn": [ { "turn": 1, "context_tokens": 50703, "ttft_p50_s": 7.378, "ttft_max_s": 11.189, "decode_tok_s_per_stream": 88.1, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58769, "ttft_p50_s": 1.575, "ttft_max_s": 1.753, "decode_tok_s_per_stream": 79.7, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59627, "ttft_p50_s": 0.354, "ttft_max_s": 0.472, "decode_tok_s_per_stream": 142.6, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 59995, "ttft_p50_s": 0.539, "ttft_max_s": 0.638, "decode_tok_s_per_stream": 160.5, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60346, "ttft_p50_s": 0.702, "ttft_max_s": 0.793, "decode_tok_s_per_stream": 160.1, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 60966, "ttft_p50_s": 1.094, "ttft_max_s": 1.1, "decode_tok_s_per_stream": 158.1, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62317, "ttft_p50_s": 0.701, "ttft_max_s": 0.79, "decode_tok_s_per_stream": 154.8, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63614, "ttft_p50_s": 1.218, "ttft_max_s": 1.258, "decode_tok_s_per_stream": 160.3, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1272, "ttft_p50_s": 0.273, "ttft_max_s": 0.32, "decode_tok_s_per_stream": 152.5, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 44897, "ttft_p50_s": 6.204, "ttft_max_s": 9.339, "decode_tok_s_per_stream": 54.4, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 45190, "ttft_p50_s": 0.335, "ttft_max_s": 0.408, "decode_tok_s_per_stream": 134.2, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 45452, "ttft_p50_s": 0.497, "ttft_max_s": 0.539, "decode_tok_s_per_stream": 152.4, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49422, "ttft_p50_s": 1.842, "ttft_max_s": 1.921, "decode_tok_s_per_stream": 162.4, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50484, "ttft_p50_s": 1.042, "ttft_max_s": 1.071, "decode_tok_s_per_stream": 167.1, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54341, "ttft_p50_s": 1.51, "ttft_max_s": 1.623, "decode_tok_s_per_stream": 159.3, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1276, "ttft_p50_s": 0.273, "ttft_max_s": 0.275, "decode_tok_s_per_stream": 169.2, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 44786, "ttft_p50_s": 6.173, "ttft_max_s": 9.261, "decode_tok_s_per_stream": 50.0, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60481, "ttft_p50_s": 3.02, "ttft_max_s": 4.74, "decode_tok_s_per_stream": 92.6, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60842, "ttft_p50_s": 0.977, "ttft_max_s": 1.024, "decode_tok_s_per_stream": 163.9, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61367, "ttft_p50_s": 0.552, "ttft_max_s": 0.581, "decode_tok_s_per_stream": 166.3, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61622, "ttft_p50_s": 0.475, "ttft_max_s": 0.53, "decode_tok_s_per_stream": 159.1, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62224, "ttft_p50_s": 0.58, "ttft_max_s": 0.656, "decode_tok_s_per_stream": 158.4, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 63972, "ttft_p50_s": 1.321, "ttft_max_s": 1.38, "decode_tok_s_per_stream": 158.7, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64736, "ttft_p50_s": 0.701, "ttft_max_s": 0.797, "decode_tok_s_per_stream": 147.6, "target_context": 64770, "output_tokens": 172 }, { "turn": 26, "context_tokens": 53213, "ttft_p50_s": 7.735, "ttft_max_s": 11.907, "decode_tok_s_per_stream": 30.7, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 54845, "ttft_p50_s": 1.031, "ttft_max_s": 1.525, "decode_tok_s_per_stream": 117.7, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 62436, "ttft_p50_s": 1.495, "ttft_max_s": 1.579, "decode_tok_s_per_stream": 72.4, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 64177, "ttft_p50_s": 0.734, "ttft_max_s": 0.802, "decode_tok_s_per_stream": 128.9, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 64542, "ttft_p50_s": 0.438, "ttft_max_s": 0.54, "decode_tok_s_per_stream": 106.0, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 65254, "ttft_p50_s": 0.529, "ttft_max_s": 0.642, "decode_tok_s_per_stream": 109.4, "target_context": 65290, "output_tokens": 106 }, { "turn": 34, "context_tokens": 55741, "ttft_p50_s": 8.316, "ttft_max_s": 12.77, "decode_tok_s_per_stream": 38.0, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 56033, "ttft_p50_s": 1.157, "ttft_max_s": 1.44, "decode_tok_s_per_stream": 98.8, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.8001, "prefix_cache_queried_tokens": 6899768 }, { "agents": 4, "mode": "cold", "turns_requested": 128, "turns_ok": 128, "status": "ok", "wall_s": 526.03, "turns_over_window": 12, "ttft_first_turn_s": 7.368, "ttft_last_turn_s": 3.323, "ttft_mean_s": 5.611, "ttft_p95_s": 10.101, "context_tokens_last": 55925, "prompt_tokens_total": 6896096, "turns_per_minute": 14.6, "per_turn": [ { "turn": 1, "context_tokens": 50709, "ttft_p50_s": 7.368, "ttft_max_s": 11.124, "decode_tok_s_per_stream": 88.4, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58775, "ttft_p50_s": 8.65, "ttft_max_s": 10.126, "decode_tok_s_per_stream": 31.7, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59633, "ttft_p50_s": 7.71, "ttft_max_s": 8.795, "decode_tok_s_per_stream": 22.0, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 60001, "ttft_p50_s": 7.508, "ttft_max_s": 8.395, "decode_tok_s_per_stream": 43.5, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60352, "ttft_p50_s": 7.882, "ttft_max_s": 9.009, "decode_tok_s_per_stream": 50.7, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 60972, "ttft_p50_s": 8.3, "ttft_max_s": 18.506, "decode_tok_s_per_stream": 89.9, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62323, "ttft_p50_s": 6.522, "ttft_max_s": 6.746, "decode_tok_s_per_stream": 33.3, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63620, "ttft_p50_s": 6.335, "ttft_max_s": 9.207, "decode_tok_s_per_stream": 117.2, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.112, "ttft_max_s": 0.417, "decode_tok_s_per_stream": 130.5, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 44675, "ttft_p50_s": 2.822, "ttft_max_s": 4.975, "decode_tok_s_per_stream": 81.6, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 44968, "ttft_p50_s": 3.958, "ttft_max_s": 5.791, "decode_tok_s_per_stream": 76.6, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 45230, "ttft_p50_s": 3.872, "ttft_max_s": 6.219, "decode_tok_s_per_stream": 35.3, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49431, "ttft_p50_s": 4.471, "ttft_max_s": 6.422, "decode_tok_s_per_stream": 63.9, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50493, "ttft_p50_s": 4.595, "ttft_max_s": 6.183, "decode_tok_s_per_stream": 128.5, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54350, "ttft_p50_s": 4.966, "ttft_max_s": 8.057, "decode_tok_s_per_stream": 117.4, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1283, "ttft_p50_s": 0.11, "ttft_max_s": 0.213, "decode_tok_s_per_stream": 130.0, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 44564, "ttft_p50_s": 2.999, "ttft_max_s": 5.114, "decode_tok_s_per_stream": 117.5, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60489, "ttft_p50_s": 5.773, "ttft_max_s": 10.87, "decode_tok_s_per_stream": 56.1, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60850, "ttft_p50_s": 5.83, "ttft_max_s": 8.456, "decode_tok_s_per_stream": 56.9, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61375, "ttft_p50_s": 6.022, "ttft_max_s": 8.575, "decode_tok_s_per_stream": 41.5, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61630, "ttft_p50_s": 6.06, "ttft_max_s": 8.683, "decode_tok_s_per_stream": 56.3, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62232, "ttft_p50_s": 6.135, "ttft_max_s": 8.784, "decode_tok_s_per_stream": 32.5, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 63980, "ttft_p50_s": 6.117, "ttft_max_s": 8.368, "decode_tok_s_per_stream": 54.6, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64744, "ttft_p50_s": 6.486, "ttft_max_s": 8.978, "decode_tok_s_per_stream": 28.5, "target_context": 64770, "output_tokens": 172 }, { "turn": 26, "context_tokens": 53220, "ttft_p50_s": 7.562, "ttft_max_s": 10.456, "decode_tok_s_per_stream": 33.6, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 54852, "ttft_p50_s": 5.637, "ttft_max_s": 7.319, "decode_tok_s_per_stream": 54.5, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 62443, "ttft_p50_s": 6.173, "ttft_max_s": 10.567, "decode_tok_s_per_stream": 37.1, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 64184, "ttft_p50_s": 6.407, "ttft_max_s": 9.343, "decode_tok_s_per_stream": 26.9, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 64549, "ttft_p50_s": 6.951, "ttft_max_s": 7.82, "decode_tok_s_per_stream": 37.6, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 65261, "ttft_p50_s": 6.434, "ttft_max_s": 16.48, "decode_tok_s_per_stream": 14.4, "target_context": 65290, "output_tokens": 106 }, { "turn": 34, "context_tokens": 55633, "ttft_p50_s": 3.267, "ttft_max_s": 5.794, "decode_tok_s_per_stream": 31.2, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 55925, "ttft_p50_s": 3.323, "ttft_max_s": 3.367, "decode_tok_s_per_stream": 176.3, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 6896096 } ], "dimensions": { "model": "Qwen3.6-35B-A3B", "shape": "typical", "seqs": "32", "graphs": "on" } }, { "label": "qwen3.6-35b-a3b · heavy session", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "profile": "docs/usage.json", "session": "heavy", "turns": 199, "turns_in_window": 55, "context_min": 927, "context_max": 274585, "output_tokens_p50": 560.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 55, "turns_ok": 55, "status": "ok", "wall_s": 176.69, "turns_over_window": 144, "ttft_first_turn_s": 2.714, "ttft_last_turn_s": 0.515, "ttft_mean_s": 0.622, "ttft_p95_s": 2.369, "context_tokens_last": 46561, "prompt_tokens_total": 2421847, "turns_per_minute": 18.7, "per_turn": [ { "turn": 4, "context_tokens": 48994, "ttft_p50_s": 2.714, "ttft_max_s": 2.714, "decode_tok_s_per_stream": 268.3, "target_context": 55938, "output_tokens": 202 }, { "turn": 6, "context_tokens": 56853, "ttft_p50_s": 3.358, "ttft_max_s": 3.358, "decode_tok_s_per_stream": 265.2, "target_context": 56909, "output_tokens": 398 }, { "turn": 7, "context_tokens": 58690, "ttft_p50_s": 0.356, "ttft_max_s": 0.356, "decode_tok_s_per_stream": 268.1, "target_context": 58822, "output_tokens": 124 }, { "turn": 19, "context_tokens": 1276, "ttft_p50_s": 0.072, "ttft_max_s": 0.072, "decode_tok_s_per_stream": 293.8, "target_context": 961, "output_tokens": 16 }, { "turn": 20, "context_tokens": 44788, "ttft_p50_s": 2.363, "ttft_max_s": 2.363, "decode_tok_s_per_stream": 268.7, "target_context": 33996, "output_tokens": 279 }, { "turn": 21, "context_tokens": 45096, "ttft_p50_s": 0.252, "ttft_max_s": 0.252, "decode_tok_s_per_stream": 269.4, "target_context": 34455, "output_tokens": 223 }, { "turn": 22, "context_tokens": 45378, "ttft_p50_s": 0.277, "ttft_max_s": 0.277, "decode_tok_s_per_stream": 267.6, "target_context": 34900, "output_tokens": 578 }, { "turn": 23, "context_tokens": 45815, "ttft_p50_s": 0.339, "ttft_max_s": 0.339, "decode_tok_s_per_stream": 267.2, "target_context": 35765, "output_tokens": 1164 }, { "turn": 24, "context_tokens": 46512, "ttft_p50_s": 0.226, "ttft_max_s": 0.226, "decode_tok_s_per_stream": 267.2, "target_context": 37099, "output_tokens": 890 }, { "turn": 25, "context_tokens": 47093, "ttft_p50_s": 0.273, "ttft_max_s": 0.273, "decode_tok_s_per_stream": 266.4, "target_context": 38060, "output_tokens": 1787 }, { "turn": 26, "context_tokens": 48078, "ttft_p50_s": 0.19, "ttft_max_s": 0.19, "decode_tok_s_per_stream": 267.3, "target_context": 39965, "output_tokens": 502 }, { "turn": 27, "context_tokens": 48491, "ttft_p50_s": 0.192, "ttft_max_s": 0.192, "decode_tok_s_per_stream": 266.6, "target_context": 40530, "output_tokens": 867 }, { "turn": 28, "context_tokens": 49073, "ttft_p50_s": 0.239, "ttft_max_s": 0.239, "decode_tok_s_per_stream": 266.4, "target_context": 41685, "output_tokens": 996 }, { "turn": 29, "context_tokens": 49716, "ttft_p50_s": 0.291, "ttft_max_s": 0.291, "decode_tok_s_per_stream": 265.6, "target_context": 42730, "output_tokens": 1944 }, { "turn": 30, "context_tokens": 50805, "ttft_p50_s": 0.233, "ttft_max_s": 0.233, "decode_tok_s_per_stream": 265.2, "target_context": 44830, "output_tokens": 2348 }, { "turn": 31, "context_tokens": 52101, "ttft_p50_s": 0.333, "ttft_max_s": 0.333, "decode_tok_s_per_stream": 265.4, "target_context": 47231, "output_tokens": 985 }, { "turn": 32, "context_tokens": 52761, "ttft_p50_s": 0.204, "ttft_max_s": 0.204, "decode_tok_s_per_stream": 264.6, "target_context": 48267, "output_tokens": 2720 }, { "turn": 33, "context_tokens": 54268, "ttft_p50_s": 0.347, "ttft_max_s": 0.347, "decode_tok_s_per_stream": 266.4, "target_context": 51031, "output_tokens": 318 }, { "turn": 34, "context_tokens": 1276, "ttft_p50_s": 0.07, "ttft_max_s": 0.07, "decode_tok_s_per_stream": 293.6, "target_context": 927, "output_tokens": 13 }, { "turn": 35, "context_tokens": 39732, "ttft_p50_s": 1.988, "ttft_max_s": 1.988, "decode_tok_s_per_stream": 274.7, "target_context": 29988, "output_tokens": 207 }, { "turn": 36, "context_tokens": 40007, "ttft_p50_s": 0.183, "ttft_max_s": 0.183, "decode_tok_s_per_stream": 274.4, "target_context": 35061, "output_tokens": 206 }, { "turn": 37, "context_tokens": 40280, "ttft_p50_s": 0.216, "ttft_max_s": 0.216, "decode_tok_s_per_stream": 274.6, "target_context": 36124, "output_tokens": 307 }, { "turn": 38, "context_tokens": 40599, "ttft_p50_s": 0.251, "ttft_max_s": 0.251, "decode_tok_s_per_stream": 273.4, "target_context": 36598, "output_tokens": 405 }, { "turn": 39, "context_tokens": 1276, "ttft_p50_s": 0.071, "ttft_max_s": 0.071, "decode_tok_s_per_stream": 289.4, "target_context": 957, "output_tokens": 23 }, { "turn": 40, "context_tokens": 44908, "ttft_p50_s": 2.385, "ttft_max_s": 2.385, "decode_tok_s_per_stream": 269.3, "target_context": 34132, "output_tokens": 217 }, { "turn": 41, "context_tokens": 45187, "ttft_p50_s": 0.246, "ttft_max_s": 0.246, "decode_tok_s_per_stream": 269.3, "target_context": 35247, "output_tokens": 240 }, { "turn": 42, "context_tokens": 45476, "ttft_p50_s": 0.275, "ttft_max_s": 0.275, "decode_tok_s_per_stream": 267.6, "target_context": 44130, "output_tokens": 589 }, { "turn": 43, "context_tokens": 45919, "ttft_p50_s": 0.188, "ttft_max_s": 0.188, "decode_tok_s_per_stream": 268.6, "target_context": 45079, "output_tokens": 312 }, { "turn": 44, "context_tokens": 13801, "ttft_p50_s": 0.548, "ttft_max_s": 0.548, "decode_tok_s_per_stream": 284.8, "target_context": 14112, "output_tokens": 1229 }, { "turn": 45, "context_tokens": 47861, "ttft_p50_s": 2.155, "ttft_max_s": 2.155, "decode_tok_s_per_stream": 266.2, "target_context": 47335, "output_tokens": 2465 }, { "turn": 46, "context_tokens": 49142, "ttft_p50_s": 0.399, "ttft_max_s": 0.399, "decode_tok_s_per_stream": 266.3, "target_context": 49984, "output_tokens": 2228 }, { "turn": 47, "context_tokens": 51376, "ttft_p50_s": 0.249, "ttft_max_s": 0.249, "decode_tok_s_per_stream": 265.6, "target_context": 52506, "output_tokens": 1819 }, { "turn": 48, "context_tokens": 54245, "ttft_p50_s": 0.476, "ttft_max_s": 0.476, "decode_tok_s_per_stream": 265.5, "target_context": 55007, "output_tokens": 589 }, { "turn": 49, "context_tokens": 55971, "ttft_p50_s": 0.306, "ttft_max_s": 0.306, "decode_tok_s_per_stream": 265.3, "target_context": 56050, "output_tokens": 566 }, { "turn": 50, "context_tokens": 57571, "ttft_p50_s": 0.439, "ttft_max_s": 0.439, "decode_tok_s_per_stream": 264.4, "target_context": 57643, "output_tokens": 737 }, { "turn": 51, "context_tokens": 58448, "ttft_p50_s": 0.32, "ttft_max_s": 0.32, "decode_tok_s_per_stream": 264.7, "target_context": 58729, "output_tokens": 478 }, { "turn": 52, "context_tokens": 59204, "ttft_p50_s": 0.359, "ttft_max_s": 0.359, "decode_tok_s_per_stream": 264.0, "target_context": 59363, "output_tokens": 560 }, { "turn": 53, "context_tokens": 64251, "ttft_p50_s": 0.649, "ttft_max_s": 0.649, "decode_tok_s_per_stream": 263.4, "target_context": 64045, "output_tokens": 218 }, { "turn": 54, "context_tokens": 64882, "ttft_p50_s": 0.318, "ttft_max_s": 0.318, "decode_tok_s_per_stream": 260.9, "target_context": 64837, "output_tokens": 464 }, { "turn": 83, "context_tokens": 39769, "ttft_p50_s": 1.99, "ttft_max_s": 1.99, "decode_tok_s_per_stream": 274.5, "target_context": 37006, "output_tokens": 178 }, { "turn": 84, "context_tokens": 40031, "ttft_p50_s": 0.184, "ttft_max_s": 0.184, "decode_tok_s_per_stream": 280.8, "target_context": 37308, "output_tokens": 62 }, { "turn": 145, "context_tokens": 1280, "ttft_p50_s": 0.108, "ttft_max_s": 0.108, "decode_tok_s_per_stream": 290.1, "target_context": 954, "output_tokens": 17 }, { "turn": 146, "context_tokens": 44715, "ttft_p50_s": 2.348, "ttft_max_s": 2.348, "decode_tok_s_per_stream": 269.0, "target_context": 33809, "output_tokens": 349 }, { "turn": 147, "context_tokens": 45055, "ttft_p50_s": 0.265, "ttft_max_s": 0.265, "decode_tok_s_per_stream": 271.2, "target_context": 34921, "output_tokens": 186 }, { "turn": 148, "context_tokens": 45323, "ttft_p50_s": 0.274, "ttft_max_s": 0.274, "decode_tok_s_per_stream": 267.3, "target_context": 38701, "output_tokens": 2701 }, { "turn": 149, "context_tokens": 46686, "ttft_p50_s": 0.381, "ttft_max_s": 0.381, "decode_tok_s_per_stream": 267.5, "target_context": 41416, "output_tokens": 995 }, { "turn": 150, "context_tokens": 47319, "ttft_p50_s": 0.276, "ttft_max_s": 0.276, "decode_tok_s_per_stream": 271.5, "target_context": 43422, "output_tokens": 145 }, { "turn": 151, "context_tokens": 47573, "ttft_p50_s": 0.284, "ttft_max_s": 0.284, "decode_tok_s_per_stream": 268.5, "target_context": 44550, "output_tokens": 317 }, { "turn": 193, "context_tokens": 40112, "ttft_p50_s": 2.027, "ttft_max_s": 2.027, "decode_tok_s_per_stream": 273.0, "target_context": 38824, "output_tokens": 427 }, { "turn": 194, "context_tokens": 40487, "ttft_p50_s": 0.222, "ttft_max_s": 0.222, "decode_tok_s_per_stream": 273.0, "target_context": 40325, "output_tokens": 442 }, { "turn": 195, "context_tokens": 41216, "ttft_p50_s": 0.291, "ttft_max_s": 0.291, "decode_tok_s_per_stream": 272.8, "target_context": 41396, "output_tokens": 339 }, { "turn": 196, "context_tokens": 42365, "ttft_p50_s": 0.22, "ttft_max_s": 0.22, "decode_tok_s_per_stream": 272.0, "target_context": 42468, "output_tokens": 455 }, { "turn": 197, "context_tokens": 42869, "ttft_p50_s": 0.195, "ttft_max_s": 0.195, "decode_tok_s_per_stream": 271.6, "target_context": 42987, "output_tokens": 520 }, { "turn": 198, "context_tokens": 43286, "ttft_p50_s": 0.271, "ttft_max_s": 0.271, "decode_tok_s_per_stream": 271.1, "target_context": 43517, "output_tokens": 426 }, { "turn": 199, "context_tokens": 46561, "ttft_p50_s": 0.515, "ttft_max_s": 0.515, "decode_tok_s_per_stream": 269.2, "target_context": 46592, "output_tokens": 281 } ], "prefix_cache_hit_rate": 0.7979, "prefix_cache_queried_tokens": 2421847 }, { "agents": 1, "mode": "cold", "turns_requested": 55, "turns_ok": 55, "status": "ok", "wall_s": 275.13, "turns_over_window": 144, "ttft_first_turn_s": 2.703, "ttft_last_turn_s": 2.518, "ttft_mean_s": 2.411, "ttft_p95_s": 3.536, "context_tokens_last": 46569, "prompt_tokens_total": 2415595, "turns_per_minute": 12.0, "per_turn": [ { "turn": 4, "context_tokens": 49000, "ttft_p50_s": 2.703, "ttft_max_s": 2.703, "decode_tok_s_per_stream": 268.8, "target_context": 55938, "output_tokens": 202 }, { "turn": 6, "context_tokens": 56859, "ttft_p50_s": 3.337, "ttft_max_s": 3.337, "decode_tok_s_per_stream": 264.8, "target_context": 56909, "output_tokens": 398 }, { "turn": 7, "context_tokens": 58696, "ttft_p50_s": 3.525, "ttft_max_s": 3.525, "decode_tok_s_per_stream": 267.6, "target_context": 58822, "output_tokens": 124 }, { "turn": 19, "context_tokens": 1283, "ttft_p50_s": 0.056, "ttft_max_s": 0.056, "decode_tok_s_per_stream": 291.1, "target_context": 961, "output_tokens": 16 }, { "turn": 20, "context_tokens": 44566, "ttft_p50_s": 2.35, "ttft_max_s": 2.35, "decode_tok_s_per_stream": 269.2, "target_context": 33996, "output_tokens": 279 }, { "turn": 21, "context_tokens": 44874, "ttft_p50_s": 2.365, "ttft_max_s": 2.365, "decode_tok_s_per_stream": 270.0, "target_context": 34455, "output_tokens": 223 }, { "turn": 22, "context_tokens": 45156, "ttft_p50_s": 2.367, "ttft_max_s": 2.367, "decode_tok_s_per_stream": 267.9, "target_context": 34900, "output_tokens": 578 }, { "turn": 23, "context_tokens": 45593, "ttft_p50_s": 2.448, "ttft_max_s": 2.448, "decode_tok_s_per_stream": 267.3, "target_context": 35765, "output_tokens": 1164 }, { "turn": 24, "context_tokens": 46290, "ttft_p50_s": 2.475, "ttft_max_s": 2.475, "decode_tok_s_per_stream": 267.0, "target_context": 37099, "output_tokens": 890 }, { "turn": 25, "context_tokens": 46871, "ttft_p50_s": 2.521, "ttft_max_s": 2.521, "decode_tok_s_per_stream": 266.9, "target_context": 38060, "output_tokens": 1787 }, { "turn": 26, "context_tokens": 47855, "ttft_p50_s": 2.605, "ttft_max_s": 2.605, "decode_tok_s_per_stream": 267.6, "target_context": 39965, "output_tokens": 502 }, { "turn": 27, "context_tokens": 48267, "ttft_p50_s": 2.657, "ttft_max_s": 2.657, "decode_tok_s_per_stream": 266.2, "target_context": 40530, "output_tokens": 867 }, { "turn": 28, "context_tokens": 48849, "ttft_p50_s": 2.716, "ttft_max_s": 2.716, "decode_tok_s_per_stream": 266.6, "target_context": 41685, "output_tokens": 996 }, { "turn": 29, "context_tokens": 49492, "ttft_p50_s": 2.764, "ttft_max_s": 2.764, "decode_tok_s_per_stream": 265.6, "target_context": 42730, "output_tokens": 1944 }, { "turn": 30, "context_tokens": 50581, "ttft_p50_s": 2.833, "ttft_max_s": 2.833, "decode_tok_s_per_stream": 265.6, "target_context": 44830, "output_tokens": 2348 }, { "turn": 31, "context_tokens": 51876, "ttft_p50_s": 2.938, "ttft_max_s": 2.938, "decode_tok_s_per_stream": 265.9, "target_context": 47231, "output_tokens": 985 }, { "turn": 32, "context_tokens": 52534, "ttft_p50_s": 2.978, "ttft_max_s": 2.978, "decode_tok_s_per_stream": 264.7, "target_context": 48267, "output_tokens": 2720 }, { "turn": 33, "context_tokens": 54041, "ttft_p50_s": 3.097, "ttft_max_s": 3.097, "decode_tok_s_per_stream": 266.0, "target_context": 51031, "output_tokens": 318 }, { "turn": 34, "context_tokens": 1283, "ttft_p50_s": 0.069, "ttft_max_s": 0.069, "decode_tok_s_per_stream": 290.7, "target_context": 927, "output_tokens": 13 }, { "turn": 35, "context_tokens": 39509, "ttft_p50_s": 1.982, "ttft_max_s": 1.982, "decode_tok_s_per_stream": 274.4, "target_context": 29988, "output_tokens": 207 }, { "turn": 36, "context_tokens": 39784, "ttft_p50_s": 1.992, "ttft_max_s": 1.992, "decode_tok_s_per_stream": 274.4, "target_context": 35061, "output_tokens": 206 }, { "turn": 37, "context_tokens": 40057, "ttft_p50_s": 2.024, "ttft_max_s": 2.024, "decode_tok_s_per_stream": 273.4, "target_context": 36124, "output_tokens": 307 }, { "turn": 38, "context_tokens": 40376, "ttft_p50_s": 2.041, "ttft_max_s": 2.041, "decode_tok_s_per_stream": 273.2, "target_context": 36598, "output_tokens": 405 }, { "turn": 39, "context_tokens": 1283, "ttft_p50_s": 0.07, "ttft_max_s": 0.07, "decode_tok_s_per_stream": 290.6, "target_context": 957, "output_tokens": 23 }, { "turn": 40, "context_tokens": 44686, "ttft_p50_s": 2.358, "ttft_max_s": 2.358, "decode_tok_s_per_stream": 269.4, "target_context": 34132, "output_tokens": 217 }, { "turn": 41, "context_tokens": 44965, "ttft_p50_s": 2.365, "ttft_max_s": 2.365, "decode_tok_s_per_stream": 268.7, "target_context": 35247, "output_tokens": 240 }, { "turn": 42, "context_tokens": 45254, "ttft_p50_s": 2.405, "ttft_max_s": 2.405, "decode_tok_s_per_stream": 268.2, "target_context": 44130, "output_tokens": 589 }, { "turn": 43, "context_tokens": 45697, "ttft_p50_s": 2.385, "ttft_max_s": 2.385, "decode_tok_s_per_stream": 268.8, "target_context": 45079, "output_tokens": 312 }, { "turn": 44, "context_tokens": 13808, "ttft_p50_s": 0.552, "ttft_max_s": 0.552, "decode_tok_s_per_stream": 284.6, "target_context": 14112, "output_tokens": 1229 }, { "turn": 45, "context_tokens": 47752, "ttft_p50_s": 2.607, "ttft_max_s": 2.607, "decode_tok_s_per_stream": 266.0, "target_context": 47335, "output_tokens": 2465 }, { "turn": 46, "context_tokens": 49034, "ttft_p50_s": 2.717, "ttft_max_s": 2.717, "decode_tok_s_per_stream": 266.0, "target_context": 49984, "output_tokens": 2228 }, { "turn": 47, "context_tokens": 51384, "ttft_p50_s": 2.882, "ttft_max_s": 2.882, "decode_tok_s_per_stream": 265.4, "target_context": 52506, "output_tokens": 1819 }, { "turn": 48, "context_tokens": 54255, "ttft_p50_s": 3.11, "ttft_max_s": 3.11, "decode_tok_s_per_stream": 265.6, "target_context": 55007, "output_tokens": 589 }, { "turn": 49, "context_tokens": 55867, "ttft_p50_s": 3.309, "ttft_max_s": 3.309, "decode_tok_s_per_stream": 264.9, "target_context": 56050, "output_tokens": 566 }, { "turn": 50, "context_tokens": 57583, "ttft_p50_s": 3.431, "ttft_max_s": 3.431, "decode_tok_s_per_stream": 263.7, "target_context": 57643, "output_tokens": 737 }, { "turn": 51, "context_tokens": 58461, "ttft_p50_s": 3.495, "ttft_max_s": 3.495, "decode_tok_s_per_stream": 264.3, "target_context": 58729, "output_tokens": 478 }, { "turn": 52, "context_tokens": 59217, "ttft_p50_s": 3.561, "ttft_max_s": 3.561, "decode_tok_s_per_stream": 263.9, "target_context": 59363, "output_tokens": 560 }, { "turn": 53, "context_tokens": 64264, "ttft_p50_s": 4.037, "ttft_max_s": 4.037, "decode_tok_s_per_stream": 263.0, "target_context": 64045, "output_tokens": 218 }, { "turn": 54, "context_tokens": 64782, "ttft_p50_s": 4.084, "ttft_max_s": 4.084, "decode_tok_s_per_stream": 261.2, "target_context": 64837, "output_tokens": 464 }, { "turn": 83, "context_tokens": 39891, "ttft_p50_s": 2.033, "ttft_max_s": 2.033, "decode_tok_s_per_stream": 274.7, "target_context": 37006, "output_tokens": 178 }, { "turn": 84, "context_tokens": 40153, "ttft_p50_s": 2.033, "ttft_max_s": 2.033, "decode_tok_s_per_stream": 280.9, "target_context": 37308, "output_tokens": 62 }, { "turn": 145, "context_tokens": 1288, "ttft_p50_s": 0.109, "ttft_max_s": 0.109, "decode_tok_s_per_stream": 290.0, "target_context": 954, "output_tokens": 17 }, { "turn": 146, "context_tokens": 44491, "ttft_p50_s": 2.359, "ttft_max_s": 2.359, "decode_tok_s_per_stream": 269.0, "target_context": 33809, "output_tokens": 349 }, { "turn": 147, "context_tokens": 44831, "ttft_p50_s": 2.374, "ttft_max_s": 2.374, "decode_tok_s_per_stream": 270.1, "target_context": 34921, "output_tokens": 186 }, { "turn": 148, "context_tokens": 45099, "ttft_p50_s": 2.404, "ttft_max_s": 2.404, "decode_tok_s_per_stream": 266.5, "target_context": 38701, "output_tokens": 2701 }, { "turn": 149, "context_tokens": 46462, "ttft_p50_s": 2.512, "ttft_max_s": 2.512, "decode_tok_s_per_stream": 267.6, "target_context": 41416, "output_tokens": 995 }, { "turn": 150, "context_tokens": 47095, "ttft_p50_s": 2.561, "ttft_max_s": 2.561, "decode_tok_s_per_stream": 270.8, "target_context": 43422, "output_tokens": 145 }, { "turn": 151, "context_tokens": 47349, "ttft_p50_s": 2.558, "ttft_max_s": 2.558, "decode_tok_s_per_stream": 268.0, "target_context": 44550, "output_tokens": 317 }, { "turn": 193, "context_tokens": 40120, "ttft_p50_s": 2.034, "ttft_max_s": 2.034, "decode_tok_s_per_stream": 273.1, "target_context": 38824, "output_tokens": 427 }, { "turn": 194, "context_tokens": 40495, "ttft_p50_s": 2.047, "ttft_max_s": 2.047, "decode_tok_s_per_stream": 272.2, "target_context": 40325, "output_tokens": 442 }, { "turn": 195, "context_tokens": 41224, "ttft_p50_s": 2.086, "ttft_max_s": 2.086, "decode_tok_s_per_stream": 272.5, "target_context": 41396, "output_tokens": 339 }, { "turn": 196, "context_tokens": 42373, "ttft_p50_s": 2.219, "ttft_max_s": 2.219, "decode_tok_s_per_stream": 271.6, "target_context": 42468, "output_tokens": 455 }, { "turn": 197, "context_tokens": 42877, "ttft_p50_s": 2.265, "ttft_max_s": 2.265, "decode_tok_s_per_stream": 271.0, "target_context": 42987, "output_tokens": 520 }, { "turn": 198, "context_tokens": 43294, "ttft_p50_s": 2.273, "ttft_max_s": 2.273, "decode_tok_s_per_stream": 270.6, "target_context": 43517, "output_tokens": 426 }, { "turn": 199, "context_tokens": 46569, "ttft_p50_s": 2.518, "ttft_max_s": 2.518, "decode_tok_s_per_stream": 268.6, "target_context": 46592, "output_tokens": 281 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 2415595 } ], "dimensions": { "model": "Qwen3.6-35B-A3B", "shape": "heavy", "seqs": "32", "graphs": "on" } }, { "label": "qwen3.6-35b-a3b 256K · typical session", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 262144, "workload": { "profile": "docs/usage.json", "session": "typical", "turns": 35, "turns_in_window": 35, "context_min": 919, "context_max": 65699, "output_tokens_p50": 252.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 35, "turns_ok": 35, "status": "ok", "wall_s": 96.62, "turns_over_window": 0, "ttft_first_turn_s": 2.853, "ttft_last_turn_s": 0.288, "ttft_mean_s": 0.541, "ttft_p95_s": 2.332, "context_tokens_last": 68776, "prompt_tokens_total": 1989065, "turns_per_minute": 21.7, "per_turn": [ { "turn": 1, "context_tokens": 50703, "ttft_p50_s": 2.853, "ttft_max_s": 2.853, "decode_tok_s_per_stream": 268.0, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58769, "ttft_p50_s": 0.833, "ttft_max_s": 0.833, "decode_tok_s_per_stream": 267.0, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59627, "ttft_p50_s": 0.237, "ttft_max_s": 0.237, "decode_tok_s_per_stream": 268.2, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 59995, "ttft_p50_s": 0.274, "ttft_max_s": 0.274, "decode_tok_s_per_stream": 265.8, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60346, "ttft_p50_s": 0.317, "ttft_max_s": 0.317, "decode_tok_s_per_stream": 265.6, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 60966, "ttft_p50_s": 0.315, "ttft_max_s": 0.315, "decode_tok_s_per_stream": 263.6, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62317, "ttft_p50_s": 0.318, "ttft_max_s": 0.318, "decode_tok_s_per_stream": 267.2, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63614, "ttft_p50_s": 0.414, "ttft_max_s": 0.414, "decode_tok_s_per_stream": 262.9, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1272, "ttft_p50_s": 0.134, "ttft_max_s": 0.134, "decode_tok_s_per_stream": 293.8, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 44897, "ttft_p50_s": 2.348, "ttft_max_s": 2.348, "decode_tok_s_per_stream": 270.8, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 45190, "ttft_p50_s": 0.249, "ttft_max_s": 0.249, "decode_tok_s_per_stream": 271.6, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 45452, "ttft_p50_s": 0.27, "ttft_max_s": 0.27, "decode_tok_s_per_stream": 270.7, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49422, "ttft_p50_s": 0.576, "ttft_max_s": 0.576, "decode_tok_s_per_stream": 267.0, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50484, "ttft_p50_s": 0.343, "ttft_max_s": 0.343, "decode_tok_s_per_stream": 267.2, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54341, "ttft_p50_s": 0.507, "ttft_max_s": 0.507, "decode_tok_s_per_stream": 264.5, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1276, "ttft_p50_s": 0.136, "ttft_max_s": 0.136, "decode_tok_s_per_stream": 297.6, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 44786, "ttft_p50_s": 2.325, "ttft_max_s": 2.325, "decode_tok_s_per_stream": 270.9, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60481, "ttft_p50_s": 1.515, "ttft_max_s": 1.515, "decode_tok_s_per_stream": 264.7, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60842, "ttft_p50_s": 0.364, "ttft_max_s": 0.364, "decode_tok_s_per_stream": 265.7, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61367, "ttft_p50_s": 0.201, "ttft_max_s": 0.201, "decode_tok_s_per_stream": 267.2, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61622, "ttft_p50_s": 0.253, "ttft_max_s": 0.253, "decode_tok_s_per_stream": 263.2, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62224, "ttft_p50_s": 0.275, "ttft_max_s": 0.275, "decode_tok_s_per_stream": 263.5, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 63972, "ttft_p50_s": 0.466, "ttft_max_s": 0.466, "decode_tok_s_per_stream": 264.3, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64736, "ttft_p50_s": 0.319, "ttft_max_s": 0.319, "decode_tok_s_per_stream": 265.7, "target_context": 64770, "output_tokens": 172 }, { "turn": 25, "context_tokens": 64995, "ttft_p50_s": 0.33, "ttft_max_s": 0.33, "decode_tok_s_per_stream": 259.2, "target_context": 48486, "output_tokens": 2201 }, { "turn": 26, "context_tokens": 66156, "ttft_p50_s": 0.274, "ttft_max_s": 0.274, "decode_tok_s_per_stream": 263.9, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 66411, "ttft_p50_s": 0.272, "ttft_max_s": 0.272, "decode_tok_s_per_stream": 259.6, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 66885, "ttft_p50_s": 0.338, "ttft_max_s": 0.338, "decode_tok_s_per_stream": 265.2, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 67134, "ttft_p50_s": 0.335, "ttft_max_s": 0.335, "decode_tok_s_per_stream": 260.0, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 67503, "ttft_p50_s": 0.341, "ttft_max_s": 0.341, "decode_tok_s_per_stream": 263.4, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 67758, "ttft_p50_s": 0.184, "ttft_max_s": 0.184, "decode_tok_s_per_stream": 265.0, "target_context": 65290, "output_tokens": 106 }, { "turn": 32, "context_tokens": 67991, "ttft_p50_s": 0.205, "ttft_max_s": 0.205, "decode_tok_s_per_stream": 261.6, "target_context": 65395, "output_tokens": 220 }, { "turn": 33, "context_tokens": 68274, "ttft_p50_s": 0.272, "ttft_max_s": 0.272, "decode_tok_s_per_stream": 276.9, "target_context": 65699, "output_tokens": 50 }, { "turn": 34, "context_tokens": 68481, "ttft_p50_s": 0.26, "ttft_max_s": 0.26, "decode_tok_s_per_stream": 261.4, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 68776, "ttft_p50_s": 0.288, "ttft_max_s": 0.288, "decode_tok_s_per_stream": 269.3, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.8905, "prefix_cache_queried_tokens": 1989065 }, { "agents": 1, "mode": "cold", "turns_requested": 35, "turns_ok": 35, "status": "ok", "wall_s": 198.58, "turns_over_window": 0, "ttft_first_turn_s": 2.781, "ttft_last_turn_s": 4.459, "ttft_mean_s": 3.448, "ttft_p95_s": 4.438, "context_tokens_last": 68782, "prompt_tokens_total": 1988398, "turns_per_minute": 10.6, "per_turn": [ { "turn": 1, "context_tokens": 50709, "ttft_p50_s": 2.781, "ttft_max_s": 2.781, "decode_tok_s_per_stream": 267.5, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58775, "ttft_p50_s": 3.421, "ttft_max_s": 3.421, "decode_tok_s_per_stream": 266.5, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59633, "ttft_p50_s": 3.556, "ttft_max_s": 3.556, "decode_tok_s_per_stream": 267.1, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 60001, "ttft_p50_s": 3.558, "ttft_max_s": 3.558, "decode_tok_s_per_stream": 264.4, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60352, "ttft_p50_s": 3.622, "ttft_max_s": 3.622, "decode_tok_s_per_stream": 264.5, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 60972, "ttft_p50_s": 3.657, "ttft_max_s": 3.657, "decode_tok_s_per_stream": 262.6, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62323, "ttft_p50_s": 3.852, "ttft_max_s": 3.852, "decode_tok_s_per_stream": 265.2, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63620, "ttft_p50_s": 3.922, "ttft_max_s": 3.922, "decode_tok_s_per_stream": 261.9, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.135, "ttft_max_s": 0.135, "decode_tok_s_per_stream": 294.3, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 44675, "ttft_p50_s": 2.327, "ttft_max_s": 2.327, "decode_tok_s_per_stream": 269.9, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 44968, "ttft_p50_s": 2.375, "ttft_max_s": 2.375, "decode_tok_s_per_stream": 270.9, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 45230, "ttft_p50_s": 2.378, "ttft_max_s": 2.378, "decode_tok_s_per_stream": 269.8, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49431, "ttft_p50_s": 2.729, "ttft_max_s": 2.729, "decode_tok_s_per_stream": 266.3, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50493, "ttft_p50_s": 2.786, "ttft_max_s": 2.786, "decode_tok_s_per_stream": 266.6, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54350, "ttft_p50_s": 3.104, "ttft_max_s": 3.104, "decode_tok_s_per_stream": 264.1, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1283, "ttft_p50_s": 0.134, "ttft_max_s": 0.134, "decode_tok_s_per_stream": 291.7, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 44564, "ttft_p50_s": 2.325, "ttft_max_s": 2.325, "decode_tok_s_per_stream": 269.8, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60489, "ttft_p50_s": 3.622, "ttft_max_s": 3.622, "decode_tok_s_per_stream": 264.1, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60850, "ttft_p50_s": 3.686, "ttft_max_s": 3.686, "decode_tok_s_per_stream": 264.2, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61375, "ttft_p50_s": 3.74, "ttft_max_s": 3.74, "decode_tok_s_per_stream": 266.2, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61630, "ttft_p50_s": 3.795, "ttft_max_s": 3.795, "decode_tok_s_per_stream": 262.4, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62232, "ttft_p50_s": 3.824, "ttft_max_s": 3.824, "decode_tok_s_per_stream": 262.2, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 63980, "ttft_p50_s": 3.96, "ttft_max_s": 3.96, "decode_tok_s_per_stream": 262.9, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64744, "ttft_p50_s": 4.057, "ttft_max_s": 4.057, "decode_tok_s_per_stream": 264.7, "target_context": 64770, "output_tokens": 172 }, { "turn": 25, "context_tokens": 65003, "ttft_p50_s": 4.057, "ttft_max_s": 4.057, "decode_tok_s_per_stream": 258.7, "target_context": 48486, "output_tokens": 2201 }, { "turn": 26, "context_tokens": 66163, "ttft_p50_s": 4.192, "ttft_max_s": 4.192, "decode_tok_s_per_stream": 262.6, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 66418, "ttft_p50_s": 4.186, "ttft_max_s": 4.186, "decode_tok_s_per_stream": 258.9, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 66892, "ttft_p50_s": 4.256, "ttft_max_s": 4.256, "decode_tok_s_per_stream": 263.4, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 67141, "ttft_p50_s": 4.251, "ttft_max_s": 4.251, "decode_tok_s_per_stream": 259.3, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 67510, "ttft_p50_s": 4.298, "ttft_max_s": 4.298, "decode_tok_s_per_stream": 262.8, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 67765, "ttft_p50_s": 4.355, "ttft_max_s": 4.355, "decode_tok_s_per_stream": 264.6, "target_context": 65290, "output_tokens": 106 }, { "turn": 32, "context_tokens": 67998, "ttft_p50_s": 4.383, "ttft_max_s": 4.383, "decode_tok_s_per_stream": 260.8, "target_context": 65395, "output_tokens": 220 }, { "turn": 33, "context_tokens": 68281, "ttft_p50_s": 4.436, "ttft_max_s": 4.436, "decode_tok_s_per_stream": 274.4, "target_context": 65699, "output_tokens": 50 }, { "turn": 34, "context_tokens": 68488, "ttft_p50_s": 4.444, "ttft_max_s": 4.444, "decode_tok_s_per_stream": 260.2, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 68782, "ttft_p50_s": 4.459, "ttft_max_s": 4.459, "decode_tok_s_per_stream": 267.6, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 1988398 }, { "agents": 4, "mode": "warm", "turns_requested": 140, "turns_ok": 140, "status": "ok", "wall_s": 190.73, "turns_over_window": 0, "ttft_first_turn_s": 7.329, "ttft_last_turn_s": 0.622, "ttft_mean_s": 1.251, "ttft_p95_s": 5.14, "context_tokens_last": 68776, "prompt_tokens_total": 7956260, "turns_per_minute": 44.0, "per_turn": [ { "turn": 1, "context_tokens": 50703, "ttft_p50_s": 7.329, "ttft_max_s": 11.119, "decode_tok_s_per_stream": 88.5, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58769, "ttft_p50_s": 1.569, "ttft_max_s": 1.745, "decode_tok_s_per_stream": 80.2, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59627, "ttft_p50_s": 0.36, "ttft_max_s": 0.479, "decode_tok_s_per_stream": 142.8, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 59995, "ttft_p50_s": 0.538, "ttft_max_s": 0.634, "decode_tok_s_per_stream": 159.1, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60346, "ttft_p50_s": 0.699, "ttft_max_s": 0.79, "decode_tok_s_per_stream": 158.3, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 60966, "ttft_p50_s": 1.096, "ttft_max_s": 1.102, "decode_tok_s_per_stream": 157.8, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62317, "ttft_p50_s": 0.699, "ttft_max_s": 0.788, "decode_tok_s_per_stream": 155.1, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63614, "ttft_p50_s": 1.214, "ttft_max_s": 1.254, "decode_tok_s_per_stream": 160.6, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1272, "ttft_p50_s": 0.274, "ttft_max_s": 0.327, "decode_tok_s_per_stream": 150.5, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 44897, "ttft_p50_s": 6.153, "ttft_max_s": 9.28, "decode_tok_s_per_stream": 54.8, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 45190, "ttft_p50_s": 0.362, "ttft_max_s": 0.477, "decode_tok_s_per_stream": 128.7, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 45452, "ttft_p50_s": 0.367, "ttft_max_s": 0.459, "decode_tok_s_per_stream": 117.3, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49422, "ttft_p50_s": 1.003, "ttft_max_s": 1.115, "decode_tok_s_per_stream": 156.3, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50484, "ttft_p50_s": 1.034, "ttft_max_s": 1.059, "decode_tok_s_per_stream": 167.2, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54341, "ttft_p50_s": 1.504, "ttft_max_s": 1.616, "decode_tok_s_per_stream": 159.7, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1276, "ttft_p50_s": 0.275, "ttft_max_s": 0.276, "decode_tok_s_per_stream": 162.8, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 44786, "ttft_p50_s": 6.124, "ttft_max_s": 9.192, "decode_tok_s_per_stream": 57.7, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60481, "ttft_p50_s": 3.501, "ttft_max_s": 4.687, "decode_tok_s_per_stream": 92.0, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60842, "ttft_p50_s": 0.855, "ttft_max_s": 0.977, "decode_tok_s_per_stream": 154.7, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61367, "ttft_p50_s": 0.389, "ttft_max_s": 0.425, "decode_tok_s_per_stream": 161.8, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61622, "ttft_p50_s": 0.465, "ttft_max_s": 0.52, "decode_tok_s_per_stream": 159.4, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62224, "ttft_p50_s": 0.581, "ttft_max_s": 0.656, "decode_tok_s_per_stream": 159.2, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 63972, "ttft_p50_s": 1.315, "ttft_max_s": 1.373, "decode_tok_s_per_stream": 159.2, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64736, "ttft_p50_s": 0.736, "ttft_max_s": 0.832, "decode_tok_s_per_stream": 152.0, "target_context": 64770, "output_tokens": 172 }, { "turn": 25, "context_tokens": 64995, "ttft_p50_s": 0.79, "ttft_max_s": 0.903, "decode_tok_s_per_stream": 159.4, "target_context": 48486, "output_tokens": 2201 }, { "turn": 26, "context_tokens": 66156, "ttft_p50_s": 0.496, "ttft_max_s": 0.557, "decode_tok_s_per_stream": 155.2, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 66411, "ttft_p50_s": 0.595, "ttft_max_s": 0.672, "decode_tok_s_per_stream": 158.2, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 66885, "ttft_p50_s": 0.753, "ttft_max_s": 0.851, "decode_tok_s_per_stream": 149.1, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 67134, "ttft_p50_s": 0.803, "ttft_max_s": 0.918, "decode_tok_s_per_stream": 154.8, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 67503, "ttft_p50_s": 1.17, "ttft_max_s": 1.177, "decode_tok_s_per_stream": 163.8, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 67758, "ttft_p50_s": 0.366, "ttft_max_s": 0.383, "decode_tok_s_per_stream": 161.8, "target_context": 65290, "output_tokens": 106 }, { "turn": 32, "context_tokens": 67991, "ttft_p50_s": 0.392, "ttft_max_s": 0.433, "decode_tok_s_per_stream": 158.1, "target_context": 65395, "output_tokens": 220 }, { "turn": 33, "context_tokens": 68274, "ttft_p50_s": 0.549, "ttft_max_s": 0.609, "decode_tok_s_per_stream": 146.9, "target_context": 65699, "output_tokens": 50 }, { "turn": 34, "context_tokens": 68481, "ttft_p50_s": 0.585, "ttft_max_s": 0.663, "decode_tok_s_per_stream": 155.3, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 68776, "ttft_p50_s": 0.622, "ttft_max_s": 0.703, "decode_tok_s_per_stream": 146.2, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.8886, "prefix_cache_queried_tokens": 7956260 }, { "agents": 4, "mode": "cold", "turns_requested": 140, "turns_ok": 140, "status": "ok", "wall_s": 610.07, "turns_over_window": 0, "ttft_first_turn_s": 7.269, "ttft_last_turn_s": 7.063, "ttft_mean_s": 6.119, "ttft_p95_s": 9.984, "context_tokens_last": 68782, "prompt_tokens_total": 7953592, "turns_per_minute": 13.8, "per_turn": [ { "turn": 1, "context_tokens": 50709, "ttft_p50_s": 7.269, "ttft_max_s": 11.029, "decode_tok_s_per_stream": 89.0, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58775, "ttft_p50_s": 8.584, "ttft_max_s": 10.048, "decode_tok_s_per_stream": 31.9, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59633, "ttft_p50_s": 7.659, "ttft_max_s": 8.738, "decode_tok_s_per_stream": 22.1, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 60001, "ttft_p50_s": 7.458, "ttft_max_s": 8.34, "decode_tok_s_per_stream": 43.8, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60352, "ttft_p50_s": 7.824, "ttft_max_s": 8.942, "decode_tok_s_per_stream": 51.1, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 60972, "ttft_p50_s": 8.245, "ttft_max_s": 18.414, "decode_tok_s_per_stream": 90.3, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62323, "ttft_p50_s": 6.476, "ttft_max_s": 6.701, "decode_tok_s_per_stream": 33.5, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63620, "ttft_p50_s": 6.292, "ttft_max_s": 9.152, "decode_tok_s_per_stream": 117.3, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.109, "ttft_max_s": 0.419, "decode_tok_s_per_stream": 133.1, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 44675, "ttft_p50_s": 2.822, "ttft_max_s": 4.986, "decode_tok_s_per_stream": 82.1, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 44968, "ttft_p50_s": 3.929, "ttft_max_s": 5.751, "decode_tok_s_per_stream": 69.8, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 45230, "ttft_p50_s": 3.837, "ttft_max_s": 5.825, "decode_tok_s_per_stream": 34.5, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49431, "ttft_p50_s": 4.431, "ttft_max_s": 5.938, "decode_tok_s_per_stream": 64.2, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50493, "ttft_p50_s": 4.572, "ttft_max_s": 6.627, "decode_tok_s_per_stream": 128.5, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54350, "ttft_p50_s": 4.926, "ttft_max_s": 7.995, "decode_tok_s_per_stream": 117.9, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1283, "ttft_p50_s": 0.112, "ttft_max_s": 0.225, "decode_tok_s_per_stream": 130.0, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 44564, "ttft_p50_s": 2.975, "ttft_max_s": 5.073, "decode_tok_s_per_stream": 118.0, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60489, "ttft_p50_s": 5.731, "ttft_max_s": 10.799, "decode_tok_s_per_stream": 56.3, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60850, "ttft_p50_s": 5.789, "ttft_max_s": 8.401, "decode_tok_s_per_stream": 57.2, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61375, "ttft_p50_s": 5.982, "ttft_max_s": 8.526, "decode_tok_s_per_stream": 41.7, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61630, "ttft_p50_s": 6.021, "ttft_max_s": 8.634, "decode_tok_s_per_stream": 56.6, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62232, "ttft_p50_s": 6.099, "ttft_max_s": 9.214, "decode_tok_s_per_stream": 35.5, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 63980, "ttft_p50_s": 6.079, "ttft_max_s": 8.313, "decode_tok_s_per_stream": 54.9, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64744, "ttft_p50_s": 6.444, "ttft_max_s": 8.925, "decode_tok_s_per_stream": 42.6, "target_context": 64770, "output_tokens": 172 }, { "turn": 25, "context_tokens": 65003, "ttft_p50_s": 6.363, "ttft_max_s": 9.529, "decode_tok_s_per_stream": 62.3, "target_context": 48486, "output_tokens": 2201 }, { "turn": 26, "context_tokens": 66163, "ttft_p50_s": 8.699, "ttft_max_s": 9.322, "decode_tok_s_per_stream": 24.3, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 66418, "ttft_p50_s": 8.426, "ttft_max_s": 8.773, "decode_tok_s_per_stream": 66.7, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 66892, "ttft_p50_s": 8.81, "ttft_max_s": 9.533, "decode_tok_s_per_stream": 29.0, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 67141, "ttft_p50_s": 6.525, "ttft_max_s": 8.824, "decode_tok_s_per_stream": 64.2, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 67510, "ttft_p50_s": 6.877, "ttft_max_s": 9.136, "decode_tok_s_per_stream": 33.8, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 67765, "ttft_p50_s": 8.75, "ttft_max_s": 13.916, "decode_tok_s_per_stream": 17.1, "target_context": 65290, "output_tokens": 106 }, { "turn": 32, "context_tokens": 67998, "ttft_p50_s": 8.795, "ttft_max_s": 9.254, "decode_tok_s_per_stream": 22.6, "target_context": 65395, "output_tokens": 220 }, { "turn": 33, "context_tokens": 68281, "ttft_p50_s": 9.012, "ttft_max_s": 9.88, "decode_tok_s_per_stream": 169.7, "target_context": 65699, "output_tokens": 50 }, { "turn": 34, "context_tokens": 68488, "ttft_p50_s": 8.921, "ttft_max_s": 9.135, "decode_tok_s_per_stream": 39.8, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 68782, "ttft_p50_s": 7.063, "ttft_max_s": 12.345, "decode_tok_s_per_stream": 103.6, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 7953592 } ], "dimensions": { "model": "Qwen3.6-35B-A3B", "shape": "typical", "seqs": "32", "graphs": "on", "ctx": "256K" } }, { "label": "qwen3.6-35b-a3b 256K · heavy session", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 262144, "workload": { "profile": "docs/usage.json", "session": "heavy", "turns": 199, "turns_in_window": 188, "context_min": 927, "context_max": 274585, "output_tokens_p50": 560.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 188, "turns_ok": 188, "status": "ok", "wall_s": 1039.27, "turns_over_window": 11, "ttft_first_turn_s": 3.337, "ttft_last_turn_s": 0.512, "ttft_mean_s": 0.862, "ttft_p95_s": 2.31, "context_tokens_last": 46560, "prompt_tokens_total": 20653703, "turns_per_minute": 10.9, "per_turn": [ { "turn": 1, "context_tokens": 56967, "ttft_p50_s": 3.337, "ttft_max_s": 3.337, "decode_tok_s_per_stream": 264.4, "target_context": 64988, "output_tokens": 1739 }, { "turn": 2, "context_tokens": 66459, "ttft_p50_s": 1.064, "ttft_max_s": 1.064, "decode_tok_s_per_stream": 258.7, "target_context": 67299, "output_tokens": 1219 }, { "turn": 3, "context_tokens": 67983, "ttft_p50_s": 0.438, "ttft_max_s": 0.438, "decode_tok_s_per_stream": 258.3, "target_context": 68580, "output_tokens": 1387 }, { "turn": 4, "context_tokens": 68790, "ttft_p50_s": 0.293, "ttft_max_s": 0.293, "decode_tok_s_per_stream": 262.4, "target_context": 55938, "output_tokens": 202 }, { "turn": 5, "context_tokens": 70542, "ttft_p50_s": 0.501, "ttft_max_s": 0.501, "decode_tok_s_per_stream": 257.2, "target_context": 70499, "output_tokens": 1861 }, { "turn": 6, "context_tokens": 71571, "ttft_p50_s": 0.362, "ttft_max_s": 0.362, "decode_tok_s_per_stream": 259.0, "target_context": 56909, "output_tokens": 398 }, { "turn": 7, "context_tokens": 71935, "ttft_p50_s": 0.431, "ttft_max_s": 0.431, "decode_tok_s_per_stream": 263.0, "target_context": 58822, "output_tokens": 124 }, { "turn": 8, "context_tokens": 72402, "ttft_p50_s": 0.209, "ttft_max_s": 0.209, "decode_tok_s_per_stream": 256.6, "target_context": 72414, "output_tokens": 1619 }, { "turn": 9, "context_tokens": 73329, "ttft_p50_s": 0.357, "ttft_max_s": 0.357, "decode_tok_s_per_stream": 256.0, "target_context": 74118, "output_tokens": 2252 }, { "turn": 10, "context_tokens": 75596, "ttft_p50_s": 0.3, "ttft_max_s": 0.3, "decode_tok_s_per_stream": 257.6, "target_context": 76664, "output_tokens": 384 }, { "turn": 11, "context_tokens": 77571, "ttft_p50_s": 0.601, "ttft_max_s": 0.601, "decode_tok_s_per_stream": 254.6, "target_context": 77578, "output_tokens": 1155 }, { "turn": 12, "context_tokens": 78531, "ttft_p50_s": 0.453, "ttft_max_s": 0.453, "decode_tok_s_per_stream": 254.0, "target_context": 79065, "output_tokens": 1134 }, { "turn": 13, "context_tokens": 80519, "ttft_p50_s": 0.385, "ttft_max_s": 0.385, "decode_tok_s_per_stream": 252.6, "target_context": 80967, "output_tokens": 2228 }, { "turn": 14, "context_tokens": 82342, "ttft_p50_s": 0.383, "ttft_max_s": 0.383, "decode_tok_s_per_stream": 254.6, "target_context": 83411, "output_tokens": 310 }, { "turn": 15, "context_tokens": 83940, "ttft_p50_s": 0.541, "ttft_max_s": 0.541, "decode_tok_s_per_stream": 252.7, "target_context": 83903, "output_tokens": 818 }, { "turn": 16, "context_tokens": 85095, "ttft_p50_s": 0.478, "ttft_max_s": 0.478, "decode_tok_s_per_stream": 252.3, "target_context": 85329, "output_tokens": 853 }, { "turn": 17, "context_tokens": 86152, "ttft_p50_s": 0.326, "ttft_max_s": 0.326, "decode_tok_s_per_stream": 254.9, "target_context": 86443, "output_tokens": 243 }, { "turn": 18, "context_tokens": 86799, "ttft_p50_s": 0.336, "ttft_max_s": 0.336, "decode_tok_s_per_stream": 251.6, "target_context": 86751, "output_tokens": 642 }, { "turn": 19, "context_tokens": 1276, "ttft_p50_s": 0.069, "ttft_max_s": 0.069, "decode_tok_s_per_stream": 290.8, "target_context": 961, "output_tokens": 16 }, { "turn": 20, "context_tokens": 45018, "ttft_p50_s": 2.367, "ttft_max_s": 2.367, "decode_tok_s_per_stream": 269.7, "target_context": 33996, "output_tokens": 279 }, { "turn": 21, "context_tokens": 45326, "ttft_p50_s": 0.254, "ttft_max_s": 0.254, "decode_tok_s_per_stream": 270.2, "target_context": 34455, "output_tokens": 223 }, { "turn": 22, "context_tokens": 45608, "ttft_p50_s": 0.305, "ttft_max_s": 0.305, "decode_tok_s_per_stream": 268.5, "target_context": 34900, "output_tokens": 578 }, { "turn": 23, "context_tokens": 46045, "ttft_p50_s": 0.187, "ttft_max_s": 0.187, "decode_tok_s_per_stream": 267.9, "target_context": 35765, "output_tokens": 1164 }, { "turn": 24, "context_tokens": 46742, "ttft_p50_s": 0.23, "ttft_max_s": 0.23, "decode_tok_s_per_stream": 267.9, "target_context": 37099, "output_tokens": 890 }, { "turn": 25, "context_tokens": 47323, "ttft_p50_s": 0.276, "ttft_max_s": 0.276, "decode_tok_s_per_stream": 267.0, "target_context": 38060, "output_tokens": 1787 }, { "turn": 26, "context_tokens": 48308, "ttft_p50_s": 0.124, "ttft_max_s": 0.124, "decode_tok_s_per_stream": 267.9, "target_context": 39965, "output_tokens": 502 }, { "turn": 27, "context_tokens": 48721, "ttft_p50_s": 0.229, "ttft_max_s": 0.229, "decode_tok_s_per_stream": 267.3, "target_context": 40530, "output_tokens": 867 }, { "turn": 28, "context_tokens": 49303, "ttft_p50_s": 0.279, "ttft_max_s": 0.279, "decode_tok_s_per_stream": 266.9, "target_context": 41685, "output_tokens": 996 }, { "turn": 29, "context_tokens": 49946, "ttft_p50_s": 0.326, "ttft_max_s": 0.326, "decode_tok_s_per_stream": 266.3, "target_context": 42730, "output_tokens": 1944 }, { "turn": 30, "context_tokens": 51035, "ttft_p50_s": 0.242, "ttft_max_s": 0.242, "decode_tok_s_per_stream": 265.8, "target_context": 44830, "output_tokens": 2348 }, { "turn": 31, "context_tokens": 52330, "ttft_p50_s": 0.353, "ttft_max_s": 0.353, "decode_tok_s_per_stream": 265.9, "target_context": 47231, "output_tokens": 985 }, { "turn": 32, "context_tokens": 52988, "ttft_p50_s": 0.241, "ttft_max_s": 0.241, "decode_tok_s_per_stream": 265.5, "target_context": 48267, "output_tokens": 2720 }, { "turn": 33, "context_tokens": 54495, "ttft_p50_s": 0.367, "ttft_max_s": 0.367, "decode_tok_s_per_stream": 267.0, "target_context": 51031, "output_tokens": 318 }, { "turn": 34, "context_tokens": 1276, "ttft_p50_s": 0.069, "ttft_max_s": 0.069, "decode_tok_s_per_stream": 292.5, "target_context": 927, "output_tokens": 13 }, { "turn": 35, "context_tokens": 39732, "ttft_p50_s": 1.967, "ttft_max_s": 1.967, "decode_tok_s_per_stream": 274.7, "target_context": 29988, "output_tokens": 207 }, { "turn": 36, "context_tokens": 40007, "ttft_p50_s": 0.184, "ttft_max_s": 0.184, "decode_tok_s_per_stream": 274.7, "target_context": 35061, "output_tokens": 206 }, { "turn": 37, "context_tokens": 40280, "ttft_p50_s": 0.215, "ttft_max_s": 0.215, "decode_tok_s_per_stream": 274.1, "target_context": 36124, "output_tokens": 307 }, { "turn": 38, "context_tokens": 40599, "ttft_p50_s": 0.185, "ttft_max_s": 0.185, "decode_tok_s_per_stream": 273.5, "target_context": 36598, "output_tokens": 405 }, { "turn": 39, "context_tokens": 1276, "ttft_p50_s": 0.07, "ttft_max_s": 0.07, "decode_tok_s_per_stream": 290.4, "target_context": 957, "output_tokens": 23 }, { "turn": 40, "context_tokens": 44908, "ttft_p50_s": 2.367, "ttft_max_s": 2.367, "decode_tok_s_per_stream": 270.2, "target_context": 34132, "output_tokens": 217 }, { "turn": 41, "context_tokens": 45187, "ttft_p50_s": 0.248, "ttft_max_s": 0.248, "decode_tok_s_per_stream": 269.6, "target_context": 35247, "output_tokens": 240 }, { "turn": 42, "context_tokens": 45476, "ttft_p50_s": 0.279, "ttft_max_s": 0.279, "decode_tok_s_per_stream": 268.6, "target_context": 44130, "output_tokens": 589 }, { "turn": 43, "context_tokens": 45919, "ttft_p50_s": 0.19, "ttft_max_s": 0.19, "decode_tok_s_per_stream": 269.4, "target_context": 45079, "output_tokens": 312 }, { "turn": 44, "context_tokens": 13801, "ttft_p50_s": 0.542, "ttft_max_s": 0.542, "decode_tok_s_per_stream": 284.9, "target_context": 14112, "output_tokens": 1229 }, { "turn": 45, "context_tokens": 47861, "ttft_p50_s": 2.139, "ttft_max_s": 2.139, "decode_tok_s_per_stream": 266.7, "target_context": 47335, "output_tokens": 2465 }, { "turn": 46, "context_tokens": 49142, "ttft_p50_s": 0.398, "ttft_max_s": 0.398, "decode_tok_s_per_stream": 266.5, "target_context": 49984, "output_tokens": 2228 }, { "turn": 47, "context_tokens": 51376, "ttft_p50_s": 0.183, "ttft_max_s": 0.183, "decode_tok_s_per_stream": 265.8, "target_context": 52506, "output_tokens": 1819 }, { "turn": 48, "context_tokens": 54247, "ttft_p50_s": 0.475, "ttft_max_s": 0.475, "decode_tok_s_per_stream": 265.5, "target_context": 55007, "output_tokens": 589 }, { "turn": 49, "context_tokens": 55973, "ttft_p50_s": 0.305, "ttft_max_s": 0.305, "decode_tok_s_per_stream": 265.5, "target_context": 56050, "output_tokens": 566 }, { "turn": 50, "context_tokens": 57573, "ttft_p50_s": 0.439, "ttft_max_s": 0.439, "decode_tok_s_per_stream": 264.2, "target_context": 57643, "output_tokens": 737 }, { "turn": 51, "context_tokens": 58450, "ttft_p50_s": 0.32, "ttft_max_s": 0.32, "decode_tok_s_per_stream": 264.6, "target_context": 58729, "output_tokens": 478 }, { "turn": 52, "context_tokens": 59206, "ttft_p50_s": 0.394, "ttft_max_s": 0.394, "decode_tok_s_per_stream": 263.6, "target_context": 59363, "output_tokens": 560 }, { "turn": 53, "context_tokens": 64253, "ttft_p50_s": 0.647, "ttft_max_s": 0.647, "decode_tok_s_per_stream": 263.6, "target_context": 64045, "output_tokens": 218 }, { "turn": 54, "context_tokens": 64884, "ttft_p50_s": 0.316, "ttft_max_s": 0.316, "decode_tok_s_per_stream": 261.5, "target_context": 64837, "output_tokens": 464 }, { "turn": 55, "context_tokens": 78386, "ttft_p50_s": 1.667, "ttft_max_s": 1.667, "decode_tok_s_per_stream": 254.8, "target_context": 77430, "output_tokens": 757 }, { "turn": 56, "context_tokens": 84898, "ttft_p50_s": 0.84, "ttft_max_s": 0.84, "decode_tok_s_per_stream": 253.5, "target_context": 84792, "output_tokens": 453 }, { "turn": 57, "context_tokens": 88050, "ttft_p50_s": 0.595, "ttft_max_s": 0.595, "decode_tok_s_per_stream": 250.5, "target_context": 88014, "output_tokens": 662 }, { "turn": 58, "context_tokens": 88543, "ttft_p50_s": 0.402, "ttft_max_s": 0.402, "decode_tok_s_per_stream": 248.6, "target_context": 88805, "output_tokens": 3447 }, { "turn": 59, "context_tokens": 93099, "ttft_p50_s": 0.696, "ttft_max_s": 0.696, "decode_tok_s_per_stream": 246.5, "target_context": 94604, "output_tokens": 5536 }, { "turn": 60, "context_tokens": 97868, "ttft_p50_s": 0.794, "ttft_max_s": 0.794, "decode_tok_s_per_stream": 244.9, "target_context": 100477, "output_tokens": 6852 }, { "turn": 61, "context_tokens": 104267, "ttft_p50_s": 0.765, "ttft_max_s": 0.765, "decode_tok_s_per_stream": 242.2, "target_context": 107360, "output_tokens": 7445 }, { "turn": 62, "context_tokens": 111982, "ttft_p50_s": 0.991, "ttft_max_s": 0.991, "decode_tok_s_per_stream": 245.4, "target_context": 115070, "output_tokens": 207 }, { "turn": 63, "context_tokens": 115834, "ttft_p50_s": 0.919, "ttft_max_s": 0.919, "decode_tok_s_per_stream": 246.7, "target_context": 115310, "output_tokens": 152 }, { "turn": 64, "context_tokens": 116096, "ttft_p50_s": 0.36, "ttft_max_s": 0.36, "decode_tok_s_per_stream": 240.9, "target_context": 115511, "output_tokens": 576 }, { "turn": 65, "context_tokens": 116577, "ttft_p50_s": 0.339, "ttft_max_s": 0.339, "decode_tok_s_per_stream": 239.7, "target_context": 116418, "output_tokens": 1052 }, { "turn": 66, "context_tokens": 117418, "ttft_p50_s": 0.516, "ttft_max_s": 0.516, "decode_tok_s_per_stream": 239.3, "target_context": 117793, "output_tokens": 1211 }, { "turn": 67, "context_tokens": 119031, "ttft_p50_s": 0.487, "ttft_max_s": 0.487, "decode_tok_s_per_stream": 240.1, "target_context": 119395, "output_tokens": 510 }, { "turn": 68, "context_tokens": 123501, "ttft_p50_s": 1.163, "ttft_max_s": 1.163, "decode_tok_s_per_stream": 238.4, "target_context": 122992, "output_tokens": 590 }, { "turn": 69, "context_tokens": 123990, "ttft_p50_s": 0.594, "ttft_max_s": 0.594, "decode_tok_s_per_stream": 234.7, "target_context": 123791, "output_tokens": 9989 }, { "turn": 70, "context_tokens": 129323, "ttft_p50_s": 0.417, "ttft_max_s": 0.417, "decode_tok_s_per_stream": 234.4, "target_context": 133841, "output_tokens": 3278 }, { "turn": 71, "context_tokens": 136775, "ttft_p50_s": 1.269, "ttft_max_s": 1.269, "decode_tok_s_per_stream": 237.1, "target_context": 137168, "output_tokens": 207 }, { "turn": 72, "context_tokens": 138448, "ttft_p50_s": 0.845, "ttft_max_s": 0.845, "decode_tok_s_per_stream": 235.0, "target_context": 138157, "output_tokens": 301 }, { "turn": 73, "context_tokens": 138794, "ttft_p50_s": 0.544, "ttft_max_s": 0.544, "decode_tok_s_per_stream": 232.3, "target_context": 138661, "output_tokens": 800 }, { "turn": 74, "context_tokens": 139406, "ttft_p50_s": 0.686, "ttft_max_s": 0.686, "decode_tok_s_per_stream": 234.9, "target_context": 139603, "output_tokens": 272 }, { "turn": 75, "context_tokens": 139966, "ttft_p50_s": 0.372, "ttft_max_s": 0.372, "decode_tok_s_per_stream": 231.3, "target_context": 139981, "output_tokens": 1244 }, { "turn": 76, "context_tokens": 141851, "ttft_p50_s": 0.723, "ttft_max_s": 0.723, "decode_tok_s_per_stream": 232.2, "target_context": 142096, "output_tokens": 528 }, { "turn": 77, "context_tokens": 143353, "ttft_p50_s": 0.568, "ttft_max_s": 0.568, "decode_tok_s_per_stream": 232.7, "target_context": 143292, "output_tokens": 418 }, { "turn": 78, "context_tokens": 143760, "ttft_p50_s": 0.696, "ttft_max_s": 0.696, "decode_tok_s_per_stream": 230.8, "target_context": 143789, "output_tokens": 973 }, { "turn": 79, "context_tokens": 144467, "ttft_p50_s": 0.447, "ttft_max_s": 0.447, "decode_tok_s_per_stream": 231.2, "target_context": 144773, "output_tokens": 677 }, { "turn": 80, "context_tokens": 145361, "ttft_p50_s": 0.566, "ttft_max_s": 0.566, "decode_tok_s_per_stream": 233.8, "target_context": 145546, "output_tokens": 267 }, { "turn": 81, "context_tokens": 145919, "ttft_p50_s": 0.656, "ttft_max_s": 0.656, "decode_tok_s_per_stream": 230.3, "target_context": 145928, "output_tokens": 907 }, { "turn": 82, "context_tokens": 146592, "ttft_p50_s": 0.444, "ttft_max_s": 0.444, "decode_tok_s_per_stream": 232.4, "target_context": 146943, "output_tokens": 345 }, { "turn": 83, "context_tokens": 45285, "ttft_p50_s": 2.383, "ttft_max_s": 2.383, "decode_tok_s_per_stream": 270.6, "target_context": 37006, "output_tokens": 178 }, { "turn": 84, "context_tokens": 45547, "ttft_p50_s": 0.255, "ttft_max_s": 0.255, "decode_tok_s_per_stream": 276.1, "target_context": 37308, "output_tokens": 62 }, { "turn": 85, "context_tokens": 148370, "ttft_p50_s": 13.391, "ttft_max_s": 13.391, "decode_tok_s_per_stream": 229.7, "target_context": 147329, "output_tokens": 696 }, { "turn": 86, "context_tokens": 148859, "ttft_p50_s": 0.458, "ttft_max_s": 0.458, "decode_tok_s_per_stream": 228.9, "target_context": 148072, "output_tokens": 1084 }, { "turn": 87, "context_tokens": 149516, "ttft_p50_s": 0.574, "ttft_max_s": 0.574, "decode_tok_s_per_stream": 236.6, "target_context": 149399, "output_tokens": 139 }, { "turn": 88, "context_tokens": 149762, "ttft_p50_s": 0.571, "ttft_max_s": 0.571, "decode_tok_s_per_stream": 233.8, "target_context": 149577, "output_tokens": 198 }, { "turn": 89, "context_tokens": 150032, "ttft_p50_s": 0.68, "ttft_max_s": 0.68, "decode_tok_s_per_stream": 236.1, "target_context": 149787, "output_tokens": 141 }, { "turn": 90, "context_tokens": 150279, "ttft_p50_s": 0.708, "ttft_max_s": 0.708, "decode_tok_s_per_stream": 248.6, "target_context": 150025, "output_tokens": 61 }, { "turn": 91, "context_tokens": 150489, "ttft_p50_s": 0.289, "ttft_max_s": 0.289, "decode_tok_s_per_stream": 227.5, "target_context": 150151, "output_tokens": 2321 }, { "turn": 92, "context_tokens": 151691, "ttft_p50_s": 0.573, "ttft_max_s": 0.573, "decode_tok_s_per_stream": 229.2, "target_context": 152644, "output_tokens": 534 }, { "turn": 93, "context_tokens": 153144, "ttft_p50_s": 0.919, "ttft_max_s": 0.919, "decode_tok_s_per_stream": 229.5, "target_context": 153291, "output_tokens": 315 }, { "turn": 94, "context_tokens": 153926, "ttft_p50_s": 0.53, "ttft_max_s": 0.53, "decode_tok_s_per_stream": 226.5, "target_context": 154030, "output_tokens": 911 }, { "turn": 95, "context_tokens": 154859, "ttft_p50_s": 0.775, "ttft_max_s": 0.775, "decode_tok_s_per_stream": 228.3, "target_context": 155245, "output_tokens": 383 }, { "turn": 96, "context_tokens": 155672, "ttft_p50_s": 0.474, "ttft_max_s": 0.474, "decode_tok_s_per_stream": 226.9, "target_context": 155756, "output_tokens": 622 }, { "turn": 97, "context_tokens": 156132, "ttft_p50_s": 0.594, "ttft_max_s": 0.594, "decode_tok_s_per_stream": 234.4, "target_context": 156407, "output_tokens": 134 }, { "turn": 98, "context_tokens": 156607, "ttft_p50_s": 0.695, "ttft_max_s": 0.695, "decode_tok_s_per_stream": 228.0, "target_context": 156597, "output_tokens": 366 }, { "turn": 99, "context_tokens": 157184, "ttft_p50_s": 0.359, "ttft_max_s": 0.359, "decode_tok_s_per_stream": 226.6, "target_context": 157336, "output_tokens": 633 }, { "turn": 100, "context_tokens": 158118, "ttft_p50_s": 0.593, "ttft_max_s": 0.593, "decode_tok_s_per_stream": 229.8, "target_context": 158372, "output_tokens": 229 }, { "turn": 101, "context_tokens": 158639, "ttft_p50_s": 0.603, "ttft_max_s": 0.603, "decode_tok_s_per_stream": 225.7, "target_context": 158633, "output_tokens": 794 }, { "turn": 102, "context_tokens": 160106, "ttft_p50_s": 0.485, "ttft_max_s": 0.485, "decode_tok_s_per_stream": 228.1, "target_context": 160400, "output_tokens": 292 }, { "turn": 103, "context_tokens": 160656, "ttft_p50_s": 0.599, "ttft_max_s": 0.599, "decode_tok_s_per_stream": 238.3, "target_context": 160736, "output_tokens": 90 }, { "turn": 104, "context_tokens": 161001, "ttft_p50_s": 0.701, "ttft_max_s": 0.701, "decode_tok_s_per_stream": 224.5, "target_context": 160944, "output_tokens": 1044 }, { "turn": 105, "context_tokens": 161769, "ttft_p50_s": 0.429, "ttft_max_s": 0.429, "decode_tok_s_per_stream": 222.9, "target_context": 162243, "output_tokens": 4442 }, { "turn": 106, "context_tokens": 164406, "ttft_p50_s": 0.493, "ttft_max_s": 0.493, "decode_tok_s_per_stream": 225.3, "target_context": 166793, "output_tokens": 442 }, { "turn": 107, "context_tokens": 167344, "ttft_p50_s": 1.122, "ttft_max_s": 1.122, "decode_tok_s_per_stream": 221.4, "target_context": 167320, "output_tokens": 4287 }, { "turn": 108, "context_tokens": 169475, "ttft_p50_s": 0.64, "ttft_max_s": 0.64, "decode_tok_s_per_stream": 221.1, "target_context": 171776, "output_tokens": 2797 }, { "turn": 109, "context_tokens": 173496, "ttft_p50_s": 1.149, "ttft_max_s": 1.149, "decode_tok_s_per_stream": 221.3, "target_context": 174769, "output_tokens": 952 }, { "turn": 110, "context_tokens": 175514, "ttft_p50_s": 1.156, "ttft_max_s": 1.156, "decode_tok_s_per_stream": 221.3, "target_context": 175840, "output_tokens": 495 }, { "turn": 111, "context_tokens": 176163, "ttft_p50_s": 0.778, "ttft_max_s": 0.778, "decode_tok_s_per_stream": 221.2, "target_context": 176359, "output_tokens": 459 }, { "turn": 112, "context_tokens": 181431, "ttft_p50_s": 1.464, "ttft_max_s": 1.464, "decode_tok_s_per_stream": 217.7, "target_context": 181231, "output_tokens": 1241 }, { "turn": 113, "context_tokens": 183235, "ttft_p50_s": 0.931, "ttft_max_s": 0.931, "decode_tok_s_per_stream": 222.6, "target_context": 183733, "output_tokens": 207 }, { "turn": 114, "context_tokens": 185258, "ttft_p50_s": 0.864, "ttft_max_s": 0.864, "decode_tok_s_per_stream": 219.6, "target_context": 185143, "output_tokens": 355 }, { "turn": 115, "context_tokens": 185611, "ttft_p50_s": 0.405, "ttft_max_s": 0.405, "decode_tok_s_per_stream": 219.1, "target_context": 185537, "output_tokens": 392 }, { "turn": 116, "context_tokens": 187605, "ttft_p50_s": 0.95, "ttft_max_s": 0.95, "decode_tok_s_per_stream": 215.9, "target_context": 187617, "output_tokens": 2149 }, { "turn": 117, "context_tokens": 189255, "ttft_p50_s": 0.829, "ttft_max_s": 0.829, "decode_tok_s_per_stream": 216.5, "target_context": 190260, "output_tokens": 910 }, { "turn": 118, "context_tokens": 191027, "ttft_p50_s": 0.702, "ttft_max_s": 0.702, "decode_tok_s_per_stream": 219.2, "target_context": 191378, "output_tokens": 301 }, { "turn": 119, "context_tokens": 191704, "ttft_p50_s": 0.877, "ttft_max_s": 0.877, "decode_tok_s_per_stream": 217.0, "target_context": 191712, "output_tokens": 522 }, { "turn": 120, "context_tokens": 192137, "ttft_p50_s": 0.416, "ttft_max_s": 0.416, "decode_tok_s_per_stream": 214.2, "target_context": 192298, "output_tokens": 3828 }, { "turn": 121, "context_tokens": 194694, "ttft_p50_s": 0.561, "ttft_max_s": 0.561, "decode_tok_s_per_stream": 213.7, "target_context": 196608, "output_tokens": 2076 }, { "turn": 122, "context_tokens": 198297, "ttft_p50_s": 1.373, "ttft_max_s": 1.373, "decode_tok_s_per_stream": 214.6, "target_context": 199085, "output_tokens": 447 }, { "turn": 123, "context_tokens": 199624, "ttft_p50_s": 0.719, "ttft_max_s": 0.719, "decode_tok_s_per_stream": 218.1, "target_context": 199661, "output_tokens": 209 }, { "turn": 124, "context_tokens": 199911, "ttft_p50_s": 0.723, "ttft_max_s": 0.723, "decode_tok_s_per_stream": 214.5, "target_context": 199912, "output_tokens": 426 }, { "turn": 125, "context_tokens": 200300, "ttft_p50_s": 0.901, "ttft_max_s": 0.901, "decode_tok_s_per_stream": 213.4, "target_context": 200352, "output_tokens": 680 }, { "turn": 126, "context_tokens": 200926, "ttft_p50_s": 0.569, "ttft_max_s": 0.569, "decode_tok_s_per_stream": 246.1, "target_context": 201202, "output_tokens": 44 }, { "turn": 127, "context_tokens": 201367, "ttft_p50_s": 0.56, "ttft_max_s": 0.56, "decode_tok_s_per_stream": 214.0, "target_context": 201264, "output_tokens": 486 }, { "turn": 128, "context_tokens": 201785, "ttft_p50_s": 0.726, "ttft_max_s": 0.726, "decode_tok_s_per_stream": 216.0, "target_context": 201793, "output_tokens": 269 }, { "turn": 129, "context_tokens": 202217, "ttft_p50_s": 0.737, "ttft_max_s": 0.737, "decode_tok_s_per_stream": 213.9, "target_context": 202225, "output_tokens": 447 }, { "turn": 130, "context_tokens": 202617, "ttft_p50_s": 0.343, "ttft_max_s": 0.343, "decode_tok_s_per_stream": 235.5, "target_context": 202746, "output_tokens": 61 }, { "turn": 131, "context_tokens": 202950, "ttft_p50_s": 0.418, "ttft_max_s": 0.418, "decode_tok_s_per_stream": 212.2, "target_context": 202882, "output_tokens": 860 }, { "turn": 132, "context_tokens": 203662, "ttft_p50_s": 0.596, "ttft_max_s": 0.596, "decode_tok_s_per_stream": 210.8, "target_context": 203961, "output_tokens": 2214 }, { "turn": 133, "context_tokens": 205829, "ttft_p50_s": 0.595, "ttft_max_s": 0.595, "decode_tok_s_per_stream": 210.8, "target_context": 206847, "output_tokens": 1213 }, { "turn": 134, "context_tokens": 224331, "ttft_p50_s": 5.211, "ttft_max_s": 5.211, "decode_tok_s_per_stream": 204.1, "target_context": 223128, "output_tokens": 5934 }, { "turn": 135, "context_tokens": 227338, "ttft_p50_s": 0.621, "ttft_max_s": 0.621, "decode_tok_s_per_stream": 204.6, "target_context": 229612, "output_tokens": 1495 }, { "turn": 136, "context_tokens": 233229, "ttft_p50_s": 2.153, "ttft_max_s": 2.153, "decode_tok_s_per_stream": 202.4, "target_context": 233361, "output_tokens": 6644 }, { "turn": 137, "context_tokens": 238471, "ttft_p50_s": 1.366, "ttft_max_s": 1.366, "decode_tok_s_per_stream": 199.8, "target_context": 241651, "output_tokens": 10824 }, { "turn": 138, "context_tokens": 249850, "ttft_p50_s": 2.144, "ttft_max_s": 2.144, "decode_tok_s_per_stream": 199.9, "target_context": 254665, "output_tokens": 791 }, { "turn": 139, "context_tokens": 256460, "ttft_p50_s": 2.204, "ttft_max_s": 2.204, "decode_tok_s_per_stream": 199.2, "target_context": 255958, "output_tokens": 627 }, { "turn": 140, "context_tokens": 256959, "ttft_p50_s": 0.406, "ttft_max_s": 0.406, "decode_tok_s_per_stream": 197.6, "target_context": 257125, "output_tokens": 1788 }, { "turn": 141, "context_tokens": 259543, "ttft_p50_s": 1.286, "ttft_max_s": 1.286, "decode_tok_s_per_stream": 196.7, "target_context": 260098, "output_tokens": 1960 }, { "turn": 145, "context_tokens": 1280, "ttft_p50_s": 0.158, "ttft_max_s": 0.158, "decode_tok_s_per_stream": 513.0, "target_context": 954, "output_tokens": 17 }, { "turn": 146, "context_tokens": 45063, "ttft_p50_s": 2.376, "ttft_max_s": 2.376, "decode_tok_s_per_stream": 268.4, "target_context": 33809, "output_tokens": 349 }, { "turn": 147, "context_tokens": 45403, "ttft_p50_s": 0.277, "ttft_max_s": 0.277, "decode_tok_s_per_stream": 270.8, "target_context": 34921, "output_tokens": 186 }, { "turn": 148, "context_tokens": 45671, "ttft_p50_s": 0.31, "ttft_max_s": 0.31, "decode_tok_s_per_stream": 266.6, "target_context": 38701, "output_tokens": 2701 }, { "turn": 149, "context_tokens": 47034, "ttft_p50_s": 0.238, "ttft_max_s": 0.238, "decode_tok_s_per_stream": 267.1, "target_context": 41416, "output_tokens": 995 }, { "turn": 150, "context_tokens": 47667, "ttft_p50_s": 0.285, "ttft_max_s": 0.285, "decode_tok_s_per_stream": 271.3, "target_context": 43422, "output_tokens": 145 }, { "turn": 151, "context_tokens": 47921, "ttft_p50_s": 0.266, "ttft_max_s": 0.266, "decode_tok_s_per_stream": 268.2, "target_context": 44550, "output_tokens": 317 }, { "turn": 152, "context_tokens": 63322, "ttft_p50_s": 1.473, "ttft_max_s": 1.473, "decode_tok_s_per_stream": 259.5, "target_context": 62731, "output_tokens": 2841 }, { "turn": 153, "context_tokens": 64781, "ttft_p50_s": 0.335, "ttft_max_s": 0.335, "decode_tok_s_per_stream": 260.5, "target_context": 65734, "output_tokens": 691 }, { "turn": 154, "context_tokens": 67139, "ttft_p50_s": 0.55, "ttft_max_s": 0.55, "decode_tok_s_per_stream": 258.7, "target_context": 67342, "output_tokens": 505 }, { "turn": 155, "context_tokens": 68252, "ttft_p50_s": 0.487, "ttft_max_s": 0.487, "decode_tok_s_per_stream": 258.2, "target_context": 68386, "output_tokens": 681 }, { "turn": 156, "context_tokens": 68868, "ttft_p50_s": 0.34, "ttft_max_s": 0.34, "decode_tok_s_per_stream": 260.7, "target_context": 69150, "output_tokens": 231 }, { "turn": 157, "context_tokens": 70089, "ttft_p50_s": 0.445, "ttft_max_s": 0.445, "decode_tok_s_per_stream": 257.5, "target_context": 70102, "output_tokens": 646 }, { "turn": 158, "context_tokens": 70923, "ttft_p50_s": 0.298, "ttft_max_s": 0.298, "decode_tok_s_per_stream": 258.4, "target_context": 71149, "output_tokens": 397 }, { "turn": 159, "context_tokens": 71643, "ttft_p50_s": 0.361, "ttft_max_s": 0.361, "decode_tok_s_per_stream": 256.3, "target_context": 71759, "output_tokens": 1162 }, { "turn": 160, "context_tokens": 74802, "ttft_p50_s": 0.485, "ttft_max_s": 0.485, "decode_tok_s_per_stream": 255.0, "target_context": 75212, "output_tokens": 1401 }, { "turn": 161, "context_tokens": 76220, "ttft_p50_s": 0.379, "ttft_max_s": 0.379, "decode_tok_s_per_stream": 255.5, "target_context": 76846, "output_tokens": 605 }, { "turn": 162, "context_tokens": 77387, "ttft_p50_s": 0.311, "ttft_max_s": 0.311, "decode_tok_s_per_stream": 253.1, "target_context": 77587, "output_tokens": 3572 }, { "turn": 163, "context_tokens": 79716, "ttft_p50_s": 0.307, "ttft_max_s": 0.307, "decode_tok_s_per_stream": 254.4, "target_context": 81505, "output_tokens": 536 }, { "turn": 164, "context_tokens": 82248, "ttft_p50_s": 0.633, "ttft_max_s": 0.633, "decode_tok_s_per_stream": 258.9, "target_context": 82241, "output_tokens": 125 }, { "turn": 165, "context_tokens": 83076, "ttft_p50_s": 0.467, "ttft_max_s": 0.467, "decode_tok_s_per_stream": 254.4, "target_context": 82999, "output_tokens": 308 }, { "turn": 166, "context_tokens": 89207, "ttft_p50_s": 0.958, "ttft_max_s": 0.958, "decode_tok_s_per_stream": 249.8, "target_context": 88687, "output_tokens": 546 }, { "turn": 167, "context_tokens": 96958, "ttft_p50_s": 1.118, "ttft_max_s": 1.118, "decode_tok_s_per_stream": 247.6, "target_context": 96508, "output_tokens": 521 }, { "turn": 168, "context_tokens": 97394, "ttft_p50_s": 0.425, "ttft_max_s": 0.425, "decode_tok_s_per_stream": 247.7, "target_context": 97123, "output_tokens": 520 }, { "turn": 171, "context_tokens": 107460, "ttft_p50_s": 9.072, "ttft_max_s": 9.072, "decode_tok_s_per_stream": 244.6, "target_context": 98111, "output_tokens": 418 }, { "turn": 173, "context_tokens": 98534, "ttft_p50_s": 7.798, "ttft_max_s": 7.798, "decode_tok_s_per_stream": 247.0, "target_context": 98606, "output_tokens": 686 }, { "turn": 179, "context_tokens": 99229, "ttft_p50_s": 7.885, "ttft_max_s": 7.885, "decode_tok_s_per_stream": 248.5, "target_context": 99302, "output_tokens": 294 }, { "turn": 180, "context_tokens": 106036, "ttft_p50_s": 1.272, "ttft_max_s": 1.272, "decode_tok_s_per_stream": 243.6, "target_context": 106032, "output_tokens": 1153 }, { "turn": 181, "context_tokens": 107419, "ttft_p50_s": 0.685, "ttft_max_s": 0.685, "decode_tok_s_per_stream": 243.1, "target_context": 107953, "output_tokens": 1055 }, { "turn": 182, "context_tokens": 108762, "ttft_p50_s": 0.535, "ttft_max_s": 0.535, "decode_tok_s_per_stream": 241.2, "target_context": 109319, "output_tokens": 2083 }, { "turn": 183, "context_tokens": 110557, "ttft_p50_s": 0.461, "ttft_max_s": 0.461, "decode_tok_s_per_stream": 241.0, "target_context": 111659, "output_tokens": 1316 }, { "turn": 184, "context_tokens": 112369, "ttft_p50_s": 0.46, "ttft_max_s": 0.46, "decode_tok_s_per_stream": 242.3, "target_context": 113014, "output_tokens": 445 }, { "turn": 185, "context_tokens": 113913, "ttft_p50_s": 0.722, "ttft_max_s": 0.722, "decode_tok_s_per_stream": 242.4, "target_context": 114031, "output_tokens": 421 }, { "turn": 186, "context_tokens": 114404, "ttft_p50_s": 0.333, "ttft_max_s": 0.333, "decode_tok_s_per_stream": 241.7, "target_context": 114589, "output_tokens": 442 }, { "turn": 187, "context_tokens": 115136, "ttft_p50_s": 0.48, "ttft_max_s": 0.48, "decode_tok_s_per_stream": 244.1, "target_context": 115301, "output_tokens": 215 }, { "turn": 188, "context_tokens": 117738, "ttft_p50_s": 0.88, "ttft_max_s": 0.88, "decode_tok_s_per_stream": 239.2, "target_context": 117721, "output_tokens": 932 }, { "turn": 189, "context_tokens": 118342, "ttft_p50_s": 0.384, "ttft_max_s": 0.384, "decode_tok_s_per_stream": 243.9, "target_context": 118719, "output_tokens": 177 }, { "turn": 190, "context_tokens": 118957, "ttft_p50_s": 0.471, "ttft_max_s": 0.471, "decode_tok_s_per_stream": 240.1, "target_context": 118935, "output_tokens": 453 }, { "turn": 191, "context_tokens": 119347, "ttft_p50_s": 0.491, "ttft_max_s": 0.491, "decode_tok_s_per_stream": 244.2, "target_context": 119390, "output_tokens": 167 }, { "turn": 192, "context_tokens": 119609, "ttft_p50_s": 0.565, "ttft_max_s": 0.565, "decode_tok_s_per_stream": 238.4, "target_context": 119622, "output_tokens": 1266 }, { "turn": 193, "context_tokens": 39764, "ttft_p50_s": 1.976, "ttft_max_s": 1.976, "decode_tok_s_per_stream": 273.4, "target_context": 38824, "output_tokens": 427 }, { "turn": 194, "context_tokens": 40139, "ttft_p50_s": 0.211, "ttft_max_s": 0.211, "decode_tok_s_per_stream": 273.5, "target_context": 40325, "output_tokens": 442 }, { "turn": 195, "context_tokens": 41215, "ttft_p50_s": 0.293, "ttft_max_s": 0.293, "decode_tok_s_per_stream": 273.1, "target_context": 41396, "output_tokens": 339 }, { "turn": 196, "context_tokens": 42364, "ttft_p50_s": 0.222, "ttft_max_s": 0.222, "decode_tok_s_per_stream": 272.4, "target_context": 42468, "output_tokens": 455 }, { "turn": 197, "context_tokens": 42868, "ttft_p50_s": 0.262, "ttft_max_s": 0.262, "decode_tok_s_per_stream": 271.9, "target_context": 42987, "output_tokens": 520 }, { "turn": 198, "context_tokens": 43285, "ttft_p50_s": 0.272, "ttft_max_s": 0.272, "decode_tok_s_per_stream": 271.2, "target_context": 43517, "output_tokens": 426 }, { "turn": 199, "context_tokens": 46560, "ttft_p50_s": 0.512, "ttft_max_s": 0.512, "decode_tok_s_per_stream": 269.2, "target_context": 46592, "output_tokens": 281 } ], "prefix_cache_hit_rate": 0.9408, "prefix_cache_queried_tokens": 20653703 }, { "agents": 1, "mode": "cold", "turns_requested": 188, "turns_ok": 188, "status": "ok", "wall_s": 2991.21, "turns_over_window": 11, "ttft_first_turn_s": 3.277, "ttft_last_turn_s": 2.496, "ttft_mean_s": 11.243, "ttft_p95_s": 27.127, "context_tokens_last": 46568, "prompt_tokens_total": 20647499, "turns_per_minute": 3.8, "per_turn": [ { "turn": 1, "context_tokens": 56973, "ttft_p50_s": 3.277, "ttft_max_s": 3.277, "decode_tok_s_per_stream": 264.1, "target_context": 64988, "output_tokens": 1739 }, { "turn": 2, "context_tokens": 66464, "ttft_p50_s": 4.185, "ttft_max_s": 4.185, "decode_tok_s_per_stream": 258.2, "target_context": 67299, "output_tokens": 1219 }, { "turn": 3, "context_tokens": 67987, "ttft_p50_s": 4.378, "ttft_max_s": 4.378, "decode_tok_s_per_stream": 258.1, "target_context": 68580, "output_tokens": 1387 }, { "turn": 4, "context_tokens": 68794, "ttft_p50_s": 4.41, "ttft_max_s": 4.41, "decode_tok_s_per_stream": 261.0, "target_context": 55938, "output_tokens": 202 }, { "turn": 5, "context_tokens": 70546, "ttft_p50_s": 4.625, "ttft_max_s": 4.625, "decode_tok_s_per_stream": 257.0, "target_context": 70499, "output_tokens": 1861 }, { "turn": 6, "context_tokens": 71573, "ttft_p50_s": 4.677, "ttft_max_s": 4.677, "decode_tok_s_per_stream": 257.6, "target_context": 56909, "output_tokens": 398 }, { "turn": 7, "context_tokens": 71937, "ttft_p50_s": 4.761, "ttft_max_s": 4.761, "decode_tok_s_per_stream": 262.8, "target_context": 58822, "output_tokens": 124 }, { "turn": 8, "context_tokens": 72404, "ttft_p50_s": 4.742, "ttft_max_s": 4.742, "decode_tok_s_per_stream": 256.6, "target_context": 72414, "output_tokens": 1619 }, { "turn": 9, "context_tokens": 73331, "ttft_p50_s": 4.915, "ttft_max_s": 4.915, "decode_tok_s_per_stream": 255.2, "target_context": 74118, "output_tokens": 2252 }, { "turn": 10, "context_tokens": 75598, "ttft_p50_s": 5.151, "ttft_max_s": 5.151, "decode_tok_s_per_stream": 256.7, "target_context": 76664, "output_tokens": 384 }, { "turn": 11, "context_tokens": 77573, "ttft_p50_s": 5.396, "ttft_max_s": 5.396, "decode_tok_s_per_stream": 254.1, "target_context": 77578, "output_tokens": 1155 }, { "turn": 12, "context_tokens": 78533, "ttft_p50_s": 5.455, "ttft_max_s": 5.455, "decode_tok_s_per_stream": 253.8, "target_context": 79065, "output_tokens": 1134 }, { "turn": 13, "context_tokens": 80521, "ttft_p50_s": 5.658, "ttft_max_s": 5.658, "decode_tok_s_per_stream": 252.5, "target_context": 80967, "output_tokens": 2228 }, { "turn": 14, "context_tokens": 82343, "ttft_p50_s": 5.912, "ttft_max_s": 5.912, "decode_tok_s_per_stream": 254.5, "target_context": 83411, "output_tokens": 310 }, { "turn": 15, "context_tokens": 83941, "ttft_p50_s": 6.062, "ttft_max_s": 6.062, "decode_tok_s_per_stream": 252.4, "target_context": 83903, "output_tokens": 818 }, { "turn": 16, "context_tokens": 85095, "ttft_p50_s": 6.146, "ttft_max_s": 6.146, "decode_tok_s_per_stream": 251.9, "target_context": 85329, "output_tokens": 853 }, { "turn": 17, "context_tokens": 86152, "ttft_p50_s": 6.268, "ttft_max_s": 6.268, "decode_tok_s_per_stream": 253.9, "target_context": 86443, "output_tokens": 243 }, { "turn": 18, "context_tokens": 86799, "ttft_p50_s": 6.338, "ttft_max_s": 6.338, "decode_tok_s_per_stream": 251.4, "target_context": 86751, "output_tokens": 642 }, { "turn": 19, "context_tokens": 1283, "ttft_p50_s": 0.071, "ttft_max_s": 0.071, "decode_tok_s_per_stream": 291.1, "target_context": 961, "output_tokens": 16 }, { "turn": 20, "context_tokens": 44795, "ttft_p50_s": 2.355, "ttft_max_s": 2.355, "decode_tok_s_per_stream": 269.3, "target_context": 33996, "output_tokens": 279 }, { "turn": 21, "context_tokens": 45103, "ttft_p50_s": 2.399, "ttft_max_s": 2.399, "decode_tok_s_per_stream": 270.4, "target_context": 34455, "output_tokens": 223 }, { "turn": 22, "context_tokens": 45385, "ttft_p50_s": 2.399, "ttft_max_s": 2.399, "decode_tok_s_per_stream": 268.3, "target_context": 34900, "output_tokens": 578 }, { "turn": 23, "context_tokens": 45822, "ttft_p50_s": 2.453, "ttft_max_s": 2.453, "decode_tok_s_per_stream": 267.5, "target_context": 35765, "output_tokens": 1164 }, { "turn": 24, "context_tokens": 46519, "ttft_p50_s": 2.461, "ttft_max_s": 2.461, "decode_tok_s_per_stream": 267.3, "target_context": 37099, "output_tokens": 890 }, { "turn": 25, "context_tokens": 47100, "ttft_p50_s": 2.546, "ttft_max_s": 2.546, "decode_tok_s_per_stream": 267.0, "target_context": 38060, "output_tokens": 1787 }, { "turn": 26, "context_tokens": 48085, "ttft_p50_s": 2.642, "ttft_max_s": 2.642, "decode_tok_s_per_stream": 267.9, "target_context": 39965, "output_tokens": 502 }, { "turn": 27, "context_tokens": 48498, "ttft_p50_s": 2.657, "ttft_max_s": 2.657, "decode_tok_s_per_stream": 267.3, "target_context": 40530, "output_tokens": 867 }, { "turn": 28, "context_tokens": 49080, "ttft_p50_s": 2.702, "ttft_max_s": 2.702, "decode_tok_s_per_stream": 266.5, "target_context": 41685, "output_tokens": 996 }, { "turn": 29, "context_tokens": 49723, "ttft_p50_s": 2.754, "ttft_max_s": 2.754, "decode_tok_s_per_stream": 266.2, "target_context": 42730, "output_tokens": 1944 }, { "turn": 30, "context_tokens": 50812, "ttft_p50_s": 2.812, "ttft_max_s": 2.812, "decode_tok_s_per_stream": 265.5, "target_context": 44830, "output_tokens": 2348 }, { "turn": 31, "context_tokens": 52107, "ttft_p50_s": 2.948, "ttft_max_s": 2.948, "decode_tok_s_per_stream": 266.1, "target_context": 47231, "output_tokens": 985 }, { "turn": 32, "context_tokens": 52765, "ttft_p50_s": 2.974, "ttft_max_s": 2.974, "decode_tok_s_per_stream": 264.7, "target_context": 48267, "output_tokens": 2720 }, { "turn": 33, "context_tokens": 54272, "ttft_p50_s": 3.117, "ttft_max_s": 3.117, "decode_tok_s_per_stream": 266.2, "target_context": 51031, "output_tokens": 318 }, { "turn": 34, "context_tokens": 1283, "ttft_p50_s": 0.071, "ttft_max_s": 0.071, "decode_tok_s_per_stream": 293.1, "target_context": 927, "output_tokens": 13 }, { "turn": 35, "context_tokens": 39509, "ttft_p50_s": 1.969, "ttft_max_s": 1.969, "decode_tok_s_per_stream": 274.8, "target_context": 29988, "output_tokens": 207 }, { "turn": 36, "context_tokens": 39784, "ttft_p50_s": 1.977, "ttft_max_s": 1.977, "decode_tok_s_per_stream": 274.1, "target_context": 35061, "output_tokens": 206 }, { "turn": 37, "context_tokens": 40057, "ttft_p50_s": 2.009, "ttft_max_s": 2.009, "decode_tok_s_per_stream": 273.6, "target_context": 36124, "output_tokens": 307 }, { "turn": 38, "context_tokens": 40376, "ttft_p50_s": 1.993, "ttft_max_s": 1.993, "decode_tok_s_per_stream": 273.1, "target_context": 36598, "output_tokens": 405 }, { "turn": 39, "context_tokens": 1283, "ttft_p50_s": 0.07, "ttft_max_s": 0.07, "decode_tok_s_per_stream": 290.2, "target_context": 957, "output_tokens": 23 }, { "turn": 40, "context_tokens": 44686, "ttft_p50_s": 2.339, "ttft_max_s": 2.339, "decode_tok_s_per_stream": 269.7, "target_context": 34132, "output_tokens": 217 }, { "turn": 41, "context_tokens": 44965, "ttft_p50_s": 2.38, "ttft_max_s": 2.38, "decode_tok_s_per_stream": 269.8, "target_context": 35247, "output_tokens": 240 }, { "turn": 42, "context_tokens": 45254, "ttft_p50_s": 2.39, "ttft_max_s": 2.39, "decode_tok_s_per_stream": 268.3, "target_context": 44130, "output_tokens": 589 }, { "turn": 43, "context_tokens": 45696, "ttft_p50_s": 2.361, "ttft_max_s": 2.361, "decode_tok_s_per_stream": 268.6, "target_context": 45079, "output_tokens": 312 }, { "turn": 44, "context_tokens": 13808, "ttft_p50_s": 0.546, "ttft_max_s": 0.546, "decode_tok_s_per_stream": 285.0, "target_context": 14112, "output_tokens": 1229 }, { "turn": 45, "context_tokens": 47752, "ttft_p50_s": 2.586, "ttft_max_s": 2.586, "decode_tok_s_per_stream": 266.4, "target_context": 47335, "output_tokens": 2465 }, { "turn": 46, "context_tokens": 49034, "ttft_p50_s": 2.698, "ttft_max_s": 2.698, "decode_tok_s_per_stream": 265.9, "target_context": 49984, "output_tokens": 2228 }, { "turn": 47, "context_tokens": 51384, "ttft_p50_s": 2.862, "ttft_max_s": 2.862, "decode_tok_s_per_stream": 266.0, "target_context": 52506, "output_tokens": 1819 }, { "turn": 48, "context_tokens": 54255, "ttft_p50_s": 3.08, "ttft_max_s": 3.08, "decode_tok_s_per_stream": 265.3, "target_context": 55007, "output_tokens": 589 }, { "turn": 49, "context_tokens": 55867, "ttft_p50_s": 3.285, "ttft_max_s": 3.285, "decode_tok_s_per_stream": 265.5, "target_context": 56050, "output_tokens": 566 }, { "turn": 50, "context_tokens": 57583, "ttft_p50_s": 3.408, "ttft_max_s": 3.408, "decode_tok_s_per_stream": 264.5, "target_context": 57643, "output_tokens": 737 }, { "turn": 51, "context_tokens": 58461, "ttft_p50_s": 3.473, "ttft_max_s": 3.473, "decode_tok_s_per_stream": 264.3, "target_context": 58729, "output_tokens": 478 }, { "turn": 52, "context_tokens": 59217, "ttft_p50_s": 3.536, "ttft_max_s": 3.536, "decode_tok_s_per_stream": 263.5, "target_context": 59363, "output_tokens": 560 }, { "turn": 53, "context_tokens": 64264, "ttft_p50_s": 4.011, "ttft_max_s": 4.011, "decode_tok_s_per_stream": 263.5, "target_context": 64045, "output_tokens": 218 }, { "turn": 54, "context_tokens": 64782, "ttft_p50_s": 4.058, "ttft_max_s": 4.058, "decode_tok_s_per_stream": 261.8, "target_context": 64837, "output_tokens": 464 }, { "turn": 55, "context_tokens": 78400, "ttft_p50_s": 5.388, "ttft_max_s": 5.388, "decode_tok_s_per_stream": 254.4, "target_context": 77430, "output_tokens": 757 }, { "turn": 56, "context_tokens": 84914, "ttft_p50_s": 6.122, "ttft_max_s": 6.122, "decode_tok_s_per_stream": 253.1, "target_context": 84792, "output_tokens": 453 }, { "turn": 57, "context_tokens": 88068, "ttft_p50_s": 6.602, "ttft_max_s": 6.602, "decode_tok_s_per_stream": 249.9, "target_context": 88014, "output_tokens": 662 }, { "turn": 58, "context_tokens": 88562, "ttft_p50_s": 6.674, "ttft_max_s": 6.674, "decode_tok_s_per_stream": 248.3, "target_context": 88805, "output_tokens": 3447 }, { "turn": 59, "context_tokens": 93007, "ttft_p50_s": 7.121, "ttft_max_s": 7.121, "decode_tok_s_per_stream": 246.3, "target_context": 94604, "output_tokens": 5536 }, { "turn": 60, "context_tokens": 97900, "ttft_p50_s": 7.804, "ttft_max_s": 7.804, "decode_tok_s_per_stream": 244.7, "target_context": 100477, "output_tokens": 6852 }, { "turn": 61, "context_tokens": 104311, "ttft_p50_s": 8.599, "ttft_max_s": 8.599, "decode_tok_s_per_stream": 242.0, "target_context": 107360, "output_tokens": 7445 }, { "turn": 62, "context_tokens": 111925, "ttft_p50_s": 9.695, "ttft_max_s": 9.695, "decode_tok_s_per_stream": 245.2, "target_context": 115070, "output_tokens": 207 }, { "turn": 63, "context_tokens": 115893, "ttft_p50_s": 10.265, "ttft_max_s": 10.265, "decode_tok_s_per_stream": 246.1, "target_context": 115310, "output_tokens": 152 }, { "turn": 64, "context_tokens": 116155, "ttft_p50_s": 10.311, "ttft_max_s": 10.311, "decode_tok_s_per_stream": 240.8, "target_context": 115511, "output_tokens": 576 }, { "turn": 65, "context_tokens": 116636, "ttft_p50_s": 10.387, "ttft_max_s": 10.387, "decode_tok_s_per_stream": 239.3, "target_context": 116418, "output_tokens": 1052 }, { "turn": 66, "context_tokens": 117364, "ttft_p50_s": 10.484, "ttft_max_s": 10.484, "decode_tok_s_per_stream": 239.0, "target_context": 117793, "output_tokens": 1211 }, { "turn": 67, "context_tokens": 119094, "ttft_p50_s": 10.772, "ttft_max_s": 10.772, "decode_tok_s_per_stream": 239.9, "target_context": 119395, "output_tokens": 510 }, { "turn": 68, "context_tokens": 123450, "ttft_p50_s": 11.434, "ttft_max_s": 11.434, "decode_tok_s_per_stream": 238.0, "target_context": 122992, "output_tokens": 590 }, { "turn": 69, "context_tokens": 123939, "ttft_p50_s": 11.529, "ttft_max_s": 11.529, "decode_tok_s_per_stream": 234.6, "target_context": 123791, "output_tokens": 9989 }, { "turn": 70, "context_tokens": 129289, "ttft_p50_s": 12.383, "ttft_max_s": 12.383, "decode_tok_s_per_stream": 234.3, "target_context": 133841, "output_tokens": 3278 }, { "turn": 71, "context_tokens": 136746, "ttft_p50_s": 13.589, "ttft_max_s": 13.589, "decode_tok_s_per_stream": 237.2, "target_context": 137168, "output_tokens": 207 }, { "turn": 72, "context_tokens": 138419, "ttft_p50_s": 13.816, "ttft_max_s": 13.816, "decode_tok_s_per_stream": 234.6, "target_context": 138157, "output_tokens": 301 }, { "turn": 73, "context_tokens": 138765, "ttft_p50_s": 13.899, "ttft_max_s": 13.899, "decode_tok_s_per_stream": 231.8, "target_context": 138661, "output_tokens": 800 }, { "turn": 74, "context_tokens": 139378, "ttft_p50_s": 14.094, "ttft_max_s": 14.094, "decode_tok_s_per_stream": 234.6, "target_context": 139603, "output_tokens": 272 }, { "turn": 75, "context_tokens": 140053, "ttft_p50_s": 14.256, "ttft_max_s": 14.256, "decode_tok_s_per_stream": 230.8, "target_context": 139981, "output_tokens": 1244 }, { "turn": 76, "context_tokens": 141825, "ttft_p50_s": 14.467, "ttft_max_s": 14.467, "decode_tok_s_per_stream": 232.0, "target_context": 142096, "output_tokens": 528 }, { "turn": 77, "context_tokens": 143328, "ttft_p50_s": 14.723, "ttft_max_s": 14.723, "decode_tok_s_per_stream": 232.3, "target_context": 143292, "output_tokens": 418 }, { "turn": 78, "context_tokens": 143737, "ttft_p50_s": 14.775, "ttft_max_s": 14.775, "decode_tok_s_per_stream": 230.8, "target_context": 143789, "output_tokens": 973 }, { "turn": 79, "context_tokens": 144445, "ttft_p50_s": 14.952, "ttft_max_s": 14.952, "decode_tok_s_per_stream": 230.9, "target_context": 144773, "output_tokens": 677 }, { "turn": 80, "context_tokens": 145454, "ttft_p50_s": 15.083, "ttft_max_s": 15.083, "decode_tok_s_per_stream": 233.6, "target_context": 145546, "output_tokens": 267 }, { "turn": 81, "context_tokens": 145897, "ttft_p50_s": 15.227, "ttft_max_s": 15.227, "decode_tok_s_per_stream": 230.1, "target_context": 145928, "output_tokens": 907 }, { "turn": 82, "context_tokens": 146686, "ttft_p50_s": 15.422, "ttft_max_s": 15.422, "decode_tok_s_per_stream": 231.9, "target_context": 146943, "output_tokens": 345 }, { "turn": 83, "context_tokens": 45407, "ttft_p50_s": 2.403, "ttft_max_s": 2.403, "decode_tok_s_per_stream": 270.2, "target_context": 37006, "output_tokens": 178 }, { "turn": 84, "context_tokens": 45669, "ttft_p50_s": 2.43, "ttft_max_s": 2.43, "decode_tok_s_per_stream": 275.2, "target_context": 37308, "output_tokens": 62 }, { "turn": 85, "context_tokens": 148377, "ttft_p50_s": 15.602, "ttft_max_s": 15.602, "decode_tok_s_per_stream": 229.7, "target_context": 147329, "output_tokens": 696 }, { "turn": 86, "context_tokens": 148866, "ttft_p50_s": 15.769, "ttft_max_s": 15.769, "decode_tok_s_per_stream": 228.9, "target_context": 148072, "output_tokens": 1084 }, { "turn": 87, "context_tokens": 149523, "ttft_p50_s": 15.886, "ttft_max_s": 15.886, "decode_tok_s_per_stream": 236.4, "target_context": 149399, "output_tokens": 139 }, { "turn": 88, "context_tokens": 149769, "ttft_p50_s": 15.873, "ttft_max_s": 15.873, "decode_tok_s_per_stream": 233.4, "target_context": 149577, "output_tokens": 198 }, { "turn": 89, "context_tokens": 150039, "ttft_p50_s": 15.986, "ttft_max_s": 15.986, "decode_tok_s_per_stream": 235.5, "target_context": 149787, "output_tokens": 141 }, { "turn": 90, "context_tokens": 150286, "ttft_p50_s": 15.878, "ttft_max_s": 15.878, "decode_tok_s_per_stream": 248.0, "target_context": 150025, "output_tokens": 61 }, { "turn": 91, "context_tokens": 150496, "ttft_p50_s": 15.972, "ttft_max_s": 15.972, "decode_tok_s_per_stream": 227.5, "target_context": 150151, "output_tokens": 2321 }, { "turn": 92, "context_tokens": 151697, "ttft_p50_s": 16.212, "ttft_max_s": 16.212, "decode_tok_s_per_stream": 229.1, "target_context": 152644, "output_tokens": 534 }, { "turn": 93, "context_tokens": 153035, "ttft_p50_s": 16.598, "ttft_max_s": 16.598, "decode_tok_s_per_stream": 229.5, "target_context": 153291, "output_tokens": 315 }, { "turn": 94, "context_tokens": 153932, "ttft_p50_s": 16.688, "ttft_max_s": 16.688, "decode_tok_s_per_stream": 226.5, "target_context": 154030, "output_tokens": 911 }, { "turn": 95, "context_tokens": 154865, "ttft_p50_s": 16.828, "ttft_max_s": 16.828, "decode_tok_s_per_stream": 228.1, "target_context": 155245, "output_tokens": 383 }, { "turn": 96, "context_tokens": 155678, "ttft_p50_s": 16.983, "ttft_max_s": 16.983, "decode_tok_s_per_stream": 226.8, "target_context": 155756, "output_tokens": 622 }, { "turn": 97, "context_tokens": 156139, "ttft_p50_s": 17.074, "ttft_max_s": 17.074, "decode_tok_s_per_stream": 234.0, "target_context": 156407, "output_tokens": 134 }, { "turn": 98, "context_tokens": 156614, "ttft_p50_s": 17.201, "ttft_max_s": 17.201, "decode_tok_s_per_stream": 227.8, "target_context": 156597, "output_tokens": 366 }, { "turn": 99, "context_tokens": 157191, "ttft_p50_s": 17.205, "ttft_max_s": 17.205, "decode_tok_s_per_stream": 226.4, "target_context": 157336, "output_tokens": 633 }, { "turn": 100, "context_tokens": 158126, "ttft_p50_s": 17.481, "ttft_max_s": 17.481, "decode_tok_s_per_stream": 229.6, "target_context": 158372, "output_tokens": 229 }, { "turn": 101, "context_tokens": 158647, "ttft_p50_s": 17.489, "ttft_max_s": 17.489, "decode_tok_s_per_stream": 225.7, "target_context": 158633, "output_tokens": 794 }, { "turn": 102, "context_tokens": 160114, "ttft_p50_s": 17.829, "ttft_max_s": 17.829, "decode_tok_s_per_stream": 228.1, "target_context": 160400, "output_tokens": 292 }, { "turn": 103, "context_tokens": 160664, "ttft_p50_s": 17.982, "ttft_max_s": 17.982, "decode_tok_s_per_stream": 238.3, "target_context": 160736, "output_tokens": 90 }, { "turn": 104, "context_tokens": 161009, "ttft_p50_s": 18.084, "ttft_max_s": 18.084, "decode_tok_s_per_stream": 224.5, "target_context": 160944, "output_tokens": 1044 }, { "turn": 105, "context_tokens": 161777, "ttft_p50_s": 18.234, "ttft_max_s": 18.234, "decode_tok_s_per_stream": 223.1, "target_context": 162243, "output_tokens": 4442 }, { "turn": 106, "context_tokens": 164414, "ttft_p50_s": 18.59, "ttft_max_s": 18.59, "decode_tok_s_per_stream": 225.3, "target_context": 166793, "output_tokens": 442 }, { "turn": 107, "context_tokens": 167236, "ttft_p50_s": 19.28, "ttft_max_s": 19.28, "decode_tok_s_per_stream": 221.4, "target_context": 167320, "output_tokens": 4287 }, { "turn": 108, "context_tokens": 169483, "ttft_p50_s": 19.673, "ttft_max_s": 19.673, "decode_tok_s_per_stream": 220.9, "target_context": 171776, "output_tokens": 2797 }, { "turn": 109, "context_tokens": 173505, "ttft_p50_s": 20.604, "ttft_max_s": 20.604, "decode_tok_s_per_stream": 221.1, "target_context": 174769, "output_tokens": 952 }, { "turn": 110, "context_tokens": 175523, "ttft_p50_s": 21.021, "ttft_max_s": 21.021, "decode_tok_s_per_stream": 220.8, "target_context": 175840, "output_tokens": 495 }, { "turn": 111, "context_tokens": 176172, "ttft_p50_s": 21.164, "ttft_max_s": 21.164, "decode_tok_s_per_stream": 220.8, "target_context": 176359, "output_tokens": 459 }, { "turn": 112, "context_tokens": 181440, "ttft_p50_s": 22.277, "ttft_max_s": 22.277, "decode_tok_s_per_stream": 217.7, "target_context": 181231, "output_tokens": 1241 }, { "turn": 113, "context_tokens": 183246, "ttft_p50_s": 22.572, "ttft_max_s": 22.572, "decode_tok_s_per_stream": 222.7, "target_context": 183733, "output_tokens": 207 }, { "turn": 114, "context_tokens": 185269, "ttft_p50_s": 23.02, "ttft_max_s": 23.02, "decode_tok_s_per_stream": 219.3, "target_context": 185143, "output_tokens": 355 }, { "turn": 115, "context_tokens": 185622, "ttft_p50_s": 23.144, "ttft_max_s": 23.144, "decode_tok_s_per_stream": 219.1, "target_context": 185537, "output_tokens": 392 }, { "turn": 116, "context_tokens": 187616, "ttft_p50_s": 23.576, "ttft_max_s": 23.576, "decode_tok_s_per_stream": 215.7, "target_context": 187617, "output_tokens": 2149 }, { "turn": 117, "context_tokens": 189266, "ttft_p50_s": 23.986, "ttft_max_s": 23.986, "decode_tok_s_per_stream": 216.5, "target_context": 190260, "output_tokens": 910 }, { "turn": 118, "context_tokens": 191038, "ttft_p50_s": 24.295, "ttft_max_s": 24.295, "decode_tok_s_per_stream": 218.8, "target_context": 191378, "output_tokens": 301 }, { "turn": 119, "context_tokens": 191715, "ttft_p50_s": 24.506, "ttft_max_s": 24.506, "decode_tok_s_per_stream": 217.0, "target_context": 191712, "output_tokens": 522 }, { "turn": 120, "context_tokens": 192148, "ttft_p50_s": 24.606, "ttft_max_s": 24.606, "decode_tok_s_per_stream": 214.2, "target_context": 192298, "output_tokens": 3828 }, { "turn": 121, "context_tokens": 194704, "ttft_p50_s": 25.183, "ttft_max_s": 25.183, "decode_tok_s_per_stream": 213.7, "target_context": 196608, "output_tokens": 2076 }, { "turn": 122, "context_tokens": 198307, "ttft_p50_s": 26.025, "ttft_max_s": 26.025, "decode_tok_s_per_stream": 214.4, "target_context": 199085, "output_tokens": 447 }, { "turn": 123, "context_tokens": 199634, "ttft_p50_s": 26.403, "ttft_max_s": 26.403, "decode_tok_s_per_stream": 217.6, "target_context": 199661, "output_tokens": 209 }, { "turn": 124, "context_tokens": 199921, "ttft_p50_s": 26.403, "ttft_max_s": 26.403, "decode_tok_s_per_stream": 214.3, "target_context": 199912, "output_tokens": 426 }, { "turn": 125, "context_tokens": 200310, "ttft_p50_s": 26.46, "ttft_max_s": 26.46, "decode_tok_s_per_stream": 213.2, "target_context": 200352, "output_tokens": 680 }, { "turn": 126, "context_tokens": 200935, "ttft_p50_s": 26.706, "ttft_max_s": 26.706, "decode_tok_s_per_stream": 246.1, "target_context": 201202, "output_tokens": 44 }, { "turn": 127, "context_tokens": 201376, "ttft_p50_s": 26.662, "ttft_max_s": 26.662, "decode_tok_s_per_stream": 213.8, "target_context": 201264, "output_tokens": 486 }, { "turn": 128, "context_tokens": 201794, "ttft_p50_s": 26.865, "ttft_max_s": 26.865, "decode_tok_s_per_stream": 215.8, "target_context": 201793, "output_tokens": 269 }, { "turn": 129, "context_tokens": 202226, "ttft_p50_s": 26.875, "ttft_max_s": 26.875, "decode_tok_s_per_stream": 213.5, "target_context": 202225, "output_tokens": 447 }, { "turn": 130, "context_tokens": 202626, "ttft_p50_s": 26.949, "ttft_max_s": 26.949, "decode_tok_s_per_stream": 235.1, "target_context": 202746, "output_tokens": 61 }, { "turn": 131, "context_tokens": 202959, "ttft_p50_s": 27.019, "ttft_max_s": 27.019, "decode_tok_s_per_stream": 212.0, "target_context": 202882, "output_tokens": 860 }, { "turn": 132, "context_tokens": 203555, "ttft_p50_s": 27.185, "ttft_max_s": 27.185, "decode_tok_s_per_stream": 210.9, "target_context": 203961, "output_tokens": 2214 }, { "turn": 133, "context_tokens": 205838, "ttft_p50_s": 27.769, "ttft_max_s": 27.769, "decode_tok_s_per_stream": 210.8, "target_context": 206847, "output_tokens": 1213 }, { "turn": 134, "context_tokens": 224340, "ttft_p50_s": 32.493, "ttft_max_s": 32.493, "decode_tok_s_per_stream": 204.3, "target_context": 223128, "output_tokens": 5934 }, { "turn": 135, "context_tokens": 227347, "ttft_p50_s": 33.263, "ttft_max_s": 33.263, "decode_tok_s_per_stream": 204.6, "target_context": 229612, "output_tokens": 1495 }, { "turn": 136, "context_tokens": 233238, "ttft_p50_s": 34.787, "ttft_max_s": 34.787, "decode_tok_s_per_stream": 202.1, "target_context": 233361, "output_tokens": 6644 }, { "turn": 137, "context_tokens": 238479, "ttft_p50_s": 36.255, "ttft_max_s": 36.255, "decode_tok_s_per_stream": 199.9, "target_context": 241651, "output_tokens": 10824 }, { "turn": 138, "context_tokens": 249856, "ttft_p50_s": 39.368, "ttft_max_s": 39.368, "decode_tok_s_per_stream": 199.8, "target_context": 254665, "output_tokens": 791 }, { "turn": 139, "context_tokens": 256465, "ttft_p50_s": 41.285, "ttft_max_s": 41.285, "decode_tok_s_per_stream": 199.1, "target_context": 255958, "output_tokens": 627 }, { "turn": 140, "context_tokens": 256964, "ttft_p50_s": 41.546, "ttft_max_s": 41.546, "decode_tok_s_per_stream": 197.5, "target_context": 257125, "output_tokens": 1788 }, { "turn": 141, "context_tokens": 259431, "ttft_p50_s": 42.242, "ttft_max_s": 42.242, "decode_tok_s_per_stream": 197.0, "target_context": 260098, "output_tokens": 1960 }, { "turn": 145, "context_tokens": 1288, "ttft_p50_s": 0.071, "ttft_max_s": 0.071, "decode_tok_s_per_stream": 292.0, "target_context": 954, "output_tokens": 17 }, { "turn": 146, "context_tokens": 44839, "ttft_p50_s": 2.353, "ttft_max_s": 2.353, "decode_tok_s_per_stream": 268.8, "target_context": 33809, "output_tokens": 349 }, { "turn": 147, "context_tokens": 45179, "ttft_p50_s": 2.393, "ttft_max_s": 2.393, "decode_tok_s_per_stream": 270.1, "target_context": 34921, "output_tokens": 186 }, { "turn": 148, "context_tokens": 45447, "ttft_p50_s": 2.394, "ttft_max_s": 2.394, "decode_tok_s_per_stream": 267.1, "target_context": 38701, "output_tokens": 2701 }, { "turn": 149, "context_tokens": 46809, "ttft_p50_s": 2.5, "ttft_max_s": 2.5, "decode_tok_s_per_stream": 267.5, "target_context": 41416, "output_tokens": 995 }, { "turn": 150, "context_tokens": 47441, "ttft_p50_s": 2.512, "ttft_max_s": 2.512, "decode_tok_s_per_stream": 270.8, "target_context": 43422, "output_tokens": 145 }, { "turn": 151, "context_tokens": 47695, "ttft_p50_s": 2.545, "ttft_max_s": 2.545, "decode_tok_s_per_stream": 268.4, "target_context": 44550, "output_tokens": 317 }, { "turn": 152, "context_tokens": 63327, "ttft_p50_s": 3.871, "ttft_max_s": 3.871, "decode_tok_s_per_stream": 259.8, "target_context": 62731, "output_tokens": 2841 }, { "turn": 153, "context_tokens": 64785, "ttft_p50_s": 4.062, "ttft_max_s": 4.062, "decode_tok_s_per_stream": 260.1, "target_context": 65734, "output_tokens": 691 }, { "turn": 154, "context_tokens": 67143, "ttft_p50_s": 4.227, "ttft_max_s": 4.227, "decode_tok_s_per_stream": 258.6, "target_context": 67342, "output_tokens": 505 }, { "turn": 155, "context_tokens": 68256, "ttft_p50_s": 4.405, "ttft_max_s": 4.405, "decode_tok_s_per_stream": 258.1, "target_context": 68386, "output_tokens": 681 }, { "turn": 156, "context_tokens": 68872, "ttft_p50_s": 4.506, "ttft_max_s": 4.506, "decode_tok_s_per_stream": 260.6, "target_context": 69150, "output_tokens": 231 }, { "turn": 157, "context_tokens": 70093, "ttft_p50_s": 4.582, "ttft_max_s": 4.582, "decode_tok_s_per_stream": 258.4, "target_context": 70102, "output_tokens": 646 }, { "turn": 158, "context_tokens": 70927, "ttft_p50_s": 4.622, "ttft_max_s": 4.622, "decode_tok_s_per_stream": 258.4, "target_context": 71149, "output_tokens": 397 }, { "turn": 159, "context_tokens": 71647, "ttft_p50_s": 4.726, "ttft_max_s": 4.726, "decode_tok_s_per_stream": 256.8, "target_context": 71759, "output_tokens": 1162 }, { "turn": 160, "context_tokens": 74806, "ttft_p50_s": 5.117, "ttft_max_s": 5.117, "decode_tok_s_per_stream": 255.3, "target_context": 75212, "output_tokens": 1401 }, { "turn": 161, "context_tokens": 76224, "ttft_p50_s": 5.173, "ttft_max_s": 5.173, "decode_tok_s_per_stream": 255.7, "target_context": 76846, "output_tokens": 605 }, { "turn": 162, "context_tokens": 77391, "ttft_p50_s": 5.339, "ttft_max_s": 5.339, "decode_tok_s_per_stream": 253.3, "target_context": 77587, "output_tokens": 3572 }, { "turn": 163, "context_tokens": 79717, "ttft_p50_s": 5.612, "ttft_max_s": 5.612, "decode_tok_s_per_stream": 254.6, "target_context": 81505, "output_tokens": 536 }, { "turn": 164, "context_tokens": 82248, "ttft_p50_s": 5.874, "ttft_max_s": 5.874, "decode_tok_s_per_stream": 258.6, "target_context": 82241, "output_tokens": 125 }, { "turn": 165, "context_tokens": 83076, "ttft_p50_s": 5.922, "ttft_max_s": 5.922, "decode_tok_s_per_stream": 254.8, "target_context": 82999, "output_tokens": 308 }, { "turn": 166, "context_tokens": 89207, "ttft_p50_s": 6.706, "ttft_max_s": 6.706, "decode_tok_s_per_stream": 249.8, "target_context": 88687, "output_tokens": 546 }, { "turn": 167, "context_tokens": 96958, "ttft_p50_s": 7.629, "ttft_max_s": 7.629, "decode_tok_s_per_stream": 247.7, "target_context": 96508, "output_tokens": 521 }, { "turn": 168, "context_tokens": 97393, "ttft_p50_s": 7.741, "ttft_max_s": 7.741, "decode_tok_s_per_stream": 247.6, "target_context": 97123, "output_tokens": 520 }, { "turn": 171, "context_tokens": 107236, "ttft_p50_s": 9.037, "ttft_max_s": 9.037, "decode_tok_s_per_stream": 244.6, "target_context": 98111, "output_tokens": 418 }, { "turn": 173, "context_tokens": 98542, "ttft_p50_s": 7.815, "ttft_max_s": 7.815, "decode_tok_s_per_stream": 246.8, "target_context": 98606, "output_tokens": 686 }, { "turn": 179, "context_tokens": 99353, "ttft_p50_s": 7.967, "ttft_max_s": 7.967, "decode_tok_s_per_stream": 248.4, "target_context": 99302, "output_tokens": 294 }, { "turn": 180, "context_tokens": 105928, "ttft_p50_s": 8.859, "ttft_max_s": 8.859, "decode_tok_s_per_stream": 243.4, "target_context": 106032, "output_tokens": 1153 }, { "turn": 181, "context_tokens": 107427, "ttft_p50_s": 9.131, "ttft_max_s": 9.131, "decode_tok_s_per_stream": 243.1, "target_context": 107953, "output_tokens": 1055 }, { "turn": 182, "context_tokens": 108770, "ttft_p50_s": 9.308, "ttft_max_s": 9.308, "decode_tok_s_per_stream": 241.2, "target_context": 109319, "output_tokens": 2083 }, { "turn": 183, "context_tokens": 110565, "ttft_p50_s": 9.468, "ttft_max_s": 9.468, "decode_tok_s_per_stream": 241.0, "target_context": 111659, "output_tokens": 1316 }, { "turn": 184, "context_tokens": 112378, "ttft_p50_s": 9.784, "ttft_max_s": 9.784, "decode_tok_s_per_stream": 242.4, "target_context": 113014, "output_tokens": 445 }, { "turn": 185, "context_tokens": 113921, "ttft_p50_s": 10.069, "ttft_max_s": 10.069, "decode_tok_s_per_stream": 241.9, "target_context": 114031, "output_tokens": 421 }, { "turn": 186, "context_tokens": 114412, "ttft_p50_s": 10.08, "ttft_max_s": 10.08, "decode_tok_s_per_stream": 241.8, "target_context": 114589, "output_tokens": 442 }, { "turn": 187, "context_tokens": 115144, "ttft_p50_s": 10.181, "ttft_max_s": 10.181, "decode_tok_s_per_stream": 243.5, "target_context": 115301, "output_tokens": 215 }, { "turn": 188, "context_tokens": 117746, "ttft_p50_s": 10.507, "ttft_max_s": 10.507, "decode_tok_s_per_stream": 239.0, "target_context": 117721, "output_tokens": 932 }, { "turn": 189, "context_tokens": 118350, "ttft_p50_s": 10.666, "ttft_max_s": 10.666, "decode_tok_s_per_stream": 244.5, "target_context": 118719, "output_tokens": 177 }, { "turn": 190, "context_tokens": 118965, "ttft_p50_s": 10.743, "ttft_max_s": 10.743, "decode_tok_s_per_stream": 240.1, "target_context": 118935, "output_tokens": 453 }, { "turn": 191, "context_tokens": 119355, "ttft_p50_s": 10.734, "ttft_max_s": 10.734, "decode_tok_s_per_stream": 243.9, "target_context": 119390, "output_tokens": 167 }, { "turn": 192, "context_tokens": 119617, "ttft_p50_s": 10.837, "ttft_max_s": 10.837, "decode_tok_s_per_stream": 238.2, "target_context": 119622, "output_tokens": 1266 }, { "turn": 193, "context_tokens": 39772, "ttft_p50_s": 1.991, "ttft_max_s": 1.991, "decode_tok_s_per_stream": 273.3, "target_context": 38824, "output_tokens": 427 }, { "turn": 194, "context_tokens": 40147, "ttft_p50_s": 2.027, "ttft_max_s": 2.027, "decode_tok_s_per_stream": 273.0, "target_context": 40325, "output_tokens": 442 }, { "turn": 195, "context_tokens": 41223, "ttft_p50_s": 2.104, "ttft_max_s": 2.104, "decode_tok_s_per_stream": 272.6, "target_context": 41396, "output_tokens": 339 }, { "turn": 196, "context_tokens": 42372, "ttft_p50_s": 2.169, "ttft_max_s": 2.169, "decode_tok_s_per_stream": 272.0, "target_context": 42468, "output_tokens": 455 }, { "turn": 197, "context_tokens": 42876, "ttft_p50_s": 2.246, "ttft_max_s": 2.246, "decode_tok_s_per_stream": 271.2, "target_context": 42987, "output_tokens": 520 }, { "turn": 198, "context_tokens": 43293, "ttft_p50_s": 2.256, "ttft_max_s": 2.256, "decode_tok_s_per_stream": 270.6, "target_context": 43517, "output_tokens": 426 }, { "turn": 199, "context_tokens": 46568, "ttft_p50_s": 2.496, "ttft_max_s": 2.496, "decode_tok_s_per_stream": 268.8, "target_context": 46592, "output_tokens": 281 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 20647499 } ], "dimensions": { "model": "Qwen3.6-35B-A3B", "shape": "heavy", "seqs": "32", "graphs": "on", "ctx": "256K" } }, { "label": "nemotron-3.5-lightning · typical session", "model": "Nemotron-3.5-Lightning-30B-A3B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "profile": "docs/usage.json", "session": "typical", "turns": 35, "turns_in_window": 33, "context_min": 919, "context_max": 65699, "output_tokens_p50": 252.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 32, "turns_ok": 32, "status": "ok", "wall_s": 70.7, "turns_over_window": 3, "ttft_first_turn_s": 2.63, "ttft_last_turn_s": 0.176, "ttft_mean_s": 0.717, "ttft_p95_s": 2.816, "context_tokens_last": 56114, "prompt_tokens_total": 1709041, "turns_per_minute": 27.2, "per_turn": [ { "turn": 1, "context_tokens": 46803, "ttft_p50_s": 2.63, "ttft_max_s": 2.63, "decode_tok_s_per_stream": 381.7, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58804, "ttft_p50_s": 1.109, "ttft_max_s": 1.109, "decode_tok_s_per_stream": 388.0, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59617, "ttft_p50_s": 0.246, "ttft_max_s": 0.246, "decode_tok_s_per_stream": 394.1, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 59972, "ttft_p50_s": 0.138, "ttft_max_s": 0.138, "decode_tok_s_per_stream": 380.1, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60326, "ttft_p50_s": 0.179, "ttft_max_s": 0.179, "decode_tok_s_per_stream": 379.7, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 61038, "ttft_p50_s": 0.232, "ttft_max_s": 0.232, "decode_tok_s_per_stream": 375.7, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62402, "ttft_p50_s": 0.332, "ttft_max_s": 0.332, "decode_tok_s_per_stream": 385.2, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63614, "ttft_p50_s": 0.235, "ttft_max_s": 0.235, "decode_tok_s_per_stream": 376.6, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.125, "ttft_max_s": 0.125, "decode_tok_s_per_stream": 1441.9, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 41490, "ttft_p50_s": 2.25, "ttft_max_s": 2.25, "decode_tok_s_per_stream": 392.0, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 41783, "ttft_p50_s": 0.176, "ttft_max_s": 0.176, "decode_tok_s_per_stream": 397.6, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 43418, "ttft_p50_s": 0.278, "ttft_max_s": 0.278, "decode_tok_s_per_stream": 386.0, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49430, "ttft_p50_s": 0.564, "ttft_max_s": 0.564, "decode_tok_s_per_stream": 377.9, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50533, "ttft_p50_s": 0.2, "ttft_max_s": 0.2, "decode_tok_s_per_stream": 378.4, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54377, "ttft_p50_s": 0.493, "ttft_max_s": 0.493, "decode_tok_s_per_stream": 376.0, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1282, "ttft_p50_s": 0.115, "ttft_max_s": 0.115, "decode_tok_s_per_stream": 1411.3, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 41388, "ttft_p50_s": 2.248, "ttft_max_s": 2.248, "decode_tok_s_per_stream": 391.7, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60537, "ttft_p50_s": 1.546, "ttft_max_s": 1.546, "decode_tok_s_per_stream": 381.5, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60901, "ttft_p50_s": 0.198, "ttft_max_s": 0.198, "decode_tok_s_per_stream": 383.1, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61302, "ttft_p50_s": 0.232, "ttft_max_s": 0.232, "decode_tok_s_per_stream": 388.0, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61551, "ttft_p50_s": 0.238, "ttft_max_s": 0.238, "decode_tok_s_per_stream": 376.4, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62174, "ttft_p50_s": 0.304, "ttft_max_s": 0.304, "decode_tok_s_per_stream": 376.9, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 64047, "ttft_p50_s": 0.309, "ttft_max_s": 0.309, "decode_tok_s_per_stream": 381.5, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64669, "ttft_p50_s": 0.19, "ttft_max_s": 0.19, "decode_tok_s_per_stream": 387.3, "target_context": 64770, "output_tokens": 172 }, { "turn": 26, "context_tokens": 53176, "ttft_p50_s": 3.044, "ttft_max_s": 3.044, "decode_tok_s_per_stream": 396.3, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 54800, "ttft_p50_s": 0.361, "ttft_max_s": 0.361, "decode_tok_s_per_stream": 380.5, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 62375, "ttft_p50_s": 0.8, "ttft_max_s": 0.8, "decode_tok_s_per_stream": 395.5, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 64204, "ttft_p50_s": 0.301, "ttft_max_s": 0.301, "decode_tok_s_per_stream": 378.4, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 64571, "ttft_p50_s": 0.188, "ttft_max_s": 0.188, "decode_tok_s_per_stream": 388.3, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 65241, "ttft_p50_s": 0.231, "ttft_max_s": 0.231, "decode_tok_s_per_stream": 395.4, "target_context": 65290, "output_tokens": 106 }, { "turn": 34, "context_tokens": 55824, "ttft_p50_s": 3.279, "ttft_max_s": 3.279, "decode_tok_s_per_stream": 387.2, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 56114, "ttft_p50_s": 0.176, "ttft_max_s": 0.176, "decode_tok_s_per_stream": 416.4, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.8056, "prefix_cache_queried_tokens": 1709041 }, { "agents": 1, "mode": "cold", "turns_requested": 32, "turns_ok": 32, "status": "ok", "wall_s": 150.09, "turns_over_window": 3, "ttft_first_turn_s": 2.631, "ttft_last_turn_s": 3.324, "ttft_mean_s": 3.19, "ttft_p95_s": 4.001, "context_tokens_last": 56015, "prompt_tokens_total": 1708209, "turns_per_minute": 12.8, "per_turn": [ { "turn": 1, "context_tokens": 46809, "ttft_p50_s": 2.631, "ttft_max_s": 2.631, "decode_tok_s_per_stream": 381.0, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58810, "ttft_p50_s": 3.5, "ttft_max_s": 3.5, "decode_tok_s_per_stream": 386.3, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59623, "ttft_p50_s": 3.557, "ttft_max_s": 3.557, "decode_tok_s_per_stream": 393.4, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 59978, "ttft_p50_s": 3.604, "ttft_max_s": 3.604, "decode_tok_s_per_stream": 379.0, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60332, "ttft_p50_s": 3.646, "ttft_max_s": 3.646, "decode_tok_s_per_stream": 377.7, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 61044, "ttft_p50_s": 3.698, "ttft_max_s": 3.698, "decode_tok_s_per_stream": 374.8, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62408, "ttft_p50_s": 3.825, "ttft_max_s": 3.825, "decode_tok_s_per_stream": 387.9, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63620, "ttft_p50_s": 3.88, "ttft_max_s": 3.88, "decode_tok_s_per_stream": 376.1, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1284, "ttft_p50_s": 0.126, "ttft_max_s": 0.126, "decode_tok_s_per_stream": 1468.2, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 41286, "ttft_p50_s": 2.257, "ttft_max_s": 2.257, "decode_tok_s_per_stream": 391.0, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 41579, "ttft_p50_s": 2.273, "ttft_max_s": 2.273, "decode_tok_s_per_stream": 396.0, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 43425, "ttft_p50_s": 2.403, "ttft_max_s": 2.403, "decode_tok_s_per_stream": 390.3, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49437, "ttft_p50_s": 2.828, "ttft_max_s": 2.828, "decode_tok_s_per_stream": 378.0, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50540, "ttft_p50_s": 2.888, "ttft_max_s": 2.888, "decode_tok_s_per_stream": 377.2, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54384, "ttft_p50_s": 3.186, "ttft_max_s": 3.186, "decode_tok_s_per_stream": 374.8, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1289, "ttft_p50_s": 0.116, "ttft_max_s": 0.116, "decode_tok_s_per_stream": 1427.2, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 41184, "ttft_p50_s": 2.253, "ttft_max_s": 2.253, "decode_tok_s_per_stream": 391.2, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60439, "ttft_p50_s": 3.66, "ttft_max_s": 3.66, "decode_tok_s_per_stream": 380.6, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60803, "ttft_p50_s": 3.673, "ttft_max_s": 3.673, "decode_tok_s_per_stream": 385.4, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61310, "ttft_p50_s": 3.703, "ttft_max_s": 3.703, "decode_tok_s_per_stream": 387.5, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61559, "ttft_p50_s": 3.922, "ttft_max_s": 3.922, "decode_tok_s_per_stream": 351.7, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62183, "ttft_p50_s": 3.791, "ttft_max_s": 3.791, "decode_tok_s_per_stream": 375.9, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 64056, "ttft_p50_s": 3.959, "ttft_max_s": 3.959, "decode_tok_s_per_stream": 380.3, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64678, "ttft_p50_s": 3.999, "ttft_max_s": 3.999, "decode_tok_s_per_stream": 385.9, "target_context": 64770, "output_tokens": 172 }, { "turn": 26, "context_tokens": 53183, "ttft_p50_s": 3.065, "ttft_max_s": 3.065, "decode_tok_s_per_stream": 395.5, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 54807, "ttft_p50_s": 3.209, "ttft_max_s": 3.209, "decode_tok_s_per_stream": 378.8, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 62382, "ttft_p50_s": 3.804, "ttft_max_s": 3.804, "decode_tok_s_per_stream": 394.6, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 64211, "ttft_p50_s": 3.967, "ttft_max_s": 3.967, "decode_tok_s_per_stream": 379.8, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 64578, "ttft_p50_s": 4.002, "ttft_max_s": 4.002, "decode_tok_s_per_stream": 391.3, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 65248, "ttft_p50_s": 4.048, "ttft_max_s": 4.048, "decode_tok_s_per_stream": 401.1, "target_context": 65290, "output_tokens": 106 }, { "turn": 34, "context_tokens": 55725, "ttft_p50_s": 3.29, "ttft_max_s": 3.29, "decode_tok_s_per_stream": 386.8, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 56015, "ttft_p50_s": 3.324, "ttft_max_s": 3.324, "decode_tok_s_per_stream": 416.2, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 1708227 }, { "agents": 4, "mode": "warm", "turns_requested": 128, "turns_ok": 128, "status": "ok", "wall_s": 159.07, "turns_over_window": 12, "ttft_first_turn_s": 8.106, "ttft_last_turn_s": 0.165, "ttft_mean_s": 1.692, "ttft_p95_s": 8.715, "context_tokens_last": 56114, "prompt_tokens_total": 6836164, "turns_per_minute": 48.3, "per_turn": [ { "turn": 1, "context_tokens": 46803, "ttft_p50_s": 8.106, "ttft_max_s": 10.005, "decode_tok_s_per_stream": 128.1, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58804, "ttft_p50_s": 1.984, "ttft_max_s": 2.24, "decode_tok_s_per_stream": 124.2, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59617, "ttft_p50_s": 0.445, "ttft_max_s": 0.468, "decode_tok_s_per_stream": 184.3, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 59972, "ttft_p50_s": 0.193, "ttft_max_s": 0.235, "decode_tok_s_per_stream": 227.7, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60326, "ttft_p50_s": 0.237, "ttft_max_s": 0.278, "decode_tok_s_per_stream": 198.2, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 61038, "ttft_p50_s": 0.297, "ttft_max_s": 0.373, "decode_tok_s_per_stream": 238.3, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62402, "ttft_p50_s": 0.675, "ttft_max_s": 0.713, "decode_tok_s_per_stream": 177.8, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63614, "ttft_p50_s": 0.461, "ttft_max_s": 0.655, "decode_tok_s_per_stream": 207.2, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.279, "ttft_max_s": 0.328, "decode_tok_s_per_stream": 514.1, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 41490, "ttft_p50_s": 6.921, "ttft_max_s": 8.454, "decode_tok_s_per_stream": 102.7, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 41783, "ttft_p50_s": 0.162, "ttft_max_s": 0.165, "decode_tok_s_per_stream": 151.6, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 43418, "ttft_p50_s": 0.287, "ttft_max_s": 0.308, "decode_tok_s_per_stream": 127.2, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49430, "ttft_p50_s": 0.623, "ttft_max_s": 0.688, "decode_tok_s_per_stream": 232.6, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50533, "ttft_p50_s": 0.26, "ttft_max_s": 0.287, "decode_tok_s_per_stream": 207.0, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54377, "ttft_p50_s": 0.957, "ttft_max_s": 1.25, "decode_tok_s_per_stream": 220.6, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1282, "ttft_p50_s": 0.256, "ttft_max_s": 0.688, "decode_tok_s_per_stream": 392.3, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 41388, "ttft_p50_s": 6.883, "ttft_max_s": 7.642, "decode_tok_s_per_stream": 56.7, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60537, "ttft_p50_s": 2.794, "ttft_max_s": 3.594, "decode_tok_s_per_stream": 131.0, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60901, "ttft_p50_s": 0.23, "ttft_max_s": 0.302, "decode_tok_s_per_stream": 180.9, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61302, "ttft_p50_s": 0.239, "ttft_max_s": 0.371, "decode_tok_s_per_stream": 144.5, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61551, "ttft_p50_s": 0.251, "ttft_max_s": 0.391, "decode_tok_s_per_stream": 211.2, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62174, "ttft_p50_s": 0.63, "ttft_max_s": 0.705, "decode_tok_s_per_stream": 207.2, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 64047, "ttft_p50_s": 0.761, "ttft_max_s": 0.85, "decode_tok_s_per_stream": 212.8, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64669, "ttft_p50_s": 0.386, "ttft_max_s": 0.42, "decode_tok_s_per_stream": 222.0, "target_context": 64770, "output_tokens": 172 }, { "turn": 26, "context_tokens": 53176, "ttft_p50_s": 9.352, "ttft_max_s": 11.667, "decode_tok_s_per_stream": 52.5, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 54800, "ttft_p50_s": 0.363, "ttft_max_s": 0.387, "decode_tok_s_per_stream": 159.4, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 62375, "ttft_p50_s": 0.889, "ttft_max_s": 1.545, "decode_tok_s_per_stream": 78.2, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 64204, "ttft_p50_s": 0.38, "ttft_max_s": 0.49, "decode_tok_s_per_stream": 178.9, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 64571, "ttft_p50_s": 0.239, "ttft_max_s": 0.311, "decode_tok_s_per_stream": 172.9, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 65241, "ttft_p50_s": 0.241, "ttft_max_s": 0.372, "decode_tok_s_per_stream": 163.8, "target_context": 65290, "output_tokens": 106 }, { "turn": 34, "context_tokens": 55824, "ttft_p50_s": 9.955, "ttft_max_s": 12.332, "decode_tok_s_per_stream": 77.7, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 56114, "ttft_p50_s": 0.165, "ttft_max_s": 0.196, "decode_tok_s_per_stream": 191.5, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.8056, "prefix_cache_queried_tokens": 6836182 }, { "agents": 4, "mode": "cold", "turns_requested": 128, "turns_ok": 128, "status": "ok", "wall_s": 464.38, "turns_over_window": 12, "ttft_first_turn_s": 8.022, "ttft_last_turn_s": 5.223, "ttft_mean_s": 5.326, "ttft_p95_s": 8.022, "context_tokens_last": 56015, "prompt_tokens_total": 6832836, "turns_per_minute": 16.5, "per_turn": [ { "turn": 1, "context_tokens": 46809, "ttft_p50_s": 8.022, "ttft_max_s": 9.958, "decode_tok_s_per_stream": 128.7, "target_context": 57978, "output_tokens": 830 }, { "turn": 2, "context_tokens": 58810, "ttft_p50_s": 9.477, "ttft_max_s": 10.463, "decode_tok_s_per_stream": 38.0, "target_context": 59129, "output_tokens": 247 }, { "turn": 3, "context_tokens": 59623, "ttft_p50_s": 7.854, "ttft_max_s": 8.642, "decode_tok_s_per_stream": 23.3, "target_context": 59678, "output_tokens": 169 }, { "turn": 4, "context_tokens": 59978, "ttft_p50_s": 6.69, "ttft_max_s": 7.662, "decode_tok_s_per_stream": 42.2, "target_context": 59986, "output_tokens": 383 }, { "turn": 5, "context_tokens": 60332, "ttft_p50_s": 6.304, "ttft_max_s": 7.245, "decode_tok_s_per_stream": 48.2, "target_context": 60410, "output_tokens": 476 }, { "turn": 6, "context_tokens": 61044, "ttft_p50_s": 6.186, "ttft_max_s": 6.505, "decode_tok_s_per_stream": 105.0, "target_context": 61198, "output_tokens": 1601 }, { "turn": 7, "context_tokens": 62408, "ttft_p50_s": 6.713, "ttft_max_s": 7.243, "decode_tok_s_per_stream": 23.0, "target_context": 63120, "output_tokens": 204 }, { "turn": 8, "context_tokens": 63620, "ttft_p50_s": 6.52, "ttft_max_s": 6.96, "decode_tok_s_per_stream": 72.9, "target_context": 63620, "output_tokens": 670 }, { "turn": 9, "context_tokens": 1284, "ttft_p50_s": 1.173, "ttft_max_s": 1.24, "decode_tok_s_per_stream": 29.8, "target_context": 974, "output_tokens": 12 }, { "turn": 10, "context_tokens": 41286, "ttft_p50_s": 2.499, "ttft_max_s": 2.808, "decode_tok_s_per_stream": 35.5, "target_context": 33953, "output_tokens": 251 }, { "turn": 11, "context_tokens": 41579, "ttft_p50_s": 2.663, "ttft_max_s": 3.116, "decode_tok_s_per_stream": 27.0, "target_context": 35108, "output_tokens": 180 }, { "turn": 12, "context_tokens": 43425, "ttft_p50_s": 2.864, "ttft_max_s": 3.605, "decode_tok_s_per_stream": 34.0, "target_context": 43405, "output_tokens": 252 }, { "turn": 13, "context_tokens": 49437, "ttft_p50_s": 3.662, "ttft_max_s": 4.069, "decode_tok_s_per_stream": 127.2, "target_context": 49415, "output_tokens": 1739 }, { "turn": 14, "context_tokens": 50540, "ttft_p50_s": 4.003, "ttft_max_s": 5.186, "decode_tok_s_per_stream": 89.2, "target_context": 51386, "output_tokens": 1024 }, { "turn": 15, "context_tokens": 54384, "ttft_p50_s": 4.813, "ttft_max_s": 5.815, "decode_tok_s_per_stream": 177.0, "target_context": 54752, "output_tokens": 4908 }, { "turn": 16, "context_tokens": 1289, "ttft_p50_s": 1.131, "ttft_max_s": 1.346, "decode_tok_s_per_stream": 42.5, "target_context": 919, "output_tokens": 12 }, { "turn": 17, "context_tokens": 41184, "ttft_p50_s": 3.051, "ttft_max_s": 3.802, "decode_tok_s_per_stream": 39.6, "target_context": 33987, "output_tokens": 261 }, { "turn": 18, "context_tokens": 60439, "ttft_p50_s": 6.501, "ttft_max_s": 7.737, "decode_tok_s_per_stream": 43.9, "target_context": 60470, "output_tokens": 403 }, { "turn": 19, "context_tokens": 60803, "ttft_p50_s": 6.219, "ttft_max_s": 7.215, "decode_tok_s_per_stream": 28.9, "target_context": 36543, "output_tokens": 256 }, { "turn": 20, "context_tokens": 61310, "ttft_p50_s": 6.191, "ttft_max_s": 7.125, "decode_tok_s_per_stream": 18.7, "target_context": 61390, "output_tokens": 162 }, { "turn": 21, "context_tokens": 61559, "ttft_p50_s": 6.095, "ttft_max_s": 6.495, "decode_tok_s_per_stream": 79.6, "target_context": 43213, "output_tokens": 950 }, { "turn": 22, "context_tokens": 62183, "ttft_p50_s": 6.4, "ttft_max_s": 6.748, "decode_tok_s_per_stream": 60.4, "target_context": 61948, "output_tokens": 677 }, { "turn": 23, "context_tokens": 64056, "ttft_p50_s": 6.743, "ttft_max_s": 7.145, "decode_tok_s_per_stream": 28.0, "target_context": 64293, "output_tokens": 272 }, { "turn": 24, "context_tokens": 64678, "ttft_p50_s": 6.552, "ttft_max_s": 6.976, "decode_tok_s_per_stream": 20.4, "target_context": 64770, "output_tokens": 172 }, { "turn": 26, "context_tokens": 53183, "ttft_p50_s": 4.62, "ttft_max_s": 5.642, "decode_tok_s_per_stream": 20.6, "target_context": 52665, "output_tokens": 158 }, { "turn": 27, "context_tokens": 54807, "ttft_p50_s": 5.114, "ttft_max_s": 5.882, "decode_tok_s_per_stream": 63.9, "target_context": 54844, "output_tokens": 643 }, { "turn": 28, "context_tokens": 62382, "ttft_p50_s": 7.045, "ttft_max_s": 7.891, "decode_tok_s_per_stream": 17.4, "target_context": 62629, "output_tokens": 142 }, { "turn": 29, "context_tokens": 64211, "ttft_p50_s": 7.019, "ttft_max_s": 7.52, "decode_tok_s_per_stream": 40.4, "target_context": 64188, "output_tokens": 409 }, { "turn": 30, "context_tokens": 64578, "ttft_p50_s": 7.01, "ttft_max_s": 7.715, "decode_tok_s_per_stream": 17.3, "target_context": 64597, "output_tokens": 158 }, { "turn": 31, "context_tokens": 65248, "ttft_p50_s": 7.089, "ttft_max_s": 7.175, "decode_tok_s_per_stream": 12.8, "target_context": 65290, "output_tokens": 106 }, { "turn": 34, "context_tokens": 55725, "ttft_p50_s": 5.409, "ttft_max_s": 6.576, "decode_tok_s_per_stream": 29.2, "target_context": 55488, "output_tokens": 248 }, { "turn": 35, "context_tokens": 56015, "ttft_p50_s": 5.223, "ttft_max_s": 5.672, "decode_tok_s_per_stream": 22.9, "target_context": 55834, "output_tokens": 79 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 6832893 } ], "dimensions": { "model": "Nemotron-3.5-Lightning-30B-A3B", "shape": "typical", "seqs": "32", "graphs": "on" } }, { "label": "nemotron-3.5-lightning · heavy session", "model": "Nemotron-3.5-Lightning-30B-A3B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "profile": "docs/usage.json", "session": "heavy", "turns": 199, "turns_in_window": 55, "context_min": 927, "context_max": 274585, "output_tokens_p50": 560.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 55, "turns_ok": 55, "status": "ok", "wall_s": 131.08, "turns_over_window": 144, "ttft_first_turn_s": 2.53, "ttft_last_turn_s": 0.342, "ttft_mean_s": 0.559, "ttft_p95_s": 2.267, "context_tokens_last": 46459, "prompt_tokens_total": 2331388, "turns_per_minute": 25.2, "per_turn": [ { "turn": 4, "context_tokens": 45229, "ttft_p50_s": 2.53, "ttft_max_s": 2.53, "decode_tok_s_per_stream": 392.3, "target_context": 55938, "output_tokens": 202 }, { "turn": 6, "context_tokens": 56873, "ttft_p50_s": 3.348, "ttft_max_s": 3.348, "decode_tok_s_per_stream": 381.5, "target_context": 56909, "output_tokens": 398 }, { "turn": 7, "context_tokens": 58701, "ttft_p50_s": 0.349, "ttft_max_s": 0.349, "decode_tok_s_per_stream": 398.8, "target_context": 58822, "output_tokens": 124 }, { "turn": 19, "context_tokens": 1282, "ttft_p50_s": 0.08, "ttft_max_s": 0.08, "decode_tok_s_per_stream": 824.6, "target_context": 961, "output_tokens": 16 }, { "turn": 20, "context_tokens": 41390, "ttft_p50_s": 2.267, "ttft_max_s": 2.267, "decode_tok_s_per_stream": 389.6, "target_context": 33996, "output_tokens": 279 }, { "turn": 21, "context_tokens": 41695, "ttft_p50_s": 0.173, "ttft_max_s": 0.173, "decode_tok_s_per_stream": 392.9, "target_context": 34455, "output_tokens": 223 }, { "turn": 22, "context_tokens": 41974, "ttft_p50_s": 0.186, "ttft_max_s": 0.186, "decode_tok_s_per_stream": 381.5, "target_context": 34900, "output_tokens": 578 }, { "turn": 23, "context_tokens": 42421, "ttft_p50_s": 0.197, "ttft_max_s": 0.197, "decode_tok_s_per_stream": 379.8, "target_context": 35765, "output_tokens": 1164 }, { "turn": 24, "context_tokens": 43148, "ttft_p50_s": 0.143, "ttft_max_s": 0.143, "decode_tok_s_per_stream": 379.7, "target_context": 37099, "output_tokens": 890 }, { "turn": 25, "context_tokens": 43750, "ttft_p50_s": 0.178, "ttft_max_s": 0.178, "decode_tok_s_per_stream": 378.3, "target_context": 38060, "output_tokens": 1787 }, { "turn": 26, "context_tokens": 44792, "ttft_p50_s": 0.258, "ttft_max_s": 0.258, "decode_tok_s_per_stream": 380.4, "target_context": 39965, "output_tokens": 502 }, { "turn": 27, "context_tokens": 45214, "ttft_p50_s": 0.1, "ttft_max_s": 0.1, "decode_tok_s_per_stream": 378.9, "target_context": 40530, "output_tokens": 867 }, { "turn": 28, "context_tokens": 45817, "ttft_p50_s": 0.179, "ttft_max_s": 0.179, "decode_tok_s_per_stream": 379.0, "target_context": 41685, "output_tokens": 996 }, { "turn": 29, "context_tokens": 46488, "ttft_p50_s": 0.216, "ttft_max_s": 0.216, "decode_tok_s_per_stream": 377.3, "target_context": 42730, "output_tokens": 1944 }, { "turn": 30, "context_tokens": 47639, "ttft_p50_s": 0.126, "ttft_max_s": 0.126, "decode_tok_s_per_stream": 376.7, "target_context": 44830, "output_tokens": 2348 }, { "turn": 31, "context_tokens": 49014, "ttft_p50_s": 0.266, "ttft_max_s": 0.266, "decode_tok_s_per_stream": 378.0, "target_context": 47231, "output_tokens": 985 }, { "turn": 32, "context_tokens": 49702, "ttft_p50_s": 0.17, "ttft_max_s": 0.17, "decode_tok_s_per_stream": 376.3, "target_context": 48267, "output_tokens": 2720 }, { "turn": 33, "context_tokens": 51307, "ttft_p50_s": 0.091, "ttft_max_s": 0.091, "decode_tok_s_per_stream": 380.4, "target_context": 51031, "output_tokens": 318 }, { "turn": 34, "context_tokens": 1282, "ttft_p50_s": 0.08, "ttft_max_s": 0.08, "decode_tok_s_per_stream": 1172.6, "target_context": 927, "output_tokens": 13 }, { "turn": 35, "context_tokens": 36730, "ttft_p50_s": 1.956, "ttft_max_s": 1.956, "decode_tok_s_per_stream": 395.8, "target_context": 29988, "output_tokens": 207 }, { "turn": 36, "context_tokens": 37001, "ttft_p50_s": 0.149, "ttft_max_s": 0.149, "decode_tok_s_per_stream": 395.9, "target_context": 35061, "output_tokens": 206 }, { "turn": 37, "context_tokens": 37272, "ttft_p50_s": 0.154, "ttft_max_s": 0.154, "decode_tok_s_per_stream": 385.1, "target_context": 36124, "output_tokens": 307 }, { "turn": 38, "context_tokens": 37591, "ttft_p50_s": 0.176, "ttft_max_s": 0.176, "decode_tok_s_per_stream": 384.2, "target_context": 36598, "output_tokens": 405 }, { "turn": 39, "context_tokens": 1282, "ttft_p50_s": 0.081, "ttft_max_s": 0.081, "decode_tok_s_per_stream": 608.2, "target_context": 957, "output_tokens": 23 }, { "turn": 40, "context_tokens": 41501, "ttft_p50_s": 2.268, "ttft_max_s": 2.268, "decode_tok_s_per_stream": 393.1, "target_context": 34132, "output_tokens": 217 }, { "turn": 41, "context_tokens": 41776, "ttft_p50_s": 0.14, "ttft_max_s": 0.14, "decode_tok_s_per_stream": 391.5, "target_context": 35247, "output_tokens": 240 }, { "turn": 42, "context_tokens": 44074, "ttft_p50_s": 0.326, "ttft_max_s": 0.326, "decode_tok_s_per_stream": 380.2, "target_context": 44130, "output_tokens": 589 }, { "turn": 43, "context_tokens": 44845, "ttft_p50_s": 0.253, "ttft_max_s": 0.253, "decode_tok_s_per_stream": 382.4, "target_context": 45079, "output_tokens": 312 }, { "turn": 44, "context_tokens": 13885, "ttft_p50_s": 0.681, "ttft_max_s": 0.681, "decode_tok_s_per_stream": 388.3, "target_context": 14112, "output_tokens": 1229 }, { "turn": 45, "context_tokens": 47589, "ttft_p50_s": 2.116, "ttft_max_s": 2.116, "decode_tok_s_per_stream": 378.2, "target_context": 47335, "output_tokens": 2465 }, { "turn": 46, "context_tokens": 48948, "ttft_p50_s": 0.217, "ttft_max_s": 0.217, "decode_tok_s_per_stream": 378.2, "target_context": 49984, "output_tokens": 2228 }, { "turn": 47, "context_tokens": 51490, "ttft_p50_s": 0.279, "ttft_max_s": 0.279, "decode_tok_s_per_stream": 377.6, "target_context": 52506, "output_tokens": 1819 }, { "turn": 48, "context_tokens": 54271, "ttft_p50_s": 0.323, "ttft_max_s": 0.323, "decode_tok_s_per_stream": 378.3, "target_context": 55007, "output_tokens": 589 }, { "turn": 49, "context_tokens": 55910, "ttft_p50_s": 0.301, "ttft_max_s": 0.301, "decode_tok_s_per_stream": 378.1, "target_context": 56050, "output_tokens": 566 }, { "turn": 50, "context_tokens": 57537, "ttft_p50_s": 0.258, "ttft_max_s": 0.258, "decode_tok_s_per_stream": 376.9, "target_context": 57643, "output_tokens": 737 }, { "turn": 51, "context_tokens": 58510, "ttft_p50_s": 0.203, "ttft_max_s": 0.203, "decode_tok_s_per_stream": 378.4, "target_context": 58729, "output_tokens": 478 }, { "turn": 52, "context_tokens": 59247, "ttft_p50_s": 0.238, "ttft_max_s": 0.238, "decode_tok_s_per_stream": 377.7, "target_context": 59363, "output_tokens": 560 }, { "turn": 53, "context_tokens": 64261, "ttft_p50_s": 0.647, "ttft_max_s": 0.647, "decode_tok_s_per_stream": 383.0, "target_context": 64045, "output_tokens": 218 }, { "turn": 54, "context_tokens": 64862, "ttft_p50_s": 0.193, "ttft_max_s": 0.193, "decode_tok_s_per_stream": 377.7, "target_context": 64837, "output_tokens": 464 }, { "turn": 83, "context_tokens": 39726, "ttft_p50_s": 2.153, "ttft_max_s": 2.153, "decode_tok_s_per_stream": 396.9, "target_context": 37006, "output_tokens": 178 }, { "turn": 84, "context_tokens": 39983, "ttft_p50_s": 0.193, "ttft_max_s": 0.193, "decode_tok_s_per_stream": 435.3, "target_context": 37308, "output_tokens": 62 }, { "turn": 145, "context_tokens": 1286, "ttft_p50_s": 0.116, "ttft_max_s": 0.116, "decode_tok_s_per_stream": 772.1, "target_context": 954, "output_tokens": 17 }, { "turn": 146, "context_tokens": 41353, "ttft_p50_s": 2.248, "ttft_max_s": 2.248, "decode_tok_s_per_stream": 387.5, "target_context": 33809, "output_tokens": 349 }, { "turn": 147, "context_tokens": 41695, "ttft_p50_s": 0.137, "ttft_max_s": 0.137, "decode_tok_s_per_stream": 395.6, "target_context": 34921, "output_tokens": 186 }, { "turn": 148, "context_tokens": 41959, "ttft_p50_s": 0.184, "ttft_max_s": 0.184, "decode_tok_s_per_stream": 378.6, "target_context": 38701, "output_tokens": 2701 }, { "turn": 149, "context_tokens": 43408, "ttft_p50_s": 0.147, "ttft_max_s": 0.147, "decode_tok_s_per_stream": 378.7, "target_context": 41416, "output_tokens": 995 }, { "turn": 150, "context_tokens": 44068, "ttft_p50_s": 0.202, "ttft_max_s": 0.202, "decode_tok_s_per_stream": 388.0, "target_context": 43422, "output_tokens": 145 }, { "turn": 151, "context_tokens": 44529, "ttft_p50_s": 0.173, "ttft_max_s": 0.173, "decode_tok_s_per_stream": 382.6, "target_context": 44550, "output_tokens": 317 }, { "turn": 193, "context_tokens": 40204, "ttft_p50_s": 2.158, "ttft_max_s": 2.158, "decode_tok_s_per_stream": 386.4, "target_context": 38824, "output_tokens": 427 }, { "turn": 194, "context_tokens": 40582, "ttft_p50_s": 0.219, "ttft_max_s": 0.219, "decode_tok_s_per_stream": 386.3, "target_context": 40325, "output_tokens": 442 }, { "turn": 195, "context_tokens": 41288, "ttft_p50_s": 0.121, "ttft_max_s": 0.121, "decode_tok_s_per_stream": 388.1, "target_context": 41396, "output_tokens": 339 }, { "turn": 196, "context_tokens": 42375, "ttft_p50_s": 0.199, "ttft_max_s": 0.199, "decode_tok_s_per_stream": 381.9, "target_context": 42468, "output_tokens": 455 }, { "turn": 197, "context_tokens": 42874, "ttft_p50_s": 0.126, "ttft_max_s": 0.126, "decode_tok_s_per_stream": 380.8, "target_context": 42987, "output_tokens": 520 }, { "turn": 198, "context_tokens": 43299, "ttft_p50_s": 0.156, "ttft_max_s": 0.156, "decode_tok_s_per_stream": 381.8, "target_context": 43517, "output_tokens": 426 }, { "turn": 199, "context_tokens": 46459, "ttft_p50_s": 0.342, "ttft_max_s": 0.342, "decode_tok_s_per_stream": 383.2, "target_context": 46592, "output_tokens": 281 } ], "prefix_cache_hit_rate": 0.7987, "prefix_cache_queried_tokens": 2331388 }, { "agents": 1, "mode": "cold", "turns_requested": 55, "turns_ok": 55, "status": "ok", "wall_s": 231.19, "turns_over_window": 144, "ttft_first_turn_s": 2.524, "ttft_last_turn_s": 2.592, "ttft_mean_s": 2.382, "ttft_p95_s": 3.528, "context_tokens_last": 46465, "prompt_tokens_total": 2326146, "turns_per_minute": 14.3, "per_turn": [ { "turn": 4, "context_tokens": 45235, "ttft_p50_s": 2.524, "ttft_max_s": 2.524, "decode_tok_s_per_stream": 392.2, "target_context": 55938, "output_tokens": 202 }, { "turn": 6, "context_tokens": 56879, "ttft_p50_s": 3.37, "ttft_max_s": 3.37, "decode_tok_s_per_stream": 382.0, "target_context": 56909, "output_tokens": 398 }, { "turn": 7, "context_tokens": 58707, "ttft_p50_s": 3.512, "ttft_max_s": 3.512, "decode_tok_s_per_stream": 399.7, "target_context": 58822, "output_tokens": 124 }, { "turn": 19, "context_tokens": 1289, "ttft_p50_s": 0.08, "ttft_max_s": 0.08, "decode_tok_s_per_stream": 824.0, "target_context": 961, "output_tokens": 16 }, { "turn": 20, "context_tokens": 41186, "ttft_p50_s": 2.258, "ttft_max_s": 2.258, "decode_tok_s_per_stream": 390.3, "target_context": 33996, "output_tokens": 279 }, { "turn": 21, "context_tokens": 41491, "ttft_p50_s": 2.272, "ttft_max_s": 2.272, "decode_tok_s_per_stream": 392.5, "target_context": 34455, "output_tokens": 223 }, { "turn": 22, "context_tokens": 41769, "ttft_p50_s": 2.281, "ttft_max_s": 2.281, "decode_tok_s_per_stream": 383.9, "target_context": 34900, "output_tokens": 578 }, { "turn": 23, "context_tokens": 42216, "ttft_p50_s": 2.296, "ttft_max_s": 2.296, "decode_tok_s_per_stream": 379.7, "target_context": 35765, "output_tokens": 1164 }, { "turn": 24, "context_tokens": 42943, "ttft_p50_s": 2.379, "ttft_max_s": 2.379, "decode_tok_s_per_stream": 381.5, "target_context": 37099, "output_tokens": 890 }, { "turn": 25, "context_tokens": 43545, "ttft_p50_s": 2.411, "ttft_max_s": 2.411, "decode_tok_s_per_stream": 378.4, "target_context": 38060, "output_tokens": 1787 }, { "turn": 26, "context_tokens": 44586, "ttft_p50_s": 2.457, "ttft_max_s": 2.457, "decode_tok_s_per_stream": 383.1, "target_context": 39965, "output_tokens": 502 }, { "turn": 27, "context_tokens": 45008, "ttft_p50_s": 2.516, "ttft_max_s": 2.516, "decode_tok_s_per_stream": 380.3, "target_context": 40530, "output_tokens": 867 }, { "turn": 28, "context_tokens": 45611, "ttft_p50_s": 2.533, "ttft_max_s": 2.533, "decode_tok_s_per_stream": 379.6, "target_context": 41685, "output_tokens": 996 }, { "turn": 29, "context_tokens": 46282, "ttft_p50_s": 2.584, "ttft_max_s": 2.584, "decode_tok_s_per_stream": 377.4, "target_context": 42730, "output_tokens": 1944 }, { "turn": 30, "context_tokens": 47433, "ttft_p50_s": 2.686, "ttft_max_s": 2.686, "decode_tok_s_per_stream": 377.7, "target_context": 44830, "output_tokens": 2348 }, { "turn": 31, "context_tokens": 48808, "ttft_p50_s": 2.755, "ttft_max_s": 2.755, "decode_tok_s_per_stream": 379.1, "target_context": 47231, "output_tokens": 985 }, { "turn": 32, "context_tokens": 49496, "ttft_p50_s": 2.82, "ttft_max_s": 2.82, "decode_tok_s_per_stream": 376.2, "target_context": 48267, "output_tokens": 2720 }, { "turn": 33, "context_tokens": 51100, "ttft_p50_s": 2.928, "ttft_max_s": 2.928, "decode_tok_s_per_stream": 385.7, "target_context": 51031, "output_tokens": 318 }, { "turn": 34, "context_tokens": 1289, "ttft_p50_s": 0.079, "ttft_max_s": 0.079, "decode_tok_s_per_stream": 1149.2, "target_context": 927, "output_tokens": 13 }, { "turn": 35, "context_tokens": 36525, "ttft_p50_s": 1.954, "ttft_max_s": 1.954, "decode_tok_s_per_stream": 395.7, "target_context": 29988, "output_tokens": 207 }, { "turn": 36, "context_tokens": 36796, "ttft_p50_s": 1.968, "ttft_max_s": 1.968, "decode_tok_s_per_stream": 395.0, "target_context": 35061, "output_tokens": 206 }, { "turn": 37, "context_tokens": 37067, "ttft_p50_s": 1.986, "ttft_max_s": 1.986, "decode_tok_s_per_stream": 390.5, "target_context": 36124, "output_tokens": 307 }, { "turn": 38, "context_tokens": 37386, "ttft_p50_s": 1.993, "ttft_max_s": 1.993, "decode_tok_s_per_stream": 383.8, "target_context": 36598, "output_tokens": 405 }, { "turn": 39, "context_tokens": 1289, "ttft_p50_s": 0.081, "ttft_max_s": 0.081, "decode_tok_s_per_stream": 613.2, "target_context": 957, "output_tokens": 23 }, { "turn": 40, "context_tokens": 41297, "ttft_p50_s": 2.238, "ttft_max_s": 2.238, "decode_tok_s_per_stream": 392.5, "target_context": 34132, "output_tokens": 217 }, { "turn": 41, "context_tokens": 41572, "ttft_p50_s": 2.281, "ttft_max_s": 2.281, "decode_tok_s_per_stream": 391.2, "target_context": 35247, "output_tokens": 240 }, { "turn": 42, "context_tokens": 44082, "ttft_p50_s": 2.448, "ttft_max_s": 2.448, "decode_tok_s_per_stream": 382.5, "target_context": 44130, "output_tokens": 589 }, { "turn": 43, "context_tokens": 44853, "ttft_p50_s": 2.523, "ttft_max_s": 2.523, "decode_tok_s_per_stream": 386.5, "target_context": 45079, "output_tokens": 312 }, { "turn": 44, "context_tokens": 13892, "ttft_p50_s": 0.68, "ttft_max_s": 0.68, "decode_tok_s_per_stream": 387.8, "target_context": 14112, "output_tokens": 1229 }, { "turn": 45, "context_tokens": 47489, "ttft_p50_s": 2.662, "ttft_max_s": 2.662, "decode_tok_s_per_stream": 377.2, "target_context": 47335, "output_tokens": 2465 }, { "turn": 46, "context_tokens": 48849, "ttft_p50_s": 2.752, "ttft_max_s": 2.752, "decode_tok_s_per_stream": 376.5, "target_context": 49984, "output_tokens": 2228 }, { "turn": 47, "context_tokens": 51498, "ttft_p50_s": 2.983, "ttft_max_s": 2.983, "decode_tok_s_per_stream": 377.0, "target_context": 52506, "output_tokens": 1819 }, { "turn": 48, "context_tokens": 54279, "ttft_p50_s": 3.189, "ttft_max_s": 3.189, "decode_tok_s_per_stream": 379.9, "target_context": 55007, "output_tokens": 589 }, { "turn": 49, "context_tokens": 55918, "ttft_p50_s": 3.311, "ttft_max_s": 3.311, "decode_tok_s_per_stream": 379.5, "target_context": 56050, "output_tokens": 566 }, { "turn": 50, "context_tokens": 57545, "ttft_p50_s": 3.464, "ttft_max_s": 3.464, "decode_tok_s_per_stream": 378.2, "target_context": 57643, "output_tokens": 737 }, { "turn": 51, "context_tokens": 58518, "ttft_p50_s": 3.51, "ttft_max_s": 3.51, "decode_tok_s_per_stream": 377.6, "target_context": 58729, "output_tokens": 478 }, { "turn": 52, "context_tokens": 59255, "ttft_p50_s": 3.567, "ttft_max_s": 3.567, "decode_tok_s_per_stream": 378.1, "target_context": 59363, "output_tokens": 560 }, { "turn": 53, "context_tokens": 64163, "ttft_p50_s": 3.967, "ttft_max_s": 3.967, "decode_tok_s_per_stream": 381.9, "target_context": 64045, "output_tokens": 218 }, { "turn": 54, "context_tokens": 64869, "ttft_p50_s": 4.031, "ttft_max_s": 4.031, "decode_tok_s_per_stream": 377.9, "target_context": 64837, "output_tokens": 464 }, { "turn": 83, "context_tokens": 39733, "ttft_p50_s": 2.159, "ttft_max_s": 2.159, "decode_tok_s_per_stream": 396.9, "target_context": 37006, "output_tokens": 178 }, { "turn": 84, "context_tokens": 39990, "ttft_p50_s": 2.163, "ttft_max_s": 2.163, "decode_tok_s_per_stream": 434.9, "target_context": 37308, "output_tokens": 62 }, { "turn": 145, "context_tokens": 1294, "ttft_p50_s": 0.116, "ttft_max_s": 0.116, "decode_tok_s_per_stream": 774.1, "target_context": 954, "output_tokens": 17 }, { "turn": 146, "context_tokens": 41147, "ttft_p50_s": 2.248, "ttft_max_s": 2.248, "decode_tok_s_per_stream": 387.5, "target_context": 33809, "output_tokens": 349 }, { "turn": 147, "context_tokens": 41488, "ttft_p50_s": 2.262, "ttft_max_s": 2.262, "decode_tok_s_per_stream": 395.7, "target_context": 34921, "output_tokens": 186 }, { "turn": 148, "context_tokens": 41752, "ttft_p50_s": 2.244, "ttft_max_s": 2.244, "decode_tok_s_per_stream": 379.3, "target_context": 38701, "output_tokens": 2701 }, { "turn": 149, "context_tokens": 43200, "ttft_p50_s": 2.364, "ttft_max_s": 2.364, "decode_tok_s_per_stream": 379.3, "target_context": 41416, "output_tokens": 995 }, { "turn": 150, "context_tokens": 43860, "ttft_p50_s": 2.421, "ttft_max_s": 2.421, "decode_tok_s_per_stream": 399.0, "target_context": 43422, "output_tokens": 145 }, { "turn": 151, "context_tokens": 44535, "ttft_p50_s": 2.445, "ttft_max_s": 2.445, "decode_tok_s_per_stream": 386.3, "target_context": 44550, "output_tokens": 317 }, { "turn": 193, "context_tokens": 40212, "ttft_p50_s": 2.136, "ttft_max_s": 2.136, "decode_tok_s_per_stream": 386.6, "target_context": 38824, "output_tokens": 427 }, { "turn": 194, "context_tokens": 40590, "ttft_p50_s": 2.224, "ttft_max_s": 2.224, "decode_tok_s_per_stream": 385.1, "target_context": 40325, "output_tokens": 442 }, { "turn": 195, "context_tokens": 41296, "ttft_p50_s": 2.255, "ttft_max_s": 2.255, "decode_tok_s_per_stream": 388.0, "target_context": 41396, "output_tokens": 339 }, { "turn": 196, "context_tokens": 42382, "ttft_p50_s": 2.295, "ttft_max_s": 2.295, "decode_tok_s_per_stream": 381.8, "target_context": 42468, "output_tokens": 455 }, { "turn": 197, "context_tokens": 42881, "ttft_p50_s": 2.377, "ttft_max_s": 2.377, "decode_tok_s_per_stream": 384.4, "target_context": 42987, "output_tokens": 520 }, { "turn": 198, "context_tokens": 43306, "ttft_p50_s": 2.397, "ttft_max_s": 2.397, "decode_tok_s_per_stream": 385.0, "target_context": 43517, "output_tokens": 426 }, { "turn": 199, "context_tokens": 46465, "ttft_p50_s": 2.592, "ttft_max_s": 2.592, "decode_tok_s_per_stream": 387.6, "target_context": 46592, "output_tokens": 281 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 2326146 } ], "dimensions": { "model": "Nemotron-3.5-Lightning-30B-A3B", "shape": "heavy", "seqs": "32", "graphs": "on" } }, { "label": "qwen3.8-27b text-only, CUDA graphs · typical session", "model": "Qwen3.8-27B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "profile": "docs/usage.json", "session": "typical", "turns": 35, "turns_in_window": 2, "context_min": 919, "context_max": 65699, "output_tokens_p50": 252.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 2, "turns_ok": 2, "status": "ok", "wall_s": 0.59, "turns_over_window": 33, "ttft_first_turn_s": 0.122, "ttft_last_turn_s": 0.234, "ttft_mean_s": 0.178, "ttft_p95_s": 0.228, "context_tokens_last": 1276, "prompt_tokens_total": 2548, "turns_per_minute": 202.1, "per_turn": [ { "turn": 9, "context_tokens": 1272, "ttft_p50_s": 0.122, "ttft_max_s": 0.122, "decode_tok_s_per_stream": 63.1, "target_context": 974, "output_tokens": 12 }, { "turn": 16, "context_tokens": 1276, "ttft_p50_s": 0.234, "ttft_max_s": 0.234, "decode_tok_s_per_stream": 174.2, "target_context": 919, "output_tokens": 12 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 2548 }, { "agents": 1, "mode": "cold", "turns_requested": 2, "turns_ok": 2, "status": "ok", "wall_s": 0.6, "turns_over_window": 33, "ttft_first_turn_s": 0.122, "ttft_last_turn_s": 0.241, "ttft_mean_s": 0.182, "ttft_p95_s": 0.235, "context_tokens_last": 1283, "prompt_tokens_total": 2561, "turns_per_minute": 199.6, "per_turn": [ { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.122, "ttft_max_s": 0.122, "decode_tok_s_per_stream": 63.2, "target_context": 974, "output_tokens": 12 }, { "turn": 16, "context_tokens": 1283, "ttft_p50_s": 0.241, "ttft_max_s": 0.241, "decode_tok_s_per_stream": 174.7, "target_context": 919, "output_tokens": 12 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 2561 }, { "agents": 4, "mode": "warm", "turns_requested": 8, "turns_ok": 8, "status": "ok", "wall_s": 1.33, "turns_over_window": 132, "ttft_first_turn_s": 0.45, "ttft_last_turn_s": 0.468, "ttft_mean_s": 0.438, "ttft_p95_s": 0.602, "context_tokens_last": 1276, "prompt_tokens_total": 10192, "turns_per_minute": 360.4, "per_turn": [ { "turn": 9, "context_tokens": 1272, "ttft_p50_s": 0.45, "ttft_max_s": 0.608, "decode_tok_s_per_stream": 40.0, "target_context": 974, "output_tokens": 12 }, { "turn": 16, "context_tokens": 1276, "ttft_p50_s": 0.468, "ttft_max_s": 0.58, "decode_tok_s_per_stream": 102.6, "target_context": 919, "output_tokens": 12 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 10192 }, { "agents": 4, "mode": "cold", "turns_requested": 8, "turns_ok": 8, "status": "ok", "wall_s": 1.37, "turns_over_window": 132, "ttft_first_turn_s": 0.369, "ttft_last_turn_s": 0.361, "ttft_mean_s": 0.384, "ttft_p95_s": 0.571, "context_tokens_last": 1283, "prompt_tokens_total": 10244, "turns_per_minute": 349.6, "per_turn": [ { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.369, "ttft_max_s": 0.488, "decode_tok_s_per_stream": 31.0, "target_context": 974, "output_tokens": 12 }, { "turn": 16, "context_tokens": 1283, "ttft_p50_s": 0.361, "ttft_max_s": 0.615, "decode_tok_s_per_stream": 51.0, "target_context": 919, "output_tokens": 12 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 10244 } ], "dimensions": { "model": "Qwen3.8-27B", "shape": "typical", "seqs": "32", "vision": "off", "graphs": "on" } }, { "label": "qwen3.8-27b text-only, CUDA graphs · heavy session", "model": "Qwen3.8-27B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "profile": "docs/usage.json", "session": "heavy", "turns": 199, "turns_in_window": 6, "context_min": 927, "context_max": 274585, "output_tokens_p50": 560.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 5, "turns_ok": 5, "status": "ok", "wall_s": 23.59, "turns_over_window": 194, "ttft_first_turn_s": 0.23, "ttft_last_turn_s": 0.235, "ttft_mean_s": 0.613, "ttft_p95_s": 1.844, "context_tokens_last": 1280, "prompt_tokens_total": 23735, "turns_per_minute": 12.7, "per_turn": [ { "turn": 19, "context_tokens": 1276, "ttft_p50_s": 0.23, "ttft_max_s": 0.23, "decode_tok_s_per_stream": 118.8, "target_context": 961, "output_tokens": 16 }, { "turn": 34, "context_tokens": 1276, "ttft_p50_s": 0.122, "ttft_max_s": 0.122, "decode_tok_s_per_stream": 63.6, "target_context": 927, "output_tokens": 13 }, { "turn": 39, "context_tokens": 1276, "ttft_p50_s": 0.232, "ttft_max_s": 0.232, "decode_tok_s_per_stream": 92.7, "target_context": 957, "output_tokens": 23 }, { "turn": 44, "context_tokens": 18627, "ttft_p50_s": 2.247, "ttft_max_s": 2.247, "decode_tok_s_per_stream": 62.2, "target_context": 14112, "output_tokens": 1229 }, { "turn": 145, "context_tokens": 1280, "ttft_p50_s": 0.235, "ttft_max_s": 0.235, "decode_tok_s_per_stream": 111.9, "target_context": 954, "output_tokens": 17 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 23735 }, { "agents": 1, "mode": "cold", "turns_requested": 5, "turns_ok": 5, "status": "ok", "wall_s": 23.64, "turns_over_window": 194, "ttft_first_turn_s": 0.131, "ttft_last_turn_s": 0.243, "ttft_mean_s": 0.599, "ttft_p95_s": 1.844, "context_tokens_last": 1288, "prompt_tokens_total": 23656, "turns_per_minute": 12.7, "per_turn": [ { "turn": 19, "context_tokens": 1283, "ttft_p50_s": 0.131, "ttft_max_s": 0.131, "decode_tok_s_per_stream": 63.5, "target_context": 961, "output_tokens": 16 }, { "turn": 34, "context_tokens": 1283, "ttft_p50_s": 0.243, "ttft_max_s": 0.243, "decode_tok_s_per_stream": 151.2, "target_context": 927, "output_tokens": 13 }, { "turn": 39, "context_tokens": 1283, "ttft_p50_s": 0.132, "ttft_max_s": 0.132, "decode_tok_s_per_stream": 63.4, "target_context": 957, "output_tokens": 23 }, { "turn": 44, "context_tokens": 18519, "ttft_p50_s": 2.244, "ttft_max_s": 2.244, "decode_tok_s_per_stream": 62.2, "target_context": 14112, "output_tokens": 1229 }, { "turn": 145, "context_tokens": 1288, "ttft_p50_s": 0.243, "ttft_max_s": 0.243, "decode_tok_s_per_stream": 112.3, "target_context": 954, "output_tokens": 17 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 23656 } ], "dimensions": { "model": "Qwen3.8-27B", "shape": "heavy", "seqs": "32", "vision": "off", "graphs": "on" } }, { "label": "qwen3.8-27b default · typical session", "model": "Qwen3.8-27B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "profile": "docs/usage.json", "session": "typical", "turns": 35, "turns_in_window": 2, "context_min": 919, "context_max": 65699, "output_tokens_p50": 252.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 2, "turns_ok": 2, "status": "ok", "wall_s": 1.11, "turns_over_window": 33, "ttft_first_turn_s": 0.436, "ttft_last_turn_s": 0.161, "ttft_mean_s": 0.298, "ttft_p95_s": 0.422, "context_tokens_last": 1276, "prompt_tokens_total": 2548, "turns_per_minute": 108.4, "per_turn": [ { "turn": 9, "context_tokens": 1272, "ttft_p50_s": 0.436, "ttft_max_s": 0.436, "decode_tok_s_per_stream": 92.7, "target_context": 974, "output_tokens": 12 }, { "turn": 16, "context_tokens": 1276, "ttft_p50_s": 0.161, "ttft_max_s": 0.161, "decode_tok_s_per_stream": 28.1, "target_context": 919, "output_tokens": 12 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 2548 }, { "agents": 1, "mode": "cold", "turns_requested": 2, "turns_ok": 2, "status": "ok", "wall_s": 1.12, "turns_over_window": 33, "ttft_first_turn_s": 0.16, "ttft_last_turn_s": 0.168, "ttft_mean_s": 0.164, "ttft_p95_s": 0.167, "context_tokens_last": 1283, "prompt_tokens_total": 2561, "turns_per_minute": 107.4, "per_turn": [ { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.16, "ttft_max_s": 0.16, "decode_tok_s_per_stream": 28.1, "target_context": 974, "output_tokens": 12 }, { "turn": 16, "context_tokens": 1283, "ttft_p50_s": 0.168, "ttft_max_s": 0.168, "decode_tok_s_per_stream": 27.7, "target_context": 919, "output_tokens": 12 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 2561 }, { "agents": 4, "mode": "warm", "turns_requested": 8, "turns_ok": 8, "status": "ok", "wall_s": 2.27, "turns_over_window": 132, "ttft_first_turn_s": 1.011, "ttft_last_turn_s": 0.491, "ttft_mean_s": 0.703, "ttft_p95_s": 1.122, "context_tokens_last": 1276, "prompt_tokens_total": 10192, "turns_per_minute": 211.3, "per_turn": [ { "turn": 9, "context_tokens": 1272, "ttft_p50_s": 1.011, "ttft_max_s": 1.136, "decode_tok_s_per_stream": 42.5, "target_context": 974, "output_tokens": 12 }, { "turn": 16, "context_tokens": 1276, "ttft_p50_s": 0.491, "ttft_max_s": 0.705, "decode_tok_s_per_stream": 43.5, "target_context": 919, "output_tokens": 12 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 10192 }, { "agents": 4, "mode": "cold", "turns_requested": 8, "turns_ok": 8, "status": "ok", "wall_s": 1.95, "turns_over_window": 132, "ttft_first_turn_s": 0.542, "ttft_last_turn_s": 0.408, "ttft_mean_s": 0.519, "ttft_p95_s": 0.777, "context_tokens_last": 1283, "prompt_tokens_total": 10244, "turns_per_minute": 246.4, "per_turn": [ { "turn": 9, "context_tokens": 1278, "ttft_p50_s": 0.542, "ttft_max_s": 0.737, "decode_tok_s_per_stream": 19.9, "target_context": 974, "output_tokens": 12 }, { "turn": 16, "context_tokens": 1283, "ttft_p50_s": 0.408, "ttft_max_s": 0.799, "decode_tok_s_per_stream": 25.1, "target_context": 919, "output_tokens": 12 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 10244 } ], "dimensions": { "model": "Qwen3.8-27B", "shape": "typical", "seqs": "8", "vision": "on", "graphs": "off" } }, { "label": "qwen3.8-27b default · heavy session", "model": "Qwen3.8-27B", "generated": "2026-09-07", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "profile": "docs/usage.json", "session": "heavy", "turns": 199, "turns_in_window": 6, "context_min": 927, "context_max": 274585, "output_tokens_p50": 560.0, "ignore_eos": true, "thinking": false }, "levels": [ { "agents": 1, "mode": "warm", "turns_requested": 5, "turns_ok": 5, "status": "ok", "wall_s": 54.47, "turns_over_window": 194, "ttft_first_turn_s": 0.161, "ttft_last_turn_s": 0.442, "ttft_mean_s": 0.683, "ttft_p95_s": 2.078, "context_tokens_last": 1280, "prompt_tokens_total": 23735, "turns_per_minute": 5.5, "per_turn": [ { "turn": 19, "context_tokens": 1276, "ttft_p50_s": 0.161, "ttft_max_s": 0.161, "decode_tok_s_per_stream": 26.8, "target_context": 961, "output_tokens": 16 }, { "turn": 34, "context_tokens": 1276, "ttft_p50_s": 0.162, "ttft_max_s": 0.162, "decode_tok_s_per_stream": 27.4, "target_context": 927, "output_tokens": 13 }, { "turn": 39, "context_tokens": 1276, "ttft_p50_s": 0.162, "ttft_max_s": 0.162, "decode_tok_s_per_stream": 26.4, "target_context": 957, "output_tokens": 23 }, { "turn": 44, "context_tokens": 18627, "ttft_p50_s": 2.487, "ttft_max_s": 2.487, "decode_tok_s_per_stream": 25.1, "target_context": 14112, "output_tokens": 1229 }, { "turn": 145, "context_tokens": 1280, "ttft_p50_s": 0.442, "ttft_max_s": 0.442, "decode_tok_s_per_stream": 49.9, "target_context": 954, "output_tokens": 17 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 23735 }, { "agents": 1, "mode": "cold", "turns_requested": 5, "turns_ok": 5, "status": "ok", "wall_s": 54.64, "turns_over_window": 194, "ttft_first_turn_s": 0.447, "ttft_last_turn_s": 0.446, "ttft_mean_s": 0.739, "ttft_p95_s": 2.062, "context_tokens_last": 1288, "prompt_tokens_total": 23656, "turns_per_minute": 5.5, "per_turn": [ { "turn": 19, "context_tokens": 1283, "ttft_p50_s": 0.447, "ttft_max_s": 0.447, "decode_tok_s_per_stream": 53.5, "target_context": 961, "output_tokens": 16 }, { "turn": 34, "context_tokens": 1283, "ttft_p50_s": 0.169, "ttft_max_s": 0.169, "decode_tok_s_per_stream": 27.4, "target_context": 927, "output_tokens": 13 }, { "turn": 39, "context_tokens": 1283, "ttft_p50_s": 0.17, "ttft_max_s": 0.17, "decode_tok_s_per_stream": 26.3, "target_context": 957, "output_tokens": 23 }, { "turn": 44, "context_tokens": 18519, "ttft_p50_s": 2.466, "ttft_max_s": 2.466, "decode_tok_s_per_stream": 25.1, "target_context": 14112, "output_tokens": 1229 }, { "turn": 145, "context_tokens": 1288, "ttft_p50_s": 0.446, "ttft_max_s": 0.446, "decode_tok_s_per_stream": 50.3, "target_context": 954, "output_tokens": 17 } ], "prefix_cache_hit_rate": 0.0, "prefix_cache_queried_tokens": 23656 } ], "dimensions": { "model": "Qwen3.8-27B", "shape": "heavy", "seqs": "8", "vision": "on", "graphs": "off" } } ] }