{ "schema_version": 1, "generated": "2026-09-07", "hardware": { "gpu": "NVIDIA GeForce RTX 5090", "arch": "Blackwell sm120", "vram_total_mib": 32607, "vram_usable_gib": 31.4, "vram_baseline_mib": 950, "vram_baseline_note": "desktop compositor (kwin_wayland + Xwayland)", "driver": "610.57.04", "host_ram_gib": 30, "cpu_threads": 16, "os": "Arch Linux", "runtime": "podman 6.1.0, CDI nvidia.com/gpu=all" }, "software": { "image": "docker.io/vllm/vllm-openai:v0.28.0", "vllm": "0.28.0", "torch": "2.13.0+cu130", "transformers": "5.15.1" }, "method": { "workload": "single request, 300 output tokens cut from a longer answer (no ignore_eos, so speculative decoding sees natural text), temperature 0, thinking disabled, idle server", "prompt": "Write a detailed essay of about 600 words on the sea: its physical nature, its role in climate, and what it has meant to people.", "warmup": "first request after startup discarded as cold; reported value is the median of the 3 that follow", "kv_tokens": "kv_cache_size_tokens from the vllm:cache_config_info gauge on /metrics", "acceptance": "vllm:spec_decode_num_accepted_tokens_total / vllm:spec_decode_num_draft_tokens_total, over the samples" }, "runs": [ { "id": "qwen36-moe-default", "status": "ok", "profile": "qwen3.6-35b-a3b", "overrides": "", "model": "Qwen3.6-35B-A3B", "checkpoint": "nvidia/Qwen3.6-35B-A3B-NVFP4", "shape": "MoE, 35B total / 3B active", "precision": "NVFP4", "weights_gb": 21.9, "weights_source": "loaded on the GPU, from vLLM's load report", "max_model_len": 65536, "vision": true, "cuda_graphs": true, "spec_decode": false, "decode_tok_s": 289.0, "decode_cold_sample": 289.5, "decode_samples": [ 289.1, 289.0, 289.0 ], "generated": "2026-09-07", "vllm_version": "0.28.0", "kv_tokens": 370189, "max_concurrency": 5.65, "kv_cache_dtype": "fp8" }, { "id": "qwen36-moe-mtp", "status": "ok", "profile": "qwen3.6-35b-a3b", "overrides": "SPEC_DECODE=1", "model": "Qwen3.6-35B-A3B", "checkpoint": "nvidia/Qwen3.6-35B-A3B-NVFP4", "shape": "MoE, 35B total / 3B active", "precision": "NVFP4", "weights_gb": 23.6, "weights_source": "loaded on the GPU, from vLLM's load report", "max_model_len": 65536, "vision": true, "cuda_graphs": true, "spec_decode": true, "decode_tok_s": 358.0, "decode_cold_sample": 359.8, "decode_samples": [ 361.9, 358.0, 355.3 ], "generated": "2026-09-07", "vllm_version": "0.28.0", "kv_tokens": 111009, "max_concurrency": 1.69, "kv_cache_dtype": "fp8", "spec_acceptance": 0.39 }, { "id": "qwen38-dense-mtp", "status": "ok", "profile": "qwen3.8-27b", "overrides": "SPEC_DECODE=1 LANGUAGE_MODEL_ONLY=1", "model": "Qwen3.8-27B", "checkpoint": "Inferact/Qwen3.8-27B-NVFP4", "shape": "dense 27B", "precision": "NVFP4", "weights_gb": 25.9, "weights_source": "loaded on the GPU, from vLLM's load report", "max_model_len": 32768, "vision": false, "cuda_graphs": false, "spec_decode": true, "decode_tok_s": 53.1, "decode_cold_sample": 53.0, "decode_samples": [ 53.0, 53.2, 53.1 ], "generated": "2026-09-07", "vllm_version": "0.28.0", "kv_tokens": 70087, "max_concurrency": 2.14, "kv_cache_dtype": "fp8", "spec_acceptance": 0.48 }, { "id": "qwen38-fp8-official", "status": "ok", "profile": "qwen3.8-27b", "overrides": "MODEL=Qwen/Qwen3.8-27B-FP8 MAX_MODEL_LEN=4096 MAX_NUM_SEQS=2 GPU_MEMORY_UTILIZATION=0.93 LANGUAGE_MODEL_ONLY=1 EXTRA_ARGS=--max-num-batched-tokens 1024", "model": "Qwen3.8-27B", "checkpoint": "Qwen/Qwen3.8-27B-FP8", "shape": "dense 27B", "precision": "FP8 (official)", "weights_gb": 29.7, "weights_source": "loaded on the GPU, from vLLM's load report", "max_model_len": 4096, "vision": false, "cuda_graphs": false, "spec_decode": false, "decode_tok_s": 40.5, "decode_cold_sample": 40.3, "decode_samples": [ 40.1, 40.5, 40.5 ], "generated": "2026-09-07", "vllm_version": "0.28.0", "kv_tokens": 9102, "max_concurrency": 2.22, "kv_cache_dtype": "fp8" }, { "id": "qwen38-dense-default", "status": "ok", "profile": "qwen3.8-27b", "overrides": "", "model": "Qwen3.8-27B", "checkpoint": "Inferact/Qwen3.8-27B-NVFP4", "shape": "dense 27B", "precision": "NVFP4", "weights_gb": 26.0, "weights_source": "loaded on the GPU, from vLLM's load report", "max_model_len": 32768, "vision": true, "cuda_graphs": false, "spec_decode": false, "decode_tok_s": 25.6, "decode_cold_sample": 25.5, "decode_samples": [ 25.6, 25.6, 25.7 ], "generated": "2026-09-07", "vllm_version": "0.28.0", "kv_tokens": 60681, "max_concurrency": 1.85, "kv_cache_dtype": "fp8" }, { "id": "nemotron-default", "status": "ok", "profile": "nemotron-3.5-lightning", "overrides": "", "model": "Nemotron-3.5-Lightning-30B-A3B", "checkpoint": "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", "shape": "MoE, 30B total / 3B active, Mamba hybrid", "precision": "NVFP4", "weights_gb": 19.2, "weights_source": "loaded on the GPU, from vLLM's load report", "max_model_len": 65536, "vision": true, "cuda_graphs": true, "spec_decode": false, "decode_tok_s": 392.4, "decode_cold_sample": 392.7, "decode_samples": [ 392.4, 392.3, 392.4 ], "generated": "2026-09-07", "vllm_version": "0.28.0", "kv_tokens": 1497245, "max_concurrency": 22.85, "kv_cache_dtype": "fp8" } ], "failures": [ { "id": "qwen38-bf16", "checkpoint": "Qwen/Qwen3.8-27B", "precision": "BF16 (official)", "weights_gb": 55.6, "status": "not_attempted", "reason": "55.6 GB of weights against 31.4 GiB of VRAM and 30 GiB of host RAM. Roughly 24 GB would have to be CPU-offloaded onto a host that cannot hold it either." }, { "id": "qwen38-fp8-8k", "checkpoint": "Qwen/Qwen3.8-27B-FP8", "precision": "FP8 (official)", "status": "oom_at_first_request", "config": "MAX_MODEL_LEN=8192 MAX_NUM_SEQS=4 GPU_MEMORY_UTILIZATION=0.95 LANGUAGE_MODEL_ONLY=1", "reason": "Profiles fine, reports a 21,845-token KV pool, prints 'Application startup complete', then dies on the first real request: 'Tried to allocate 394.00 MiB. GPU 0 has a total capacity of 31.39 GiB of which 151.75 MiB is free.' Memory profiling underestimates the true activation peak when weights occupy 91% of the card." }, { "id": "qwen38-mtp-with-vision", "checkpoint": "Inferact/Qwen3.8-27B-NVFP4", "precision": "NVFP4", "status": "oom_at_startup", "config": "SPEC_DECODE=1 with the vision tower loaded", "reason": "Needs exactly 2.37 GiB = 248,320 x 5,120 in BF16. vllm/model_executor/models/qwen3_5_mtp.py gives the draft head its own VocabParallelEmbedding even though the checkpoint sets mtp_use_dedicated_embeddings: false. Lowering GPU_MEMORY_UTILIZATION does not help: the allocation happens after profiling, outside the budget." }, { "id": "qwen38-mtp-plus-cudagraphs", "checkpoint": "Inferact/Qwen3.8-27B-NVFP4", "precision": "NVFP4", "status": "insufficient_kv", "config": "SPEC_DECODE=1 LANGUAGE_MODEL_ONLY=1 ENFORCE_EAGER=0", "reason": "MTP and CUDA graphs want the same couple of gigabytes and cannot both fit. At 32K context the KV cache is squeezed to 0.81 GiB against the 1.87 GiB one full-length sequence needs; at 16K context with graph capture capped at 16 it is 1.2 GiB against 1.35 GiB. The engine refuses to start with a clear ValueError rather than failing later. Pick one: MTP for single-stream latency, graphs for concurrency." } ], "sweeps": [ { "label": "qwen3.6-35b-a3b baseline", "model": "Qwen3.6-35B-A3B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "prompt_words": 700, "output_tokens": 256, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.0669, "ttft_p95_s": 0.0679, "decode_tok_s_per_stream": 280.1, "total_tok_s": 261.9, "latency_p50_s": 0.977, "latency_p95_s": 0.979 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1313, "ttft_p95_s": 0.1321, "decode_tok_s_per_stream": 215.4, "total_tok_s": 388.7, "latency_p50_s": 1.315, "latency_p95_s": 1.319 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1404, "ttft_p95_s": 0.142, "decode_tok_s_per_stream": 206.9, "total_tok_s": 746.4, "latency_p50_s": 1.369, "latency_p95_s": 1.376 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.2253, "ttft_p95_s": 0.2279, "decode_tok_s_per_stream": 173.9, "total_tok_s": 1223.0, "latency_p50_s": 1.67, "latency_p95_s": 1.691 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 1.0163, "ttft_p95_s": 1.9177, "decode_tok_s_per_stream": 169.2, "total_tok_s": 1210.9, "latency_p50_s": 2.558, "latency_p95_s": 3.402 } ], "notes": "profiles/qwen3.6-35b-a3b.env unchanged; no MTP", "dimensions": { "model": "Qwen3.6-35B-A3B", "prompt": "786 tok", "output": "256 tok", "seqs": "8", "vision": "on", "graphs": "on", "mtp": "off" } }, { "label": "qwen3.6-35b-a3b max-num-seqs 32", "model": "Qwen3.6-35B-A3B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "prompt_words": 700, "output_tokens": 256, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.0673, "ttft_p95_s": 0.0684, "decode_tok_s_per_stream": 276.6, "total_tok_s": 258.7, "latency_p50_s": 0.989, "latency_p95_s": 0.991 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1643, "ttft_p95_s": 0.1996, "decode_tok_s_per_stream": 213.4, "total_tok_s": 376.0, "latency_p50_s": 1.359, "latency_p95_s": 1.397 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.2073, "ttft_p95_s": 0.208, "decode_tok_s_per_stream": 205.8, "total_tok_s": 708.5, "latency_p50_s": 1.442, "latency_p95_s": 1.449 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.227, "ttft_p95_s": 0.2844, "decode_tok_s_per_stream": 172.0, "total_tok_s": 1179.5, "latency_p50_s": 1.732, "latency_p95_s": 1.76 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.4016, "ttft_p95_s": 0.4864, "decode_tok_s_per_stream": 140.9, "total_tok_s": 1860.8, "latency_p50_s": 2.194, "latency_p95_s": 2.245 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.6586, "ttft_p95_s": 0.8837, "decode_tok_s_per_stream": 100.1, "total_tok_s": 2546.4, "latency_p50_s": 3.2, "latency_p95_s": 3.228 } ], "notes": "profiles/qwen3.6-35b-a3b.env with MAX_NUM_SEQS=32; no MTP. KV pool 370,189 tokens.", "dimensions": { "model": "Qwen3.6-35B-A3B", "prompt": "786 tok", "output": "256 tok", "seqs": "32", "vision": "on", "graphs": "on", "mtp": "off" } }, { "label": "qwen3.6-35b-a3b 16K prompt, seqs 8", "model": "Qwen3.6-35B-A3B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "prompt_words": 14500, "output_tokens": 256, "rounds": 1, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 1, "succeeded": 1, "failed": 0, "status": "ok", "prompt_tokens": 15647, "output_tokens": 256, "ttft_p50_s": 0.6444, "ttft_p95_s": 0.6444, "decode_tok_s_per_stream": 262.1, "total_tok_s": 158.2, "latency_p50_s": 1.617, "latency_p95_s": 1.617 }, { "concurrency": 2, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 15647, "output_tokens": 256, "ttft_p50_s": 1.1512, "ttft_p95_s": 1.24, "decode_tok_s_per_stream": 188.0, "total_tok_s": 203.3, "latency_p50_s": 2.514, "latency_p95_s": 2.517 }, { "concurrency": 4, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 15647, "output_tokens": 256, "ttft_p50_s": 1.9288, "ttft_p95_s": 2.5088, "decode_tok_s_per_stream": 129.8, "total_tok_s": 259.6, "latency_p50_s": 3.927, "latency_p95_s": 3.941 }, { "concurrency": 8, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 15647, "output_tokens": 256, "ttft_p50_s": 3.2206, "ttft_p95_s": 4.957, "decode_tok_s_per_stream": 70.9, "total_tok_s": 297.5, "latency_p50_s": 6.839, "latency_p95_s": 6.877 }, { "concurrency": 16, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 15648, "output_tokens": 256, "ttft_p50_s": 6.1885, "ttft_p95_s": 11.3195, "decode_tok_s_per_stream": 48.4, "total_tok_s": 309.2, "latency_p50_s": 11.837, "latency_p95_s": 13.236 }, { "concurrency": 32, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 15648, "output_tokens": 256, "ttft_p50_s": 13.1467, "ttft_p95_s": 24.5552, "decode_tok_s_per_stream": 47.7, "total_tok_s": 304.4, "latency_p50_s": 18.946, "latency_p95_s": 26.89 } ], "notes": "long-context sweep; profile default MAX_NUM_SEQS=8; KV pool 377,274 tokens", "dimensions": { "model": "Qwen3.6-35B-A3B", "prompt": "16K tok", "output": "256 tok", "seqs": "8", "vision": "on", "graphs": "on", "mtp": "off" } }, { "label": "qwen3.6-35b-a3b 16K prompt, seqs 32", "model": "Qwen3.6-35B-A3B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "prompt_words": 14500, "output_tokens": 256, "rounds": 1, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 1, "succeeded": 1, "failed": 0, "status": "ok", "prompt_tokens": 15647, "output_tokens": 256, "ttft_p50_s": 0.6325, "ttft_p95_s": 0.6325, "decode_tok_s_per_stream": 255.8, "total_tok_s": 157.0, "latency_p50_s": 1.63, "latency_p95_s": 1.63 }, { "concurrency": 2, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 15647, "output_tokens": 256, "ttft_p50_s": 1.2148, "ttft_p95_s": 1.3058, "decode_tok_s_per_stream": 181.4, "total_tok_s": 194.6, "latency_p50_s": 2.627, "latency_p95_s": 2.63 }, { "concurrency": 4, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 15647, "output_tokens": 256, "ttft_p50_s": 1.9403, "ttft_p95_s": 2.5505, "decode_tok_s_per_stream": 125.1, "total_tok_s": 254.0, "latency_p50_s": 4.014, "latency_p95_s": 4.029 }, { "concurrency": 8, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 15647, "output_tokens": 256, "ttft_p50_s": 3.1907, "ttft_p95_s": 4.9215, "decode_tok_s_per_stream": 73.0, "total_tok_s": 303.6, "latency_p50_s": 6.704, "latency_p95_s": 6.739 }, { "concurrency": 16, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 15648, "output_tokens": 256, "ttft_p50_s": 5.485, "ttft_p95_s": 11.7309, "decode_tok_s_per_stream": 39.3, "total_tok_s": 306.0, "latency_p50_s": 11.979, "latency_p95_s": 13.374 }, { "concurrency": 32, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 15648, "output_tokens": 256, "ttft_p50_s": 12.1443, "ttft_p95_s": 22.0892, "decode_tok_s_per_stream": 31.2, "total_tok_s": 339.0, "latency_p50_s": 20.323, "latency_p95_s": 24.142 } ], "notes": "long-context sweep; MAX_NUM_SEQS=32", "dimensions": { "model": "Qwen3.6-35B-A3B", "prompt": "16K tok", "output": "256 tok", "seqs": "32", "vision": "on", "graphs": "on", "mtp": "off" } }, { "label": "qwen3.8-27b default", "model": "Qwen3.8-27B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "prompt_words": 700, "output_tokens": 256, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1208, "ttft_p95_s": 0.1211, "decode_tok_s_per_stream": 24.9, "total_tok_s": 24.7, "latency_p50_s": 10.363, "latency_p95_s": 10.438 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1927, "ttft_p95_s": 0.5233, "decode_tok_s_per_stream": 24.1, "total_tok_s": 46.7, "latency_p50_s": 10.961, "latency_p95_s": 11.188 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.3311, "ttft_p95_s": 0.3811, "decode_tok_s_per_stream": 23.9, "total_tok_s": 93.3, "latency_p50_s": 10.975, "latency_p95_s": 10.985 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.5703, "ttft_p95_s": 0.7444, "decode_tok_s_per_stream": 23.6, "total_tok_s": 179.5, "latency_p50_s": 11.366, "latency_p95_s": 11.439 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 6.105, "ttft_p95_s": 12.089, "decode_tok_s_per_stream": 23.4, "total_tok_s": 180.4, "latency_p50_s": 17.224, "latency_p95_s": 22.804 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 17.3485, "ttft_p95_s": 34.4664, "decode_tok_s_per_stream": 23.3, "total_tok_s": 181.4, "latency_p50_s": 28.417, "latency_p95_s": 45.177 } ], "notes": "profiles/qwen3.8-27b.env unchanged; vision on, eager, MAX_NUM_SEQS=8; KV pool 61,895 tokens", "dimensions": { "model": "Qwen3.8-27B", "prompt": "786 tok", "output": "256 tok", "seqs": "8", "vision": "on", "graphs": "off", "mtp": "off" } }, { "label": "qwen3.8-27b seqs 32", "model": "Qwen3.8-27B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "prompt_words": 700, "output_tokens": 256, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.121, "ttft_p95_s": 0.1219, "decode_tok_s_per_stream": 25.0, "total_tok_s": 24.8, "latency_p50_s": 10.306, "latency_p95_s": 10.343 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.188, "ttft_p95_s": 0.5352, "decode_tok_s_per_stream": 24.0, "total_tok_s": 46.5, "latency_p50_s": 11.01, "latency_p95_s": 11.187 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.3321, "ttft_p95_s": 0.3818, "decode_tok_s_per_stream": 23.9, "total_tok_s": 93.1, "latency_p50_s": 10.999, "latency_p95_s": 11.0 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.564, "ttft_p95_s": 0.7424, "decode_tok_s_per_stream": 23.6, "total_tok_s": 179.7, "latency_p50_s": 11.347, "latency_p95_s": 11.437 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.9171, "ttft_p95_s": 11.9222, "decode_tok_s_per_stream": 23.1, "total_tok_s": 181.2, "latency_p50_s": 11.939, "latency_p95_s": 22.62 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 12.1507, "ttft_p95_s": 23.7732, "decode_tok_s_per_stream": 22.8, "total_tok_s": 238.1, "latency_p50_s": 23.32, "latency_p95_s": 34.438 } ], "notes": "profiles/qwen3.8-27b.env with MAX_NUM_SEQS=32; vision on, eager; KV pool 60,681 tokens", "dimensions": { "model": "Qwen3.8-27B", "prompt": "786 tok", "output": "256 tok", "seqs": "32", "vision": "on", "graphs": "off", "mtp": "off" } }, { "label": "qwen3.8-27b MTP, text-only", "model": "Qwen3.8-27B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "prompt_words": 700, "output_tokens": 256, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1501, "ttft_p95_s": 0.1505, "decode_tok_s_per_stream": 75.5, "total_tok_s": 72.3, "latency_p50_s": 3.539, "latency_p95_s": 3.719 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.3841, "ttft_p95_s": 0.8485, "decode_tok_s_per_stream": 68.3, "total_tok_s": 119.4, "latency_p50_s": 4.12, "latency_p95_s": 4.704 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.3601, "ttft_p95_s": 0.3971, "decode_tok_s_per_stream": 74.6, "total_tok_s": 257.9, "latency_p50_s": 3.777, "latency_p95_s": 3.997 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.5124, "ttft_p95_s": 4.3128, "decode_tok_s_per_stream": 71.2, "total_tok_s": 260.2, "latency_p50_s": 4.174, "latency_p95_s": 7.816 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 4.3367, "ttft_p95_s": 9.8728, "decode_tok_s_per_stream": 69.9, "total_tok_s": 269.0, "latency_p50_s": 7.974, "latency_p95_s": 13.386 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 11.7399, "ttft_p95_s": 22.9684, "decode_tok_s_per_stream": 69.5, "total_tok_s": 305.0, "latency_p50_s": 15.352, "latency_p95_s": 26.465 } ], "notes": "SPEC_DECODE=1 LANGUAGE_MODEL_ONLY=1, MAX_NUM_SEQS=8 (profile default); KV pool 70,087 tokens", "dimensions": { "model": "Qwen3.8-27B", "prompt": "786 tok", "output": "256 tok", "seqs": "8", "vision": "off", "graphs": "off", "mtp": "on" } }, { "label": "qwen3.8-27b text-only, CUDA graphs", "model": "Qwen3.8-27B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "prompt_words": 700, "output_tokens": 256, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.0856, "ttft_p95_s": 0.086, "decode_tok_s_per_stream": 62.2, "total_tok_s": 61.1, "latency_p50_s": 4.187, "latency_p95_s": 4.187 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1537, "ttft_p95_s": 0.1656, "decode_tok_s_per_stream": 55.2, "total_tok_s": 107.0, "latency_p50_s": 4.774, "latency_p95_s": 4.784 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.2776, "ttft_p95_s": 0.3148, "decode_tok_s_per_stream": 53.7, "total_tok_s": 203.1, "latency_p50_s": 5.025, "latency_p95_s": 5.04 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.4669, "ttft_p95_s": 0.6186, "decode_tok_s_per_stream": 53.9, "total_tok_s": 391.5, "latency_p50_s": 5.195, "latency_p95_s": 5.228 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.738, "ttft_p95_s": 5.8675, "decode_tok_s_per_stream": 49.9, "total_tok_s": 393.7, "latency_p50_s": 5.851, "latency_p95_s": 10.393 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 6.0525, "ttft_p95_s": 15.6831, "decode_tok_s_per_stream": 49.8, "total_tok_s": 388.8, "latency_p50_s": 11.17, "latency_p95_s": 20.358 } ], "notes": "ENFORCE_EAGER=0 LANGUAGE_MODEL_ONLY=1 MAX_NUM_SEQS=32; no MTP; KV pool 53,399 tokens", "dimensions": { "model": "Qwen3.8-27B", "prompt": "786 tok", "output": "256 tok", "seqs": "32", "vision": "off", "graphs": "on", "mtp": "off" } }, { "label": "qwen3.8-27b text-only, eager", "model": "Qwen3.8-27B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "prompt_words": 700, "output_tokens": 256, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1161, "ttft_p95_s": 0.1161, "decode_tok_s_per_stream": 26.5, "total_tok_s": 26.3, "latency_p50_s": 9.728, "latency_p95_s": 9.749 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.1822, "ttft_p95_s": 0.5125, "decode_tok_s_per_stream": 25.3, "total_tok_s": 49.1, "latency_p50_s": 10.434, "latency_p95_s": 10.638 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 256, "ttft_p50_s": 0.325, "ttft_p95_s": 0.3736, "decode_tok_s_per_stream": 25.3, "total_tok_s": 98.3, "latency_p50_s": 10.406, "latency_p95_s": 10.416 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.5514, "ttft_p95_s": 0.7255, "decode_tok_s_per_stream": 24.9, "total_tok_s": 189.4, "latency_p50_s": 10.771, "latency_p95_s": 10.853 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 0.8925, "ttft_p95_s": 1.3693, "decode_tok_s_per_stream": 24.2, "total_tok_s": 354.4, "latency_p50_s": 11.439, "latency_p95_s": 11.589 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 256, "ttft_p50_s": 1.5847, "ttft_p95_s": 12.6853, "decode_tok_s_per_stream": 22.7, "total_tok_s": 358.9, "latency_p50_s": 12.81, "latency_p95_s": 22.845 } ], "notes": "control for the CUDA graphs test: LANGUAGE_MODEL_ONLY=1 MAX_NUM_SEQS=32, ENFORCE_EAGER=1; no MTP; KV pool 121,362 tokens", "dimensions": { "model": "Qwen3.8-27B", "prompt": "786 tok", "output": "256 tok", "seqs": "32", "vision": "off", "graphs": "off", "mtp": "off" } }, { "label": "nemotron-3.5-lightning", "model": "Nemotron-3.5-Lightning-30B-A3B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "prompt_words": 700, "output_tokens": 256, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 808, "output_tokens": 256, "ttft_p50_s": 0.0468, "ttft_p95_s": 0.0469, "decode_tok_s_per_stream": 387.5, "total_tok_s": 362.9, "latency_p50_s": 0.705, "latency_p95_s": 0.705 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 808, "output_tokens": 256, "ttft_p50_s": 0.188, "ttft_p95_s": 0.2841, "decode_tok_s_per_stream": 295.2, "total_tok_s": 486.2, "latency_p50_s": 1.052, "latency_p95_s": 1.153 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 808, "output_tokens": 256, "ttft_p50_s": 0.18, "ttft_p95_s": 0.192, "decode_tok_s_per_stream": 246.0, "total_tok_s": 837.1, "latency_p50_s": 1.22, "latency_p95_s": 1.226 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 809, "output_tokens": 256, "ttft_p50_s": 0.1891, "ttft_p95_s": 0.2838, "decode_tok_s_per_stream": 186.8, "total_tok_s": 1292.8, "latency_p50_s": 1.577, "latency_p95_s": 1.608 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 809, "output_tokens": 256, "ttft_p50_s": 0.3013, "ttft_p95_s": 0.4995, "decode_tok_s_per_stream": 136.2, "total_tok_s": 1873.4, "latency_p50_s": 2.174, "latency_p95_s": 2.184 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 809, "output_tokens": 256, "ttft_p50_s": 0.5213, "ttft_p95_s": 0.9858, "decode_tok_s_per_stream": 100.7, "total_tok_s": 2652.4, "latency_p50_s": 3.057, "latency_p95_s": 3.087 } ], "dimensions": { "model": "Nemotron-3.5-Lightning-30B-A3B", "prompt": "808 tok", "output": "256 tok", "seqs": "32", "vision": "off", "graphs": "on", "mtp": "off" }, "notes": "profiles/nemotron-3.5-lightning.env; MoE 30B/3B active, Mamba hybrid; KV pool 1,497,245 tokens" }, { "label": "qwen3.6-35b-a3b 4K output, seqs 32", "model": "Qwen3.6-35B-A3B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "prompt_words": 700, "output_tokens": 4096, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 4096, "ttft_p50_s": 0.0686, "ttft_p95_s": 0.0695, "decode_tok_s_per_stream": 279.9, "total_tok_s": 278.6, "latency_p50_s": 14.7, "latency_p95_s": 14.707 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 4096, "ttft_p50_s": 0.1659, "ttft_p95_s": 0.2005, "decode_tok_s_per_stream": 214.0, "total_tok_s": 424.3, "latency_p50_s": 19.306, "latency_p95_s": 19.391 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 4096, "ttft_p50_s": 0.2081, "ttft_p95_s": 0.2096, "decode_tok_s_per_stream": 200.6, "total_tok_s": 794.4, "latency_p50_s": 20.621, "latency_p95_s": 20.739 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 4096, "ttft_p50_s": 0.2281, "ttft_p95_s": 0.2864, "decode_tok_s_per_stream": 167.1, "total_tok_s": 1324.7, "latency_p50_s": 24.731, "latency_p95_s": 25.233 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 4096, "ttft_p50_s": 0.407, "ttft_p95_s": 0.4898, "decode_tok_s_per_stream": 129.1, "total_tok_s": 2044.4, "latency_p50_s": 32.048, "latency_p95_s": 32.707 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 4096, "ttft_p50_s": 0.6676, "ttft_p95_s": 0.8956, "decode_tok_s_per_stream": 93.1, "total_tok_s": 2934.2, "latency_p50_s": 44.635, "latency_p95_s": 44.824 } ], "dimensions": { "model": "Qwen3.6-35B-A3B", "prompt": "786 tok", "output": "4096 tok", "seqs": "32", "vision": "on", "graphs": "on", "mtp": "off" } }, { "label": "nemotron-3.5-lightning 4K output", "model": "Nemotron-3.5-Lightning-30B-A3B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 65536, "workload": { "prompt_words": 700, "output_tokens": 4096, "rounds": 2, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 808, "output_tokens": 4096, "ttft_p50_s": 0.0416, "ttft_p95_s": 0.0419, "decode_tok_s_per_stream": 385.6, "total_tok_s": 384.2, "latency_p50_s": 10.661, "latency_p95_s": 10.662 }, { "concurrency": 2, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 808, "output_tokens": 4096, "ttft_p50_s": 0.1588, "ttft_p95_s": 0.2421, "decode_tok_s_per_stream": 292.8, "total_tok_s": 579.1, "latency_p50_s": 14.144, "latency_p95_s": 14.277 }, { "concurrency": 4, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 808, "output_tokens": 4096, "ttft_p50_s": 0.1699, "ttft_p95_s": 0.1709, "decode_tok_s_per_stream": 241.6, "total_tok_s": 959.5, "latency_p50_s": 17.072, "latency_p95_s": 17.171 }, { "concurrency": 8, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 809, "output_tokens": 4096, "ttft_p50_s": 0.1909, "ttft_p95_s": 0.2697, "decode_tok_s_per_stream": 184.1, "total_tok_s": 1461.3, "latency_p50_s": 22.417, "latency_p95_s": 22.513 }, { "concurrency": 16, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 809, "output_tokens": 4096, "ttft_p50_s": 0.3054, "ttft_p95_s": 0.5039, "decode_tok_s_per_stream": 148.1, "total_tok_s": 2343.1, "latency_p50_s": 27.956, "latency_p95_s": 28.59 }, { "concurrency": 32, "requests": 64, "succeeded": 64, "failed": 0, "status": "ok", "prompt_tokens": 809, "output_tokens": 4096, "ttft_p50_s": 0.5285, "ttft_p95_s": 0.9919, "decode_tok_s_per_stream": 116.7, "total_tok_s": 3676.8, "latency_p50_s": 35.605, "latency_p95_s": 35.93 } ], "dimensions": { "model": "Nemotron-3.5-Lightning-30B-A3B", "prompt": "808 tok", "output": "4096 tok", "seqs": "32", "vision": "off", "graphs": "on", "mtp": "off" } }, { "label": "qwen3.8-27b 4K output, text-only, CUDA graphs", "model": "Qwen3.8-27B", "base_url": "http://127.0.0.1:8000", "generated": "2026-09-06", "vllm_version": "0.28.0", "max_model_len": 32768, "workload": { "prompt_words": 700, "output_tokens": 4096, "rounds": 1, "ignore_eos": true, "thinking": false }, "levels": [ { "concurrency": 1, "requests": 1, "succeeded": 1, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 4096, "ttft_p50_s": 0.0901, "ttft_p95_s": 0.0901, "decode_tok_s_per_stream": 62.0, "total_tok_s": 62.0, "latency_p50_s": 66.103, "latency_p95_s": 66.103 }, { "concurrency": 2, "requests": 2, "succeeded": 2, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 4096, "ttft_p50_s": 0.1572, "ttft_p95_s": 0.1698, "decode_tok_s_per_stream": 55.0, "total_tok_s": 109.7, "latency_p50_s": 74.641, "latency_p95_s": 74.648 }, { "concurrency": 4, "requests": 4, "succeeded": 4, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 4096, "ttft_p50_s": 0.2843, "ttft_p95_s": 0.3173, "decode_tok_s_per_stream": 53.5, "total_tok_s": 213.2, "latency_p50_s": 76.849, "latency_p95_s": 76.862 }, { "concurrency": 8, "requests": 8, "succeeded": 8, "failed": 0, "status": "ok", "prompt_tokens": 786, "output_tokens": 4096, "ttft_p50_s": 0.4829, "ttft_p95_s": 0.6251, "decode_tok_s_per_stream": 53.5, "total_tok_s": 300.0, "latency_p50_s": 76.995, "latency_p95_s": 109.2 }, { "concurrency": 16, "requests": 16, "succeeded": 16, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 4096, "ttft_p50_s": 0.757, "ttft_p95_s": 89.477, "decode_tok_s_per_stream": 50.7, "total_tok_s": 327.9, "latency_p50_s": 126.635, "latency_p95_s": 185.058 }, { "concurrency": 32, "requests": 32, "succeeded": 32, "failed": 0, "status": "ok", "prompt_tokens": 787, "output_tokens": 4096, "ttft_p50_s": 126.8696, "ttft_p95_s": 243.1332, "decode_tok_s_per_stream": 52.5, "total_tok_s": 349.8, "latency_p50_s": 212.59, "latency_p95_s": 358.201 } ], "dimensions": { "model": "Qwen3.8-27B", "prompt": "786 tok", "output": "4096 tok", "seqs": "32", "vision": "off", "graphs": "on", "mtp": "off" } } ], "jitter": [ { "label": "Qwen3.6-35B-A3B, bf16 KV", "model": "Qwen3.6-35B-A3B", "vllm_version": "0.28.0", "generated": "2026-09-06", "prompts": 8, "repeats": 4, "positions": 3072, "max_tokens": 128, "top_k": 20, "thinking": true, "kld_mean": 0.002291, "kld_p50": 8e-06, "kld_p99": 0.039499, "kld_max": 0.148018, "top1_flip_rate": 0.01009, "notes": "profile default with KV_CACHE_DTYPE=auto; the root reference for the Qwen3.6 comparisons" }, { "label": "Qwen3.6-35B-A3B, fp8 KV (profile default)", "model": "Qwen3.6-35B-A3B", "vllm_version": "0.28.0", "generated": "2026-09-06", "prompts": 8, "repeats": 4, "positions": 3072, "max_tokens": 128, "top_k": 20, "thinking": true, "kld_mean": 0.002388, "kld_p50": 8e-06, "kld_p99": 0.044859, "kld_max": 0.121335, "top1_flip_rate": 0.00749 }, { "label": "Qwen3.8-27B, official FP8, bf16 KV", "model": "Qwen3.8-27B", "vllm_version": "0.28.0", "generated": "2026-09-06", "prompts": 8, "repeats": 4, "positions": 3072, "max_tokens": 128, "top_k": 20, "thinking": true, "kld_mean": 0.0, "kld_p50": 0.0, "kld_p99": 0.0, "kld_max": 0.0, "top1_flip_rate": 0.0, "notes": "Qwen/Qwen3.8-27B-FP8, text-only, 4K ctx; the root reference for the Qwen3.8 comparisons" }, { "label": "Qwen3.8-27B, NVFP4, bf16 KV", "model": "Qwen3.8-27B", "vllm_version": "0.28.0", "generated": "2026-09-06", "prompts": 8, "repeats": 4, "positions": 3072, "max_tokens": 128, "top_k": 20, "thinking": true, "kld_mean": 0.0, "kld_p50": 0.0, "kld_p99": 0.0, "kld_max": 0.0, "top1_flip_rate": 0.0 }, { "label": "Nemotron 3.5 Lightning, fp8 KV (profile default)", "model": "Nemotron-3.5-Lightning-30B-A3B", "vllm_version": "0.28.0", "generated": "2026-09-06", "prompts": 8, "repeats": 4, "positions": 3072, "max_tokens": 128, "top_k": 20, "thinking": true, "kld_mean": 0.008675, "kld_p50": 1e-06, "kld_p99": 0.185469, "kld_max": 1.152156, "top1_flip_rate": 0.01335 } ], "divergence": [ { "label": "Qwen3.6 bf16 KV \u2192 same server again", "a": "q36-root", "b": "q36-root-rerun", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 4 }, "teacher_forced": { "positions": 24576, "kld_mean": 0.004159, "kld_p50": 2e-05, "kld_p90": 0.011055, "kld_p99": 0.058235, "kld_p999": 0.171327, "kld_max": 1.702063, "top1_agree": 0.98612, "dp_mean_abs": 0.011923, "dp_p1": -0.10571, "dp_p5": -0.04237, "dp_p50": 0.0, "dp_p95": 0.04103, "dp_p99": 0.10541, "support_mass_mean": 0.998, "ppl_a": 1.1268, "ppl_b": 1.1272, "prompts": 96, "kld_mean_ci95": [ 0.003735, 0.004601 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.003105, "top1_agree": 0.99414, "dp_mean_abs": 0.00895 }, "code-other": { "positions": 2560, "kld_mean": 0.003919, "top1_agree": 0.98594, "dp_mean_abs": 0.012434 }, "code-py": { "positions": 3072, "kld_mean": 0.003477, "top1_agree": 0.99186, "dp_mean_abs": 0.00999 }, "finnish": { "positions": 2048, "kld_mean": 0.005261, "top1_agree": 0.97803, "dp_mean_abs": 0.01497 }, "instruction": { "positions": 2560, "kld_mean": 0.004866, "top1_agree": 0.98555, "dp_mean_abs": 0.012257 }, "math": { "positions": 3072, "kld_mean": 0.002222, "top1_agree": 0.99154, "dp_mean_abs": 0.008169 }, "multilingual": { "positions": 1024, "kld_mean": 0.005367, "top1_agree": 0.98047, "dp_mean_abs": 0.016688 }, "reasoning": { "positions": 2048, "kld_mean": 0.003757, "top1_agree": 0.98926, "dp_mean_abs": 0.011121 }, "structured": { "positions": 2560, "kld_mean": 0.003793, "top1_agree": 0.98828, "dp_mean_abs": 0.010639 }, "summarise": { "positions": 2048, "kld_mean": 0.005997, "top1_agree": 0.97852, "dp_mean_abs": 0.016109 }, "writing": { "positions": 2048, "kld_mean": 0.005613, "top1_agree": 0.97705, "dp_mean_abs": 0.01442 } } }, "generation": { "positions": 8779, "kld_mean": 0.0025, "kld_p50": 9e-06, "kld_p90": 0.006135, "kld_p99": 0.043219, "kld_p999": 0.11566, "kld_max": 0.225399, "top1_agree": 1.0, "dp_mean_abs": 0.007962, "dp_p1": -0.08579, "dp_p5": -0.02556, "dp_p50": 0.0, "dp_p95": 0.02698, "dp_p99": 0.08698, "support_mass_mean": 0.99899, "ppl_a": 1.0752, "ppl_b": 1.0749, "prompts": 96, "identical": 3, "identical_rate": 0.0312, "diverged": 93, "first_divergence_p10": 16.4, "first_divergence_p50": 72.0, "first_divergence_min": 0, "gen_tokens_a": 24576, "gen_tokens_b": 24576 }, "dimensions": { "model": "Qwen3.6-35B-A3B", "kind": "floor" }, "notes": "run-to-run floor at full prompt-set scale" }, { "label": "Qwen3.6 bf16 KV \u2192 fp8 KV", "a": "q36-root", "b": "q36-kvfp8", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 4 }, "teacher_forced": { "positions": 24576, "kld_mean": 0.004865, "kld_p50": 2.1e-05, "kld_p90": 0.013553, "kld_p99": 0.06941, "kld_p999": 0.188693, "kld_max": 0.786505, "top1_agree": 0.98604, "dp_mean_abs": 0.012972, "dp_p1": -0.11689, "dp_p5": -0.0464, "dp_p50": 0.0, "dp_p95": 0.04624, "dp_p99": 0.11685, "support_mass_mean": 0.99799, "ppl_a": 1.1268, "ppl_b": 1.1269, "prompts": 96, "kld_mean_ci95": [ 0.004338, 0.005378 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.003183, "top1_agree": 0.99349, "dp_mean_abs": 0.008898 }, "code-other": { "positions": 2560, "kld_mean": 0.005131, "top1_agree": 0.98281, "dp_mean_abs": 0.014447 }, "code-py": { "positions": 3072, "kld_mean": 0.003898, "top1_agree": 0.99186, "dp_mean_abs": 0.011034 }, "finnish": { "positions": 2048, "kld_mean": 0.005875, "top1_agree": 0.98389, "dp_mean_abs": 0.016154 }, "instruction": { "positions": 2560, "kld_mean": 0.005179, "top1_agree": 0.9832, "dp_mean_abs": 0.012437 }, "math": { "positions": 3072, "kld_mean": 0.002504, "top1_agree": 0.99154, "dp_mean_abs": 0.00858 }, "multilingual": { "positions": 1024, "kld_mean": 0.006068, "top1_agree": 0.98535, "dp_mean_abs": 0.017153 }, "reasoning": { "positions": 2048, "kld_mean": 0.004499, "top1_agree": 0.98828, "dp_mean_abs": 0.012328 }, "structured": { "positions": 2560, "kld_mean": 0.004638, "top1_agree": 0.98828, "dp_mean_abs": 0.011862 }, "summarise": { "positions": 2048, "kld_mean": 0.006842, "top1_agree": 0.97949, "dp_mean_abs": 0.016768 }, "writing": { "positions": 2048, "kld_mean": 0.007453, "top1_agree": 0.9751, "dp_mean_abs": 0.01731 } } }, "generation": { "positions": 9017, "kld_mean": 0.002648, "kld_p50": 8e-06, "kld_p90": 0.005784, "kld_p99": 0.04677, "kld_p999": 0.134141, "kld_max": 0.309301, "top1_agree": 1.0, "dp_mean_abs": 0.007817, "dp_p1": -0.08231, "dp_p5": -0.02495, "dp_p50": 0.0, "dp_p95": 0.0266, "dp_p99": 0.08973, "support_mass_mean": 0.99959, "ppl_a": 1.069, "ppl_b": 1.0681, "prompts": 96, "identical": 6, "identical_rate": 0.0625, "diverged": 90, "first_divergence_p10": 14.0, "first_divergence_p50": 73.5, "first_divergence_min": 0, "gen_tokens_a": 24576, "gen_tokens_b": 24542 }, "dimensions": { "model": "Qwen3.6-35B-A3B", "kv": "fp8" } }, { "label": "Qwen3.6 bf16 KV \u2192 + MTP", "a": "q36-root", "b": "q36-mtp", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 4 }, "teacher_forced": { "positions": 24576, "kld_mean": 2.54301, "kld_p50": 0.0005, "kld_p90": 10.298756, "kld_p99": 19.885825, "kld_p999": 23.728873, "kld_max": 29.49199, "top1_agree": 0.74007, "dp_mean_abs": 0.244073, "dp_p1": -1.0, "dp_p5": -0.99994, "dp_p50": -1e-05, "dp_p95": 0.03092, "dp_p99": 0.09965, "support_mass_mean": 0.98475, "ppl_a": 1.1268, "ppl_b": 17.6353, "prompts": 96, "kld_mean_ci95": [ 1.916554, 3.160386 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.002616, "top1_agree": 0.99544, "dp_mean_abs": 0.008351 }, "code-other": { "positions": 2560, "kld_mean": 2.405152, "top1_agree": 0.76797, "dp_mean_abs": 0.207819 }, "code-py": { "positions": 3072, "kld_mean": 1.023526, "top1_agree": 0.92415, "dp_mean_abs": 0.075187 }, "finnish": { "positions": 2048, "kld_mean": 2.383726, "top1_agree": 0.76807, "dp_mean_abs": 0.214544 }, "instruction": { "positions": 2560, "kld_mean": 4.093699, "top1_agree": 0.53164, "dp_mean_abs": 0.446974 }, "math": { "positions": 3072, "kld_mean": 4.995286, "top1_agree": 0.47526, "dp_mean_abs": 0.51637 }, "multilingual": { "positions": 1024, "kld_mean": 4.367544, "top1_agree": 0.6748, "dp_mean_abs": 0.298432 }, "reasoning": { "positions": 2048, "kld_mean": 3.09917, "top1_agree": 0.72266, "dp_mean_abs": 0.267766 }, "structured": { "positions": 2560, "kld_mean": 0.574564, "top1_agree": 0.94844, "dp_mean_abs": 0.04745 }, "summarise": { "positions": 2048, "kld_mean": 0.00619, "top1_agree": 0.97852, "dp_mean_abs": 0.016425 }, "writing": { "positions": 2048, "kld_mean": 4.971311, "top1_agree": 0.41846, "dp_mean_abs": 0.509523 } } }, "generation": { "positions": 8836, "kld_mean": 0.002475, "kld_p50": 9e-06, "kld_p90": 0.005609, "kld_p99": 0.045015, "kld_p999": 0.111432, "kld_max": 0.417871, "top1_agree": 1.0, "dp_mean_abs": 0.007708, "dp_p1": -0.0774, "dp_p5": -0.02437, "dp_p50": 0.0, "dp_p95": 0.02704, "dp_p99": 0.09248, "support_mass_mean": 0.99957, "ppl_a": 1.0671, "ppl_b": 1.066, "prompts": 96, "identical": 4, "identical_rate": 0.0417, "diverged": 92, "first_divergence_p10": 14.0, "first_divergence_p50": 72.0, "first_divergence_min": 0, "gen_tokens_a": 24576, "gen_tokens_b": 24576 }, "dimensions": { "model": "Qwen3.6-35B-A3B", "mtp": "on" }, "suspect": "prompt_logprobs under MTP returns the draft head's predictions (vLLM 0.28.0); generation side is at the floor" }, { "label": "Qwen3.6 bf16 KV \u2192 eager", "a": "q36-root", "b": "q36-eager", "model": "Qwen3.6-35B-A3B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 4 }, "teacher_forced": { "positions": 24576, "kld_mean": 0.004447, "kld_p50": 2e-05, "kld_p90": 0.011944, "kld_p99": 0.064486, "kld_p999": 0.174233, "kld_max": 0.74343, "top1_agree": 0.98661, "dp_mean_abs": 0.012243, "dp_p1": -0.11198, "dp_p5": -0.04284, "dp_p50": 0.0, "dp_p95": 0.04379, "dp_p99": 0.1101, "support_mass_mean": 0.99799, "ppl_a": 1.1268, "ppl_b": 1.1267, "prompts": 96, "kld_mean_ci95": [ 0.003977, 0.004926 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.002354, "top1_agree": 0.99544, "dp_mean_abs": 0.008149 }, "code-other": { "positions": 2560, "kld_mean": 0.004683, "top1_agree": 0.9832, "dp_mean_abs": 0.01374 }, "code-py": { "positions": 3072, "kld_mean": 0.00315, "top1_agree": 0.99284, "dp_mean_abs": 0.009855 }, "finnish": { "positions": 2048, "kld_mean": 0.005085, "top1_agree": 0.98389, "dp_mean_abs": 0.014643 }, "instruction": { "positions": 2560, "kld_mean": 0.005204, "top1_agree": 0.98555, "dp_mean_abs": 0.012149 }, "math": { "positions": 3072, "kld_mean": 0.002496, "top1_agree": 0.99219, "dp_mean_abs": 0.008755 }, "multilingual": { "positions": 1024, "kld_mean": 0.006908, "top1_agree": 0.97949, "dp_mean_abs": 0.018428 }, "reasoning": { "positions": 2048, "kld_mean": 0.004482, "top1_agree": 0.98779, "dp_mean_abs": 0.011586 }, "structured": { "positions": 2560, "kld_mean": 0.00363, "top1_agree": 0.9875, "dp_mean_abs": 0.01037 }, "summarise": { "positions": 2048, "kld_mean": 0.005978, "top1_agree": 0.98193, "dp_mean_abs": 0.01552 }, "writing": { "positions": 2048, "kld_mean": 0.00723, "top1_agree": 0.97656, "dp_mean_abs": 0.016604 } } }, "generation": { "positions": 9083, "kld_mean": 0.00257, "kld_p50": 9e-06, "kld_p90": 0.006585, "kld_p99": 0.041465, "kld_p999": 0.1015, "kld_max": 0.383368, "top1_agree": 1.0, "dp_mean_abs": 0.008181, "dp_p1": -0.07717, "dp_p5": -0.02624, "dp_p50": 0.0, "dp_p95": 0.03029, "dp_p99": 0.09036, "support_mass_mean": 0.999, "ppl_a": 1.0763, "ppl_b": 1.0753, "prompts": 96, "identical": 6, "identical_rate": 0.0625, "diverged": 90, "first_divergence_p10": 18.8, "first_divergence_p50": 70.5, "first_divergence_min": 5, "gen_tokens_a": 24576, "gen_tokens_b": 24576 }, "dimensions": { "model": "Qwen3.6-35B-A3B", "graphs": "off" } }, { "label": "Qwen3.8 FP8 \u2192 same server again", "a": "q38-fp8", "b": "q38-fp8-rerun", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 1 }, "teacher_forced": { "positions": 23856, "kld_mean": 0.0, "kld_p50": 0.0, "kld_p90": 0.0, "kld_p99": 0.0, "kld_p999": 0.0, "kld_max": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0, "dp_p1": 0.0, "dp_p5": 0.0, "dp_p50": 0.0, "dp_p95": 0.0, "dp_p99": 0.0, "support_mass_mean": 0.99328, "ppl_a": 1.2783, "ppl_b": 1.2783, "prompts": 96, "kld_mean_ci95": [ 0.0, 0.0 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "code-other": { "positions": 2560, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "code-py": { "positions": 3072, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "finnish": { "positions": 2048, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "instruction": { "positions": 2218, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "math": { "positions": 3072, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "multilingual": { "positions": 1024, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "reasoning": { "positions": 1936, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "structured": { "positions": 2379, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "summarise": { "positions": 2048, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 }, "writing": { "positions": 1963, "kld_mean": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0 } } }, "generation": { "positions": 23856, "kld_mean": 0.0, "kld_p50": 0.0, "kld_p90": 0.0, "kld_p99": 0.0, "kld_p999": 0.0, "kld_max": 0.0, "top1_agree": 1.0, "dp_mean_abs": 0.0, "dp_p1": 0.0, "dp_p5": 0.0, "dp_p50": 0.0, "dp_p95": 0.0, "dp_p99": 0.0, "support_mass_mean": 0.9933, "ppl_a": 1.2754, "ppl_b": 1.2754, "prompts": 96, "identical": 96, "identical_rate": 1.0, "diverged": 0, "first_divergence_p10": null, "first_divergence_p50": null, "first_divergence_min": null, "gen_tokens_a": 23856, "gen_tokens_b": 23856 }, "dimensions": { "model": "Qwen3.8-27B", "kind": "floor" }, "notes": "run-to-run floor at full prompt-set scale" }, { "label": "Qwen3.8 FP8 \u2192 NVFP4", "a": "q38-fp8", "b": "q38-nvfp4", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 1 }, "teacher_forced": { "positions": 23856, "kld_mean": 0.102945, "kld_p50": 0.007728, "kld_p90": 0.298918, "kld_p99": 1.048897, "kld_p999": 2.614167, "kld_max": 7.900331, "top1_agree": 0.89902, "dp_mean_abs": 0.070102, "dp_p1": -0.5094, "dp_p5": -0.29424, "dp_p50": -2e-05, "dp_p95": 0.1338, "dp_p99": 0.29122, "support_mass_mean": 0.98936, "ppl_a": 1.2783, "ppl_b": 1.4257, "prompts": 96, "kld_mean_ci95": [ 0.094588, 0.113446 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.131357, "top1_agree": 0.88477, "dp_mean_abs": 0.075865 }, "code-other": { "positions": 2560, "kld_mean": 0.118687, "top1_agree": 0.88672, "dp_mean_abs": 0.077302 }, "code-py": { "positions": 3072, "kld_mean": 0.098874, "top1_agree": 0.89779, "dp_mean_abs": 0.070391 }, "finnish": { "positions": 2048, "kld_mean": 0.119779, "top1_agree": 0.88916, "dp_mean_abs": 0.079286 }, "instruction": { "positions": 2218, "kld_mean": 0.11175, "top1_agree": 0.91073, "dp_mean_abs": 0.070718 }, "math": { "positions": 3072, "kld_mean": 0.060169, "top1_agree": 0.93034, "dp_mean_abs": 0.052046 }, "multilingual": { "positions": 1024, "kld_mean": 0.142805, "top1_agree": 0.87012, "dp_mean_abs": 0.09133 }, "reasoning": { "positions": 1936, "kld_mean": 0.078374, "top1_agree": 0.90548, "dp_mean_abs": 0.062347 }, "structured": { "positions": 2379, "kld_mean": 0.088063, "top1_agree": 0.91887, "dp_mean_abs": 0.056893 }, "summarise": { "positions": 2048, "kld_mean": 0.091472, "top1_agree": 0.90479, "dp_mean_abs": 0.066952 }, "writing": { "positions": 1963, "kld_mean": 0.13943, "top1_agree": 0.85481, "dp_mean_abs": 0.089595 } } }, "generation": { "positions": 1858, "kld_mean": 0.030317, "kld_p50": 0.000119, "kld_p90": 0.083416, "kld_p99": 0.531389, "kld_p999": 1.007337, "kld_max": 1.174222, "top1_agree": 1.0, "dp_mean_abs": 0.023981, "dp_p1": -0.29438, "dp_p5": -0.08629, "dp_p50": 0.0, "dp_p95": 0.07304, "dp_p99": 0.22435, "support_mass_mean": 0.99957, "ppl_a": 1.0576, "ppl_b": 1.0614, "prompts": 96, "identical": 0, "identical_rate": 0.0, "diverged": 96, "first_divergence_p10": 2.0, "first_divergence_p50": 8.5, "first_divergence_min": 0, "gen_tokens_a": 23856, "gen_tokens_b": 24199 }, "dimensions": { "model": "Qwen3.8-27B", "quant": "nvfp4" } }, { "label": "Qwen3.8 FP8 \u2192 NVFP4 + fp8 KV", "a": "q38-fp8", "b": "q38-nvfp4-kvfp8", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 1 }, "teacher_forced": { "positions": 23856, "kld_mean": 0.100724, "kld_p50": 0.007736, "kld_p90": 0.296099, "kld_p99": 1.011475, "kld_p999": 2.320015, "kld_max": 7.198331, "top1_agree": 0.89701, "dp_mean_abs": 0.0693, "dp_p1": -0.49746, "dp_p5": -0.29459, "dp_p50": -3e-05, "dp_p95": 0.13083, "dp_p99": 0.2869, "support_mass_mean": 0.98932, "ppl_a": 1.2783, "ppl_b": 1.4246, "prompts": 96, "kld_mean_ci95": [ 0.092225, 0.110782 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.129009, "top1_agree": 0.88997, "dp_mean_abs": 0.074679 }, "code-other": { "positions": 2560, "kld_mean": 0.117647, "top1_agree": 0.88906, "dp_mean_abs": 0.076588 }, "code-py": { "positions": 3072, "kld_mean": 0.096873, "top1_agree": 0.89551, "dp_mean_abs": 0.069285 }, "finnish": { "positions": 2048, "kld_mean": 0.120038, "top1_agree": 0.88867, "dp_mean_abs": 0.078361 }, "instruction": { "positions": 2218, "kld_mean": 0.101323, "top1_agree": 0.90848, "dp_mean_abs": 0.068307 }, "math": { "positions": 3072, "kld_mean": 0.059808, "top1_agree": 0.92578, "dp_mean_abs": 0.052377 }, "multilingual": { "positions": 1024, "kld_mean": 0.143274, "top1_agree": 0.84961, "dp_mean_abs": 0.09278 }, "reasoning": { "positions": 1936, "kld_mean": 0.074937, "top1_agree": 0.91684, "dp_mean_abs": 0.060528 }, "structured": { "positions": 2379, "kld_mean": 0.085891, "top1_agree": 0.91341, "dp_mean_abs": 0.056866 }, "summarise": { "positions": 2048, "kld_mean": 0.090197, "top1_agree": 0.89795, "dp_mean_abs": 0.066355 }, "writing": { "positions": 1963, "kld_mean": 0.137947, "top1_agree": 0.85023, "dp_mean_abs": 0.088301 } } }, "generation": { "positions": 1934, "kld_mean": 0.032756, "kld_p50": 0.000133, "kld_p90": 0.085851, "kld_p99": 0.526475, "kld_p999": 1.273776, "kld_max": 2.040129, "top1_agree": 1.0, "dp_mean_abs": 0.025732, "dp_p1": -0.30355, "dp_p5": -0.1009, "dp_p50": 0.0, "dp_p95": 0.08026, "dp_p99": 0.25999, "support_mass_mean": 0.9996, "ppl_a": 1.0617, "ppl_b": 1.0646, "prompts": 96, "identical": 0, "identical_rate": 0.0, "diverged": 96, "first_divergence_p10": 2.0, "first_divergence_p50": 9.0, "first_divergence_min": 0, "gen_tokens_a": 23856, "gen_tokens_b": 24218 }, "dimensions": { "model": "Qwen3.8-27B", "quant": "nvfp4", "kv": "fp8" } }, { "label": "Qwen3.8 NVFP4 bf16 KV \u2192 fp8 KV", "a": "q38-nvfp4", "b": "q38-nvfp4-kvfp8", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 4 }, "teacher_forced": { "positions": 23856, "kld_mean": 0.026533, "kld_p50": 0.002261, "kld_p90": 0.072227, "kld_p99": 0.261739, "kld_p999": 0.807648, "kld_max": 9.134898, "top1_agree": 0.94668, "dp_mean_abs": 0.032231, "dp_p1": -0.20229, "dp_p5": -0.1042, "dp_p50": 0.0, "dp_p95": 0.09957, "dp_p99": 0.20536, "support_mass_mean": 0.9909, "ppl_a": 1.4257, "ppl_b": 1.4246, "prompts": 96, "kld_mean_ci95": [ 0.024482, 0.029667 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.03304, "top1_agree": 0.94531, "dp_mean_abs": 0.032682 }, "code-other": { "positions": 2560, "kld_mean": 0.027754, "top1_agree": 0.94648, "dp_mean_abs": 0.032969 }, "code-py": { "positions": 3072, "kld_mean": 0.023632, "top1_agree": 0.94271, "dp_mean_abs": 0.031604 }, "finnish": { "positions": 2048, "kld_mean": 0.031209, "top1_agree": 0.93506, "dp_mean_abs": 0.036862 }, "instruction": { "positions": 2218, "kld_mean": 0.03303, "top1_agree": 0.95582, "dp_mean_abs": 0.033746 }, "math": { "positions": 3072, "kld_mean": 0.014916, "top1_agree": 0.96354, "dp_mean_abs": 0.025304 }, "multilingual": { "positions": 1024, "kld_mean": 0.036313, "top1_agree": 0.9375, "dp_mean_abs": 0.042486 }, "reasoning": { "positions": 1936, "kld_mean": 0.021527, "top1_agree": 0.9561, "dp_mean_abs": 0.028771 }, "structured": { "positions": 2379, "kld_mean": 0.022415, "top1_agree": 0.9504, "dp_mean_abs": 0.026348 }, "summarise": { "positions": 2048, "kld_mean": 0.024864, "top1_agree": 0.94629, "dp_mean_abs": 0.032152 }, "writing": { "positions": 1963, "kld_mean": 0.036914, "top1_agree": 0.92104, "dp_mean_abs": 0.041467 } } }, "generation": { "positions": 3468, "kld_mean": 0.015163, "kld_p50": 3.1e-05, "kld_p90": 0.035223, "kld_p99": 0.236664, "kld_p999": 0.929232, "kld_max": 2.325681, "top1_agree": 1.0, "dp_mean_abs": 0.018213, "dp_p1": -0.15582, "dp_p5": -0.06382, "dp_p50": 0.0, "dp_p95": 0.06435, "dp_p99": 0.18966, "support_mass_mean": 0.99945, "ppl_a": 1.084, "ppl_b": 1.0827, "prompts": 96, "identical": 0, "identical_rate": 0.0, "diverged": 96, "first_divergence_p10": 2.0, "first_divergence_p50": 26.0, "first_divergence_min": 0, "gen_tokens_a": 24199, "gen_tokens_b": 24218 }, "dimensions": { "model": "Qwen3.8-27B", "quant": "nvfp4", "kv": "fp8" }, "notes": "Both sides NVFP4; isolates the fp8 KV cache cost on the dense model" }, { "label": "Qwen3.8 FP8 \u2192 NVFP4, CUDA graphs", "a": "q38-fp8", "b": "q38-nvfp4-graphs", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 1 }, "teacher_forced": { "positions": 23856, "kld_mean": 0.10176, "kld_p50": 0.007864, "kld_p90": 0.296179, "kld_p99": 1.032517, "kld_p999": 2.592781, "kld_max": 6.75846, "top1_agree": 0.89801, "dp_mean_abs": 0.069232, "dp_p1": -0.50956, "dp_p5": -0.29211, "dp_p50": -3e-05, "dp_p95": 0.13212, "dp_p99": 0.28174, "support_mass_mean": 0.98937, "ppl_a": 1.2783, "ppl_b": 1.4238, "prompts": 96, "kld_mean_ci95": [ 0.093006, 0.111755 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.131699, "top1_agree": 0.87956, "dp_mean_abs": 0.077165 }, "code-other": { "positions": 2560, "kld_mean": 0.118864, "top1_agree": 0.89453, "dp_mean_abs": 0.076472 }, "code-py": { "positions": 3072, "kld_mean": 0.097956, "top1_agree": 0.89681, "dp_mean_abs": 0.069487 }, "finnish": { "positions": 2048, "kld_mean": 0.121675, "top1_agree": 0.88965, "dp_mean_abs": 0.07922 }, "instruction": { "positions": 2218, "kld_mean": 0.103467, "top1_agree": 0.91389, "dp_mean_abs": 0.066707 }, "math": { "positions": 3072, "kld_mean": 0.062024, "top1_agree": 0.92969, "dp_mean_abs": 0.051315 }, "multilingual": { "positions": 1024, "kld_mean": 0.144221, "top1_agree": 0.85156, "dp_mean_abs": 0.093524 }, "reasoning": { "positions": 1936, "kld_mean": 0.076249, "top1_agree": 0.91167, "dp_mean_abs": 0.061172 }, "structured": { "positions": 2379, "kld_mean": 0.086496, "top1_agree": 0.91341, "dp_mean_abs": 0.056754 }, "summarise": { "positions": 2048, "kld_mean": 0.087472, "top1_agree": 0.89844, "dp_mean_abs": 0.065852 }, "writing": { "positions": 1963, "kld_mean": 0.13787, "top1_agree": 0.85176, "dp_mean_abs": 0.087587 } } }, "generation": { "positions": 1993, "kld_mean": 0.030772, "kld_p50": 0.000126, "kld_p90": 0.076665, "kld_p99": 0.506391, "kld_p999": 1.129355, "kld_max": 1.366878, "top1_agree": 1.0, "dp_mean_abs": 0.024209, "dp_p1": -0.24925, "dp_p5": -0.08655, "dp_p50": 0.0, "dp_p95": 0.0819, "dp_p99": 0.23101, "support_mass_mean": 0.99958, "ppl_a": 1.0635, "ppl_b": 1.0657, "prompts": 96, "identical": 0, "identical_rate": 0.0, "diverged": 96, "first_divergence_p10": 2.0, "first_divergence_p50": 9.0, "first_divergence_min": 0, "gen_tokens_a": 23856, "gen_tokens_b": 24039 }, "dimensions": { "model": "Qwen3.8-27B", "quant": "nvfp4", "graphs": "on" } }, { "label": "Qwen3.8 NVFP4 eager \u2192 CUDA graphs", "a": "q38-nvfp4", "b": "q38-nvfp4-graphs", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 4 }, "teacher_forced": { "positions": 23856, "kld_mean": 0.026583, "kld_p50": 0.002184, "kld_p90": 0.071308, "kld_p99": 0.265732, "kld_p999": 1.061518, "kld_max": 4.014652, "top1_agree": 0.94593, "dp_mean_abs": 0.031957, "dp_p1": -0.20518, "dp_p5": -0.09819, "dp_p50": 0.0, "dp_p95": 0.10059, "dp_p99": 0.20607, "support_mass_mean": 0.9909, "ppl_a": 1.4257, "ppl_b": 1.4238, "prompts": 96, "kld_mean_ci95": [ 0.024634, 0.030305 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.029216, "top1_agree": 0.94076, "dp_mean_abs": 0.032069 }, "code-other": { "positions": 2560, "kld_mean": 0.02543, "top1_agree": 0.94805, "dp_mean_abs": 0.031658 }, "code-py": { "positions": 3072, "kld_mean": 0.024443, "top1_agree": 0.94173, "dp_mean_abs": 0.031651 }, "finnish": { "positions": 2048, "kld_mean": 0.034167, "top1_agree": 0.93457, "dp_mean_abs": 0.037777 }, "instruction": { "positions": 2218, "kld_mean": 0.03339, "top1_agree": 0.95311, "dp_mean_abs": 0.033885 }, "math": { "positions": 3072, "kld_mean": 0.015767, "top1_agree": 0.96452, "dp_mean_abs": 0.025405 }, "multilingual": { "positions": 1024, "kld_mean": 0.037247, "top1_agree": 0.93555, "dp_mean_abs": 0.041333 }, "reasoning": { "positions": 1936, "kld_mean": 0.022417, "top1_agree": 0.95403, "dp_mean_abs": 0.028389 }, "structured": { "positions": 2379, "kld_mean": 0.024053, "top1_agree": 0.95292, "dp_mean_abs": 0.026561 }, "summarise": { "positions": 2048, "kld_mean": 0.021748, "top1_agree": 0.94434, "dp_mean_abs": 0.030867 }, "writing": { "positions": 1963, "kld_mean": 0.03736, "top1_agree": 0.919, "dp_mean_abs": 0.041044 } } }, "generation": { "positions": 3454, "kld_mean": 0.015706, "kld_p50": 5.5e-05, "kld_p90": 0.035679, "kld_p99": 0.233722, "kld_p999": 1.284242, "kld_max": 2.46449, "top1_agree": 1.0, "dp_mean_abs": 0.018381, "dp_p1": -0.17067, "dp_p5": -0.05907, "dp_p50": 0.0, "dp_p95": 0.06713, "dp_p99": 0.16839, "support_mass_mean": 0.99898, "ppl_a": 1.0971, "ppl_b": 1.0961, "prompts": 96, "identical": 0, "identical_rate": 0.0, "diverged": 96, "first_divergence_p10": 2.0, "first_divergence_p50": 22.0, "first_divergence_min": 0, "gen_tokens_a": 24199, "gen_tokens_b": 24039 }, "dimensions": { "model": "Qwen3.8-27B", "quant": "nvfp4", "graphs": "on", "kind": "floor" }, "notes": "Both sides NVFP4, bf16 KV; same weights, different kernel path (torch.compile + CUDA graphs, text-only, seqs 32). The server is deterministic so this is the between-configuration floor on the W4A4 model, not run-to-run noise." }, { "label": "Qwen3.8 FP8 \u2192 NVFP4 + MTP", "a": "q38-fp8", "b": "q38-nvfp4-mtp", "model": "Qwen3.8-27B", "generated": "2026-09-06", "vllm_version": "0.28.0", "prompt_set": "quality/prompts.jsonl", "settings": { "max_tokens": 256, "top_k": 20, "thinking": true, "temperature": 0, "seed": 0, "concurrency": 1 }, "teacher_forced": { "positions": 23856, "kld_mean": 0.102963, "kld_p50": 0.007564, "kld_p90": 0.302229, "kld_p99": 1.03499, "kld_p999": 2.506306, "kld_max": 7.7598, "top1_agree": 0.89743, "dp_mean_abs": 0.069852, "dp_p1": -0.51443, "dp_p5": -0.29314, "dp_p50": -2e-05, "dp_p95": 0.13673, "dp_p99": 0.29202, "support_mass_mean": 0.98941, "ppl_a": 1.2783, "ppl_b": 1.4261, "prompts": 96, "kld_mean_ci95": [ 0.094096, 0.113293 ], "per_category": { "agentic": { "positions": 1536, "kld_mean": 0.131658, "top1_agree": 0.88411, "dp_mean_abs": 0.077751 }, "code-other": { "positions": 2560, "kld_mean": 0.119829, "top1_agree": 0.89023, "dp_mean_abs": 0.076399 }, "code-py": { "positions": 3072, "kld_mean": 0.09656, "top1_agree": 0.89583, "dp_mean_abs": 0.069218 }, "finnish": { "positions": 2048, "kld_mean": 0.124689, "top1_agree": 0.88916, "dp_mean_abs": 0.081208 }, "instruction": { "positions": 2218, "kld_mean": 0.114011, "top1_agree": 0.90938, "dp_mean_abs": 0.071224 }, "math": { "positions": 3072, "kld_mean": 0.059061, "top1_agree": 0.92741, "dp_mean_abs": 0.0519 }, "multilingual": { "positions": 1024, "kld_mean": 0.152818, "top1_agree": 0.8584, "dp_mean_abs": 0.092401 }, "reasoning": { "positions": 1936, "kld_mean": 0.074958, "top1_agree": 0.91116, "dp_mean_abs": 0.060408 }, "structured": { "positions": 2379, "kld_mean": 0.08679, "top1_agree": 0.91593, "dp_mean_abs": 0.055851 }, "summarise": { "positions": 2048, "kld_mean": 0.088833, "top1_agree": 0.89746, "dp_mean_abs": 0.065889 }, "writing": { "positions": 1963, "kld_mean": 0.138042, "top1_agree": 0.85227, "dp_mean_abs": 0.089477 } } }, "generation": { "positions": 2068, "kld_mean": 0.029949, "kld_p50": 0.000123, "kld_p90": 0.085411, "kld_p99": 0.502315, "kld_p999": 1.310637, "kld_max": 1.643917, "top1_agree": 1.0, "dp_mean_abs": 0.023847, "dp_p1": -0.2714, "dp_p5": -0.10652, "dp_p50": 0.0, "dp_p95": 0.06503, "dp_p99": 0.20001, "support_mass_mean": 0.99964, "ppl_a": 1.0563, "ppl_b": 1.0622, "prompts": 96, "identical": 0, "identical_rate": 0.0, "diverged": 96, "first_divergence_p10": 2.0, "first_divergence_p50": 9.0, "first_divergence_min": 0, "gen_tokens_a": 23856, "gen_tokens_b": 23937 }, "dimensions": { "model": "Qwen3.8-27B", "quant": "nvfp4", "mtp": "on" } } ] }