diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh index 9f4b9e0871..fcacde4d92 100755 --- a/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_b300_sglang.sh @@ -13,16 +13,13 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, # EP_SIZE, DP_ATTENTION +# +# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. source "$(dirname "$0")/../../benchmark_lib.sh" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION -if [[ "$KV_OFFLOADING" != "none" ]]; then - echo "Error: KV_OFFLOADING=$KV_OFFLOADING is not supported by this recipe" >&2 - exit 1 -fi - if [[ -n "${SLURM_JOB_ID:-}" ]]; then echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" fi @@ -46,6 +43,45 @@ install_agentic_deps SERVER_LOG="$RESULT_DIR/server.log" mkdir -p "$RESULT_DIR" +CACHE_ARGS=() +if require_agentic_kv_offload_backend hicache; then + # HiCache extends RadixAttention: prefixes evicted from the HBM KV pool + # spill to a pinned host pool instead of being recomputed. On the + # 1M-context agentic corpus the live working set outgrows HBM past + # conc 8 (TP8) / 64 (DP8) and the radix hit rate collapses to <0.1 + # against a ~0.97 theoretical ceiling, so every turn re-prefills its + # whole history; the host tier restores those hits at C2C bandwidth. + # GLM-5.2 is DSA/MLA-family (attention_backend=dsa): every rank holds + # complete per-token KV (169.98 GB device pool per rank, replicated on + # all 8 ranks), so host capacity is controlled through the host/device + # token-capacity ratio like the DSv4 recipe, NOT a per-rank + # --hicache-size. A GB-based size of TOTAL_CPU_DRAM_GB/TP pinned the + # whole 0.80-DRAM budget (8 x 299 GB) at init on top of 465 GB of + # weights and OOM-killed the node (run 29678598595); DSv4's own + # ratio=2 default pins 2 x 170 GB x 8 = 2.7 TB here and OOMs too + # (GLM-5.2's device pool is far larger than DSv4's). Fractional 0.75 + # = ~128 GB/rank = ~1.0 TB total, matching the cluster's proven ~1 TB + # host-pool envelope; validated on-node 2026-07-19 (boot + 4.2M-token + # overflow bench forcing eviction through the DSA KV+INDEXER pools). + DEFAULT_HICACHE_RATIO=0.75 + HICACHE_RATIO="${HICACHE_RATIO:-$DEFAULT_HICACHE_RATIO}" + if awk -v r="$HICACHE_RATIO" -v cap="$DEFAULT_HICACHE_RATIO" 'BEGIN { exit !(r > cap) }'; then + echo "Error: HICACHE_RATIO=$HICACHE_RATIO exceeds configured limit $DEFAULT_HICACHE_RATIO" >&2 + exit 1 + fi + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_back}" + HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" + HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" + echo "HiCache CPU tier: ratio=$HICACHE_RATIO, capacity=${TOTAL_CPU_DRAM_GB} GB, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-ratio "$HICACHE_RATIO" + --hicache-write-policy "$HICACHE_WRITE_POLICY" + --hicache-io-backend "$HICACHE_IO_BACKEND" + --hicache-mem-layout "$HICACHE_MEM_LAYOUT" + ) +fi + # With attention-DP, front the DP ranks with sglang-router using consistent # hashing on the AIPerf correlation id so multi-turn sessions stay on the DP # rank that holds their radix-cache prefix. @@ -136,6 +172,7 @@ SGLANG_CMD=( --mem-fraction-static 0.85 --max-running-requests "$MAX_RUNNING_REQUESTS" "${GRAPH_ARGS[@]}" + "${CACHE_ARGS[@]}" --watchdog-timeout 1800 --enable-metrics ) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 4b35c30f87..3aac1b8378 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7873,5 +7873,5 @@ glm5.2-fp4-b300-sglang-agentic: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 8, dp-attn: true, kv-offloading: none, conc-list: [48, 64, 96, 128, 192, 256, 512], router: { name: sglang-router, version: "0.3.2" } } + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 16, 32] } + - { tp: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [48, 64, 96, 128, 192, 256, 512], router: { name: sglang-router, version: "0.3.2" } } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9e959c9464..e57215906e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4950,3 +4950,11 @@ description: - "Bump vLLM image to v0.25.0" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2256 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic + description: + - "Enable DRAM KV offloading (SGLang HiCache) on all points of both arms: DSv4-style host/device token-capacity ratio, tuned to 0.75 for GLM-5.2 (DSA/MLA-family: the 169.98 GB device pool is replicated on all 8 ranks, so ratio 0.75 = ~1.0 TB pinned host pools; ratio 2 and GB-based sizing both OOM-killed 3 TB nodes), io-backend direct, mem-layout page_first_direct, write policy write_back; validated on-node with an overflow bench forcing eviction through the DSA KV+INDEXER host pools" + - "Run 29651235293 showed the 1M-context corpus working set outgrowing the HBM KV pool past conc 8 (TP8) / conc 64 (DP8): gpu_kv_cache_usage pinned at 1.0 and the radix hit rate collapsed from a ~0.97 theoretical ceiling to 0.04-0.06, so every post-knee turn re-prefilled its full history and throughput fell together with interactivity" + - "HiCache spills evicted prefixes to host DRAM and restores them at C2C bandwidth instead of recomputing; sizing follows the qwen3.5-fp8-b300-sglang-agentic-hicache recipe (GLM-5.2 is plain GQA: one host pool per rank, GB-based --hicache-size)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2279