diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh index 87809f54b6..eb23d38b1a 100644 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh @@ -21,6 +21,7 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export ROCM_QUICK_REDUCE_QUANTIZATION=INT8 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} @@ -41,7 +42,7 @@ python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \ --model-loader-extra-config '{"enable_multithread_load": true}' \ --watchdog-timeout 1200 \ --disable-radix-cache \ ---enable-aiter-allreduce-fusion --max-running-requests $CONC \ +--max-running-requests $CONC \ --page-size 16 \ > $SERVER_LOG 2>&1 & diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh index 16e86e5443..6488d1380c 100755 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh @@ -21,6 +21,7 @@ export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 export SGLANG_MAMBA_SSM_DTYPE=bfloat16 +export ROCM_QUICK_REDUCE_QUANTIZATION=INT8 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} @@ -41,7 +42,7 @@ python3 -m sglang.launch_server --model-path=$MODEL --trust-remote-code \ --model-loader-extra-config '{"enable_multithread_load": true}' \ --watchdog-timeout 1200 \ --disable-radix-cache \ ---enable-aiter-allreduce-fusion --max-running-requests $CONC \ +--max-running-requests $CONC \ --page-size 16 \ --speculative-algorithm EAGLE \ --speculative-num-steps 3 \ diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b358f10bf1..4e75e3682c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4887,6 +4887,14 @@ - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201 +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + description: + - "Add ROCM_QUICK_REDUCE_QUANTIZATION=INT8 in both non-MTP and MTP benchmark scripts" + - "Remove --enable-aiter-allreduce-fusion from both non-MTP and MTP launch commands" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2265 + - config-keys: - dsv4-fp4-b200-vllm-agentic description: