Skip to content

[NV] Kimi K2.5 NVFP4 B200 disaggregated TRT-LLM (Dynamo) recipe #2249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
richardhuo-nv wants to merge 2 commits into
base: main
Choose a base branch
from
Open

[NV] Kimi K2.5 NVFP4 B200 disaggregated TRT-LLM (Dynamo) recipe #2249

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7decc36
kimik2.5-fp4-b200-dynamo-trt: Kimi K2.5 NVFP4 B200 disaggregated TRT-…
richardhuo-nv Jul 16, 2026
0869a4a
fix link
richardhuo-nv Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 128 additions & 0 deletions ...s/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
name: disagg-B200-1p1d-dep4-dep8-c308-stp
model:
path: kimik2.5-fp4
container: dynamo-trtllm
precision: fp4
identity:
model:
repo: nvidia/Kimi-K2.5-NVFP4
container:
image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21
slurm:
time_limit: 04:00:00
dynamo:
install: true
hash: 4025cb6a8992d19fd19eead36065e08ea5301f35
request_plane: tcp
health_check:
max_attempts: 540
interval_seconds: 10
resources:
gpu_type: b200
gpus_per_node: 8
prefill_nodes: 1
prefill_workers: 1
gpus_per_prefill: 4
decode_nodes: 1
decode_workers: 1
gpus_per_decode: 8
backend:
type: trtllm
prefill_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
TRTLLM_ENABLE_PDL: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
MIMALLOC_PURGE_DELAY: '0'
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
decode_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
TRTLLM_ENABLE_PDL: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
MIMALLOC_PURGE_DELAY: '0'
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
trtllm_config:
prefill:
cache_transceiver_config:
backend: DEFAULT
kv_transfer_timeout_ms: 600000
max_tokens_in_buffer: 68736
cuda_graph_config: null
disable_overlap_scheduler: true
enable_attention_dp: true
kv_cache_config:
dtype: fp8
enable_block_reuse: false
free_gpu_memory_fraction: 0.9
max_batch_size: 2
max_num_tokens: 16386
max_seq_len: 8232
moe_config:
backend: CUTEDSL
moe_expert_parallel_size: 4
nvfp4_gemm_config:
allowed_backends:
- cutlass
- cublaslt
- cutedsl
- cuda_core
pipeline_parallel_size: 1
print_iter_log: true
tensor_parallel_size: 4
trust_remote_code: true
decode:
cache_transceiver_config:
backend: DEFAULT
kv_transfer_timeout_ms: 600000
max_tokens_in_buffer: 68736
cuda_graph_config:
batch_sizes:
- 1
- 2
- 4
- 8
- 16
- 24
- 32
enable_padding: true
enable_attention_dp: true
enable_lm_head_tp_in_adp: false
kv_cache_config:
dtype: fp8
enable_block_reuse: false
free_gpu_memory_fraction: 0.92
max_batch_size: 32
max_num_tokens: 32
max_seq_len: 9256
moe_config:
backend: CUTEDSL
use_low_precision_moe_combine: true
moe_expert_parallel_size: 8
num_postprocess_workers: 4
nvfp4_gemm_config:
allowed_backends:
- cutlass
- cublaslt
- cutedsl
- cuda_core
pipeline_parallel_size: 1
print_iter_log: true
stream_interval: 100
tensor_parallel_size: 8
trust_remote_code: true
frontend:
type: dynamo
enable_multiple_frontends: true
env:
ETCD_LEASE_TTL: '120'
benchmark:
type: sa-bench
isl: 8192
osl: 1024
concurrencies:
- 308
req_rate: inf
num_prompts_mult: 20
124 changes: 124 additions & 0 deletions ...es/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
name: disagg-B200-1p4d-dep4-tep8-c24-stp
model:
path: kimik2.5-fp4
container: dynamo-trtllm
precision: fp4
identity:
model:
repo: nvidia/Kimi-K2.5-NVFP4
container:
image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21
slurm:
time_limit: 04:00:00
dynamo:
install: true
hash: 4025cb6a8992d19fd19eead36065e08ea5301f35
request_plane: tcp
health_check:
max_attempts: 540
interval_seconds: 10
resources:
gpu_type: b200
gpus_per_node: 8
prefill_nodes: 1
prefill_workers: 1
gpus_per_prefill: 4
decode_nodes: 4
decode_workers: 4
gpus_per_decode: 8
backend:
type: trtllm
prefill_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
TRTLLM_ENABLE_PDL: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
MIMALLOC_PURGE_DELAY: '0'
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
decode_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
TRTLLM_ENABLE_PDL: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
MIMALLOC_PURGE_DELAY: '0'
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
trtllm_config:
prefill:
cache_transceiver_config:
backend: DEFAULT
kv_transfer_timeout_ms: 600000
max_tokens_in_buffer: 68736
cuda_graph_config: null
disable_overlap_scheduler: true
enable_attention_dp: true
kv_cache_config:
dtype: fp8
enable_block_reuse: false
free_gpu_memory_fraction: 0.9
max_batch_size: 2
max_num_tokens: 16386
max_seq_len: 8232
moe_config:
backend: CUTEDSL
moe_expert_parallel_size: 4
nvfp4_gemm_config:
allowed_backends:
- cutlass
- cublaslt
- cutedsl
- cuda_core
pipeline_parallel_size: 1
print_iter_log: true
tensor_parallel_size: 4
trust_remote_code: true
decode:
cache_transceiver_config:
backend: DEFAULT
kv_transfer_timeout_ms: 600000
max_tokens_in_buffer: 68736
cuda_graph_config:
batch_sizes:
- 1
- 2
- 4
enable_padding: true
enable_attention_dp: false
enable_lm_head_tp_in_adp: false
kv_cache_config:
dtype: fp8
enable_block_reuse: false
free_gpu_memory_fraction: 0.9
max_batch_size: 4
max_num_tokens: 4
max_seq_len: 9256
moe_config:
backend: TRTLLM
use_low_precision_moe_combine: true
moe_expert_parallel_size: 8
num_postprocess_workers: 4
nvfp4_gemm_config:
allowed_backends:
- cutlass
- cublaslt
- cutedsl
- cuda_core
pipeline_parallel_size: 1
print_iter_log: true
stream_interval: 100
tensor_parallel_size: 8
trust_remote_code: true
frontend:
type: dynamo
enable_multiple_frontends: false
env:
ETCD_LEASE_TTL: '120'
benchmark:
type: sa-bench
isl: 8192
osl: 1024
concurrencies:
- 24
req_rate: inf
num_prompts_mult: 20
124 changes: 124 additions & 0 deletions ...pes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
name: disagg-B200-1p4d-dep4-tep8-c4-stp
model:
path: kimik2.5-fp4
container: dynamo-trtllm
precision: fp4
identity:
model:
repo: nvidia/Kimi-K2.5-NVFP4
container:
image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21
slurm:
time_limit: 04:00:00
dynamo:
install: true
hash: 4025cb6a8992d19fd19eead36065e08ea5301f35
request_plane: tcp
health_check:
max_attempts: 540
interval_seconds: 10
resources:
gpu_type: b200
gpus_per_node: 8
prefill_nodes: 1
prefill_workers: 1
gpus_per_prefill: 4
decode_nodes: 4
decode_workers: 4
gpus_per_decode: 8
backend:
type: trtllm
prefill_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
TRTLLM_ENABLE_PDL: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
MIMALLOC_PURGE_DELAY: '0'
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
decode_environment:
TLLM_LOG_LEVEL: INFO
TRTLLM_SERVER_DISABLE_GC: '1'
TRTLLM_WORKER_DISABLE_GC: '1'
TRTLLM_ENABLE_PDL: '1'
NCCL_GRAPH_MIXING_SUPPORT: '0'
MIMALLOC_PURGE_DELAY: '0'
UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp
trtllm_config:
prefill:
cache_transceiver_config:
backend: DEFAULT
kv_transfer_timeout_ms: 600000
max_tokens_in_buffer: 68736
cuda_graph_config: null
disable_overlap_scheduler: true
enable_attention_dp: true
kv_cache_config:
dtype: fp8
enable_block_reuse: false
free_gpu_memory_fraction: 0.9
max_batch_size: 2
max_num_tokens: 16386
max_seq_len: 8232
moe_config:
backend: CUTEDSL
moe_expert_parallel_size: 4
nvfp4_gemm_config:
allowed_backends:
- cutlass
- cublaslt
- cutedsl
- cuda_core
pipeline_parallel_size: 1
print_iter_log: true
tensor_parallel_size: 4
trust_remote_code: true
decode:
cache_transceiver_config:
backend: DEFAULT
kv_transfer_timeout_ms: 600000
max_tokens_in_buffer: 68736
cuda_graph_config:
batch_sizes:
- 1
- 2
- 4
enable_padding: true
enable_attention_dp: false
enable_lm_head_tp_in_adp: false
kv_cache_config:
dtype: fp8
enable_block_reuse: false
free_gpu_memory_fraction: 0.9
max_batch_size: 1
max_num_tokens: 1
max_seq_len: 9256
moe_config:
backend: TRTLLM
use_low_precision_moe_combine: true
moe_expert_parallel_size: 8
num_postprocess_workers: 4
nvfp4_gemm_config:
allowed_backends:
- cutlass
- cublaslt
- cutedsl
- cuda_core
pipeline_parallel_size: 1
print_iter_log: true
stream_interval: 100
tensor_parallel_size: 8
trust_remote_code: true
frontend:
type: dynamo
enable_multiple_frontends: false
env:
ETCD_LEASE_TTL: '120'
benchmark:
type: sa-bench
isl: 8192
osl: 1024
concurrencies:
- 4
req_rate: inf
num_prompts_mult: 20
Loading