diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml new file mode 100644 index 0000000000..1cb6ab0de4 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml @@ -0,0 +1,128 @@ +name: disagg-B200-1p1d-dep4-dep8-c308-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.92 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 308 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml new file mode 100644 index 0000000000..49a71471de --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml @@ -0,0 +1,124 @@ +name: disagg-B200-1p4d-dep4-tep8-c24-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 24 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml new file mode 100644 index 0000000000..cabec31b18 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml @@ -0,0 +1,124 @@ +name: disagg-B200-1p4d-dep4-tep8-c4-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 4 + decode_workers: 4 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 4 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c115-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c115-stp.yaml new file mode 100644 index 0000000000..298279a010 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c115-stp.yaml @@ -0,0 +1,126 @@ +name: disagg-B200-1p5d-dep4-tep4-c115-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 5 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 16 + max_num_tokens: 16 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 115 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c195-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c195-stp.yaml new file mode 100644 index 0000000000..f31a7da80e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c195-stp.yaml @@ -0,0 +1,128 @@ +name: disagg-B200-1p5d-dep4-tep4-c195-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 5 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 32 + max_num_tokens: 32 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 195 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c30-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c30-stp.yaml new file mode 100644 index 0000000000..4c4c833192 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c30-stp.yaml @@ -0,0 +1,124 @@ +name: disagg-B200-1p5d-dep4-tep4-c30-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 5 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 4 + max_num_tokens: 4 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 30 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c5-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c5-stp.yaml new file mode 100644 index 0000000000..619c1bb709 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c5-stp.yaml @@ -0,0 +1,124 @@ +name: disagg-B200-1p5d-dep4-tep4-c5-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 5 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 1 + max_num_tokens: 1 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 5 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c60-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c60-stp.yaml new file mode 100644 index 0000000000..1a8b4f2988 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-1p5d-dep4-tep4-c60-stp.yaml @@ -0,0 +1,125 @@ +name: disagg-B200-1p5d-dep4-tep4-c60-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 4 + decode_nodes: 3 + decode_workers: 5 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + enable_padding: true + enable_attention_dp: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 8 + max_num_tokens: 8 + max_seq_len: 9256 + moe_config: + backend: TRTLLM + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 60 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-2p1d-dep4-dep8-c615-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-2p1d-dep4-dep8-c615-stp.yaml new file mode 100644 index 0000000000..abcce625e7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-2p1d-dep4-dep8-c615-stp.yaml @@ -0,0 +1,132 @@ +name: disagg-B200-2p1d-dep4-dep8-c615-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.92 + max_batch_size: 64 + max_num_tokens: 64 + max_seq_len: 9256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 615 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-3p1d-dep4-dep8-c1127-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-3p1d-dep4-dep8-c1127-stp.yaml new file mode 100644 index 0000000000..316a88f23a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-3p1d-dep4-dep8-c1127-stp.yaml @@ -0,0 +1,140 @@ +name: disagg-B200-3p1d-dep4-dep8-c1127-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 3 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.92 + max_batch_size: 128 + max_num_tokens: 128 + max_seq_len: 9256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 1127 + req_rate: inf + num_prompts_mult: 20 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-4p1d-dep4-dep8-c2151-stp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-4p1d-dep4-dep8-c2151-stp.yaml new file mode 100644 index 0000000000..893a725cae --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4/disagg-B200-4p1d-dep4-dep8-c2151-stp.yaml @@ -0,0 +1,156 @@ +name: disagg-B200-4p1d-dep4-dep8-c2151-stp +model: + path: kimik2.5-fp4 + container: dynamo-trtllm + precision: fp4 +identity: + model: + repo: nvidia/Kimi-K2.5-NVFP4 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 +slurm: + time_limit: 04:00:00 +dynamo: + install: true + hash: 4025cb6a8992d19fd19eead36065e08ea5301f35 + request_plane: tcp +health_check: + max_attempts: 540 + interval_seconds: 10 +resources: + gpu_type: b200 + gpus_per_node: 8 + prefill_nodes: 2 + prefill_workers: 4 + gpus_per_prefill: 4 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + UCX_TLS: rc,cuda_ipc,cuda_copy,sm,self,tcp + trtllm_config: + prefill: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: null + disable_overlap_scheduler: true + enable_attention_dp: true + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.9 + max_batch_size: 2 + max_num_tokens: 16386 + max_seq_len: 8232 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + tensor_parallel_size: 4 + trust_remote_code: true + decode: + cache_transceiver_config: + backend: DEFAULT + kv_transfer_timeout_ms: 600000 + max_tokens_in_buffer: 68736 + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + - 136 + - 144 + - 152 + - 160 + - 168 + - 176 + - 184 + - 192 + - 200 + - 208 + - 216 + - 224 + - 232 + - 240 + - 248 + - 256 + enable_padding: true + enable_attention_dp: true + enable_lm_head_tp_in_adp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.92 + max_batch_size: 256 + max_num_tokens: 256 + max_seq_len: 9256 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + stream_interval: 100 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: true + env: + ETCD_LEASE_TTL: '120' +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: + - 2151 + req_rate: inf + num_prompts_mult: 20 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index cc31fc0e47..a167f49270 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4266,6 +4266,167 @@ dsr1-fp4-b200-dynamo-sglang-mtp: ep: 8 dp-attn: true +kimik2.5-fp4-b200-dynamo-trt: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 + model: nvidia/Kimi-K2.5-NVFP4 + model-prefix: kimik2.5 + runner: b200-multinode + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "v1.3.0rc21" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # Non-MTP configurations (STP) + - conc-list: [ 4 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-1p4d-dep4-tep8-c4-stp.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [ 24 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-1p4d-dep4-tep8-c24-stp.yaml" + decode: + num-worker: 4 + tp: 8 + ep: 8 + dp-attn: false + - conc-list: [ 5 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-1p5d-dep4-tep4-c5-stp.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 30 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-1p5d-dep4-tep4-c30-stp.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 60 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-1p5d-dep4-tep4-c60-stp.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 115 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-1p5d-dep4-tep4-c115-stp.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 195 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-1p5d-dep4-tep4-c195-stp.yaml" + decode: + num-worker: 5 + tp: 4 + ep: 4 + dp-attn: false + - conc-list: [ 308 ] + prefill: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-1p1d-dep4-dep8-c308-stp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [ 615 ] + prefill: + num-worker: 2 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-2p1d-dep4-dep8-c615-stp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [ 1127 ] + prefill: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-3p1d-dep4-dep8-c1127-stp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - conc-list: [ 2151 ] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/kimi-k25-nvfp4/b200-fp4/disagg-B200-4p1d-dep4-dep8-c2151-stp.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + kimik2.5-fp4-gb200-dynamo-trt: image: nvcr.io#nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-dev.1-cuda13 model: nvidia/Kimi-K2.5-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 1822ffcb70..e82277b6f0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5019,3 +5019,12 @@ - "dsv4-fp4-b200-vllm-agentic-mtp: MTP twins (--speculative-config {\"method\":\"mtp\",\"num_speculative_tokens\":3}) of the aggregate arms -- TP8 GPU-resident, TP8 SimpleCPU, DEP8 SimpleCPU, DEP8 Mooncake, each mirroring its non-MTP conc-list; routed via spec-decoding=mtp to dsv4_fp4_b200_vllm_mtp.sh." - "MTP capture sizes are in TOKENS: --max-cudagraph-capture-size scaled to MAX_NUM_SEQS*(1+num_speculative_tokens) (= MAX_NUM_SEQS*4) so FULL_DECODE_ONLY covers the largest decode batches." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2259 + +- config-keys: + - kimik2.5-fp4-b200-dynamo-trt + description: + - "Add Kimi K2.5 NVFP4 B200 disaggregated TRT-LLM benchmarks via Dynamo (11 STP configs, ISL 8k OSL 1k)" + - "Container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc21 with Dynamo installed from source (hash 4025cb6a8992d19fd19eead36065e08ea5301f35)" + - "Topologies: 1p4d-dep4-tep8 (conc 4/24), 1p5d-dep4-tep4 (conc 5/30/60/115/195), 1p1d-dep4-dep8 (conc 308), 2p1d-dep4-dep8 (conc 615), 3p1d-dep4-dep8 (conc 1127), 4p1d-dep4-dep8 (conc 2151)" + - "Runner updated to clone srt-slurm at sa-submission-q2-2026 and copy local recipes into recipes/trtllm/kimi-k25-nvfp4/b200-fp4/" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2249 diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 58051e6405..a276644575 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -128,6 +128,12 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git checkout a98738de9b2233459b5456e9ed71af09ce893f92 mkdir -p recipes/sglang/dsr1/b200-fp4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/dsr1/b200-fp4" recipes/sglang/dsr1/b200-fp4 + elif [[ $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "kimik2.5" && $PRECISION == "fp4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + git checkout v1.0.29 + mkdir -p recipes/trtllm/kimi-k25-nvfp4/b200-fp4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/kimi-k2.5/disagg/trtllm_dynamo/b200-fp4" recipes/trtllm/kimi-k25-nvfp4/b200-fp4 else git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1