From c2b59bbe0e6a21a211dcd86a044c8ca14487b5ef Mon Sep 17 00:00:00 2001 From: jinmanx Date: Fri, 24 Jul 2026 06:18:23 +0000 Subject: [PATCH 1/2] [cuTile] Add v3 cuTile backends for 8 Liger ops + benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add cuTile (nvidia-cuda-tile) backends, dispatched via LIGER_KERNEL_IMPL=cutile, for eight ops — each drops into the existing LigerXxxFunction autograd interface so the backend swap is transparent: rms_norm, swiglu (SiLUMul), dyt, softmax, poly_norm, fused_add_rms_norm, fused_linear_cross_entropy, grpo_loss (token- and sequence-level importance sampling, GRPO/CISPO/SAPO/VESPO) Correctness is exercised by the existing transformer suites run under LIGER_KERNEL_IMPL=cutile (which routes each op to its cuTile implementation), and test_cutile_backend.py verifies that routing. Notable correctness fixes made while validating on B200 (sm_100): - grpo_loss: round BLOCK_N up to a power of 2 (ct.arange requires it) and flatten the rank-2 vllm_is_ratio to 1-D for the flat gather — both only bit non-power-of-2 vocab sizes. - fused_add_rms_norm: cast the LLAMA/NONE Y and dX stores back to the output dtype, so an fp32 norm weight with bf16 activations no longer fails to compile. Benchmarks: Triton-vs-cuTile speed+memory data on NVIDIA B200 for all eight ops, plus a benchmark-harness fix so multiple Liger providers in one script are tagged distinctly (LIGER_BENCH_PROVIDER_TAG rewrites the leading "liger" of each provider). Note: tvd and fused_neighborhood_attention cuTile ports were prototyped but are NOT included — the open-source tileiras 13.3.36 miscompiles their kernels; they fall back to the Triton implementations until the compiler fix ships. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmark/data/all_benchmark_data_cutile.csv | 568 ++++++++ benchmark/scripts/run_cutile_compare.py | 8 + benchmark/scripts/utils.py | 11 +- src/liger_kernel/ops/cutile/ops/__init__.py | 16 + src/liger_kernel/ops/cutile/ops/dyt.py | 235 ++++ .../ops/cutile/ops/fused_add_rms_norm.py | 592 ++++++++ .../cutile/ops/fused_linear_cross_entropy.py | 350 +++++ src/liger_kernel/ops/cutile/ops/grpo_loss.py | 1228 +++++++++++++++++ src/liger_kernel/ops/cutile/ops/poly_norm.py | 412 ++++++ src/liger_kernel/ops/cutile/ops/rms_norm.py | 422 ++++++ src/liger_kernel/ops/cutile/ops/softmax.py | 353 +++++ src/liger_kernel/ops/cutile/ops/swiglu.py | 283 ++++ test/transformers/test_cutile_backend.py | 8 + 13 files changed, 4483 insertions(+), 3 deletions(-) create mode 100644 src/liger_kernel/ops/cutile/ops/dyt.py create mode 100644 src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py create mode 100644 src/liger_kernel/ops/cutile/ops/fused_linear_cross_entropy.py create mode 100644 src/liger_kernel/ops/cutile/ops/grpo_loss.py create mode 100644 src/liger_kernel/ops/cutile/ops/poly_norm.py create mode 100644 src/liger_kernel/ops/cutile/ops/rms_norm.py create mode 100644 src/liger_kernel/ops/cutile/ops/softmax.py create mode 100644 src/liger_kernel/ops/cutile/ops/swiglu.py diff --git a/benchmark/data/all_benchmark_data_cutile.csv b/benchmark/data/all_benchmark_data_cutile.csv index 2addf0025..04c3a383c 100644 --- a/benchmark/data/all_benchmark_data_cutile.csv +++ b/benchmark/data/all_benchmark_data_cutile.csv @@ -827,3 +827,571 @@ multi_token_attention,liger_cutile,full,memory,MB,L,sequence length,32,0.1435546 multi_token_attention,liger_cutile,full,memory,MB,L,sequence length,64,2.00390625,2.00390625,2.00390625,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 multi_token_attention,liger_cutile,full,memory,MB,L,sequence length,128,19.50439453125,19.50439453125,19.50439453125,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 multi_token_attention,liger_cutile,full,memory,MB,L,sequence length,256,80.00439453125,80.00439453125,80.00439453125,"{""C_in"": 4, ""C_out"": 4, ""K"": 3, ""groups"": 1, ""bias"": true, ""dtype"": ""torch.bfloat16"", ""B"": 2}",NVIDIA B200,2026-06-26 08:02:12,0.8.0 +fused_add_rms_norm,torch,forward,speed,ms,BT,total tokens,1024,0.09212800115346909,0.09203840047121048,0.09219200164079666,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:51,0.8.0 +fused_add_rms_norm,torch,forward,speed,ms,BT,total tokens,2048,0.15360000729560852,0.15353600680828094,0.1536639928817749,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:51,0.8.0 +fused_add_rms_norm,torch,forward,speed,ms,BT,total tokens,4096,0.30108800530433655,0.3010239899158478,0.30115199089050293,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:51,0.8.0 +fused_add_rms_norm,torch,forward,speed,ms,BT,total tokens,8192,0.5892959833145142,0.5878080129623413,0.5897279977798462,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:51,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,forward,speed,ms,BT,total tokens,1024,0.06780800223350525,0.06755200028419495,0.06867840141057968,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,forward,speed,ms,BT,total tokens,2048,0.12080000340938568,0.1207360029220581,0.12086399644613266,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,forward,speed,ms,BT,total tokens,4096,0.23334400355815887,0.23139839768409728,0.23346560299396515,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,forward,speed,ms,BT,total tokens,8192,0.46902400255203247,0.4689471960067749,0.4699839949607849,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,forward,speed,ms,BT,total tokens,1024,0.025520000606775284,0.02454400062561035,0.027052799612283705,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,forward,speed,ms,BT,total tokens,2048,0.02537599951028824,0.02457600086927414,0.02664319984614849,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,forward,speed,ms,BT,total tokens,4096,0.02860799990594387,0.026623999699950218,0.028704000636935234,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,forward,speed,ms,BT,total tokens,8192,0.04710400104522705,0.0470208004117012,0.0471872016787529,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,torch,backward,speed,ms,BT,total tokens,1024,0.14531199634075165,0.14441600441932678,0.14547839760780334,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,torch,backward,speed,ms,BT,total tokens,2048,0.244719997048378,0.24368000030517578,0.2456447958946228,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,torch,backward,speed,ms,BT,total tokens,4096,0.4691520035266876,0.46807678937911984,0.47003520131111143,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,torch,backward,speed,ms,BT,total tokens,8192,0.9164479970932007,0.9151360154151916,0.9176575899124145,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,backward,speed,ms,BT,total tokens,1024,0.15571199357509613,0.15303680300712585,0.15754239857196808,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,backward,speed,ms,BT,total tokens,2048,0.16489599645137787,0.1638527989387512,0.1660352021455765,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,backward,speed,ms,BT,total tokens,4096,0.30508801341056824,0.3041088044643402,0.30616320371627803,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,backward,speed,ms,BT,total tokens,8192,0.5805119872093201,0.5795711994171142,0.5815423965454101,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,backward,speed,ms,BT,total tokens,1024,0.1085439994931221,0.10649599879980087,0.11409280002117157,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,backward,speed,ms,BT,total tokens,2048,0.10465599969029427,0.10312960147857667,0.10794240236282349,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,backward,speed,ms,BT,total tokens,4096,0.1096000000834465,0.1085439994931221,0.11262080073356628,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,backward,speed,ms,BT,total tokens,8192,0.18236799538135529,0.18227200210094452,0.1834239959716797,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:52,0.8.0 +fused_add_rms_norm,torch,full,speed,ms,BT,total tokens,1024,0.24432000517845154,0.24166400730609894,0.24611200392246246,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,torch,full,speed,ms,BT,total tokens,2048,0.41065600514411926,0.40980479717254636,0.4116159975528717,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,torch,full,speed,ms,BT,total tokens,4096,0.7966399788856506,0.7956479787826538,0.7978559732437134,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,torch,full,speed,ms,BT,total tokens,8192,1.572272002696991,1.56985604763031,1.5737279653549194,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,full,speed,ms,BT,total tokens,1024,0.31407999992370605,0.30970879793167116,0.3210111975669861,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,full,speed,ms,BT,total tokens,2048,0.31192000210285187,0.3091199994087219,0.31324800848960876,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,full,speed,ms,BT,total tokens,4096,0.5643040239810944,0.564191997051239,0.5656639814376831,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,full,speed,ms,BT,total tokens,8192,1.115119993686676,1.1135551691055297,1.1162240505218506,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,full,speed,ms,BT,total tokens,1024,0.22115200012922287,0.21794559955596923,0.22759040594100954,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,full,speed,ms,BT,total tokens,2048,0.22299200296401978,0.2209663987159729,0.2297343999147415,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,full,speed,ms,BT,total tokens,4096,0.22686400264501572,0.22309760153293612,0.322086411714554,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,full,speed,ms,BT,total tokens,8192,0.2980479896068573,0.29708799719810486,0.2990399897098541,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,torch,full,memory,MB,BT,total tokens,1024,168.03515625,168.03515625,168.03515625,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,torch,full,memory,MB,BT,total tokens,2048,336.0390625,336.0390625,336.0390625,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,torch,full,memory,MB,BT,total tokens,4096,672.046875,672.046875,672.046875,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,torch,full,memory,MB,BT,total tokens,8192,1344.0625,1344.0625,1344.0625,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,full,memory,MB,BT,total tokens,1024,152.08251953125,152.08251953125,152.08251953125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,full,memory,MB,BT,total tokens,2048,304.08642578125,304.08642578125,304.08642578125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,full,memory,MB,BT,total tokens,4096,608.09423828125,608.09423828125,608.09423828125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_rms_norm,full,memory,MB,BT,total tokens,8192,1108.10986328125,1108.10986328125,1108.10986328125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,full,memory,MB,BT,total tokens,1024,74.36328125,74.36328125,74.36328125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,full,memory,MB,BT,total tokens,2048,146.3671875,146.3671875,146.3671875,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,full,memory,MB,BT,total tokens,4096,290.375,290.375,290.375,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_triton_fused_add_rms_norm,full,memory,MB,BT,total tokens,8192,578.390625,578.390625,578.390625,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:53,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,forward,speed,ms,BT,total tokens,1024,0.06961600109934807,0.0695360004901886,0.06969600170850754,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,forward,speed,ms,BT,total tokens,2048,0.12272000312805176,0.12080000340938568,0.12284799665212631,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,forward,speed,ms,BT,total tokens,4096,0.23552000522613525,0.23545600473880768,0.23631359934806823,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,forward,speed,ms,BT,total tokens,8192,0.4710080027580261,0.46997119188308717,0.4710528016090393,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,forward,speed,ms,BT,total tokens,1024,0.014336000196635723,0.014240000396966934,0.014368000440299511,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,forward,speed,ms,BT,total tokens,2048,0.020479999482631683,0.020416000857949257,0.02054399996995926,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,forward,speed,ms,BT,total tokens,4096,0.030688000842928886,0.03062400035560131,0.0307839997112751,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,forward,speed,ms,BT,total tokens,8192,0.053247999399900436,0.05315199866890907,0.05331199988722801,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,backward,speed,ms,BT,total tokens,1024,0.13788799941539764,0.13595519959926605,0.1409152030944824,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,backward,speed,ms,BT,total tokens,2048,0.16070400178432465,0.15964800119400024,0.16286079883575438,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,backward,speed,ms,BT,total tokens,4096,0.2999520003795624,0.2989311933517456,0.3010304093360901,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,backward,speed,ms,BT,total tokens,8192,0.5692800283432007,0.5682176113128662,0.569382393360138,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,backward,speed,ms,BT,total tokens,1024,0.09678400307893753,0.09414400160312653,0.09945599734783173,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,backward,speed,ms,BT,total tokens,2048,0.09459199756383896,0.09280639886856079,0.09704320132732391,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,backward,speed,ms,BT,total tokens,4096,0.09830400347709656,0.09725440293550491,0.1001279965043068,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,backward,speed,ms,BT,total tokens,8192,0.17609600722789764,0.1759680062532425,0.17618559896945954,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:57,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,full,speed,ms,BT,total tokens,1024,0.2532479912042618,0.2513599991798401,0.2568320035934448,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,full,speed,ms,BT,total tokens,2048,0.29395198822021484,0.29384319186210633,0.29594879150390624,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,full,speed,ms,BT,total tokens,4096,0.5642400085926056,0.5622400045394897,0.5643519759178162,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,full,speed,ms,BT,total tokens,8192,1.1057760119438171,1.1048128128051757,1.1060479879379272,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,full,speed,ms,BT,total tokens,1024,0.1659040004014969,0.1633407950401306,0.16894079744815826,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,full,speed,ms,BT,total tokens,2048,0.16659200191497803,0.16328960359096528,0.1718336045742035,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,full,speed,ms,BT,total tokens,4096,0.16896000504493713,0.1667456030845642,0.17103999853134155,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,full,speed,ms,BT,total tokens,8192,0.2978239953517914,0.29691519737243655,0.2980288028717041,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,full,memory,MB,BT,total tokens,1024,152.08251953125,152.08251953125,152.08251953125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,full,memory,MB,BT,total tokens,2048,304.08642578125,304.08642578125,304.08642578125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,full,memory,MB,BT,total tokens,4096,608.09423828125,608.09423828125,608.09423828125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_rms_norm,full,memory,MB,BT,total tokens,8192,1108.10986328125,1108.10986328125,1108.10986328125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,full,memory,MB,BT,total tokens,1024,74.36328125,74.36328125,74.36328125,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,full,memory,MB,BT,total tokens,2048,146.3671875,146.3671875,146.3671875,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,full,memory,MB,BT,total tokens,4096,290.375,290.375,290.375,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +fused_add_rms_norm,liger_cutile_fused_add_rms_norm,full,memory,MB,BT,total tokens,8192,578.390625,578.390625,578.390625,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:40:58,0.8.0 +dyt,liger_triton,forward,speed,ms,BT,total tokens,1024,0.020640000700950623,0.019334400072693825,0.022534400224685672,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:19,0.8.0 +dyt,liger_triton,forward,speed,ms,BT,total tokens,2048,0.02014400064945221,0.01943040043115616,0.021420799568295484,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:19,0.8.0 +dyt,liger_triton,forward,speed,ms,BT,total tokens,4096,0.02457600086927414,0.02454400062561035,0.025599999353289604,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:19,0.8.0 +dyt,liger_triton,forward,speed,ms,BT,total tokens,8192,0.043007999658584595,0.04294399917125702,0.04310400038957596,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:19,0.8.0 +dyt,torch,forward,speed,ms,BT,total tokens,1024,0.03782400116324425,0.03585920110344887,0.0379519984126091,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:20,0.8.0 +dyt,torch,forward,speed,ms,BT,total tokens,2048,0.062431998550891876,0.06047999858856201,0.06255999952554703,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:20,0.8.0 +dyt,torch,forward,speed,ms,BT,total tokens,4096,0.12406399846076965,0.1239359974861145,0.12600960433483124,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:20,0.8.0 +dyt,torch,forward,speed,ms,BT,total tokens,8192,0.24873599410057068,0.24680959582328796,0.24886399507522583,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:20,0.8.0 +dyt,torch_compile,forward,speed,ms,BT,total tokens,1024,0.042847998440265656,0.04122240096330643,0.04447999969124794,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:21,0.8.0 +dyt,torch_compile,forward,speed,ms,BT,total tokens,2048,0.05020799860358238,0.048979200422763824,0.051660798490047455,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:21,0.8.0 +dyt,torch_compile,forward,speed,ms,BT,total tokens,4096,0.05132799968123436,0.050144001841545105,0.053119998425245285,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:21,0.8.0 +dyt,torch_compile,forward,speed,ms,BT,total tokens,8192,0.05241600051522255,0.05135999992489815,0.05523199960589409,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:21,0.8.0 +dyt,liger_triton,backward,speed,ms,BT,total tokens,1024,0.15270400047302246,0.14713600277900696,0.17948800325393677,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,liger_triton,backward,speed,ms,BT,total tokens,2048,0.1462399959564209,0.13947519659996033,0.15111039876937865,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,liger_triton,backward,speed,ms,BT,total tokens,4096,0.14105600118637085,0.13720320165157318,0.14790399968624116,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,liger_triton,backward,speed,ms,BT,total tokens,8192,0.1454240009188652,0.1402943968772888,0.14958719909191132,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,torch,backward,speed,ms,BT,total tokens,1024,0.0899839997291565,0.08908800035715103,0.09014400094747543,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,torch,backward,speed,ms,BT,total tokens,2048,0.14851200580596924,0.1483840048313141,0.14950400590896606,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,torch,backward,speed,ms,BT,total tokens,4096,0.2734079957008362,0.27238398790359497,0.27452159523963926,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,torch,backward,speed,ms,BT,total tokens,8192,0.5100000202655792,0.5089920163154602,0.5129600167274475,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,torch_compile,backward,speed,ms,BT,total tokens,1024,0.11052799969911575,0.10816640257835389,0.11591040343046188,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,torch_compile,backward,speed,ms,BT,total tokens,2048,0.1157120019197464,0.11267200112342834,0.12094079703092575,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,torch_compile,backward,speed,ms,BT,total tokens,4096,0.21296000480651855,0.2109503984451294,0.21305599808692932,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,torch_compile,backward,speed,ms,BT,total tokens,8192,0.4485119879245758,0.44749439358711246,0.44895999431610106,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:23,0.8.0 +dyt,liger_triton,full,speed,ms,BT,total tokens,1024,0.2593120038509369,0.24898560345172882,0.2673792004585266,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,liger_triton,full,speed,ms,BT,total tokens,2048,0.25860799849033356,0.25122560262680055,0.2658176064491272,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,liger_triton,full,speed,ms,BT,total tokens,4096,0.25887998938560486,0.25459201335906984,0.26941441297531127,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,liger_triton,full,speed,ms,BT,total tokens,8192,0.2571680098772049,0.24769279956817628,0.2678848087787628,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch,full,speed,ms,BT,total tokens,1024,0.1300159990787506,0.12809599936008453,0.13007999956607819,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch,full,speed,ms,BT,total tokens,2048,0.22009600698947906,0.2191168040037155,0.22024319767951966,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch,full,speed,ms,BT,total tokens,4096,0.42691200971603394,0.4259839951992035,0.42710399627685547,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch,full,speed,ms,BT,total tokens,8192,0.8243839740753174,0.8232640027999878,0.8245120048522949,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch_compile,full,speed,ms,BT,total tokens,1024,0.27742399275302887,0.2689344048500061,0.29007360339164734,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch_compile,full,speed,ms,BT,total tokens,2048,0.2751999944448471,0.26848639249801637,0.2838591992855072,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch_compile,full,speed,ms,BT,total tokens,4096,0.2749439924955368,0.26566400527954104,0.2915008008480072,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch_compile,full,speed,ms,BT,total tokens,8192,0.5611199736595154,0.5599040150642395,0.5615168213844299,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,liger_triton,full,memory,MB,BT,total tokens,1024,42.365234375,42.365234375,42.365234375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,liger_triton,full,memory,MB,BT,total tokens,2048,82.365234375,82.365234375,82.365234375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,liger_triton,full,memory,MB,BT,total tokens,4096,162.365234375,162.365234375,162.365234375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,liger_triton,full,memory,MB,BT,total tokens,8192,322.365234375,322.365234375,322.365234375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch,full,memory,MB,BT,total tokens,1024,128.04833984375,128.04833984375,128.04833984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch,full,memory,MB,BT,total tokens,2048,256.04833984375,256.04833984375,256.04833984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch,full,memory,MB,BT,total tokens,4096,512.04833984375,512.04833984375,512.04833984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch,full,memory,MB,BT,total tokens,8192,960.0322265625,960.0322265625,960.0322265625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch_compile,full,memory,MB,BT,total tokens,1024,56.04833984375,56.04833984375,56.04833984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch_compile,full,memory,MB,BT,total tokens,2048,112.04833984375,112.04833984375,112.04833984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch_compile,full,memory,MB,BT,total tokens,4096,224.04833984375,224.04833984375,224.04833984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,torch_compile,full,memory,MB,BT,total tokens,8192,448.04833984375,448.04833984375,448.04833984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:24,0.8.0 +dyt,liger_triton,forward,speed,ms,BT,total tokens,1024,0.021344000473618507,0.020479999482631683,0.02271999977529049,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,liger_triton,forward,speed,ms,BT,total tokens,2048,0.022255999967455864,0.020819200202822685,0.023686399683356285,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,liger_triton,forward,speed,ms,BT,total tokens,4096,0.02457600086927414,0.02454400062561035,0.025702400133013725,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,liger_triton,forward,speed,ms,BT,total tokens,8192,0.04297599941492081,0.042931199073791504,0.04310400038957596,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,torch,forward,speed,ms,BT,total tokens,1024,0.04809600114822388,0.0480320006608963,0.04816000163555145,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,torch,forward,speed,ms,BT,total tokens,2048,0.08086399734020233,0.08083199709653854,0.0809599980711937,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,torch,forward,speed,ms,BT,total tokens,4096,0.17100800573825836,0.17092480063438414,0.17110399901866913,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,torch,forward,speed,ms,BT,total tokens,8192,0.33883199095726013,0.3369919955730438,0.3389759957790375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,torch_compile,forward,speed,ms,BT,total tokens,1024,0.051231998950242996,0.04975999891757965,0.0525440014898777,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,torch_compile,forward,speed,ms,BT,total tokens,2048,0.05183999985456467,0.05084159970283508,0.05305600166320801,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,torch_compile,forward,speed,ms,BT,total tokens,4096,0.0517439991235733,0.0505216009914875,0.053465601056814194,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,torch_compile,forward,speed,ms,BT,total tokens,8192,0.05411199852824211,0.05225599929690361,0.05626240000128746,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:27,0.8.0 +dyt,liger_triton,backward,speed,ms,BT,total tokens,1024,0.17440000176429749,0.1673919975757599,0.22333440482616426,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,backward,speed,ms,BT,total tokens,2048,0.171424001455307,0.16582399606704712,0.17801600694656372,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,backward,speed,ms,BT,total tokens,4096,0.16646400094032288,0.16158079504966735,0.1720512002706528,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,backward,speed,ms,BT,total tokens,8192,0.17254400253295898,0.16660479605197906,0.18364159762859344,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,backward,speed,ms,BT,total tokens,1024,0.10047999769449234,0.09943679720163345,0.10143999755382538,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,backward,speed,ms,BT,total tokens,2048,0.1608159989118576,0.1597247987985611,0.16089600324630737,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,backward,speed,ms,BT,total tokens,4096,0.2980479896068573,0.2969535946846008,0.29902079701423645,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,backward,speed,ms,BT,total tokens,8192,0.5499839782714844,0.5497024178504943,0.550924813747406,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,backward,speed,ms,BT,total tokens,1024,0.12862399965524673,0.12536320090293884,0.137772798538208,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,backward,speed,ms,BT,total tokens,2048,0.12619199603796005,0.12275200337171555,0.13086079955101013,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,backward,speed,ms,BT,total tokens,4096,0.14134399592876434,0.14124800264835358,0.14228479862213136,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,backward,speed,ms,BT,total tokens,8192,0.2621760070323944,0.26204800605773926,0.2622720003128052,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,full,speed,ms,BT,total tokens,1024,0.2898240089416504,0.27986560463905336,0.29681280851364134,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,full,speed,ms,BT,total tokens,2048,0.2879039943218231,0.27981439232826233,0.30645120739936826,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,full,speed,ms,BT,total tokens,4096,0.28915199637413025,0.28048000335693357,0.29864320158958435,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,full,speed,ms,BT,total tokens,8192,0.286624014377594,0.2779007971286774,0.3051328003406525,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,full,speed,ms,BT,total tokens,1024,0.1536639928817749,0.1526080071926117,0.15480959713459014,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,full,speed,ms,BT,total tokens,2048,0.25603199005126953,0.25492479205131535,0.2569792091846466,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,full,speed,ms,BT,total tokens,4096,0.4936159998178482,0.4925119876861572,0.49665920138359065,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,full,speed,ms,BT,total tokens,8192,0.9472640156745911,0.9471871972084046,0.9493375778198242,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,full,speed,ms,BT,total tokens,1024,0.30480000376701355,0.29300480484962466,0.315065586566925,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,full,speed,ms,BT,total tokens,2048,0.3031519949436188,0.29315839409828187,0.31282559633255,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,full,speed,ms,BT,total tokens,4096,0.3072480112314224,0.3012927949428558,0.319513601064682,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,full,speed,ms,BT,total tokens,8192,0.3768959939479828,0.37676799297332764,0.3778879940509796,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,full,memory,MB,BT,total tokens,1024,44.716796875,44.716796875,44.716796875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,full,memory,MB,BT,total tokens,2048,84.716796875,84.716796875,84.716796875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,full,memory,MB,BT,total tokens,4096,164.716796875,164.716796875,164.716796875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_triton,full,memory,MB,BT,total tokens,8192,324.716796875,324.716796875,324.716796875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,full,memory,MB,BT,total tokens,1024,128.07958984375,128.07958984375,128.07958984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,full,memory,MB,BT,total tokens,2048,256.07958984375,256.07958984375,256.07958984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,full,memory,MB,BT,total tokens,4096,512.07958984375,512.07958984375,512.07958984375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch,full,memory,MB,BT,total tokens,8192,960.0634765625,960.0634765625,960.0634765625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,full,memory,MB,BT,total tokens,1024,56.09521484375,56.09521484375,56.09521484375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,full,memory,MB,BT,total tokens,2048,112.09521484375,112.09521484375,112.09521484375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,full,memory,MB,BT,total tokens,4096,224.09521484375,224.09521484375,224.09521484375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,torch_compile,full,memory,MB,BT,total tokens,8192,448.09521484375,448.09521484375,448.09521484375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:30,0.8.0 +dyt,liger_cutile,forward,speed,ms,BT,total tokens,1024,0.01228800043463707,0.012230399996042252,0.012300800159573555,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:34,0.8.0 +dyt,liger_cutile,forward,speed,ms,BT,total tokens,2048,0.016416000202298164,0.01635199971497059,0.01648000068962574,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:34,0.8.0 +dyt,liger_cutile,forward,speed,ms,BT,total tokens,4096,0.02457600086927414,0.02454400062561035,0.024639999493956566,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:34,0.8.0 +dyt,liger_cutile,forward,speed,ms,BT,total tokens,8192,0.040991999208927155,0.04092799872159958,0.04105599969625473,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:34,0.8.0 +dyt,liger_cutile,backward,speed,ms,BT,total tokens,1024,0.11742400005459785,0.11516160070896148,0.13591680079698565,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:36,0.8.0 +dyt,liger_cutile,backward,speed,ms,BT,total tokens,2048,0.11161600053310394,0.11034239828586578,0.11246079802513123,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:36,0.8.0 +dyt,liger_cutile,backward,speed,ms,BT,total tokens,4096,0.11055999994277954,0.1093439981341362,0.11224959939718247,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:36,0.8.0 +dyt,liger_cutile,backward,speed,ms,BT,total tokens,8192,0.11166400089859962,0.11011200100183488,0.1138560026884079,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:36,0.8.0 +dyt,liger_cutile,full,speed,ms,BT,total tokens,1024,0.1685120016336441,0.16653439700603484,0.17010560631752014,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,full,speed,ms,BT,total tokens,2048,0.1679999977350235,0.1664896011352539,0.16960640251636505,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,full,speed,ms,BT,total tokens,4096,0.17187200486660004,0.17027199268341064,0.17824000120162964,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,full,speed,ms,BT,total tokens,8192,0.1863040030002594,0.18534399569034576,0.18643200397491455,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,full,memory,MB,BT,total tokens,1024,42.36376953125,42.36376953125,42.36376953125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,full,memory,MB,BT,total tokens,2048,82.36376953125,82.36376953125,82.36376953125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,full,memory,MB,BT,total tokens,4096,162.36376953125,162.36376953125,162.36376953125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,full,memory,MB,BT,total tokens,8192,322.36376953125,322.36376953125,322.36376953125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": false, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,forward,speed,ms,BT,total tokens,1024,0.01235199999064207,0.01228800043463707,0.014271999709308147,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,forward,speed,ms,BT,total tokens,2048,0.018432000651955605,0.01836800016462803,0.01852799952030182,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,forward,speed,ms,BT,total tokens,4096,0.028704000636935234,0.02860799990594387,0.02876799926161766,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,forward,speed,ms,BT,total tokens,8192,0.047168001532554626,0.04707200080156326,0.04902400076389313,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:37,0.8.0 +dyt,liger_cutile,backward,speed,ms,BT,total tokens,1024,0.13132800161838531,0.13014400005340576,0.1351295977830887,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,backward,speed,ms,BT,total tokens,2048,0.13172800093889236,0.1302720010280609,0.13635200262069702,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,backward,speed,ms,BT,total tokens,4096,0.13036799430847168,0.12887680530548096,0.13207039535045623,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,backward,speed,ms,BT,total tokens,8192,0.1305759996175766,0.1297280013561249,0.13205119967460632,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,full,speed,ms,BT,total tokens,1024,0.18936000019311905,0.1840767949819565,0.2539648115634918,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,full,speed,ms,BT,total tokens,2048,0.18835200369358063,0.18472319543361665,0.19355519711971284,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,full,speed,ms,BT,total tokens,4096,0.18545600026845932,0.18389120399951936,0.18887039721012117,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,full,speed,ms,BT,total tokens,8192,0.20585599541664124,0.2048639953136444,0.20684799551963806,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,full,memory,MB,BT,total tokens,1024,44.73046875,44.73046875,44.73046875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,full,memory,MB,BT,total tokens,2048,84.73046875,84.73046875,84.73046875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,full,memory,MB,BT,total tokens,4096,164.73046875,164.73046875,164.73046875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +dyt,liger_cutile,full,memory,MB,BT,total tokens,8192,324.73046875,324.73046875,324.73046875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""beta"": true, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:38,0.8.0 +fused_linear_cross_entropy,torch,forward,speed,ms,BT,total tokens,1024,0.9861760139465332,0.9847231984138489,0.9871488094329834,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:53,0.8.0 +fused_linear_cross_entropy,torch,forward,speed,ms,BT,total tokens,2048,1.7889280319213867,1.7836223602294923,1.797708821296692,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:53,0.8.0 +fused_linear_cross_entropy,torch,forward,speed,ms,BT,total tokens,4096,4.003872036933899,3.766713619232178,4.24103045463562,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:53,0.8.0 +fused_linear_cross_entropy,torch,forward,speed,ms,BT,total tokens,8192,7.220223903656006,7.220223903656006,7.220223903656006,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:43:53,0.8.0 +fused_linear_cross_entropy,liger_triton,forward,speed,ms,BT,total tokens,1024,119.86128234863281,119.86128234863281,119.86128234863281,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:06,0.8.0 +fused_linear_cross_entropy,liger_triton,forward,speed,ms,BT,total tokens,2048,120.5360336303711,120.5360336303711,120.5360336303711,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:06,0.8.0 +fused_linear_cross_entropy,liger_triton,forward,speed,ms,BT,total tokens,4096,122.66496276855469,122.66496276855469,122.66496276855469,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:06,0.8.0 +fused_linear_cross_entropy,liger_triton,forward,speed,ms,BT,total tokens,8192,128.6666259765625,128.6666259765625,128.6666259765625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:06,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,forward,speed,ms,BT,total tokens,1024,37.05548858642578,37.05548858642578,37.05548858642578,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:15,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,forward,speed,ms,BT,total tokens,2048,37.787681579589844,37.787681579589844,37.787681579589844,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:15,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,forward,speed,ms,BT,total tokens,4096,38.53916931152344,38.53916931152344,38.53916931152344,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:15,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,forward,speed,ms,BT,total tokens,8192,47.0568962097168,47.0568962097168,47.0568962097168,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:15,0.8.0 +fused_linear_cross_entropy,torch,backward,speed,ms,BT,total tokens,1024,2.1914080381393433,2.187891149520874,2.194092845916748,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:24,0.8.0 +fused_linear_cross_entropy,torch,backward,speed,ms,BT,total tokens,2048,3.7611520290374756,3.7593088150024414,3.7629952430725098,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:24,0.8.0 +fused_linear_cross_entropy,torch,backward,speed,ms,BT,total tokens,4096,7.406591892242432,7.406591892242432,7.406591892242432,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:24,0.8.0 +fused_linear_cross_entropy,torch,backward,speed,ms,BT,total tokens,8192,18.40230369567871,18.40230369567871,18.40230369567871,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:24,0.8.0 +fused_linear_cross_entropy,liger_triton,backward,speed,ms,BT,total tokens,1024,1.0541759729385376,1.0458751916885376,1.124953603744507,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:33,0.8.0 +fused_linear_cross_entropy,liger_triton,backward,speed,ms,BT,total tokens,2048,1.0459839701652527,1.0434815645217896,1.0497663974761964,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:33,0.8.0 +fused_linear_cross_entropy,liger_triton,backward,speed,ms,BT,total tokens,4096,1.0572800040245056,1.05557119846344,1.058367943763733,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:33,0.8.0 +fused_linear_cross_entropy,liger_triton,backward,speed,ms,BT,total tokens,8192,1.0839040279388428,1.0833920240402222,1.0850432157516479,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:33,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,backward,speed,ms,BT,total tokens,1024,1.0526400208473206,1.0430784225463867,1.120851182937622,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:42,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,backward,speed,ms,BT,total tokens,2048,1.0475200414657593,1.0438208103179931,1.0491904020309448,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:42,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,backward,speed,ms,BT,total tokens,4096,1.0685759782791138,1.0582208156585693,1.1299583673477174,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:42,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,backward,speed,ms,BT,total tokens,8192,1.1295360326766968,1.1220863580703735,1.1595520019531251,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:42,0.8.0 +fused_linear_cross_entropy,torch,full,speed,ms,BT,total tokens,1024,3.4397759437561035,3.3334656238555906,3.4465152263641357,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:51,0.8.0 +fused_linear_cross_entropy,torch,full,speed,ms,BT,total tokens,2048,6.611999988555908,6.611999988555908,6.611999988555908,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:51,0.8.0 +fused_linear_cross_entropy,torch,full,speed,ms,BT,total tokens,4096,13.525952339172363,13.525952339172363,13.525952339172363,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:51,0.8.0 +fused_linear_cross_entropy,torch,full,speed,ms,BT,total tokens,8192,25.48534393310547,25.48534393310547,25.48534393310547,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:44:51,0.8.0 +fused_linear_cross_entropy,liger_triton,full,speed,ms,BT,total tokens,1024,120.97638702392578,120.97638702392578,120.97638702392578,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:04,0.8.0 +fused_linear_cross_entropy,liger_triton,full,speed,ms,BT,total tokens,2048,121.56419372558594,121.56419372558594,121.56419372558594,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:04,0.8.0 +fused_linear_cross_entropy,liger_triton,full,speed,ms,BT,total tokens,4096,123.82313537597656,123.82313537597656,123.82313537597656,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:04,0.8.0 +fused_linear_cross_entropy,liger_triton,full,speed,ms,BT,total tokens,8192,132.9100799560547,132.9100799560547,132.9100799560547,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:04,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,full,speed,ms,BT,total tokens,1024,38.24332809448242,38.24332809448242,38.24332809448242,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:14,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,full,speed,ms,BT,total tokens,2048,39.10553741455078,39.10553741455078,39.10553741455078,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:14,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,full,speed,ms,BT,total tokens,4096,39.636959075927734,39.636959075927734,39.636959075927734,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:14,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,full,speed,ms,BT,total tokens,8192,48.3389778137207,48.3389778137207,48.3389778137207,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:14,0.8.0 +fused_linear_cross_entropy,torch,full,memory,MB,BT,total tokens,1024,3595.00927734375,3595.00927734375,3595.00927734375,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:23,0.8.0 +fused_linear_cross_entropy,torch,full,memory,MB,BT,total tokens,2048,4121.01708984375,4121.01708984375,4121.01708984375,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:23,0.8.0 +fused_linear_cross_entropy,torch,full,memory,MB,BT,total tokens,4096,5170.03271484375,5170.03271484375,5170.03271484375,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:23,0.8.0 +fused_linear_cross_entropy,torch,full,memory,MB,BT,total tokens,8192,8208.064453125,8208.064453125,8208.064453125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:23,0.8.0 +fused_linear_cross_entropy,liger_triton,full,memory,MB,BT,total tokens,1024,6107.8408203125,6107.8408203125,6107.8408203125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:36,0.8.0 +fused_linear_cross_entropy,liger_triton,full,memory,MB,BT,total tokens,2048,6139.681640625,6139.681640625,6139.681640625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:36,0.8.0 +fused_linear_cross_entropy,liger_triton,full,memory,MB,BT,total tokens,4096,6203.36328125,6203.36328125,6203.36328125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:36,0.8.0 +fused_linear_cross_entropy,liger_triton,full,memory,MB,BT,total tokens,8192,6330.7265625,6330.7265625,6330.7265625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:36,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,full,memory,MB,BT,total tokens,1024,5105.84130859375,5105.84130859375,5105.84130859375,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:47,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,full,memory,MB,BT,total tokens,2048,5137.68212890625,5137.68212890625,5137.68212890625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:47,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,full,memory,MB,BT,total tokens,4096,5201.36376953125,5201.36376953125,5201.36376953125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:47,0.8.0 +fused_linear_cross_entropy,liger-fp32-accum,full,memory,MB,BT,total tokens,8192,5328.72705078125,5328.72705078125,5328.72705078125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:45:47,0.8.0 +fused_linear_cross_entropy,liger_cutile,forward,speed,ms,BT,total tokens,1024,34.30713653564453,34.30713653564453,34.30713653564453,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:46:11,0.8.0 +fused_linear_cross_entropy,liger_cutile,forward,speed,ms,BT,total tokens,2048,35.989566802978516,35.989566802978516,35.989566802978516,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:46:11,0.8.0 +fused_linear_cross_entropy,liger_cutile,forward,speed,ms,BT,total tokens,4096,40.74291229248047,40.74291229248047,40.74291229248047,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:46:11,0.8.0 +fused_linear_cross_entropy,liger_cutile,forward,speed,ms,BT,total tokens,8192,54.19011306762695,54.19011306762695,54.19011306762695,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:46:11,0.8.0 +fused_linear_cross_entropy,liger_cutile,backward,speed,ms,BT,total tokens,1024,2.1478880643844604,2.1469696521759034,2.1487808227539062,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:46:40,0.8.0 +fused_linear_cross_entropy,liger_cutile,backward,speed,ms,BT,total tokens,2048,2.160640001296997,2.1605952739715577,2.1630911350250246,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:46:40,0.8.0 +fused_linear_cross_entropy,liger_cutile,backward,speed,ms,BT,total tokens,4096,2.302112102508545,2.292800045013428,2.3038400173187257,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:46:40,0.8.0 +fused_linear_cross_entropy,liger_cutile,backward,speed,ms,BT,total tokens,8192,2.678256034851074,2.570815992355347,2.6833088397979736,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:46:40,0.8.0 +fused_linear_cross_entropy,liger_cutile,full,speed,ms,BT,total tokens,1024,35.64336013793945,35.64336013793945,35.64336013793945,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:08,0.8.0 +fused_linear_cross_entropy,liger_cutile,full,speed,ms,BT,total tokens,2048,38.16342544555664,38.16342544555664,38.16342544555664,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:08,0.8.0 +fused_linear_cross_entropy,liger_cutile,full,speed,ms,BT,total tokens,4096,42.44377517700195,42.44377517700195,42.44377517700195,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:08,0.8.0 +fused_linear_cross_entropy,liger_cutile,full,speed,ms,BT,total tokens,8192,57.354270935058594,57.354270935058594,57.354270935058594,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:08,0.8.0 +fused_linear_cross_entropy,liger_cutile,full,memory,MB,BT,total tokens,1024,4104.0087890625,4104.0087890625,4104.0087890625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:39,0.8.0 +fused_linear_cross_entropy,liger_cutile,full,memory,MB,BT,total tokens,2048,4136.0166015625,4136.0166015625,4136.0166015625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:39,0.8.0 +fused_linear_cross_entropy,liger_cutile,full,memory,MB,BT,total tokens,4096,4200.0322265625,4200.0322265625,4200.0322265625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:39,0.8.0 +fused_linear_cross_entropy,liger_cutile,full,memory,MB,BT,total tokens,8192,4328.0634765625,4328.0634765625,4328.0634765625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:39,0.8.0 +poly_norm,torch,forward,speed,ms,T,Sequence length,1024,0.1823039948940277,0.18223999440670013,0.18426239490509033,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,torch,forward,speed,ms,T,Sequence length,2048,0.30617600679397583,0.305184006690979,0.3072640001773834,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,torch,forward,speed,ms,T,Sequence length,4096,0.622655987739563,0.6213696241378784,0.623583972454071,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,torch,forward,speed,ms,T,Sequence length,8192,1.1883839964866638,1.1878848314285277,1.1888064384460448,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,liger_triton,forward,speed,ms,T,Sequence length,1024,0.014976000413298607,0.013894399628043175,0.017184000462293625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,liger_triton,forward,speed,ms,T,Sequence length,2048,0.016383999958634377,0.016287999227643013,0.01648000068962574,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,liger_triton,forward,speed,ms,T,Sequence length,4096,0.026655999943614006,0.026559999212622643,0.02672000043094158,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,liger_triton,forward,speed,ms,T,Sequence length,8192,0.04505600035190582,0.043168000876903534,0.0451200008392334,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,torch,backward,speed,ms,T,Sequence length,1024,0.40143999457359314,0.4002879858016968,0.46748799085617065,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,torch,backward,speed,ms,T,Sequence length,2048,0.6995199918746948,0.6987776160240173,0.7013952255249023,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,torch,backward,speed,ms,T,Sequence length,4096,1.324031949043274,1.3232959747314452,1.324083161354065,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,torch,backward,speed,ms,T,Sequence length,8192,2.6091198921203613,2.608582305908203,2.6097727775573727,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:55,0.8.0 +poly_norm,liger_triton,backward,speed,ms,T,Sequence length,1024,0.0907519981265068,0.08938240259885788,0.09372799694538117,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,backward,speed,ms,T,Sequence length,2048,0.09107200056314468,0.09043200314044952,0.09391999989748001,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,backward,speed,ms,T,Sequence length,4096,0.09328000247478485,0.09124480187892914,0.09528320282697678,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,backward,speed,ms,T,Sequence length,8192,0.131071999669075,0.13011839985847473,0.13140479922294618,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,torch,full,speed,ms,T,Sequence length,1024,0.7730879783630371,0.7701504111289978,1.559270429611206,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,torch,full,speed,ms,T,Sequence length,2048,1.0163039565086365,1.0147199630737305,1.0177600383758545,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,torch,full,speed,ms,T,Sequence length,4096,1.9655359983444214,1.9635263919830321,1.9687488079071045,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,torch,full,speed,ms,T,Sequence length,8192,3.859871983528137,3.8595264434814456,3.8602175235748293,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,full,speed,ms,T,Sequence length,1024,0.18768000602722168,0.18467840254306794,0.20294400155544282,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,full,speed,ms,T,Sequence length,2048,0.1844480037689209,0.18095360696315765,0.19294719994068144,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,full,speed,ms,T,Sequence length,4096,0.18374399840831757,0.18049280047416688,0.1942272037267685,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,full,speed,ms,T,Sequence length,8192,0.23865599930286407,0.2376127988100052,0.23960960507392884,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,torch,full,memory,MB,T,Sequence length,1024,256.013671875,256.013671875,256.013671875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,torch,full,memory,MB,T,Sequence length,2048,512.025390625,512.025390625,512.025390625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,torch,full,memory,MB,T,Sequence length,4096,1024.048828125,1024.048828125,1024.048828125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,torch,full,memory,MB,T,Sequence length,8192,2048.095703125,2048.095703125,2048.095703125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,full,memory,MB,T,Sequence length,1024,32.017578125,32.017578125,32.017578125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,full,memory,MB,T,Sequence length,2048,64.029296875,64.029296875,64.029296875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,full,memory,MB,T,Sequence length,4096,128.052734375,128.052734375,128.052734375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_triton,full,memory,MB,T,Sequence length,8192,256.099609375,256.099609375,256.099609375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:47:56,0.8.0 +poly_norm,liger_cutile,forward,speed,ms,T,Sequence length,1024,0.010239999741315842,0.01017600018531084,0.01027199998497963,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,forward,speed,ms,T,Sequence length,2048,0.014336000196635723,0.014271999709308147,0.014399999752640724,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,forward,speed,ms,T,Sequence length,4096,0.020479999482631683,0.02038400061428547,0.02051199972629547,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,forward,speed,ms,T,Sequence length,8192,0.032735999673604965,0.03267199918627739,0.03283200040459633,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,backward,speed,ms,T,Sequence length,1024,0.08102399855852127,0.07916799932718277,0.08453120291233063,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,backward,speed,ms,T,Sequence length,2048,0.07900799810886383,0.07809919863939285,0.08090239763259888,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,backward,speed,ms,T,Sequence length,4096,0.08022399991750717,0.07945600152015686,0.08154240250587463,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,backward,speed,ms,T,Sequence length,8192,0.10140799731016159,0.10134399682283401,0.10147199779748917,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,full,speed,ms,T,Sequence length,1024,0.14368000626564026,0.14158080220222474,0.14630400240421296,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,full,speed,ms,T,Sequence length,2048,0.1422719955444336,0.14011520147323608,0.14522880017757417,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,full,speed,ms,T,Sequence length,4096,0.14284799993038177,0.1412736028432846,0.15346559882164001,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,full,speed,ms,T,Sequence length,8192,0.20073600113391876,0.2006720006465912,0.2009088009595871,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,full,memory,MB,T,Sequence length,1024,40.013671875,40.013671875,40.013671875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,full,memory,MB,T,Sequence length,2048,80.025390625,80.025390625,80.025390625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,full,memory,MB,T,Sequence length,4096,160.048828125,160.048828125,160.048828125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +poly_norm,liger_cutile,full,memory,MB,T,Sequence length,8192,320.095703125,320.095703125,320.095703125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:00,0.8.0 +rms_norm,huggingface,forward,speed,ms,T,Sequence length,1024,0.06966400146484375,0.06960000097751617,0.06966400146484375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,huggingface,forward,speed,ms,T,Sequence length,2048,0.10662399977445602,0.10649599879980087,0.10851199924945831,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,huggingface,forward,speed,ms,T,Sequence length,4096,0.20892800390720367,0.2088640034198761,0.2099200040102005,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,huggingface,forward,speed,ms,T,Sequence length,8192,0.40748798847198486,0.4065984070301056,0.4076095938682556,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,liger_triton,forward,speed,ms,T,Sequence length,1024,0.016896000131964684,0.01592320017516613,0.01847040057182312,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,liger_triton,forward,speed,ms,T,Sequence length,2048,0.017184000462293625,0.01669120043516159,0.018252799287438393,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,liger_triton,forward,speed,ms,T,Sequence length,4096,0.020479999482631683,0.020320000126957893,0.02051199972629547,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,liger_triton,forward,speed,ms,T,Sequence length,8192,0.030719999223947525,0.0306560005992651,0.030751999467611313,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,huggingface,backward,speed,ms,T,Sequence length,1024,0.13529600203037262,0.13514239490032195,0.1371392011642456,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,huggingface,backward,speed,ms,T,Sequence length,2048,0.23203200101852417,0.2314240038394928,0.23260800540447235,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,huggingface,backward,speed,ms,T,Sequence length,4096,0.43828800320625305,0.43713921308517456,0.4402943909168243,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,huggingface,backward,speed,ms,T,Sequence length,8192,0.8581439852714539,0.857088029384613,0.8582080006599426,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:04,0.8.0 +rms_norm,liger_triton,backward,speed,ms,T,Sequence length,1024,0.0820159986615181,0.08064000308513641,0.08469119966030121,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,backward,speed,ms,T,Sequence length,2048,0.08099199831485748,0.07975680083036422,0.08236800134181976,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,backward,speed,ms,T,Sequence length,4096,0.08222400024533272,0.08106879889965057,0.08299519866704941,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,backward,speed,ms,T,Sequence length,8192,0.09120000153779984,0.0910400003194809,0.09312000125646591,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,huggingface,full,speed,ms,T,Sequence length,1024,0.20787200331687927,0.20690560042858125,0.2088959962129593,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,huggingface,full,speed,ms,T,Sequence length,2048,0.35017600655555725,0.3491840064525604,0.35120640993118285,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,huggingface,full,speed,ms,T,Sequence length,4096,0.6748160123825073,0.6737279891967773,0.6749439835548401,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,huggingface,full,speed,ms,T,Sequence length,8192,1.3282560110092163,1.3281728267669677,1.3292415618896483,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,full,speed,ms,T,Sequence length,1024,0.18281599879264832,0.1813376009464264,0.18570880591869354,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,full,speed,ms,T,Sequence length,2048,0.1796640008687973,0.1773120015859604,0.1831039935350418,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,full,speed,ms,T,Sequence length,4096,0.17924799770116806,0.17640320062637327,0.18366080522537231,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,full,speed,ms,T,Sequence length,8192,0.18636800348758698,0.18522879779338836,0.1879936009645462,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,huggingface,full,memory,MB,T,Sequence length,1024,152.03515625,152.03515625,152.03515625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,huggingface,full,memory,MB,T,Sequence length,2048,304.0390625,304.0390625,304.0390625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,huggingface,full,memory,MB,T,Sequence length,4096,608.046875,608.046875,608.046875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,huggingface,full,memory,MB,T,Sequence length,8192,1216.0625,1216.0625,1216.0625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,full,memory,MB,T,Sequence length,1024,34.36328125,34.36328125,34.36328125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,full,memory,MB,T,Sequence length,2048,66.3671875,66.3671875,66.3671875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,full,memory,MB,T,Sequence length,4096,130.375,130.375,130.375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_triton,full,memory,MB,T,Sequence length,8192,258.390625,258.390625,258.390625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:05,0.8.0 +rms_norm,liger_cutile,forward,speed,ms,T,Sequence length,1024,0.012319999746978283,0.012230399996042252,0.012384000234305859,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:09,0.8.0 +rms_norm,liger_cutile,forward,speed,ms,T,Sequence length,2048,0.016416000202298164,0.01635199971497059,0.01648000068962574,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:09,0.8.0 +rms_norm,liger_cutile,forward,speed,ms,T,Sequence length,4096,0.02457600086927414,0.024512000381946564,0.024671999737620354,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:09,0.8.0 +rms_norm,liger_cutile,forward,speed,ms,T,Sequence length,8192,0.040991999208927155,0.04092799872159958,0.04102399945259094,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:09,0.8.0 +rms_norm,liger_cutile,backward,speed,ms,T,Sequence length,1024,0.078015998005867,0.07410559803247452,0.08458240330219269,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:09,0.8.0 +rms_norm,liger_cutile,backward,speed,ms,T,Sequence length,2048,0.06995200365781784,0.06801280081272125,0.07229439914226532,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:09,0.8.0 +rms_norm,liger_cutile,backward,speed,ms,T,Sequence length,4096,0.07001599669456482,0.06842879951000214,0.07150080054998398,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:09,0.8.0 +rms_norm,liger_cutile,backward,speed,ms,T,Sequence length,8192,0.0666080005466938,0.06649599969387054,0.06860800087451935,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:09,0.8.0 +rms_norm,liger_cutile,full,speed,ms,T,Sequence length,1024,0.40137600898742676,0.1788991987705231,0.49897601008415227,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:10,0.8.0 +rms_norm,liger_cutile,full,speed,ms,T,Sequence length,2048,0.1568320021033287,0.15303680300712585,0.16611839830875397,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:10,0.8.0 +rms_norm,liger_cutile,full,speed,ms,T,Sequence length,4096,0.12361600250005722,0.12227199971675873,0.12668800354003906,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:10,0.8.0 +rms_norm,liger_cutile,full,speed,ms,T,Sequence length,8192,0.17102400213479996,0.1709440052509308,0.1719936043024063,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:10,0.8.0 +rms_norm,liger_cutile,full,memory,MB,T,Sequence length,1024,34.36328125,34.36328125,34.36328125,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:10,0.8.0 +rms_norm,liger_cutile,full,memory,MB,T,Sequence length,2048,66.3671875,66.3671875,66.3671875,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:10,0.8.0 +rms_norm,liger_cutile,full,memory,MB,T,Sequence length,4096,130.375,130.375,130.375,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:10,0.8.0 +rms_norm,liger_cutile,full,memory,MB,T,Sequence length,8192,258.390625,258.390625,258.390625,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""eps"": 1e-06, ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:10,0.8.0 +softmax,torch,forward,speed,ms,BT,B*T,1024,0.02051199972629547,0.02043519951403141,0.02054399996995926,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,torch,forward,speed,ms,BT,B*T,2048,0.03280000016093254,0.03272319957613945,0.03286400064826012,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,torch,forward,speed,ms,BT,B*T,4096,0.05536000058054924,0.05526399984955788,0.057312000542879105,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,torch,forward,speed,ms,BT,B*T,8192,0.10444799810647964,0.1043199971318245,0.10454399883747101,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,liger_triton,forward,speed,ms,BT,B*T,1024,0.010239999741315842,0.01017600018531084,0.010304000228643417,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:44,0.8.0 +softmax,liger_triton,forward,speed,ms,BT,B*T,2048,0.01228800043463707,0.012191999703645706,0.012384000234305859,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:44,0.8.0 +softmax,liger_triton,forward,speed,ms,BT,B*T,4096,0.018464000895619392,0.018400000408291817,0.018559999763965607,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:44,0.8.0 +softmax,liger_triton,forward,speed,ms,BT,B*T,8192,0.028704000636935234,0.02864000014960766,0.02876799926161766,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:44,0.8.0 +softmax,torch,backward,speed,ms,BT,B*T,1024,0.024063999764621258,0.021375999599695206,0.04108800031244756,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,torch,backward,speed,ms,BT,B*T,2048,0.022463999688625336,0.021555200219154358,0.023455999791622162,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,torch,backward,speed,ms,BT,B*T,4096,0.03884800150990486,0.037856001406908035,0.039903998374938965,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,torch,backward,speed,ms,BT,B*T,8192,0.07465600222349167,0.0737600028514862,0.07478400319814682,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,liger_triton,backward,speed,ms,BT,B*T,1024,0.05273600108921528,0.051731199771165845,0.05390719994902611,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:44,0.8.0 +softmax,liger_triton,backward,speed,ms,BT,B*T,2048,0.05448000133037567,0.05331199988722801,0.05536000058054924,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:44,0.8.0 +softmax,liger_triton,backward,speed,ms,BT,B*T,4096,0.05152000114321709,0.0505216009914875,0.05271679908037186,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:44,0.8.0 +softmax,liger_triton,backward,speed,ms,BT,B*T,8192,0.05422399938106537,0.05320959910750389,0.05532800033688545,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:44,0.8.0 +softmax,torch,full,speed,ms,BT,B*T,1024,0.04095999896526337,0.04089599847793579,0.04198399931192398,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,torch,full,speed,ms,BT,B*T,2048,0.06457599997520447,0.06448639929294586,0.06547199934720993,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,torch,full,speed,ms,BT,B*T,4096,0.11984000355005264,0.11974400281906128,0.11993599683046341,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,torch,full,speed,ms,BT,B*T,8192,0.2385919988155365,0.23683840036392212,0.23862400650978088,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_triton,full,speed,ms,BT,B*T,1024,0.12308799847960472,0.12133120000362396,0.1246144026517868,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:45,0.8.0 +softmax,liger_triton,full,speed,ms,BT,B*T,2048,0.1236800029873848,0.12156800180673599,0.12519680261611937,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:45,0.8.0 +softmax,liger_triton,full,speed,ms,BT,B*T,4096,0.12054400146007538,0.11802239716053009,0.12197119742631912,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:45,0.8.0 +softmax,liger_triton,full,speed,ms,BT,B*T,8192,0.12190399691462517,0.12185599654912949,0.12390399724245071,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:45,0.8.0 +softmax,torch,full,memory,MB,BT,B*T,1024,48.0,48.0,48.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,torch,full,memory,MB,BT,B*T,2048,96.0,96.0,96.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,torch,full,memory,MB,BT,B*T,4096,192.0,192.0,192.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,torch,full,memory,MB,BT,B*T,8192,384.0,384.0,384.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_triton,full,memory,MB,BT,B*T,1024,40.0,40.0,40.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:45,0.8.0 +softmax,liger_triton,full,memory,MB,BT,B*T,2048,80.0,80.0,80.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:45,0.8.0 +softmax,liger_triton,full,memory,MB,BT,B*T,4096,160.0,160.0,160.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:45,0.8.0 +softmax,liger_triton,full,memory,MB,BT,B*T,8192,320.0,320.0,320.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:45,0.8.0 +softmax,liger_cutile,forward,speed,ms,BT,B*T,1024,0.010239999741315842,0.01017600018531084,0.01033599954098463,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,liger_cutile,forward,speed,ms,BT,B*T,2048,0.01228800043463707,0.012223999947309494,0.012384000234305859,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,liger_cutile,forward,speed,ms,BT,B*T,4096,0.016383999958634377,0.01635199971497059,0.01648000068962574,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,liger_cutile,forward,speed,ms,BT,B*T,8192,0.026655999943614006,0.026559999212622643,0.02672000043094158,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:48,0.8.0 +softmax,liger_cutile,backward,speed,ms,BT,B*T,1024,0.05145600065588951,0.04972799867391586,0.052319999784231186,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,backward,speed,ms,BT,B*T,2048,0.0530879981815815,0.052032001316547394,0.054310400038957596,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,backward,speed,ms,BT,B*T,4096,0.04947200044989586,0.04825599864125252,0.05110400170087814,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,backward,speed,ms,BT,B*T,8192,0.04982399940490723,0.04875519871711731,0.05112960040569306,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,full,speed,ms,BT,B*T,1024,0.09587199985980988,0.09455999732017517,0.09891200065612793,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,full,speed,ms,BT,B*T,2048,0.09380799904465675,0.09238400310277939,0.09596800059080124,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,full,speed,ms,BT,B*T,4096,0.09497600048780441,0.09315840154886246,0.09697920083999634,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,full,speed,ms,BT,B*T,8192,0.12383999675512314,0.12183679640293121,0.1239359974861145,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,full,memory,MB,BT,B*T,1024,40.0,40.0,40.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,full,memory,MB,BT,B*T,2048,80.0,80.0,80.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,full,memory,MB,BT,B*T,4096,160.0,160.0,160.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +softmax,liger_cutile,full,memory,MB,BT,B*T,8192,320.0,320.0,320.0,"{""hidden_size"": 4096, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-24 09:21:49,0.8.0 +swiglu,huggingface,forward,speed,ms,T,total tokens,1024,0.2969439923763275,0.29494398832321167,0.2980160117149353,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:26,0.8.0 +swiglu,huggingface,forward,speed,ms,T,total tokens,2048,0.5488480031490326,0.5479360222816467,0.5490880012512207,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:26,0.8.0 +swiglu,huggingface,forward,speed,ms,T,total tokens,4096,1.1064800024032593,1.1043519973754883,1.1075071573257447,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:26,0.8.0 +swiglu,huggingface,forward,speed,ms,T,total tokens,8192,2.147328019142151,2.14650239944458,2.1481920719146728,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:26,0.8.0 +swiglu,liger_triton,forward,speed,ms,T,total tokens,1024,0.2847200036048889,0.2836799919605255,0.28570240139961245,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:30,0.8.0 +swiglu,liger_triton,forward,speed,ms,T,total tokens,2048,0.5273280143737793,0.5253440141677856,0.528115200996399,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:30,0.8.0 +swiglu,liger_triton,forward,speed,ms,T,total tokens,4096,1.0588480234146118,1.0578943729400634,1.0612735748291016,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:30,0.8.0 +swiglu,liger_triton,forward,speed,ms,T,total tokens,8192,2.0500799417495728,2.0482624053955076,2.0526655673980714,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:30,0.8.0 +swiglu,huggingface,backward,speed,ms,T,total tokens,1024,0.7332479953765869,0.7321536183357239,0.7342912077903747,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:33,0.8.0 +swiglu,huggingface,backward,speed,ms,T,total tokens,2048,1.2575680017471313,1.256646418571472,1.261734414100647,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:33,0.8.0 +swiglu,huggingface,backward,speed,ms,T,total tokens,4096,2.374783992767334,2.374726343154907,2.3759551525115965,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:33,0.8.0 +swiglu,huggingface,backward,speed,ms,T,total tokens,8192,5.803583860397339,5.554463863372803,6.052703857421875,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:33,0.8.0 +swiglu,liger_triton,backward,speed,ms,T,total tokens,1024,0.6966880261898041,0.6932799816131592,0.7018495917320251,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:36,0.8.0 +swiglu,liger_triton,backward,speed,ms,T,total tokens,2048,1.2216639518737793,1.2188224077224732,1.2225728273391723,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:36,0.8.0 +swiglu,liger_triton,backward,speed,ms,T,total tokens,4096,2.3131680488586426,2.309126424789429,2.316057634353638,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:36,0.8.0 +swiglu,liger_triton,backward,speed,ms,T,total tokens,8192,4.690448045730591,4.689536094665527,4.691359996795654,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:36,0.8.0 +swiglu,huggingface,full,speed,ms,T,total tokens,1024,1.0556319952011108,1.0547135829925538,1.0564351558685303,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:39,0.8.0 +swiglu,huggingface,full,speed,ms,T,total tokens,2048,1.8378880023956299,1.8357312440872193,1.8379903793334962,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:39,0.8.0 +swiglu,huggingface,full,speed,ms,T,total tokens,4096,3.519424080848694,3.516371297836304,3.5224768638610837,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:39,0.8.0 +swiglu,huggingface,full,speed,ms,T,total tokens,8192,8.12838363647461,8.12838363647461,8.12838363647461,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:39,0.8.0 +swiglu,liger_triton,full,speed,ms,T,total tokens,1024,1.018880009651184,1.0183744430541992,1.0209664106369019,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:43,0.8.0 +swiglu,liger_triton,full,speed,ms,T,total tokens,2048,1.7787200212478638,1.776697564125061,1.778764772415161,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:43,0.8.0 +swiglu,liger_triton,full,speed,ms,T,total tokens,4096,3.4206080436706543,3.397587251663208,3.443628835678101,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:43,0.8.0 +swiglu,liger_triton,full,speed,ms,T,total tokens,8192,7.3051838874816895,7.3051838874816895,7.3051838874816895,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:43,0.8.0 +swiglu,huggingface,full,memory,MB,T,total tokens,1024,1056.0,1056.0,1056.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:46,0.8.0 +swiglu,huggingface,full,memory,MB,T,total tokens,2048,1264.0,1264.0,1264.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:46,0.8.0 +swiglu,huggingface,full,memory,MB,T,total tokens,4096,1680.0,1680.0,1680.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:46,0.8.0 +swiglu,huggingface,full,memory,MB,T,total tokens,8192,2560.0,2560.0,2560.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:46,0.8.0 +swiglu,liger_triton,full,memory,MB,T,total tokens,1024,992.0,992.0,992.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:49,0.8.0 +swiglu,liger_triton,full,memory,MB,T,total tokens,2048,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:49,0.8.0 +swiglu,liger_triton,full,memory,MB,T,total tokens,4096,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:49,0.8.0 +swiglu,liger_triton,full,memory,MB,T,total tokens,8192,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:49,0.8.0 +swiglu,huggingface,forward,memory,MB,T,total tokens,1024,528.0,528.0,528.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:52,0.8.0 +swiglu,huggingface,forward,memory,MB,T,total tokens,2048,656.0,656.0,656.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:52,0.8.0 +swiglu,huggingface,forward,memory,MB,T,total tokens,4096,912.0,912.0,912.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:52,0.8.0 +swiglu,huggingface,forward,memory,MB,T,total tokens,8192,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:52,0.8.0 +swiglu,liger_triton,forward,memory,MB,T,total tokens,1024,500.0,500.0,500.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:55,0.8.0 +swiglu,liger_triton,forward,memory,MB,T,total tokens,2048,600.0,600.0,600.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:55,0.8.0 +swiglu,liger_triton,forward,memory,MB,T,total tokens,4096,800.0,800.0,800.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:55,0.8.0 +swiglu,liger_triton,forward,memory,MB,T,total tokens,8192,1200.0,1200.0,1200.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:55,0.8.0 +swiglu,huggingface,backward,memory,MB,T,total tokens,1024,1056.0,1056.0,1056.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:59,0.8.0 +swiglu,huggingface,backward,memory,MB,T,total tokens,2048,1264.0,1264.0,1264.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:59,0.8.0 +swiglu,huggingface,backward,memory,MB,T,total tokens,4096,1680.0,1680.0,1680.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:59,0.8.0 +swiglu,huggingface,backward,memory,MB,T,total tokens,8192,2560.0,2560.0,2560.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:48:59,0.8.0 +swiglu,liger_triton,backward,memory,MB,T,total tokens,1024,992.0,992.0,992.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:02,0.8.0 +swiglu,liger_triton,backward,memory,MB,T,total tokens,2048,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:02,0.8.0 +swiglu,liger_triton,backward,memory,MB,T,total tokens,4096,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:02,0.8.0 +swiglu,liger_triton,backward,memory,MB,T,total tokens,8192,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:02,0.8.0 +swiglu,liger_cutile,forward,speed,ms,T,total tokens,1024,0.28572800755500793,0.2856000065803528,0.2861887991428375,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:13,0.8.0 +swiglu,liger_cutile,forward,speed,ms,T,total tokens,2048,0.5252479910850525,0.5232383847236634,0.5253440141677856,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:13,0.8.0 +swiglu,liger_cutile,forward,speed,ms,T,total tokens,4096,1.0609920024871826,1.057376003265381,1.061900806427002,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:13,0.8.0 +swiglu,liger_cutile,forward,speed,ms,T,total tokens,8192,2.049008011817932,2.047404718399048,2.050713634490967,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:13,0.8.0 +swiglu,liger_cutile,backward,speed,ms,T,total tokens,1024,0.7004159986972809,0.6958080172538756,0.7042623877525329,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:19,0.8.0 +swiglu,liger_cutile,backward,speed,ms,T,total tokens,2048,1.2256959676742554,1.2241023778915405,1.2275520324707032,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:19,0.8.0 +swiglu,liger_cutile,backward,speed,ms,T,total tokens,4096,2.31112003326416,2.308256006240845,2.314009618759155,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:19,0.8.0 +swiglu,liger_cutile,backward,speed,ms,T,total tokens,8192,5.683743953704834,5.665638256072999,5.70184965133667,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:19,0.8.0 +swiglu,liger_cutile,full,speed,ms,T,total tokens,1024,1.0230720043182373,1.0225855827331543,1.025548768043518,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:26,0.8.0 +swiglu,liger_cutile,full,speed,ms,T,total tokens,2048,1.7765120267868042,1.7745471715927124,1.7779135942459108,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:26,0.8.0 +swiglu,liger_cutile,full,speed,ms,T,total tokens,4096,3.4032639265060425,3.40295672416687,3.403571128845215,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:26,0.8.0 +swiglu,liger_cutile,full,speed,ms,T,total tokens,8192,8.091615676879883,8.091615676879883,8.091615676879883,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:26,0.8.0 +swiglu,liger_cutile,full,memory,MB,T,total tokens,1024,992.0,992.0,992.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:32,0.8.0 +swiglu,liger_cutile,full,memory,MB,T,total tokens,2048,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:32,0.8.0 +swiglu,liger_cutile,full,memory,MB,T,total tokens,4096,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:32,0.8.0 +swiglu,liger_cutile,full,memory,MB,T,total tokens,8192,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:32,0.8.0 +swiglu,liger_cutile,forward,memory,MB,T,total tokens,1024,500.0,500.0,500.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:37,0.8.0 +swiglu,liger_cutile,forward,memory,MB,T,total tokens,2048,600.0,600.0,600.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:37,0.8.0 +swiglu,liger_cutile,forward,memory,MB,T,total tokens,4096,800.0,800.0,800.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:37,0.8.0 +swiglu,liger_cutile,forward,memory,MB,T,total tokens,8192,1200.0,1200.0,1200.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:37,0.8.0 +swiglu,liger_cutile,backward,memory,MB,T,total tokens,1024,992.0,992.0,992.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:43,0.8.0 +swiglu,liger_cutile,backward,memory,MB,T,total tokens,2048,1136.0,1136.0,1136.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:43,0.8.0 +swiglu,liger_cutile,backward,memory,MB,T,total tokens,4096,1424.0,1424.0,1424.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:43,0.8.0 +swiglu,liger_cutile,backward,memory,MB,T,total tokens,8192,2000.0,2000.0,2000.0,"{""hidden_size"": 4096, ""intermediate_size"": 14336, ""dtype"": ""torch.bfloat16"", ""bsz"": 1, ""hidden_act"": ""silu"", ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:49:43,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,forward,speed,ms,B,batch size,1,18.048032760620117,18.048032760620117,18.048032760620117,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:13,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,forward,speed,ms,B,batch size,2,35.88399887084961,35.88399887084961,35.88399887084961,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:13,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,forward,speed,ms,B,batch size,4,71.44035339355469,71.44035339355469,71.44035339355469,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:13,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,forward,speed,ms,B,batch size,8,142.67996215820312,142.67996215820312,142.67996215820312,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:13,0.8.0 +fused_linear_grpo_loss_token,torch,forward,speed,ms,B,batch size,1,3.098655939102173,3.093049621582031,3.117702341079712,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:31,0.8.0 +fused_linear_grpo_loss_token,torch,forward,speed,ms,B,batch size,2,5.917632102966309,5.917632102966309,5.917632102966309,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:31,0.8.0 +fused_linear_grpo_loss_token,torch,forward,speed,ms,B,batch size,4,11.718751907348633,11.718751907348633,11.718751907348633,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:31,0.8.0 +fused_linear_grpo_loss_token,torch,forward,speed,ms,B,batch size,8,23.936031341552734,23.936031341552734,23.936031341552734,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:31,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,backward,speed,ms,B,batch size,1,2.1325119733810425,2.131167984008789,2.1334335803985596,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:48,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,backward,speed,ms,B,batch size,2,2.1431039571762085,2.1422784328460693,2.146079921722412,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:48,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,backward,speed,ms,B,batch size,4,2.170383930206299,2.1698238849639893,2.1733888149261475,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:48,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,backward,speed,ms,B,batch size,8,2.225152015686035,2.2230912208557125,2.2272128105163573,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:50:48,0.8.0 +fused_linear_grpo_loss_token,torch,backward,speed,ms,B,batch size,1,2.5600640773773193,2.556972885131836,2.5619456291198732,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:05,0.8.0 +fused_linear_grpo_loss_token,torch,backward,speed,ms,B,batch size,2,4.742624044418335,4.740473556518555,4.744774532318115,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:05,0.8.0 +fused_linear_grpo_loss_token,torch,backward,speed,ms,B,batch size,4,9.487360000610352,9.487360000610352,9.487360000610352,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:05,0.8.0 +fused_linear_grpo_loss_token,torch,backward,speed,ms,B,batch size,8,19.943391799926758,19.943391799926758,19.943391799926758,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:05,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,full,speed,ms,B,batch size,1,20.15433692932129,20.15433692932129,20.15433692932129,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:24,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,full,speed,ms,B,batch size,2,38.04972839355469,38.04972839355469,38.04972839355469,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:24,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,full,speed,ms,B,batch size,4,73.67266845703125,73.67266845703125,73.67266845703125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:24,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,full,speed,ms,B,batch size,8,145.13357543945312,145.13357543945312,145.13357543945312,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:24,0.8.0 +fused_linear_grpo_loss_token,torch,full,speed,ms,B,batch size,1,6.185984134674072,6.185984134674072,6.185984134674072,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:42,0.8.0 +fused_linear_grpo_loss_token,torch,full,speed,ms,B,batch size,2,10.671168327331543,10.671168327331543,10.671168327331543,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:42,0.8.0 +fused_linear_grpo_loss_token,torch,full,speed,ms,B,batch size,4,20.690879821777344,20.690879821777344,20.690879821777344,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:42,0.8.0 +fused_linear_grpo_loss_token,torch,full,speed,ms,B,batch size,8,44.16819381713867,44.16819381713867,44.16819381713867,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:51:42,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,full,memory,MB,B,batch size,1,7126.0302734375,7126.0302734375,7126.0302734375,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:03,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,full,memory,MB,B,batch size,2,7158.04296875,7158.04296875,7158.04296875,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:03,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,full,memory,MB,B,batch size,4,7222.06640625,7222.06640625,7222.06640625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:03,0.8.0 +fused_linear_grpo_loss_token,liger_cutile,full,memory,MB,B,batch size,8,7350.11328125,7350.11328125,7350.11328125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:03,0.8.0 +fused_linear_grpo_loss_token,torch,full,memory,MB,B,batch size,1,5098.01611328125,5098.01611328125,5098.01611328125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:20,0.8.0 +fused_linear_grpo_loss_token,torch,full,memory,MB,B,batch size,2,7126.03173828125,7126.03173828125,7126.03173828125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:20,0.8.0 +fused_linear_grpo_loss_token,torch,full,memory,MB,B,batch size,4,11182.0625,11182.0625,11182.0625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:20,0.8.0 +fused_linear_grpo_loss_token,torch,full,memory,MB,B,batch size,8,19294.125,19294.125,19294.125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:20,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,forward,speed,ms,B,batch size,1,18.05628776550293,18.05628776550293,18.05628776550293,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:45,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,forward,speed,ms,B,batch size,2,35.8757438659668,35.8757438659668,35.8757438659668,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:45,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,forward,speed,ms,B,batch size,4,71.45465850830078,71.45465850830078,71.45465850830078,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:45,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,forward,speed,ms,B,batch size,8,142.71487426757812,142.71487426757812,142.71487426757812,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:52:45,0.8.0 +fused_linear_grpo_loss_sequence,torch,forward,speed,ms,B,batch size,1,3.10262393951416,3.102048015594482,3.1045248031616213,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:03,0.8.0 +fused_linear_grpo_loss_sequence,torch,forward,speed,ms,B,batch size,2,5.704671859741211,5.704671859741211,5.704671859741211,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:03,0.8.0 +fused_linear_grpo_loss_sequence,torch,forward,speed,ms,B,batch size,4,11.705344200134277,11.705344200134277,11.705344200134277,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:03,0.8.0 +fused_linear_grpo_loss_sequence,torch,forward,speed,ms,B,batch size,8,24.104032516479492,24.104032516479492,24.104032516479492,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:03,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,backward,speed,ms,B,batch size,1,2.131424069404602,2.1284672260284423,2.1340096473693846,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:20,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,backward,speed,ms,B,batch size,2,2.1422080993652344,2.1407744884490967,2.14485764503479,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:20,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,backward,speed,ms,B,batch size,4,2.1692960262298584,2.1684096813201905,2.170195150375366,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:20,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,backward,speed,ms,B,batch size,8,2.27020800113678,2.269369602203369,2.273081636428833,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:20,0.8.0 +fused_linear_grpo_loss_sequence,torch,backward,speed,ms,B,batch size,1,2.5681920051574707,2.567519950866699,2.5725120067596436,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:38,0.8.0 +fused_linear_grpo_loss_sequence,torch,backward,speed,ms,B,batch size,2,4.39303994178772,4.386905479431153,4.399174404144287,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:38,0.8.0 +fused_linear_grpo_loss_sequence,torch,backward,speed,ms,B,batch size,4,9.642016410827637,9.642016410827637,9.642016410827637,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:38,0.8.0 +fused_linear_grpo_loss_sequence,torch,backward,speed,ms,B,batch size,8,19.719167709350586,19.719167709350586,19.719167709350586,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:38,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,full,speed,ms,B,batch size,1,20.147167205810547,20.147167205810547,20.147167205810547,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:57,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,full,speed,ms,B,batch size,2,38.04240036010742,38.04240036010742,38.04240036010742,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:57,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,full,speed,ms,B,batch size,4,73.64198303222656,73.64198303222656,73.64198303222656,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:57,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,full,speed,ms,B,batch size,8,145.0833282470703,145.0833282470703,145.0833282470703,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:53:57,0.8.0 +fused_linear_grpo_loss_sequence,torch,full,speed,ms,B,batch size,1,5.659615993499756,5.659615993499756,5.659615993499756,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:14,0.8.0 +fused_linear_grpo_loss_sequence,torch,full,speed,ms,B,batch size,2,10.763232231140137,10.763232231140137,10.763232231140137,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:14,0.8.0 +fused_linear_grpo_loss_sequence,torch,full,speed,ms,B,batch size,4,22.29961585998535,22.29961585998535,22.29961585998535,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:14,0.8.0 +fused_linear_grpo_loss_sequence,torch,full,speed,ms,B,batch size,8,44.690399169921875,44.690399169921875,44.690399169921875,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:14,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,full,memory,MB,B,batch size,1,7126.0302734375,7126.0302734375,7126.0302734375,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:34,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,full,memory,MB,B,batch size,2,7158.04296875,7158.04296875,7158.04296875,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:34,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,full,memory,MB,B,batch size,4,7222.06640625,7222.06640625,7222.06640625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:34,0.8.0 +fused_linear_grpo_loss_sequence,liger_cutile,full,memory,MB,B,batch size,8,7350.11328125,7350.11328125,7350.11328125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:34,0.8.0 +fused_linear_grpo_loss_sequence,torch,full,memory,MB,B,batch size,1,5098.01611328125,5098.01611328125,5098.01611328125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:51,0.8.0 +fused_linear_grpo_loss_sequence,torch,full,memory,MB,B,batch size,2,7126.03173828125,7126.03173828125,7126.03173828125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:51,0.8.0 +fused_linear_grpo_loss_sequence,torch,full,memory,MB,B,batch size,4,11182.0625,11182.0625,11182.0625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:51,0.8.0 +fused_linear_grpo_loss_sequence,torch,full,memory,MB,B,batch size,8,19294.125,19294.125,19294.125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""sequence"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 07:54:51,0.8.0 +fused_linear_grpo_loss_token,liger_triton,forward,speed,ms,B,batch size,1,18.05423927307129,18.05423927307129,18.05423927307129,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:22:31,0.8.0 +fused_linear_grpo_loss_token,liger_triton,forward,speed,ms,B,batch size,2,35.87571334838867,35.87571334838867,35.87571334838867,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:22:31,0.8.0 +fused_linear_grpo_loss_token,liger_triton,forward,speed,ms,B,batch size,4,71.45878601074219,71.45878601074219,71.45878601074219,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:22:31,0.8.0 +fused_linear_grpo_loss_token,liger_triton,forward,speed,ms,B,batch size,8,142.67599487304688,142.67599487304688,142.67599487304688,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:22:31,0.8.0 +fused_linear_grpo_loss_token,liger_triton,backward,speed,ms,B,batch size,1,2.1304320096969604,2.12992639541626,2.131744050979614,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:23:06,0.8.0 +fused_linear_grpo_loss_token,liger_triton,backward,speed,ms,B,batch size,2,2.1432480812072754,2.1428096294403076,2.1436352252960207,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:23:06,0.8.0 +fused_linear_grpo_loss_token,liger_triton,backward,speed,ms,B,batch size,4,2.1703840494155884,2.1694528579711916,2.1712896823883057,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:23:06,0.8.0 +fused_linear_grpo_loss_token,liger_triton,backward,speed,ms,B,batch size,8,2.225632071495056,2.223891258239746,2.2261568546295165,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:23:06,0.8.0 +fused_linear_grpo_loss_token,liger_triton,full,speed,ms,B,batch size,1,20.163776397705078,20.163776397705078,20.163776397705078,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:23:42,0.8.0 +fused_linear_grpo_loss_token,liger_triton,full,speed,ms,B,batch size,2,38.042625427246094,38.042625427246094,38.042625427246094,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:23:42,0.8.0 +fused_linear_grpo_loss_token,liger_triton,full,speed,ms,B,batch size,4,73.6685791015625,73.6685791015625,73.6685791015625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:23:42,0.8.0 +fused_linear_grpo_loss_token,liger_triton,full,speed,ms,B,batch size,8,145.08348083496094,145.08348083496094,145.08348083496094,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:23:42,0.8.0 +fused_linear_grpo_loss_token,liger_triton,full,memory,MB,B,batch size,1,7126.0302734375,7126.0302734375,7126.0302734375,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:24:19,0.8.0 +fused_linear_grpo_loss_token,liger_triton,full,memory,MB,B,batch size,2,7158.04296875,7158.04296875,7158.04296875,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:24:19,0.8.0 +fused_linear_grpo_loss_token,liger_triton,full,memory,MB,B,batch size,4,7222.06640625,7222.06640625,7222.06640625,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:24:19,0.8.0 +fused_linear_grpo_loss_token,liger_triton,full,memory,MB,B,batch size,8,7350.11328125,7350.11328125,7350.11328125,"{""hidden_size"": 4096, ""vocab_size"": 128256, ""dtype"": ""torch.bfloat16"", ""T"": 1024, ""importance_sampling_level"": ""token"", ""bsz"": 1, ""seq_len"": 8192}",NVIDIA B200,2026-07-22 08:24:19,0.8.0 diff --git a/benchmark/scripts/run_cutile_compare.py b/benchmark/scripts/run_cutile_compare.py index e65dc8599..ef1e7da82 100644 --- a/benchmark/scripts/run_cutile_compare.py +++ b/benchmark/scripts/run_cutile_compare.py @@ -20,10 +20,18 @@ CUTILE_ENABLED_KERNELS = [ "cross_entropy", + "dyt", + "fused_add_rms_norm", + "fused_linear_cross_entropy", "fused_linear_jsd", "geglu", + "grpo_loss", "jsd", "layer_norm", + "poly_norm", + "rms_norm", + "softmax", + "swiglu", ] diff --git a/benchmark/scripts/utils.py b/benchmark/scripts/utils.py index a15b184c4..e0146afad 100644 --- a/benchmark/scripts/utils.py +++ b/benchmark/scripts/utils.py @@ -452,15 +452,20 @@ def run_benchmarks( # Output routing for comparison runs. # LIGER_BENCH_TARGET selects the destination CSV (e.g. "cutile" -> all_benchmark_data_cutile.csv). - # LIGER_BENCH_PROVIDER_TAG renames the "liger" provider in this run (e.g. "liger_cutile") so + # LIGER_BENCH_PROVIDER_TAG renames Liger providers in this run (e.g. "liger_cutile") so # multiple implementations can coexist in one CSV without colliding on the dedup key. + # The tag replaces the leading "liger" of each provider, preserving any suffix so + # benchmarks with several Liger providers (e.g. fused_add_rms_norm's + # "liger_fused_add_rms_norm" and "liger_rms_norm") stay distinct: + # "liger" -> "liger_cutile" + # "liger_fused_add_rms_norm" -> "liger_cutile_fused_add_rms_norm" target = os.environ.get("LIGER_BENCH_TARGET", "").strip().lower() provider_tag = os.environ.get("LIGER_BENCH_PROVIDER_TAG", "").strip().lower() if provider_tag: for bd in benchmark_data_list: - if bd.kernel_provider == "liger": - bd.kernel_provider = provider_tag + if bd.kernel_provider == "liger" or bd.kernel_provider.startswith("liger_"): + bd.kernel_provider = provider_tag + bd.kernel_provider[len("liger") :] print_benchmark_data(benchmark_data_list) diff --git a/src/liger_kernel/ops/cutile/ops/__init__.py b/src/liger_kernel/ops/cutile/ops/__init__.py index dd6a36a55..7cccf8358 100644 --- a/src/liger_kernel/ops/cutile/ops/__init__.py +++ b/src/liger_kernel/ops/cutile/ops/__init__.py @@ -15,6 +15,9 @@ from liger_kernel.ops.cutile.ops.cross_entropy import LigerCrossEntropyFunction from liger_kernel.ops.cutile.ops.cross_entropy import cross_entropy_backward from liger_kernel.ops.cutile.ops.cross_entropy import cross_entropy_forward +from liger_kernel.ops.cutile.ops.dyt import LigerDyTFunction +from liger_kernel.ops.cutile.ops.fused_add_rms_norm import LigerFusedAddRMSNormFunction +from liger_kernel.ops.cutile.ops.fused_linear_cross_entropy import LigerFusedLinearCrossEntropyFunction from liger_kernel.ops.cutile.ops.fused_linear_jsd import LigerFusedLinearJSDFunction from liger_kernel.ops.cutile.ops.fused_linear_jsd import fused_linear_jsd_backward from liger_kernel.ops.cutile.ops.fused_linear_jsd import fused_linear_jsd_forward @@ -24,6 +27,7 @@ from liger_kernel.ops.cutile.ops.group_norm import LigerGroupNormFunction from liger_kernel.ops.cutile.ops.group_norm import group_norm_backward from liger_kernel.ops.cutile.ops.group_norm import group_norm_forward +from liger_kernel.ops.cutile.ops.grpo_loss import GrpoLossFunction from liger_kernel.ops.cutile.ops.jsd import LigerJSDFunction from liger_kernel.ops.cutile.ops.jsd import jsd_backward from liger_kernel.ops.cutile.ops.jsd import jsd_forward @@ -33,11 +37,15 @@ from liger_kernel.ops.cutile.ops.layer_norm import layer_norm_forward from liger_kernel.ops.cutile.ops.llama4_rope import LigerLlama4RopeFunction from liger_kernel.ops.cutile.ops.multi_token_attention import LigerMultiTokenAttentionFunction +from liger_kernel.ops.cutile.ops.poly_norm import LigerPolyNormFunction from liger_kernel.ops.cutile.ops.qwen2vl_mrope import LigerQwen2VLMRopeFunction +from liger_kernel.ops.cutile.ops.rms_norm import LigerRMSNormFunction from liger_kernel.ops.cutile.ops.rope import LigerRopeFunction from liger_kernel.ops.cutile.ops.rope import rope_backward from liger_kernel.ops.cutile.ops.rope import rope_forward +from liger_kernel.ops.cutile.ops.softmax import LigerSoftmaxFunction from liger_kernel.ops.cutile.ops.sparsemax import LigerSparsemaxFunction +from liger_kernel.ops.cutile.ops.swiglu import LigerSiLUMulFunction from liger_kernel.ops.cutile.ops.tiled_mlp import LigerTiledMLPFunction from liger_kernel.ops.cutile.ops.tiled_mlp import apply_tiled_mlp @@ -45,6 +53,9 @@ "LigerCrossEntropyFunction", "cross_entropy_backward", "cross_entropy_forward", + "LigerDyTFunction", + "LigerFusedAddRMSNormFunction", + "LigerFusedLinearCrossEntropyFunction", "LigerFusedLinearJSDFunction", "fused_linear_jsd_backward", "fused_linear_jsd_forward", @@ -52,6 +63,7 @@ "geglu_backward", "geglu_forward", "LigerGroupNormFunction", + "GrpoLossFunction", "group_norm_backward", "group_norm_forward", "LigerJSDFunction", @@ -63,11 +75,15 @@ "layer_norm_forward", "LigerLlama4RopeFunction", "LigerMultiTokenAttentionFunction", + "LigerPolyNormFunction", "LigerQwen2VLMRopeFunction", + "LigerRMSNormFunction", "LigerRopeFunction", "rope_backward", "rope_forward", + "LigerSoftmaxFunction", "LigerSparsemaxFunction", + "LigerSiLUMulFunction", "LigerTiledMLPFunction", "apply_tiled_mlp", ] diff --git a/src/liger_kernel/ops/cutile/ops/dyt.py b/src/liger_kernel/ops/cutile/ops/dyt.py new file mode 100644 index 000000000..d7e45a155 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/dyt.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +Dynamic Tanh (DyT) activation kernel (cuTile backend). + +Formula: y = tanh(alpha * x) * gamma + beta + +Forward uses a 2D grid (num_col_blocks, M). Each block handles BLOCK_N columns for one row +using gather/scatter with check_bounds=True to handle partial last chunks. The aligned fast +path (BLOCK_N == N, power-of-2, single col-block) compiles with check_bounds=False. + +Backward uses a persistent 2D grid (num_col_blocks, NUM_SMS): each block handles BLOCK_N +columns and strides over rows (start_row_id, start_row_id+NUM_SMS, ...), matching Triton for +row parallelism and occupancy. Per-block partials are reduced on the host (no atomics). +""" + +import cuda.tile as ct +import torch + +from cuda.tile import RoundingMode as RMd + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +MAX_FUSED_SIZE = 65536 + + +@ct.kernel +def _dyt_fwd_kernel( + x_input, # (M, N) input + y_output, # (M, N) output + alpha_tensor, # (1,) scalar alpha + gamma_tensor, # (N,) per-channel scale + beta_tensor, # (N,) per-channel bias (or dummy 1-element tensor when HAVE_BETA=0) + HAVE_BETA: ct.Constant[int], + CHECK_BOUNDS: ct.Constant[bool], + BLOCK_N: ct.Constant[int], +): + """ + DyT forward kernel. + + Grid: (num_col_blocks, M, 1). + CHECK_BOUNDS=False fast path (BLOCK_N==N, power-of-2, num_col_blocks==1) avoids + predicate-mask instructions on every gather/scatter — ~9-11% faster. + """ + row_id = ct.bid(1) + # Aligned path (CHECK_BOUNDS=False): single col-block, no col offset, no padding_value. + # CuTile DCE collapses the unused branch since CHECK_BOUNDS is a compile-time Constant. + if CHECK_BOUNDS: + col_indices = ct.arange(BLOCK_N, dtype=ct.int32) + ct.bid(0) * BLOCK_N + gamma = ct.astype(ct.gather(gamma_tensor, col_indices, check_bounds=True, padding_value=0.0), ct.float32) + x = ct.astype(ct.gather(x_input, (row_id, col_indices), check_bounds=True, padding_value=0.0), ct.float32) + else: + col_indices = ct.arange(BLOCK_N, dtype=ct.int32) + gamma = ct.astype(ct.gather(gamma_tensor, col_indices, check_bounds=False), ct.float32) + x = ct.astype(ct.gather(x_input, (row_id, col_indices), check_bounds=False), ct.float32) + + alpha = ct.astype(ct.load(alpha_tensor, 0, shape=()), ct.float32) + tanh_x = ct.tanh(alpha * x) + y = tanh_x * gamma + if HAVE_BETA: + if CHECK_BOUNDS: + beta = ct.astype(ct.gather(beta_tensor, col_indices, check_bounds=True, padding_value=0.0), ct.float32) + else: + beta = ct.astype(ct.gather(beta_tensor, col_indices, check_bounds=False), ct.float32) + y = y + beta + + ct.scatter(y_output, (row_id, col_indices), ct.astype(y, y_output.dtype), check_bounds=CHECK_BOUNDS) + + +@ct.kernel +def _dyt_bwd_kernel( + dy_input, # (M, N) upstream gradient + dx_output, # (M, N) gradient w.r.t. x + da_partial, # (NUM_SMS, num_col_blocks) partial d_alpha per block, host reduces + dg_partial, # (NUM_SMS, N) partial d_gamma per block row, host reduces + db_partial, # (NUM_SMS, N) partial d_beta when HAVE_BETA, host reduces + x_input, # (M, N) saved input + alpha_tensor, # (1,) scalar alpha + gamma_tensor, # (N,) per-channel scale + HAVE_BETA: ct.Constant[int], + M: ct.Constant[int], + BLOCK_N: ct.Constant[int], + NUM_SMS: ct.Constant[int], +): + """ + DyT backward kernel (persistent 2D grid, aligned with Triton). + + Grid: (num_col_blocks, NUM_SMS) + Block (col_block, start_row_id) strides over rows: start_row_id, start_row_id+NUM_SMS, ... + Writes DG/DB/DA to unique (start_row_id, col) so no atomics; host sums. + """ + col_block = ct.bid(0) + start_row_id = ct.bid(1) + col_start = col_block * BLOCK_N + col_indices = ct.arange(BLOCK_N, dtype=ct.int32) + col_start + + alpha = ct.astype(ct.load(alpha_tensor, 0, shape=()), ct.float32) + gamma = ct.astype(ct.gather(gamma_tensor, col_indices, check_bounds=True, padding_value=0.0), ct.float32) + + da_acc = ct.full((BLOCK_N,), 0.0, dtype=ct.float32) # tile accumulator, reduce once at end + dg_acc = ct.full((BLOCK_N,), 0.0, dtype=ct.float32) + if HAVE_BETA: + db_acc = ct.full((BLOCK_N,), 0.0, dtype=ct.float32) + + # Stride over rows assigned to this block (same as Triton: start_row_id, start_row_id+NUM_SMS, ...) + num_iters = (M + NUM_SMS - 1) // NUM_SMS + for i in range(num_iters): + row_id = start_row_id + i * NUM_SMS + if row_id < M: + x = ct.astype(ct.gather(x_input, (row_id, col_indices), check_bounds=True, padding_value=0.0), ct.float32) + dy = ct.astype(ct.gather(dy_input, (row_id, col_indices), check_bounds=True, padding_value=0.0), ct.float32) + + # APPROX tanh: ~1.6x faster, 2-4 ULP off; well within bwd tolerance 1e-2. + tanh_x = ct.tanh(alpha * x, rounding_mode=RMd.APPROX) + sech2_x = ct.full((BLOCK_N,), 1.0, dtype=ct.float32) - tanh_x * tanh_x + + if HAVE_BETA: + db_acc = db_acc + dy + + dg_acc = dg_acc + dy * tanh_x + + tmp = sech2_x * dy * gamma + da_acc = da_acc + x * tmp + dx = alpha * tmp + ct.scatter(dx_output, (row_id, col_indices), ct.astype(dx, dx_output.dtype), check_bounds=True) + + # Write to unique (start_row_id, col) so host can sum over dim 0 + row_idx_tile = ct.full((BLOCK_N,), start_row_id, dtype=ct.int32) + ct.scatter(dg_partial, (row_idx_tile, col_indices), dg_acc, check_bounds=True) + if HAVE_BETA: + ct.scatter(db_partial, (row_idx_tile, col_indices), db_acc, check_bounds=True) + # DA: one scalar per block at (start_row_id, col_block) — single reduction at end + da_scalar = ct.full((1,), ct.sum(da_acc, 0, keepdims=False), dtype=ct.float32) + ct.scatter( + da_partial, + (ct.full((1,), start_row_id, dtype=ct.int32), ct.full((1,), col_block, dtype=ct.int32)), + da_scalar, + ) + + +# nww=8 matches Triton's num_warps=8 on this bwd kernel. +_dyt_bwd_kernel_nww8 = _dyt_bwd_kernel.replace_hints(num_worker_warps=8) + + +def _dyt_forward_ct(x, alpha, gamma, beta): + HAVE_BETA = beta is not None + input_shape = x.shape + x_2d = x.view(-1, input_shape[-1]) + M, N = x_2d.shape + + BLOCK_N = min(MAX_FUSED_SIZE, _next_power_of_2(N)) + num_col_blocks = (N + BLOCK_N - 1) // BLOCK_N + # Aligned fast path: single col-block + N is power-of-2 → all gathers/scatters + # are in-bounds, kernel compiles with check_bounds=False (no predicate masks). + check_bounds = not ((BLOCK_N == N) and (num_col_blocks == 1)) + + y = torch.empty_like(x_2d) + beta_tensor = beta if HAVE_BETA else torch.empty(1, device=x.device, dtype=x.dtype) + + ct.launch( + torch.cuda.current_stream(), + (num_col_blocks, M, 1), + _dyt_fwd_kernel, + (x_2d, y, alpha, gamma, beta_tensor, int(HAVE_BETA), bool(check_bounds), int(BLOCK_N)), + ) + return y.view(input_shape) + + +def _dyt_backward_ct(dy, x, alpha, gamma, beta): + HAVE_BETA = beta is not None + input_shape = x.shape + x_2d = x.view(-1, input_shape[-1]) + dy_2d = dy.view(-1, input_shape[-1]) + M, N = x_2d.shape + + NUM_SMS = torch.cuda.get_device_properties(x.device).multi_processor_count + BLOCK_N = min(_next_power_of_2(N), 1024) + num_col_blocks = (N + BLOCK_N - 1) // BLOCK_N + + dx = torch.empty_like(dy_2d) + # Per-block partials (match Triton): host reduces over dim 0 + da_partial = torch.zeros(NUM_SMS, num_col_blocks, dtype=torch.float32, device=x.device) + dg_partial = torch.empty(NUM_SMS, N, dtype=torch.float32, device=x.device) + db_partial = torch.empty(NUM_SMS, N, dtype=torch.float32, device=x.device) if HAVE_BETA else None + + db_tensor = db_partial if HAVE_BETA else torch.empty(1, device=x.device, dtype=torch.float32) + + ct.launch( + torch.cuda.current_stream(), + (num_col_blocks, NUM_SMS, 1), + _dyt_bwd_kernel_nww8, + ( + dy_2d, + dx, + da_partial, + dg_partial, + db_tensor, + x_2d, + alpha, + gamma, + int(HAVE_BETA), + int(M), + int(BLOCK_N), + int(NUM_SMS), + ), + ) + + da = da_partial.sum().to(x.dtype).unsqueeze(0) + dg = dg_partial.sum(0).to(gamma.dtype) + db = db_partial.sum(0).to(x.dtype) if HAVE_BETA else None + return dx.view(input_shape), da, dg, db + + +class LigerDyTFunction(torch.autograd.Function): + """CuTile autograd wrapper for Dynamic Tanh: y = tanh(alpha * x) * gamma + beta.""" + + @staticmethod + def forward(ctx, x, alpha, gamma, beta): + x = x.contiguous() + alpha = alpha.contiguous() + gamma = gamma.contiguous() + if beta is not None: + beta = beta.contiguous() + y = _dyt_forward_ct(x, alpha, gamma, beta) + ctx.save_for_backward(x, alpha, gamma, beta) + return y + + @staticmethod + def backward(ctx, dy): + x, alpha, gamma, beta = ctx.saved_tensors + dy = dy.contiguous() + dx, dalpha, dgamma, dbeta = _dyt_backward_ct(dy, x, alpha, gamma, beta) + return dx, dalpha, dgamma, dbeta diff --git a/src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py b/src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py new file mode 100644 index 000000000..e2144b5c1 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py @@ -0,0 +1,592 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +Fused residual-add + RMSNorm (cuTile backend). + +Forward: S = X + R, Y = S / RMS(S) * (W + offset). Both Y and the updated residual S are +returned. Row-parallel; single-pass for one chunk, two-pass for multi-chunk; forward occupancy +is autotuned per shape. + +Backward: persistent grid (one block per SM), each block accumulates dW in registers across its +rows and writes it once. A 2-chunk variant (splitting n_cols into lo/hi halves) caps per-thread +register pressure when BLOCK_SIZE exceeds _BWD_MAX_CHUNK_SIZE. dR == dX (gradient flows equally +to X and R). + +Casting modes match the Triton implementation: "llama" (fp32 RMS only), "gemma" (full fp32), +"none" (compute in input dtype). in_place is accepted for signature parity but ignored. +""" + +import math + +from types import SimpleNamespace + +import cuda.tile as ct +import torch + +from cuda.tile.tune import exhaustive_search + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 +from liger_kernel.ops.utils import ensure_contiguous + +_CASTING_MODE_NONE = -1 +_CASTING_MODE_LLAMA = 0 +_CASTING_MODE_GEMMA = 1 + +_STR_TO_CASTING_MODE = { + "llama": _CASTING_MODE_LLAMA, + "gemma": _CASTING_MODE_GEMMA, + "none": _CASTING_MODE_NONE, +} + +# _BWD_MAX_CHUNK_SIZE: threshold for switching to the 2-chunk persistent kernel. +# When BLOCK_SIZE (= next_power_of_2(n_cols)) exceeds this value, the backward +# dispatch uses _fused_add_rms_norm_bwd_persistent_2c_ct with CHUNK_SIZE = BLOCK_SIZE//2. +# This caps per-thread register usage at CHUNK_SIZE/128 elements per tile (32 f32/thread +# at CHUNK_SIZE=4096), reducing peak pressure from 87% -> 56% of the B200 budget. +_BWD_MAX_CHUNK_SIZE = 4096 + + +def calculate_settings(n_cols): + BLOCK_SIZE = _next_power_of_2(n_cols) + if BLOCK_SIZE > 65536: + raise RuntimeError(f"Hidden dimension {n_cols} exceeds maximum supported size of 65536.") + return BLOCK_SIZE + + +@ct.kernel +def _fused_add_rms_norm_fwd_ct( + Y, # (n_rows, n_cols) normalized output + S, # (n_rows, n_cols) updated residual (X + R) + X, # (n_rows, n_cols) hidden states input + R, # (n_rows, n_cols) residual input + W, # (n_cols,) RMSNorm weight + RSTD, # (n_rows,) cached rstd (scalar per row) + n_cols: ct.Constant[int], + eps: ct.Constant[float], + offset: ct.Constant[float], + BLOCK_SIZE: ct.Constant[int], + casting_mode: ct.Constant[int], +): + """ + Forward: S = X + R, Y = S * rstd * (W + offset). + + Row-parallel: one block per row. Two passes over columns. + + Casting modes: + _CASTING_MODE_LLAMA (0): fp32 RMS; S*rstd cast back to X.dtype before W multiply + _CASTING_MODE_GEMMA (1): full fp32, Y cast back to X.dtype at end (original behavior) + _CASTING_MODE_NONE (-1): no casting, compute in X.dtype throughout + """ + row_idx = ct.bid(0) + num_chunks = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE + + if num_chunks == 1: + # Single-pass: mirrors Triton's structure — S stays in registers across the + # rstd compute, so pass 2 doesn't re-load it from DRAM. Reassigning S_tile + # through dtype transitions drops the previous register footprint. + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + S_tile = ct.add( + ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ct.gather(R, (row_idx, col_idx), check_bounds=True, padding_value=0.0), + ) + ct.scatter(S, (row_idx, col_idx), S_tile, check_bounds=True) + W_tile = ct.gather(W, col_idx, check_bounds=True, padding_value=0.0) + + if casting_mode == _CASTING_MODE_LLAMA: + S_tile = ct.astype(S_tile, ct.float32) + rstd = ct.rsqrt(ct.sum(ct.mul(S_tile, S_tile), 0, keepdims=False) / n_cols + eps) + ct.scatter(RSTD, row_idx, rstd) + S_tile = ct.astype(ct.mul(S_tile, rstd), X.dtype) + # W may be higher precision than X (e.g. fp32 norm weight, bf16 activations); + # the multiply promotes to W's dtype, so cast back to Y.dtype before storing. + ct.scatter(Y, (row_idx, col_idx), ct.astype(ct.mul(S_tile, ct.add(W_tile, offset)), Y.dtype), check_bounds=True) + elif casting_mode == _CASTING_MODE_GEMMA: + S_tile = ct.astype(S_tile, ct.float32) + rstd = ct.rsqrt(ct.sum(ct.mul(S_tile, S_tile), 0, keepdims=False) / n_cols + eps) + ct.scatter(RSTD, row_idx, rstd) + W_shifted = ct.add(ct.astype(W_tile, ct.float32), offset) + Y_f32 = ct.mul(ct.mul(S_tile, rstd), W_shifted) + ct.scatter(Y, (row_idx, col_idx), ct.astype(Y_f32, Y.dtype), check_bounds=True) + else: + # NONE: compute mean_sq in X.dtype (then promote to fp32 at division), + # store rstd as X.dtype. + mean_sq = ct.sum(ct.mul(S_tile, S_tile), 0, keepdims=False) / n_cols + rstd = ct.rsqrt(mean_sq + eps) + ct.scatter(RSTD, row_idx, rstd) + ct.scatter( + Y, + (row_idx, col_idx), + ct.astype(ct.mul(ct.mul(S_tile, rstd), ct.add(W_tile, offset)), Y.dtype), + check_bounds=True, + ) + return + + # ---- Multi-chunk path (BLOCK_SIZE chunked from n_cols, num_chunks >= 2) ---- + # Two-pass loop: pass 1 computes S, scatters S, accumulates sum(S^2). + # Pass 2 re-loads S to compute Y (unavoidable: tiles don't fit in registers across chunks). + if casting_mode == _CASTING_MODE_NONE: + sum_sq_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=X.dtype) + else: + sum_sq_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + + for ci in range(num_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + X_tile = ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=0.0) + R_tile = ct.gather(R, (row_idx, col_idx), check_bounds=True, padding_value=0.0) + S_tile = ct.add(X_tile, R_tile) + ct.scatter(S, (row_idx, col_idx), S_tile, check_bounds=True) + if casting_mode == _CASTING_MODE_NONE: + sum_sq_tile = ct.add(sum_sq_tile, ct.mul(S_tile, S_tile)) + else: + S_tile_f32 = ct.astype(S_tile, ct.float32) + sum_sq_tile = ct.add(sum_sq_tile, ct.mul(S_tile_f32, S_tile_f32)) + + mean_sq = ct.sum(sum_sq_tile, 0, keepdims=False) / n_cols + rstd = ct.rsqrt(mean_sq + eps) + ct.scatter(RSTD, row_idx, rstd) + + for ci in range(num_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + S_tile = ct.gather(S, (row_idx, col_idx), check_bounds=True, padding_value=0.0) + W_tile = ct.gather(W, col_idx, check_bounds=True, padding_value=0.0) + + if casting_mode == _CASTING_MODE_LLAMA: + S_tile = ct.astype(S_tile, ct.float32) + S_normed = ct.astype(ct.mul(S_tile, rstd), X.dtype) + ct.scatter(Y, (row_idx, col_idx), ct.astype(ct.mul(S_normed, ct.add(W_tile, offset)), Y.dtype), check_bounds=True) + elif casting_mode == _CASTING_MODE_GEMMA: + S_tile = ct.astype(S_tile, ct.float32) + W_shifted = ct.add(ct.astype(W_tile, ct.float32), offset) + Y_f32 = ct.mul(ct.mul(S_tile, rstd), W_shifted) + ct.scatter(Y, (row_idx, col_idx), ct.astype(Y_f32, Y.dtype), check_bounds=True) + else: + ct.scatter( + Y, + (row_idx, col_idx), + ct.astype(ct.mul(ct.mul(S_tile, rstd), ct.add(W_tile, offset)), Y.dtype), + check_bounds=True, + ) + + +_fused_add_rms_norm_fwd_large_ct = _fused_add_rms_norm_fwd_ct.replace_hints(num_worker_warps=8) +# Small BLOCK_SIZE: Triton's calculate_settings picks num_warps=4 for BLOCK_SIZE<2048. +# Matching that avoids over-warping when each thread only has 4 fp32/tile of work. +_fused_add_rms_norm_fwd_small_ct = _fused_add_rms_norm_fwd_ct.replace_hints(num_worker_warps=4) + + +# Per-shape autotune for fwd occupancy. Higher occupancy forces the compiler to budget +# fewer regs/thread (target: 1/occ of the SM register file). Sweet spot is shape- +# dependent — tiny tiles at small N benefit from occ>=8, large tiles spill above occ=1. +def _fwd_autotune_configs(): + for occ in (None, 2, 3, 4, 5, 6, 8): + yield SimpleNamespace(occupancy=occ) + + +_fwd_tune_cache: dict = {} + + +def _autotune_fwd_kernel(base_kernel, args, n_rows, cache_key, stream): + if cache_key in _fwd_tune_cache: + return _fwd_tune_cache[cache_key] + result = exhaustive_search( + list(_fwd_autotune_configs()), + stream, + lambda cfg: (n_rows, 1, 1), + base_kernel, + lambda cfg: args, + lambda cfg: {"occupancy": cfg.occupancy} if cfg.occupancy is not None else {}, + quiet=True, + ) + best = result.best.config + tuned = base_kernel.replace_hints(occupancy=best.occupancy) if best.occupancy is not None else base_kernel + _fwd_tune_cache[cache_key] = tuned + return tuned + + +@ct.kernel(occupancy=1) +def _fused_add_rms_norm_bwd_persistent_ct( + dY, # (n_rows, n_cols) gradient of Y + dS_out, # (n_rows, n_cols) gradient of S flowing from downstream + dX, # (n_rows, n_cols) output gradient (also used for dR) + S, # (n_rows, n_cols) saved residual S = X + R from forward + W, # (n_cols,) RMSNorm weight + RSTD, # (n_rows,) cached rstd from forward + dW_partial, # (sm_count, n_cols) per-SM dW, host reduces with sum(dim=0) + n_cols: ct.Constant[int], + offset: ct.Constant[float], + num_iters: ct.Constant[int], + sm_count: ct.Constant[int], + n_rows: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], # = next_power_of_2(n_cols) — no cap + casting_mode: ct.Constant[int], +): + """ + Persistent backward: grid = (sm_count,). + Each block handles ceil(n_rows / sm_count) rows in a blocked loop: + row_idx = sm_id * num_iters + i for i in range(num_iters) + + Casting modes: + _CASTING_MODE_LLAMA (0): m = (dY * W).to(f32); dW uses cast-back intermediate + _CASTING_MODE_GEMMA (1): dY cast to f32 first, then full fp32 (original behavior) + _CASTING_MODE_NONE (-1): compute in S.dtype throughout + """ + sm_id = ct.bid(0) + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + + # Hoist W load outside the row loop: W is row-invariant, load it once per SM. + W_tile = ct.gather(W, col_idx, check_bounds=True, padding_value=0.0) + if casting_mode == _CASTING_MODE_NONE: + W_shifted = ct.add(W_tile, offset) + else: + W_shifted = ct.add(ct.astype(W_tile, ct.float32), offset) + + # Single register-resident dW accumulator (always float32 for numerical stability). + dW_acc = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + inv_n_cols = 1.0 / n_cols + + row_start = sm_id * num_iters + for i in range(num_iters): + row_idx = row_start + i + + if row_idx < n_rows: + rstd = ct.load(RSTD, row_idx, shape=()) # scalar + + dY_tile = ct.gather(dY, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3) + S_tile = ct.gather(S, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3) + + if casting_mode == _CASTING_MODE_LLAMA: + # m = (dY * W_orig_dtype).to(f32) + m_tile = ct.astype(ct.mul(dY_tile, ct.astype(W_shifted, dY.dtype)), ct.float32) + S_tile_f32 = ct.astype(S_tile, ct.float32) + sum_mS = ct.sum(ct.mul(m_tile, S_tile_f32), 0, keepdims=False) + # dW: dY * (S * rstd).to(S.dtype) + dW_acc = ct.add( + dW_acc, ct.astype(ct.mul(dY_tile, ct.astype(ct.mul(S_tile_f32, rstd), S.dtype)), ct.float32) + ) + dS_out_tile = ct.astype( + ct.gather(dS_out, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32 + ) + rstd3_coeff = rstd * rstd * rstd * inv_n_cols * sum_mS + dX_tile = ct.add(ct.sub(ct.mul(rstd, m_tile), ct.mul(rstd3_coeff, S_tile_f32)), dS_out_tile) + ct.scatter(dX, (row_idx, col_idx), ct.astype(dX_tile, dX.dtype), check_bounds=True) + elif casting_mode == _CASTING_MODE_GEMMA: + dY_tile_f32 = ct.astype(dY_tile, ct.float32) + S_tile_f32 = ct.astype(S_tile, ct.float32) + m_tile = ct.mul(dY_tile_f32, W_shifted) + sum_mS = ct.sum(ct.mul(m_tile, S_tile_f32), 0, keepdims=False) + dW_acc = ct.add(dW_acc, ct.mul(dY_tile_f32, ct.mul(S_tile_f32, rstd))) + dS_out_tile = ct.astype( + ct.gather(dS_out, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32 + ) + rstd3_coeff = rstd * rstd * rstd * inv_n_cols * sum_mS + dX_tile = ct.add(ct.sub(ct.mul(rstd, m_tile), ct.mul(rstd3_coeff, S_tile_f32)), dS_out_tile) + ct.scatter(dX, (row_idx, col_idx), ct.astype(dX_tile, dX.dtype), check_bounds=True) + else: + # _CASTING_MODE_NONE: compute in S.dtype + m_tile = ct.mul(dY_tile, W_shifted) + sum_mS = ct.sum(ct.astype(ct.mul(m_tile, S_tile), ct.float32), 0, keepdims=False) + dW_acc = ct.add(dW_acc, ct.astype(ct.mul(dY_tile, ct.mul(S_tile, rstd)), ct.float32)) + dS_out_tile = ct.gather(dS_out, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3) + rstd3_coeff = rstd * rstd * rstd * inv_n_cols * sum_mS + dX_tile = ct.add( + ct.sub( + ct.mul(rstd, m_tile), ct.astype(ct.mul(rstd3_coeff, ct.astype(S_tile, ct.float32)), S.dtype) + ), + dS_out_tile, + ) + ct.scatter(dX, (row_idx, col_idx), ct.astype(dX_tile, dX.dtype), check_bounds=True) + + # Write register-accumulated dW to global memory — exactly ONCE per SM. + ct.scatter(dW_partial, (sm_id, col_idx), dW_acc, check_bounds=True) + + +@ct.kernel(occupancy=1) +def _fused_add_rms_norm_bwd_persistent_2c_ct( + dY, # (n_rows, n_cols) gradient of Y + dS_out, # (n_rows, n_cols) gradient of S flowing from downstream + dX, # (n_rows, n_cols) output gradient (also used for dR) + S, # (n_rows, n_cols) saved residual S = X + R from forward + W, # (n_cols,) RMSNorm weight + RSTD, # (n_rows,) cached rstd from forward + dW_partial, # (sm_count, n_cols) per-SM dW, host reduces with sum(dim=0) + n_cols: ct.Constant[int], + offset: ct.Constant[float], + num_iters: ct.Constant[int], + sm_count: ct.Constant[int], + n_rows: ct.Constant[int], + CHUNK_SIZE: ct.Constant[int], # = BLOCK_SIZE // 2 = next_power_of_2(n_cols) // 2 + casting_mode: ct.Constant[int], +): + """ + 2-chunk persistent backward: used when BLOCK_SIZE > _BWD_MAX_CHUNK_SIZE. + + Splits the n_cols dimension into two halves (lo=0..CHUNK_SIZE-1, + hi=CHUNK_SIZE..2*CHUNK_SIZE-1) processed together in a single pass per row. + Each half uses a separate set of register tiles, capping per-thread register + usage at CHUNK_SIZE/128 elements (32 f32/thread at CHUNK_SIZE=4096) vs. + BLOCK_SIZE/128 in the 1-chunk kernel (64 f32/thread for n_cols=8192). + """ + sm_id = ct.bid(0) + + # Column index tiles for each half + col_idx_lo = ct.arange(CHUNK_SIZE, dtype=ct.int32) + col_idx_hi = ct.add(ct.arange(CHUNK_SIZE, dtype=ct.int32), CHUNK_SIZE) + + # Hoist W loads outside the row loop (W is row-invariant). + W_tile_lo_raw = ct.gather(W, col_idx_lo, check_bounds=True, padding_value=0.0) + W_tile_hi_raw = ct.gather(W, col_idx_hi, check_bounds=True, padding_value=0.0) + if casting_mode == _CASTING_MODE_NONE: + W_shifted_lo = ct.add(W_tile_lo_raw, offset) + W_shifted_hi = ct.add(W_tile_hi_raw, offset) + else: + W_shifted_lo = ct.add(ct.astype(W_tile_lo_raw, ct.float32), offset) + W_shifted_hi = ct.add(ct.astype(W_tile_hi_raw, ct.float32), offset) + + # Separate dW accumulators for lo and hi halves. + dW_acc_lo = ct.full((CHUNK_SIZE,), 0.0, dtype=ct.float32) + dW_acc_hi = ct.full((CHUNK_SIZE,), 0.0, dtype=ct.float32) + inv_n_cols = 1.0 / n_cols + + row_start = sm_id * num_iters + for i in range(num_iters): + row_idx = row_start + i + + if row_idx < n_rows: + rstd = ct.load(RSTD, row_idx, shape=()) # scalar + + dY_lo_raw = ct.gather(dY, (row_idx, col_idx_lo), check_bounds=True, padding_value=0.0, latency=3) + dY_hi_raw = ct.gather(dY, (row_idx, col_idx_hi), check_bounds=True, padding_value=0.0, latency=3) + S_lo_raw = ct.gather(S, (row_idx, col_idx_lo), check_bounds=True, padding_value=0.0, latency=3) + S_hi_raw = ct.gather(S, (row_idx, col_idx_hi), check_bounds=True, padding_value=0.0, latency=3) + + if casting_mode == _CASTING_MODE_LLAMA: + m_lo = ct.astype(ct.mul(dY_lo_raw, ct.astype(W_shifted_lo, dY.dtype)), ct.float32) + m_hi = ct.astype(ct.mul(dY_hi_raw, ct.astype(W_shifted_hi, dY.dtype)), ct.float32) + S_lo = ct.astype(S_lo_raw, ct.float32) + S_hi = ct.astype(S_hi_raw, ct.float32) + sum_mS = ct.sum(ct.mul(m_lo, S_lo), 0, keepdims=False) + ct.sum(ct.mul(m_hi, S_hi), 0, keepdims=False) + dW_acc_lo = ct.add( + dW_acc_lo, ct.astype(ct.mul(dY_lo_raw, ct.astype(ct.mul(S_lo, rstd), S.dtype)), ct.float32) + ) + dW_acc_hi = ct.add( + dW_acc_hi, ct.astype(ct.mul(dY_hi_raw, ct.astype(ct.mul(S_hi, rstd), S.dtype)), ct.float32) + ) + rstd3_coeff = rstd * rstd * rstd * inv_n_cols * sum_mS + dS_out_lo = ct.astype( + ct.gather(dS_out, (row_idx, col_idx_lo), check_bounds=True, padding_value=0.0, latency=3), + ct.float32, + ) + dX_lo = ct.add(ct.sub(ct.mul(rstd, m_lo), ct.mul(rstd3_coeff, S_lo)), dS_out_lo) + ct.scatter(dX, (row_idx, col_idx_lo), ct.astype(dX_lo, dX.dtype), check_bounds=True) + dS_out_hi = ct.astype( + ct.gather(dS_out, (row_idx, col_idx_hi), check_bounds=True, padding_value=0.0, latency=3), + ct.float32, + ) + dX_hi = ct.add(ct.sub(ct.mul(rstd, m_hi), ct.mul(rstd3_coeff, S_hi)), dS_out_hi) + ct.scatter(dX, (row_idx, col_idx_hi), ct.astype(dX_hi, dX.dtype), check_bounds=True) + elif casting_mode == _CASTING_MODE_GEMMA: + dY_lo = ct.astype(dY_lo_raw, ct.float32) + dY_hi = ct.astype(dY_hi_raw, ct.float32) + S_lo = ct.astype(S_lo_raw, ct.float32) + S_hi = ct.astype(S_hi_raw, ct.float32) + m_lo = ct.mul(dY_lo, W_shifted_lo) + m_hi = ct.mul(dY_hi, W_shifted_hi) + sum_mS = ct.sum(ct.mul(m_lo, S_lo), 0, keepdims=False) + ct.sum(ct.mul(m_hi, S_hi), 0, keepdims=False) + dW_acc_lo = ct.add(dW_acc_lo, ct.mul(dY_lo, ct.mul(S_lo, rstd))) + dW_acc_hi = ct.add(dW_acc_hi, ct.mul(dY_hi, ct.mul(S_hi, rstd))) + rstd3_coeff = rstd * rstd * rstd * inv_n_cols * sum_mS + dS_out_lo = ct.astype( + ct.gather(dS_out, (row_idx, col_idx_lo), check_bounds=True, padding_value=0.0, latency=3), + ct.float32, + ) + dX_lo = ct.add(ct.sub(ct.mul(rstd, m_lo), ct.mul(rstd3_coeff, S_lo)), dS_out_lo) + ct.scatter(dX, (row_idx, col_idx_lo), ct.astype(dX_lo, dX.dtype), check_bounds=True) + dS_out_hi = ct.astype( + ct.gather(dS_out, (row_idx, col_idx_hi), check_bounds=True, padding_value=0.0, latency=3), + ct.float32, + ) + dX_hi = ct.add(ct.sub(ct.mul(rstd, m_hi), ct.mul(rstd3_coeff, S_hi)), dS_out_hi) + ct.scatter(dX, (row_idx, col_idx_hi), ct.astype(dX_hi, dX.dtype), check_bounds=True) + else: + # _CASTING_MODE_NONE: compute in S.dtype + m_lo = ct.mul(dY_lo_raw, W_shifted_lo) + m_hi = ct.mul(dY_hi_raw, W_shifted_hi) + sum_mS = ct.sum(ct.astype(ct.mul(m_lo, S_lo_raw), ct.float32), 0, keepdims=False) + ct.sum( + ct.astype(ct.mul(m_hi, S_hi_raw), ct.float32), 0, keepdims=False + ) + dW_acc_lo = ct.add(dW_acc_lo, ct.astype(ct.mul(dY_lo_raw, ct.mul(S_lo_raw, rstd)), ct.float32)) + dW_acc_hi = ct.add(dW_acc_hi, ct.astype(ct.mul(dY_hi_raw, ct.mul(S_hi_raw, rstd)), ct.float32)) + rstd3_coeff = rstd * rstd * rstd * inv_n_cols * sum_mS + dS_out_lo = ct.gather(dS_out, (row_idx, col_idx_lo), check_bounds=True, padding_value=0.0, latency=3) + dX_lo = ct.add( + ct.sub( + ct.mul(rstd, m_lo), ct.astype(ct.mul(rstd3_coeff, ct.astype(S_lo_raw, ct.float32)), S.dtype) + ), + dS_out_lo, + ) + ct.scatter(dX, (row_idx, col_idx_lo), ct.astype(dX_lo, dX.dtype), check_bounds=True) + dS_out_hi = ct.gather(dS_out, (row_idx, col_idx_hi), check_bounds=True, padding_value=0.0, latency=3) + dX_hi = ct.add( + ct.sub( + ct.mul(rstd, m_hi), ct.astype(ct.mul(rstd3_coeff, ct.astype(S_hi_raw, ct.float32)), S.dtype) + ), + dS_out_hi, + ) + ct.scatter(dX, (row_idx, col_idx_hi), ct.astype(dX_hi, dX.dtype), check_bounds=True) + + # Write accumulated dW to global memory — ONCE per SM, both halves. + ct.scatter(dW_partial, (sm_id, col_idx_lo), dW_acc_lo, check_bounds=True) + ct.scatter(dW_partial, (sm_id, col_idx_hi), dW_acc_hi, check_bounds=True) + + +_fused_add_rms_norm_bwd_persistent_ct_nww8 = _fused_add_rms_norm_bwd_persistent_ct.replace_hints(num_worker_warps=8) +_fused_add_rms_norm_bwd_persistent_2c_ct_nww8 = _fused_add_rms_norm_bwd_persistent_2c_ct.replace_hints( + num_worker_warps=8 +) + + +def _fused_add_rms_norm_forward_ct(X, R, W, eps, offset, casting_mode): + if isinstance(casting_mode, str): + casting_mode = _STR_TO_CASTING_MODE[casting_mode] + + shape = X.shape + dim = shape[-1] + X2d = X.view(-1, dim) + R2d = R.view(-1, dim) + n_rows, n_cols = X2d.shape + BLOCK_SIZE = calculate_settings(n_cols) + + Y = torch.empty_like(X2d) + S = torch.empty_like(X2d) + # RSTD dtype: float32 for llama/gemma (fp32 rstd computation), X.dtype for none + rstd_dtype = torch.float32 if casting_mode in (_CASTING_MODE_LLAMA, _CASTING_MODE_GEMMA) else X.dtype + RSTD = torch.empty(n_rows, dtype=rstd_dtype, device=X.device) + + # Fwd register pressure: for n_cols > 4096, 1 huge chunk spills; 2 chunks is the + # sweet spot (3+ adds per-iteration overhead with no register benefit). + FWD_BLOCK_SIZE = BLOCK_SIZE // 2 if BLOCK_SIZE > 4096 else BLOCK_SIZE + base_kernel = _fused_add_rms_norm_fwd_small_ct if BLOCK_SIZE < 2048 else _fused_add_rms_norm_fwd_large_ct + + stream = torch.cuda.current_stream() + args = ( + Y, + S, + X2d.contiguous(), + R2d.contiguous(), + W.contiguous(), + RSTD, + int(n_cols), + float(eps), + float(offset), + int(FWD_BLOCK_SIZE), + int(casting_mode), + ) + cache_key = (n_cols, FWD_BLOCK_SIZE, casting_mode, X.dtype, str(X.device)) + tuned_kernel = _autotune_fwd_kernel(base_kernel, args, n_rows, cache_key, stream) + + ct.launch(stream, (n_rows, 1, 1), tuned_kernel, args) + + return Y.view(*shape), S.view(*shape), RSTD, BLOCK_SIZE, casting_mode + + +def _fused_add_rms_norm_backward_ct(dY, dS_out, S, W, RSTD, offset, casting_mode, BLOCK_SIZE): + shape = dY.shape + dim = shape[-1] + dY2d = dY.view(-1, dim) + dS_out2d = dS_out.view(-1, dim) + S2d = S.view(-1, dim) + n_rows, n_cols = dY2d.shape + + # Persistent kernel: one block per SM, each handles ceil(n_rows/sm_count) rows. + # Grid = (sm_count,) -> exactly 1 block/SM -> full 256KB register file available. + # When BLOCK_SIZE > _BWD_MAX_CHUNK_SIZE, use the 2-chunk kernel to reduce + # register pressure (CHUNK_SIZE/128 elements/thread vs BLOCK_SIZE/128). + sm_count = torch.cuda.get_device_properties(W.device).multi_processor_count + num_iters = math.ceil(n_rows / sm_count) + + dX = torch.empty_like(dY2d) + # Per-SM partial dW: shape (sm_count, n_cols). + dW_partial = torch.empty(sm_count, n_cols, dtype=torch.float32, device=W.device) + + grid = (sm_count, 1, 1) + # gather+latency=3 gives explicit cp.async-style pipelining that outperforms TMA + # at every test shape on this kernel. 2-chunk variant caps per-thread register + # usage when BLOCK_SIZE > _BWD_MAX_CHUNK_SIZE (splits cols into lo/hi halves). + if BLOCK_SIZE > _BWD_MAX_CHUNK_SIZE: + CHUNK_SIZE = BLOCK_SIZE // 2 + ct.launch( + torch.cuda.current_stream(), + grid, + _fused_add_rms_norm_bwd_persistent_2c_ct_nww8, + ( + dY2d.contiguous(), + dS_out2d.contiguous(), + dX, + S2d.contiguous(), + W.contiguous(), + RSTD, + dW_partial, + int(n_cols), + float(offset), + int(num_iters), + int(sm_count), + int(n_rows), + int(CHUNK_SIZE), + int(casting_mode), + ), + ) + else: + ct.launch( + torch.cuda.current_stream(), + grid, + _fused_add_rms_norm_bwd_persistent_ct_nww8, + ( + dY2d.contiguous(), + dS_out2d.contiguous(), + dX, + S2d.contiguous(), + W.contiguous(), + RSTD, + dW_partial, + int(n_cols), + float(offset), + int(num_iters), + int(sm_count), + int(n_rows), + int(BLOCK_SIZE), + int(casting_mode), + ), + ) + + dX = dX.view(*shape) + dW = dW_partial.sum(dim=0).to(W.dtype) + return dX, dX, dW # dR == dX (gradient flows equally to X and R) + + +class LigerFusedAddRMSNormFunction(torch.autograd.Function): + """CuTile autograd wrapper for fused residual-add + RMSNorm. + + Signature-compatible with the Triton LigerFusedAddRMSNormFunction. in_place is accepted + for parity but ignored (this backend always writes fresh output buffers). + """ + + @staticmethod + @ensure_contiguous + def forward(ctx, X, R, W, eps, offset=0.0, casting_mode="llama", in_place=False): + Y, S, RSTD, BLOCK_SIZE, casting_mode_int = _fused_add_rms_norm_forward_ct(X, R, W, eps, offset, casting_mode) + ctx.offset = offset + ctx.casting_mode = casting_mode_int + ctx.BLOCK_SIZE = BLOCK_SIZE + ctx.save_for_backward(S, W, RSTD) + return Y, S + + @staticmethod + @ensure_contiguous + def backward(ctx, dY, dS_out): + S, W, RSTD = ctx.saved_tensors + dX, dR, dW = _fused_add_rms_norm_backward_ct( + dY, dS_out, S, W, RSTD, ctx.offset, ctx.casting_mode, ctx.BLOCK_SIZE + ) + return dX, dR, dW, None, None, None, None diff --git a/src/liger_kernel/ops/cutile/ops/fused_linear_cross_entropy.py b/src/liger_kernel/ops/cutile/ops/fused_linear_cross_entropy.py new file mode 100644 index 000000000..a498f49db --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/fused_linear_cross_entropy.py @@ -0,0 +1,350 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +Fused linear + cross-entropy (cuTile backend). + +Fuses the final linear projection with the cross-entropy loss, computing gradients during the +forward pass (backward-in-forward) so the full (BT, V) logit tensor is never materialised. The +token dimension BT is chunked; each chunk does one GEMM (input_chunk @ weight.T), one cuTile CE +kernel (writes d_logits in-place), then folds grad_input / grad_weight / grad_bias. + +Feature parity with the Triton LigerFusedLinearCrossEntropyFunction: ce_weight, ignore_index, +lse_square_scale, label_smoothing, reduction, softcap, return_z_loss, accum_dtype, +use_token_scaling, return_token_accuracy, return_predicted_tokens. The cuTile CE kernel +(liger_cross_entropy_kernel_ct) implements all of these; this wrapper plumbs them through. +""" + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.cross_entropy import _select_cross_entropy_block_size +from liger_kernel.ops.cutile.ops.cross_entropy import liger_cross_entropy_kernel_ct +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 +from liger_kernel.ops.utils import amp_custom_bwd +from liger_kernel.ops.utils import amp_custom_fwd + +# Cap on one chunk's logit tensor so peak logit memory stays O(chunk_size x V) rather than O(BT x V). +MAX_FUSED_SIZE = 65536 // 2 + + +def _launch_ce( + logits, + target, + ce_weight, + loss_slice, + z_loss_slice, + token_acc_slice, + pred_slice, + dummies, + V, + BLOCK_SIZE, + inv_n_non_ignore, + sum_non_ignore_weight, + weight_sum, + ignore_index, + label_smoothing, + lse_square_scale, + softcap, + has_grad, + reduction_mean, + has_weight, + has_softcap, + return_z_loss, + return_token_accuracy, + return_predicted_tokens, +): + """Launch the cuTile CE kernel for one chunk, substituting dummy tensors for disabled outputs.""" + dummy_f32, dummy_i64 = dummies + n_rows = logits.shape[0] + ct.launch( + torch.cuda.current_stream(), + (n_rows, 1, 1), + liger_cross_entropy_kernel_ct, + ( + logits, + target, + ce_weight if has_weight else dummy_f32, + loss_slice, + z_loss_slice if return_z_loss else dummy_f32, + token_acc_slice if return_token_accuracy else dummy_f32, + pred_slice if return_predicted_tokens else dummy_i64, + int(V), + float(inv_n_non_ignore), + float(sum_non_ignore_weight), + float(weight_sum), + int(ignore_index), + float(label_smoothing), + float(lse_square_scale), + float(softcap if softcap is not None else 0.0), + int(BLOCK_SIZE), + int(has_grad), + int(reduction_mean), + int(has_weight), + int(has_softcap), + int(return_z_loss), + int(return_token_accuracy), + int(return_predicted_tokens), + ), + ) + + +def fused_linear_cross_entropy_forward( + _input, + weight, + target, + ce_weight=None, + bias=None, + ignore_index=-100, + lse_square_scale=0.0, + label_smoothing=0.0, + reduction="mean", + softcap=None, + return_z_loss=False, + accum_dtype=None, + use_token_scaling=False, + return_token_accuracy=False, + return_predicted_tokens=False, +): + assert isinstance(return_z_loss, bool), f"return_z_loss must be True or False. Got: {return_z_loss}" + assert isinstance(return_token_accuracy, bool), f"return_token_accuracy must be bool. Got: {return_token_accuracy}" + assert isinstance(return_predicted_tokens, bool), ( + f"return_predicted_tokens must be bool. Got: {return_predicted_tokens}" + ) + device = _input.device + input_requires_grad = _input.requires_grad + + BT, H = _input.shape + V = weight.shape[0] + BLOCK_SIZE = _select_cross_entropy_block_size(V) + + # Chunk BT so one chunk's logit tensor is ~O(BT x H): inc_factor = ceil(V/H). + inc_factor = (V + H - 1) // H + chunk_size = _next_power_of_2((BT + inc_factor - 1) // inc_factor) + num_chunks = (BT + chunk_size - 1) // chunk_size + + grad_input = torch.zeros_like(_input, device=device) + + if input_requires_grad: + acc = weight.dtype if accum_dtype is None else accum_dtype + grad_weight = torch.zeros_like(weight, dtype=acc, device=device) if weight.requires_grad else None + grad_bias = torch.zeros_like(bias, dtype=acc, device=device) if bias is not None else None + else: + grad_weight = None + grad_bias = None + + loss_1d = torch.zeros(BT, dtype=torch.float32, device=device) + z_loss_1d = torch.zeros(BT, dtype=_input.dtype, device=device) if return_z_loss else None + token_accuracy_1d = torch.zeros(BT, dtype=torch.float32, device=device) if return_token_accuracy else None + predicted_tokens_1d = torch.full((BT,), -1, dtype=torch.int64, device=device) if return_predicted_tokens else None + + # Normalization counts (matches the Triton path). + target_mask = target != ignore_index + total_n_non_ignore = target_mask.sum().item() + inv_n_non_ignore = 1.0 / max(total_n_non_ignore, 1) + reduction_mean = int(reduction == "mean") + + sum_non_ignore_weight = float(total_n_non_ignore) + weight_sum = 0.0 + has_weight = ce_weight is not None + if has_weight: + assert ce_weight.shape[0] == V, f"If given, weight has to be a Tensor of size V. Got: {ce_weight.shape}" + assert torch.is_floating_point(ce_weight), ( + f"If given, weight has to be of floating point dtype. Got: {ce_weight.dtype}" + ) + sum_non_ignore_weight = float( + torch.gather(ce_weight, dim=0, index=target.masked_select(target_mask)).sum().item() + ) + weight_sum = float(ce_weight.sum().item()) + ce_weight = ce_weight.contiguous().float() + + has_softcap = softcap is not None + # Dummy tensors for disabled kernel outputs (cuTile requires valid tensor args). + dummies = ( + torch.zeros(1, dtype=torch.float32, device=device), + torch.zeros(1, dtype=torch.int64, device=device), + ) + + for chunk_id in range(num_chunks): + start_idx = chunk_id * chunk_size + end_idx = min((chunk_id + 1) * chunk_size, BT) + _input_chunk = _input[start_idx:end_idx] # chunk_size x H + + # Matmul in the input precision. + logits_chunk = _input_chunk @ weight.t() # chunk_size x V + if bias is not None: + logits_chunk = logits_chunk + bias + target_chunk = target[start_idx:end_idx] + + # Predicted-probability scaling factors (computed before the CE kernel overwrites logits). + if use_token_scaling: + logits_for_softmax = logits_chunk.detach().clone() + if softcap is not None: + logits_for_softmax = softcap * torch.tanh(logits_for_softmax / softcap) + probs = torch.softmax(logits_for_softmax, dim=-1) + valid_target_mask = target_chunk != ignore_index + valid_targets = target_chunk[valid_target_mask] + pred_probs = torch.zeros_like(target_chunk, dtype=probs.dtype, device=device) + if valid_targets.numel() > 0: + valid_probs = probs[valid_target_mask] + pred_probs[valid_target_mask] = torch.gather(valid_probs, -1, valid_targets.unsqueeze(-1)).squeeze(-1) + scaling_factors = pred_probs.detach() + + loss_1d_slice = loss_1d[start_idx:end_idx] + z_loss_1d_slice = z_loss_1d[start_idx:end_idx] if return_z_loss else None + token_accuracy_1d_slice = token_accuracy_1d[start_idx:end_idx] if return_token_accuracy else None + predicted_tokens_1d_slice = predicted_tokens_1d[start_idx:end_idx] if return_predicted_tokens else None + + logits_chunk = logits_chunk.contiguous() + target_chunk = target_chunk.contiguous() + + # CE kernel: computes loss (and z_loss/token_accuracy/predicted_tokens) and writes + # d_logits in-place into logits_chunk when input_requires_grad. + _launch_ce( + logits_chunk, + target_chunk, + ce_weight, + loss_1d_slice, + z_loss_1d_slice, + token_accuracy_1d_slice, + predicted_tokens_1d_slice, + dummies, + V, + BLOCK_SIZE, + inv_n_non_ignore, + sum_non_ignore_weight, + weight_sum, + ignore_index, + label_smoothing, + lse_square_scale, + softcap, + input_requires_grad, + reduction_mean, + has_weight, + has_softcap, + return_z_loss, + return_token_accuracy, + return_predicted_tokens, + ) + + # Token scaling on loss / z_loss. + if use_token_scaling: + loss_1d_slice = loss_1d_slice * scaling_factors + loss_1d[start_idx:end_idx] = loss_1d_slice + if return_z_loss: + z_loss_1d[start_idx:end_idx] = z_loss_1d_slice * scaling_factors + + grad_logits_chunk = logits_chunk # now holds d_logits + if use_token_scaling: + grad_logits_chunk = grad_logits_chunk * scaling_factors.unsqueeze(-1) + + if input_requires_grad: + grad_input[start_idx:end_idx] = grad_logits_chunk.to(_input.dtype) @ weight + if grad_weight is not None: + grad_weight += torch.mm(grad_logits_chunk.t(), _input_chunk).to(grad_weight.dtype) + if grad_bias is not None: + grad_bias += grad_logits_chunk.sum(dim=0).to(grad_bias.dtype) + + if reduction == "none": + loss = loss_1d + z_loss = z_loss_1d if return_z_loss else None + token_accuracy = token_accuracy_1d if return_token_accuracy else None + else: + loss = torch.sum(loss_1d) + z_loss = torch.sum(z_loss_1d) if return_z_loss else None + token_accuracy = torch.sum(token_accuracy_1d) / total_n_non_ignore if return_token_accuracy else None + + predicted_tokens = predicted_tokens_1d if return_predicted_tokens else None + + grad_weight = grad_weight.to(weight.dtype) if grad_weight is not None else None + grad_bias = grad_bias.to(bias.dtype) if grad_bias is not None else None + + return loss, z_loss, token_accuracy, predicted_tokens, grad_input, grad_weight, grad_bias + + +def fused_linear_cross_entropy_backward(grad_output, grad_input, grad_weight, grad_bias): + """Scale the pre-computed grads by grad_output. Out-of-place (safe for repeated backward).""" + if not torch.equal(grad_output, torch.tensor(1.0, device=grad_output.device)): + grad_input = grad_input * grad_output + if grad_weight is not None: + grad_weight = grad_weight * grad_output + if grad_bias is not None: + grad_bias = grad_bias * grad_output + return grad_input, grad_weight, grad_bias + + +class LigerFusedLinearCrossEntropyFunction(torch.autograd.Function): + @staticmethod + @amp_custom_fwd + def forward( + ctx, + _input, + weight, + target, + bias=None, + ce_weight=None, + ignore_index=-100, + lse_square_scale=0.0, + label_smoothing=0.0, + reduction="mean", + softcap=None, + return_z_loss: bool = False, + accum_dtype=None, + use_token_scaling: bool = False, + return_token_accuracy: bool = False, + return_predicted_tokens: bool = False, + ): + loss, z_loss, token_accuracy, predicted_tokens, grad_input, grad_weight, grad_bias = ( + fused_linear_cross_entropy_forward( + _input, + weight, + target, + ce_weight, + bias, + ignore_index, + lse_square_scale, + label_smoothing, + reduction, + softcap, + return_z_loss, + accum_dtype, + use_token_scaling, + return_token_accuracy, + return_predicted_tokens, + ) + ) + ctx.save_for_backward( + grad_input.detach(), + grad_weight.detach() if grad_weight is not None else None, + grad_bias.detach() if grad_bias is not None else None, + ) + ctx.return_z_loss = return_z_loss + ctx.return_token_accuracy = return_token_accuracy + ctx.return_predicted_tokens = return_predicted_tokens + return loss, z_loss, token_accuracy, predicted_tokens + + @staticmethod + @amp_custom_bwd + def backward(ctx, grad_output, grad_output2, grad_output3, grad_output4): + grad_input, grad_weight, grad_bias = ctx.saved_tensors + grad_input, grad_weight, grad_bias = fused_linear_cross_entropy_backward( + grad_output, grad_input, grad_weight, grad_bias + ) + return ( + grad_input, + grad_weight, + None, # target + grad_bias, + None, # ce_weight + None, # ignore_index + None, # lse_square_scale + None, # label_smoothing + None, # reduction + None, # softcap + None, # return_z_loss + None, # accum_dtype + None, # use_token_scaling + None, # return_token_accuracy + None, # return_predicted_tokens + ) diff --git a/src/liger_kernel/ops/cutile/ops/grpo_loss.py b/src/liger_kernel/ops/cutile/ops/grpo_loss.py new file mode 100644 index 000000000..79a7265fe --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/grpo_loss.py @@ -0,0 +1,1228 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +GRPO loss (cuTile backend). + +Token-level GRPO-family policy-gradient loss. Each grid block (b, l) computes the per-token +loss for one (batch, completion-token) position: logsumexp over the vocab, target logp, the +importance ratio coef_1 = exp(logp - old_logp), then the loss for the selected loss_type +(GRPO/DAPO/BNPO/DR-GRPO/LUSPO share the PPO-clipped form; CISPO; SAPO), optional dual-clip +(delta), optional vLLM IS-ratio reweighting, and an optional KL penalty. Backward recomputes +logp from the cached lse and writes dlogits. + +Full feature parity with the Triton GrpoLossFunction: all loss types (GRPO/DAPO/BNPO/DR-GRPO/ +LUSPO/CISPO/SAPO/VESPO), token- and sequence-level (GSPO) importance sampling, dual-clip, +vLLM IS-ratio, KL penalty, and num_items_in_batch normalization. +""" + +import math +import os + +from types import SimpleNamespace + +import cuda.tile as ct +import torch + +from cuda.tile import ByTarget +from cuda.tile.tune import exhaustive_search + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +LOG2E = 1.4426950408889634 + +_LOSS_TYPE_GRPO = 0 +_LOSS_TYPE_CISPO = 1 +_LOSS_TYPE_SAPO = 2 +_LOSS_TYPE_VESPO = 3 + +_str_to_loss_type = { + "grpo": _LOSS_TYPE_GRPO, + "dapo": _LOSS_TYPE_GRPO, + "bnpo": _LOSS_TYPE_GRPO, + "dr_grpo": _LOSS_TYPE_GRPO, + "luspo": _LOSS_TYPE_GRPO, # LUSPO uses the same per-token PPO clipping as GRPO + "cispo": _LOSS_TYPE_CISPO, + "sapo": _LOSS_TYPE_SAPO, + "vespo": _LOSS_TYPE_VESPO, +} + + +@ct.kernel +def _grpo_loss_fwd_ct( + logits_input, # (B*(L+1), N) logits (2D view of (B, L+1, N)) + old_logp_input, # (B, L) float32 or dummy (1,) + ref_logp_input, # (B, L) float32 or dummy (1,) + input_ids, # (B, L) int64 completion token ids + completion_mask, # (B, L) int32 or dummy (1,) + advantages, # (B,) float32 + vllm_is_ratio, # (B, L) or dummy (1,) float32 + phi_seq_input, # (B,) VESPO per-sequence gamma weight (or dummy (1,)) + loss_output, # (B, L) float32 output + lse_cache, # (B, L) float32 cached log-sum-exp + kl_output, # (B, L) float32 or dummy (1,) + is_clipped_output, # (B, L) float32 + B: ct.Constant[int], + L: ct.Constant[int], + N: ct.Constant[int], + BLOCK_N: ct.Constant[int], + temperature, + beta: ct.Constant[float], + eps_low, + eps_high, + loss_type: ct.Constant[int], + sapo_temp_pos, + sapo_temp_neg, + delta, + use_bias_correction_kl: ct.Constant[int], + HAS_COMPLETION_MASK: ct.Constant[int], + HAS_OLD_LOGP: ct.Constant[int], + HAS_VLLM_IS_RATIO: ct.Constant[int], + vllm_is_ratio_stride, +): + """ + GRPO forward. + + Grid: (B, L). Each block (off_b, off_l) computes the loss for one + (batch, completion-token) position. + """ + off_b = ct.bid(0) + off_l = ct.bid(1) + + # Optional completion mask: skip masked tokens + if HAS_COMPLETION_MASK: + mask_val = ct.astype(ct.load(completion_mask, (off_b, off_l), shape=()), ct.int32) + if mask_val == 0: + return + + # Logits row for this (b, l): off_b*(L+1) + off_l + logits_row = ct.add(ct.mul(off_b, L + 1), off_l) + n_chunks = (N + BLOCK_N - 1) // BLOCK_N + inv_temperature = 1.0 / temperature + + # ---- Compute logsumexp via online algorithm (fold trick) ---- + m_i = ct.full((), -math.inf, dtype=ct.float32) + l_i = ct.full((), 0.0, dtype=ct.float32) + + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_N, dtype=ct.int32), ci * BLOCK_N) + logits = ct.astype( + ct.gather(logits_input, (logits_row, col_idx), check_bounds=True, padding_value=-math.inf, latency=3), + ct.float32, + ) + logits_scaled = logits * inv_temperature + + chunk_max = ct.max(logits_scaled, 0, keepdims=False) + new_m = ct.maximum(m_i, chunk_max) + alpha = ct.exp2(ct.mul(m_i - new_m, LOG2E)) + l_i = ct.add(ct.mul(l_i, alpha), ct.sum(ct.exp2(ct.mul(logits_scaled - new_m, LOG2E)), 0, keepdims=False)) + m_i = new_m + + lse = m_i + ct.log(l_i) + + # ---- Load logit at target token ---- + idx_raw = ct.load(input_ids, (off_b, off_l), shape=()) + idx = ct.astype(idx_raw, ct.int32) + idx_tile = ct.add(ct.arange(1, dtype=ct.int32), idx) + x_tile = ct.astype(ct.gather(logits_input, (logits_row, idx_tile), check_bounds=False), ct.float32) + x = ct.sum(x_tile, 0, keepdims=False) * inv_temperature + logp = x - lse + + # ---- Load old_logp ---- + if HAS_OLD_LOGP: + old_logp = ct.astype(ct.load(old_logp_input, (off_b, off_l), shape=()), ct.float32) + else: + old_logp = logp + coef_1 = ct.exp(logp - old_logp) + advantage = ct.astype(ct.load(advantages, off_b, shape=()), ct.float32) + + # ---- Compute per-token loss based on loss_type ---- + if loss_type == 0: # GRPO: standard PPO clipping + coef_2_low = ct.maximum(coef_1, ct.full((), 1.0 - eps_low, dtype=ct.float32)) + coef_2_high = ct.minimum(coef_2_low, ct.full((), 1.0 + eps_high, dtype=ct.float32)) + is_low_clipped = ct.astype(coef_1 < (1.0 - eps_low), ct.float32) * ct.astype(advantage < 0.0, ct.float32) + is_high_clipped = ct.astype(coef_1 > (1.0 + eps_high), ct.float32) * ct.astype(advantage > 0.0, ct.float32) + is_clipped = ct.minimum(is_low_clipped + is_high_clipped, ct.full((), 1.0, dtype=ct.float32)) + # Apply delta upper-clip on importance ratio (dual-clip extension) + if delta != 0.0: + coef_1_for_loss = ct.minimum(coef_1, ct.full((), delta, dtype=ct.float32)) + else: + coef_1_for_loss = coef_1 + per_token_loss1 = coef_1_for_loss * advantage + per_token_loss2 = coef_2_high * advantage + per_token_loss = -ct.minimum(per_token_loss1, per_token_loss2) + elif loss_type == 1: # CISPO + coef_2 = ct.minimum(coef_1, ct.full((), eps_high, dtype=ct.float32)) + per_token_loss = -coef_2 * advantage * logp + is_clipped = ct.astype(coef_1 > eps_high, ct.float32) * ct.astype(advantage > 0.0, ct.float32) + elif loss_type == 2: # SAPO + temp_sapo = ct.maximum( + ct.full((), sapo_temp_pos, dtype=ct.float32), + ct.full((), sapo_temp_neg, dtype=ct.float32), + ) + if advantage > 0.0: + temp_sapo = ct.full((), sapo_temp_pos, dtype=ct.float32) + else: + temp_sapo = ct.full((), sapo_temp_neg, dtype=ct.float32) + sigmoid_input = temp_sapo * (coef_1 - 1.0) + # sigmoid(x) = 1 / (1 + exp(-x)) — cuda.tile has no ct.sigmoid. + sig = ct.truediv(1.0, 1.0 + ct.exp(0.0 - sigmoid_input)) + sapo_coef = sig * 4.0 / temp_sapo + per_token_loss = -sapo_coef * advantage + is_clipped = ct.full((), 0.0, dtype=ct.float32) + else: # loss_type == 3: VESPO — detached per-sequence gamma weight on logp + phi_seq = ct.astype(ct.load(phi_seq_input, off_b, shape=()), ct.float32) + per_token_loss = -phi_seq * advantage * logp + is_clipped = ct.full((), 0.0, dtype=ct.float32) + + # ---- Apply vLLM IS ratio (optional) ---- + if HAS_VLLM_IS_RATIO: + vllm_col = off_l % vllm_is_ratio_stride + vllm_row_base = off_b * vllm_is_ratio_stride + # scalar gather: 1-element tile + vllm_tile = ct.gather( + vllm_is_ratio, ct.arange(1, dtype=ct.int32) + vllm_row_base + vllm_col, check_bounds=False + ) + vllm_ratio = ct.astype(ct.sum(ct.astype(vllm_tile, ct.float32), 0, keepdims=False), ct.float32) + per_token_loss = per_token_loss * vllm_ratio + + # ---- KL penalty (optional, beta != 0 is compile-time) ---- + if beta != 0.0: + ref_logp = ct.astype(ct.load(ref_logp_input, (off_b, off_l), shape=()), ct.float32) + kl = ct.exp(ref_logp - logp) - (ref_logp - logp) - 1.0 + if use_bias_correction_kl: + # Importance-sampling-corrected KL (DeepSeek-V3.2): kl *= coef_1 + kl = kl * coef_1 + per_token_loss = per_token_loss + beta * kl + ct.scatter(kl_output, (off_b, off_l), ct.astype(kl, kl_output.dtype)) + + # ---- Store outputs ---- + ct.scatter(loss_output, (off_b, off_l), ct.astype(per_token_loss, loss_output.dtype)) + ct.scatter(lse_cache, (off_b, off_l), ct.astype(lse, lse_cache.dtype)) + ct.scatter(is_clipped_output, (off_b, off_l), ct.astype(is_clipped, is_clipped_output.dtype)) + + +@ct.kernel +def _grpo_loss_bwd_ct( + dloss_input, # (B, L) float32 upstream gradient + dlogits_output, # (B*(L+1), N) gradient output + logits_input, # (B*(L+1), N) saved logits + old_logp_input, # (B, L) or dummy (1,) + ref_logp_input, # (B, L) or dummy (1,) + input_ids, # (B, L) int64 + advantages, # (B,) float32 + completion_mask, # (B, L) int32 or dummy (1,) + lse_cache, # (B, L) float32 saved lse + vllm_is_ratio, # (B, L) or dummy (1,) + phi_seq_input, # (B,) VESPO per-sequence gamma weight (or dummy (1,)) + B: ct.Constant[int], + L: ct.Constant[int], + N: ct.Constant[int], + BLOCK_N: ct.Constant[int], + temperature, + beta: ct.Constant[float], + eps_low, + eps_high, + loss_type: ct.Constant[int], + sapo_temp_pos, + sapo_temp_neg, + delta, + use_bias_correction_kl: ct.Constant[int], + HAS_COMPLETION_MASK: ct.Constant[int], + HAS_OLD_LOGP: ct.Constant[int], + HAS_VLLM_IS_RATIO: ct.Constant[int], + vllm_is_ratio_stride, +): + """GRPO backward. Grid: (B, L).""" + off_b = ct.bid(0) + off_l = ct.bid(1) + + logits_row = ct.add(ct.mul(off_b, L + 1), off_l) + n_chunks = (N + BLOCK_N - 1) // BLOCK_N + inv_temperature = 1.0 / temperature + + if HAS_COMPLETION_MASK: + mask_val = ct.astype(ct.load(completion_mask, (off_b, off_l), shape=()), ct.int32) + if mask_val == 0: + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_N, dtype=ct.int32), ci * BLOCK_N) + zero_tile = ct.full((BLOCK_N,), 0.0, dtype=dlogits_output.dtype) + ct.scatter(dlogits_output, (logits_row, col_idx), zero_tile, check_bounds=True) + return + + dloss = ct.astype(ct.load(dloss_input, (off_b, off_l), shape=()), ct.float32) + lse = ct.astype(ct.load(lse_cache, (off_b, off_l), shape=()), ct.float32) + + idx_raw = ct.load(input_ids, (off_b, off_l), shape=()) + idx = ct.astype(idx_raw, ct.int32) + idx_tile = ct.add(ct.arange(1, dtype=ct.int32), idx) + x_tile = ct.astype(ct.gather(logits_input, (logits_row, idx_tile), check_bounds=False), ct.float32) + x = ct.sum(x_tile, 0, keepdims=False) * inv_temperature + logp = x - lse + + if HAS_OLD_LOGP: + old_logp = ct.astype(ct.load(old_logp_input, (off_b, off_l), shape=()), ct.float32) + else: + old_logp = logp + coef_1 = ct.exp(logp - old_logp) + advantage = ct.astype(ct.load(advantages, off_b, shape=()), ct.float32) + + if loss_type == 0: # GRPO + coef_2_low = ct.maximum(coef_1, ct.full((), 1.0 - eps_low, dtype=ct.float32)) + coef_2_high = ct.minimum(coef_2_low, ct.full((), 1.0 + eps_high, dtype=ct.float32)) + if delta != 0.0: + coef_1_for_loss = ct.minimum(coef_1, ct.full((), delta, dtype=ct.float32)) + else: + coef_1_for_loss = coef_1 + per_token_loss1 = coef_1_for_loss * advantage + per_token_loss2 = coef_2_high * advantage + # gradient flows only when unclipped (per_token_loss2 >= per_token_loss1) + grad_mask = ct.astype(per_token_loss2 >= per_token_loss1, ct.float32) + # Gradient uses original coef_1; zero when delta-clamped (constant → no gradient) + dlogp = -coef_1 * advantage * grad_mask + if delta != 0.0: + dlogp = dlogp * ct.astype(coef_1 <= ct.full((), delta, dtype=ct.float32), ct.float32) + elif loss_type == 1: # CISPO + coef_2 = ct.minimum(coef_1, ct.full((), eps_high, dtype=ct.float32)) + dlogp = -coef_2 * advantage + elif loss_type == 2: # SAPO + if advantage > 0.0: + temp_sapo = ct.full((), sapo_temp_pos, dtype=ct.float32) + else: + temp_sapo = ct.full((), sapo_temp_neg, dtype=ct.float32) + sigmoid_input = temp_sapo * (coef_1 - 1.0) + sigmoid_val = ct.truediv(1.0, 1.0 + ct.exp(0.0 - sigmoid_input)) # sigmoid via 1/(1+exp(-x)) + d_sapo_d_coef1 = 4.0 * sigmoid_val * (1.0 - sigmoid_val) + dlogp = -advantage * d_sapo_d_coef1 * coef_1 + else: # loss_type == 3: VESPO — loss = -phi_seq*advantage*logp, phi_seq detached + phi_seq = ct.astype(ct.load(phi_seq_input, off_b, shape=()), ct.float32) + dlogp = -phi_seq * advantage + + if HAS_VLLM_IS_RATIO: + vllm_col = off_l % vllm_is_ratio_stride + vllm_tile = ct.gather( + vllm_is_ratio, + ct.add(ct.arange(1, dtype=ct.int32), ct.mul(off_b, vllm_is_ratio_stride) + vllm_col), + check_bounds=False, + ) + vllm_ratio = ct.astype(ct.sum(ct.astype(vllm_tile, ct.float32), 0, keepdims=False), ct.float32) + dlogp = dlogp * vllm_ratio + + if beta != 0.0: + ref_logp = ct.astype(ct.load(ref_logp_input, (off_b, off_l), shape=()), ct.float32) + if use_bias_correction_kl: + # d(kl * coef_1)/d(logp) = coef_1 * (logp - ref_logp), where coef_1 = exp(logp - old_logp) + dlogp = dlogp + beta * coef_1 * (logp - ref_logp) + else: + dlogp = dlogp + beta * (1.0 - ct.exp(ref_logp - logp)) + + dlogp_scaled = dlogp * dloss * inv_temperature + + # Compute and store dlogits for all vocab positions + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_N, dtype=ct.int32), ci * BLOCK_N) + logits = ct.astype( + ct.gather(logits_input, (logits_row, col_idx), check_bounds=True, padding_value=-math.inf, latency=10), + ct.float32, + ) + probs = ct.exp(logits * inv_temperature - lse) + + # dlogits[j] = (indicator(j==idx) - prob[j]) * dlogp + idx_tile_b = ct.add(ct.arange(BLOCK_N, dtype=ct.int32), ci * BLOCK_N) + is_target = ct.astype(idx_tile_b == idx, ct.float32) + dlogits = (is_target - probs) * dlogp_scaled + + ct.scatter( + dlogits_output, + (logits_row, col_idx), + ct.astype(dlogits, dlogits_output.dtype), + check_bounds=True, + latency=10, + ) + + +# --- bwd occupancy: static per-batch-size selection (zero lookup overhead) --- +_bwd_occ_small = _grpo_loss_bwd_ct.replace_hints(occupancy=ByTarget(sm_100=4, default=4)) +_bwd_occ_large = _grpo_loss_bwd_ct.replace_hints(occupancy=ByTarget(sm_100=12, default=12)) + + +# --- fwd occupancy autotune (per launch shape) -------------------------------- +_FWD_OCC_CONFIGS = [ + SimpleNamespace(occupancy=1), + SimpleNamespace(occupancy=2), + SimpleNamespace(occupancy=4), + SimpleNamespace(occupancy=12), +] +_FWD_FALLBACK_OCC = 1 +_fwd_autotune_cache: dict = {} + + +def _tuned_fwd_kernel(stream, cache_key, grid, fwd_args): + if os.environ.get("DISABLE_AUTOTUNE") == "1": + return _grpo_loss_fwd_ct.replace_hints(occupancy=ByTarget(sm_100=_FWD_FALLBACK_OCC, default=_FWD_FALLBACK_OCC)) + if cache_key not in _fwd_autotune_cache: + result = exhaustive_search( + _FWD_OCC_CONFIGS, + stream, + lambda cfg: grid, + _grpo_loss_fwd_ct, + lambda cfg: fwd_args, + lambda cfg: {"occupancy": ByTarget(sm_100=cfg.occupancy, default=cfg.occupancy)}, + quiet=True, + ) + best_occ = result.best.config.occupancy + _fwd_autotune_cache[cache_key] = _grpo_loss_fwd_ct.replace_hints( + occupancy=ByTarget(sm_100=best_occ, default=best_occ) + ) + return _fwd_autotune_cache[cache_key] + + +def _grpo_loss_forward_ct( + logits, + old_logp, + ref_logp, + completion_ids, + advantages, + completion_mask, + temperature, + beta, + eps_low, + eps_high, + loss_type_int, + sapo_temperature_pos, + sapo_temperature_neg, + delta, + use_bias_correction_kl, + vllm_is_ratio, + vllm_is_ratio_stride, + phi_seq, +): + B, L_ADD_1, N = logits.shape + L = L_ADD_1 - 1 + BLOCK_N = min(8192, _next_power_of_2(N)) + + logits_2d = logits.reshape(B * L_ADD_1, N).contiguous() + + loss = torch.zeros(B, L, device=logits.device, dtype=torch.float32) + lse = torch.zeros(B, L, device=logits.device, dtype=torch.float32) + is_clipped = torch.zeros(B, L, device=logits.device, dtype=torch.float32) + + has_beta = float(beta) != 0.0 + kl = torch.zeros(B, L, device=logits.device, dtype=torch.float32) if has_beta else None + + dummy_f = torch.zeros(1, device=logits.device, dtype=torch.float32) + dummy_i = torch.zeros(1, device=logits.device, dtype=torch.int32) + + old_logp_arg = old_logp.contiguous() if old_logp is not None else dummy_f + ref_logp_arg = ref_logp.contiguous() if ref_logp is not None else dummy_f + mask_arg = completion_mask.to(torch.int32).contiguous() if completion_mask is not None else dummy_i + kl_arg = kl if kl is not None else dummy_f + # Flatten to 1-D: the kernel gathers with a flat row*stride+col index, so the + # array must be rank-1 (strict tileiras rejects a rank-2 array with a 1-tuple index). + vllm_arg = vllm_is_ratio.contiguous().view(-1) if vllm_is_ratio is not None else dummy_f + phi_seq_arg = phi_seq.contiguous() if phi_seq is not None else dummy_f + + has_mask = int(completion_mask is not None) + has_old_logp = int(old_logp is not None) + # VESPO (loss_type 3) folds the vLLM correction into phi_seq, so the kernel skips vllm. + has_vllm = int(vllm_is_ratio is not None and int(loss_type_int) != 3) + + grid = (B, L, 1) + stream = torch.cuda.current_stream() + fwd_args = ( + logits_2d, + old_logp_arg, + ref_logp_arg, + completion_ids.contiguous(), + mask_arg, + advantages.contiguous(), + vllm_arg, + phi_seq_arg, + loss, + lse, + kl_arg, + is_clipped, + int(B), + int(L), + int(N), + int(BLOCK_N), + float(temperature), + float(beta), + float(eps_low), + float(eps_high), + int(loss_type_int), + float(sapo_temperature_pos), + float(sapo_temperature_neg), + float(delta), + int(use_bias_correction_kl), + int(has_mask), + int(has_old_logp), + int(has_vllm), + int(vllm_is_ratio_stride), + ) + cache_key = ( + int(B), + int(L), + int(N), + int(BLOCK_N), + int(loss_type_int), + int(has_mask), + int(has_old_logp), + int(has_vllm), + str(logits_2d.device), + ) + kernel = _tuned_fwd_kernel(stream, cache_key, grid, fwd_args) + ct.launch(stream, grid, kernel, fwd_args) + + return loss, lse, is_clipped, kl + + +def _grpo_loss_backward_ct( + dloss, + logits, + old_logp, + ref_logp, + completion_ids, + advantages, + completion_mask, + lse, + temperature, + beta, + eps_low, + eps_high, + inplace, + loss_type_int, + sapo_temperature_pos, + sapo_temperature_neg, + delta, + use_bias_correction_kl, + vllm_is_ratio, + vllm_is_ratio_stride, + phi_seq, +): + B, L_ADD_1, N = logits.shape + L = L_ADD_1 - 1 + BLOCK_N = min(4096, _next_power_of_2(N)) + + logits_2d = logits.reshape(B * L_ADD_1, N).contiguous() + dlogits_2d = logits.data.reshape(B * L_ADD_1, N) if inplace else torch.empty_like(logits_2d) + + dummy_f = torch.zeros(1, device=logits.device, dtype=torch.float32) + dummy_i = torch.zeros(1, device=logits.device, dtype=torch.int32) + + old_logp_arg = old_logp.contiguous() if old_logp is not None else dummy_f + ref_logp_arg = ref_logp.contiguous() if ref_logp is not None else dummy_f + mask_arg = completion_mask.to(torch.int32).contiguous() if completion_mask is not None else dummy_i + # Flatten to 1-D: the kernel gathers with a flat row*stride+col index, so the + # array must be rank-1 (strict tileiras rejects a rank-2 array with a 1-tuple index). + vllm_arg = vllm_is_ratio.contiguous().view(-1) if vllm_is_ratio is not None else dummy_f + phi_seq_arg = phi_seq.contiguous() if phi_seq is not None else dummy_f + + has_mask = int(completion_mask is not None) + has_old_logp = int(old_logp is not None) + has_vllm = int(vllm_is_ratio is not None and int(loss_type_int) != 3) + + grid = (B, L, 1) + bwd_kernel = _bwd_occ_small if B <= 2 else _bwd_occ_large + ct.launch( + torch.cuda.current_stream(), + grid, + bwd_kernel, + ( + dloss.contiguous(), + dlogits_2d, + logits_2d, + old_logp_arg, + ref_logp_arg, + completion_ids.contiguous(), + advantages.contiguous(), + mask_arg, + lse, + vllm_arg, + phi_seq_arg, + int(B), + int(L), + int(N), + int(BLOCK_N), + float(temperature), + float(beta), + float(eps_low), + float(eps_high), + int(loss_type_int), + float(sapo_temperature_pos), + float(sapo_temperature_neg), + float(delta), + int(use_bias_correction_kl), + int(has_mask), + int(has_old_logp), + int(has_vllm), + int(vllm_is_ratio_stride), + ), + ) + + dlogits = dlogits_2d.reshape(B, L_ADD_1, N) + dlogits[:, -1, :] = 0 + return dlogits + + +# --------------------------------------------------------------------------- +# Sequence-level importance sampling (GSPO): per-sequence coef_1 precomputed on host. +# --------------------------------------------------------------------------- + + +@ct.kernel +def _grpo_loss_fwd_seq_ct( + logits_input, # (B*(L+1), N) + ref_logp_input, # (B, L) or dummy (1,) + input_ids, # (B, L) int64 + completion_mask, # (B, L) int32 or dummy (1,) + advantages, # (B,) float32 + coef_1, # (B,) per-sequence importance weight, post delta-clamp + coef_1_raw, # (B,) per-sequence importance weight, pre delta-clamp (for bias-corrected KL) + coef_2, # (B,) clipped coef + is_clipped_seq, # (B,) clipping indicator + vllm_is_ratio, # (B, L)/(B, 1) or dummy (1,) + loss_output, # (B, L) + lse_cache, # (B, L) + kl_output, # (B, L) or dummy (1,) + is_clipped_output, # (B, L) + B: ct.Constant[int], + L: ct.Constant[int], + N: ct.Constant[int], + BLOCK_N: ct.Constant[int], + temperature, + beta: ct.Constant[float], + use_bias_correction_kl: ct.Constant[int], + HAS_COMPLETION_MASK: ct.Constant[int], + HAS_VLLM_IS_RATIO: ct.Constant[int], + vllm_is_ratio_stride, +): + """Sequence-level GRPO forward. Grid: (B, L). Uses precomputed per-sequence coefficients.""" + off_b = ct.bid(0) + off_l = ct.bid(1) + + if HAS_COMPLETION_MASK: + mask_val = ct.astype(ct.load(completion_mask, (off_b, off_l), shape=()), ct.int32) + if mask_val == 0: + return + + logits_row = ct.add(ct.mul(off_b, L + 1), off_l) + n_chunks = (N + BLOCK_N - 1) // BLOCK_N + inv_temperature = 1.0 / temperature + + m_i = ct.full((), -math.inf, dtype=ct.float32) + l_i = ct.full((), 0.0, dtype=ct.float32) + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_N, dtype=ct.int32), ci * BLOCK_N) + logits = ct.astype( + ct.gather(logits_input, (logits_row, col_idx), check_bounds=True, padding_value=-math.inf, latency=3), + ct.float32, + ) + logits_scaled = logits * inv_temperature + chunk_max = ct.max(logits_scaled, 0, keepdims=False) + new_m = ct.maximum(m_i, chunk_max) + alpha = ct.exp2(ct.mul(m_i - new_m, LOG2E)) + l_i = ct.add(ct.mul(l_i, alpha), ct.sum(ct.exp2(ct.mul(logits_scaled - new_m, LOG2E)), 0, keepdims=False)) + m_i = new_m + lse = m_i + ct.log(l_i) + + idx = ct.astype(ct.load(input_ids, (off_b, off_l), shape=()), ct.int32) + idx_tile = ct.add(ct.arange(1, dtype=ct.int32), idx) + x = ( + ct.sum( + ct.astype(ct.gather(logits_input, (logits_row, idx_tile), check_bounds=False), ct.float32), + 0, + keepdims=False, + ) + * inv_temperature + ) + logp = x - lse + + coef_1_v = ct.astype(ct.load(coef_1, off_b, shape=()), ct.float32) + coef_2_v = ct.astype(ct.load(coef_2, off_b, shape=()), ct.float32) + is_clip_v = ct.astype(ct.load(is_clipped_seq, off_b, shape=()), ct.float32) + advantage = ct.astype(ct.load(advantages, off_b, shape=()), ct.float32) + + per_token_loss = -ct.minimum(coef_1_v * advantage, coef_2_v * advantage) + + if HAS_VLLM_IS_RATIO: + vllm_col = off_l % vllm_is_ratio_stride + vllm_tile = ct.gather( + vllm_is_ratio, + ct.add(ct.arange(1, dtype=ct.int32), ct.mul(off_b, vllm_is_ratio_stride) + vllm_col), + check_bounds=False, + ) + vllm_ratio = ct.astype(ct.sum(ct.astype(vllm_tile, ct.float32), 0, keepdims=False), ct.float32) + per_token_loss = per_token_loss * vllm_ratio + + if beta != 0.0: + ref_logp = ct.astype(ct.load(ref_logp_input, (off_b, off_l), shape=()), ct.float32) + kl = ct.exp(ref_logp - logp) - (ref_logp - logp) - 1.0 + if use_bias_correction_kl: + kl = kl * ct.astype(ct.load(coef_1_raw, off_b, shape=()), ct.float32) + per_token_loss = per_token_loss + beta * kl + ct.scatter(kl_output, (off_b, off_l), ct.astype(kl, kl_output.dtype)) + + ct.scatter(loss_output, (off_b, off_l), ct.astype(per_token_loss, loss_output.dtype)) + ct.scatter(lse_cache, (off_b, off_l), ct.astype(lse, lse_cache.dtype)) + ct.scatter(is_clipped_output, (off_b, off_l), ct.astype(is_clip_v, is_clipped_output.dtype)) + + +@ct.kernel +def _grpo_loss_bwd_seq_ct( + dloss_input, # (B, L) per-token upstream grad (for KL term) + dloss_sum_input, # (B,) per-sequence sum of dloss (for policy grad) + dlogits_output, # (B*(L+1), N) + logits_input, # (B*(L+1), N) + ref_logp_input, # (B, L) or dummy (1,) + input_ids, # (B, L) int64 + advantages, # (B,) float32 + completion_mask, # (B, L) int32 or dummy (1,) + lse_cache, # (B, L) + coef_1, # (B,) per-sequence importance weight (pre delta-clamp) + seq_len, # (B,) valid tokens per sequence + B: ct.Constant[int], + L: ct.Constant[int], + N: ct.Constant[int], + BLOCK_N: ct.Constant[int], + temperature, + beta: ct.Constant[float], + use_bias_correction_kl: ct.Constant[int], + eps_low, + eps_high, + delta, + HAS_COMPLETION_MASK: ct.Constant[int], +): + """Sequence-level GRPO backward. Grid: (B, L).""" + off_b = ct.bid(0) + off_l = ct.bid(1) + logits_row = ct.add(ct.mul(off_b, L + 1), off_l) + n_chunks = (N + BLOCK_N - 1) // BLOCK_N + inv_temperature = 1.0 / temperature + + if HAS_COMPLETION_MASK: + mask_val = ct.astype(ct.load(completion_mask, (off_b, off_l), shape=()), ct.int32) + if mask_val == 0: + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_N, dtype=ct.int32), ci * BLOCK_N) + ct.scatter( + dlogits_output, + (logits_row, col_idx), + ct.full((BLOCK_N,), 0.0, dtype=dlogits_output.dtype), + check_bounds=True, + ) + return + + dloss = ct.astype(ct.load(dloss_input, (off_b, off_l), shape=()), ct.float32) + dloss_sum = ct.astype(ct.load(dloss_sum_input, off_b, shape=()), ct.float32) + lse = ct.astype(ct.load(lse_cache, (off_b, off_l), shape=()), ct.float32) + coef_1_v = ct.astype(ct.load(coef_1, off_b, shape=()), ct.float32) + seq_len_v = ct.astype(ct.load(seq_len, off_b, shape=()), ct.float32) + + idx = ct.astype(ct.load(input_ids, (off_b, off_l), shape=()), ct.int32) + idx_tile = ct.add(ct.arange(1, dtype=ct.int32), idx) + x = ( + ct.sum( + ct.astype(ct.gather(logits_input, (logits_row, idx_tile), check_bounds=False), ct.float32), + 0, + keepdims=False, + ) + * inv_temperature + ) + logp = x - lse + advantage = ct.astype(ct.load(advantages, off_b, shape=()), ct.float32) + + coef_2 = ct.minimum(ct.maximum(coef_1_v, 1.0 - eps_low), 1.0 + eps_high) + if delta != 0.0: + coef_1_for_loss = ct.minimum(coef_1_v, ct.full((), delta, dtype=ct.float32)) + else: + coef_1_for_loss = coef_1_v + is_unclipped = ct.astype((coef_2 * advantage) >= (coef_1_for_loss * advantage), ct.float32) + + dlogp = -coef_1_v * advantage / seq_len_v * is_unclipped * dloss_sum + if delta != 0.0: + dlogp = dlogp * ct.astype(coef_1_v <= ct.full((), delta, dtype=ct.float32), ct.float32) + + if beta != 0.0: + ref_logp = ct.astype(ct.load(ref_logp_input, (off_b, off_l), shape=()), ct.float32) + if use_bias_correction_kl: + dlogp = dlogp + beta * coef_1_v * (logp - ref_logp) * dloss + else: + dlogp = dlogp + beta * (1.0 - ct.exp(ref_logp - logp)) * dloss + + dlogp_scaled = dlogp * inv_temperature + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_N, dtype=ct.int32), ci * BLOCK_N) + logits = ct.astype( + ct.gather(logits_input, (logits_row, col_idx), check_bounds=True, padding_value=-math.inf, latency=10), + ct.float32, + ) + probs = ct.exp(logits * inv_temperature - lse) + is_target = ct.astype(col_idx == idx, ct.float32) + dlogits = (is_target - probs) * dlogp_scaled + ct.scatter( + dlogits_output, + (logits_row, col_idx), + ct.astype(dlogits, dlogits_output.dtype), + check_bounds=True, + latency=10, + ) + + +_bwd_seq_occ_small = _grpo_loss_bwd_seq_ct.replace_hints(occupancy=ByTarget(sm_100=4, default=4)) +_bwd_seq_occ_large = _grpo_loss_bwd_seq_ct.replace_hints(occupancy=ByTarget(sm_100=12, default=12)) + + +def _compute_dapo_normalizer(completion_mask, num_items_in_batch=None): + """Per-process normalizer for DAPO/CISPO/VESPO (matches the Triton path).""" + world_size = 1 + if torch.distributed.is_available() and torch.distributed.is_initialized(): + world_size = torch.distributed.get_world_size() + + if num_items_in_batch is not None: + if isinstance(num_items_in_batch, torch.Tensor): + normalizer = num_items_in_batch.to(device=completion_mask.device, dtype=torch.float32) + else: + normalizer = torch.as_tensor(float(num_items_in_batch), device=completion_mask.device, dtype=torch.float32) + normalizer = normalizer / world_size + return torch.clamp(normalizer, min=1.0) + + normalizer = completion_mask.to(torch.float32).sum() + if world_size > 1: + normalizer = normalizer.clone() + torch.distributed.all_reduce(normalizer, op=torch.distributed.ReduceOp.SUM) + normalizer = normalizer / world_size + return torch.clamp(normalizer, min=1.0) + + +def _reduce_loss(per_token_loss, mask, loss_type, max_completion_length, B, L, num_items_in_batch=None): + """Apply loss reduction based on loss_type (matches the Triton path).""" + if loss_type == "grpo" or loss_type == "sapo": + return ((per_token_loss * mask).sum(-1) / mask.sum(-1).clamp(min=1.0)).mean() + elif loss_type == "bnpo": + return (per_token_loss * mask).sum() / mask.sum().clamp(min=1.0) + elif loss_type == "dr_grpo": + max_len = max_completion_length if max_completion_length is not None else L + return (per_token_loss * mask).sum() / (B * max_len) + elif loss_type == "dapo" or loss_type == "cispo" or loss_type == "vespo": + return (per_token_loss * mask).sum() / _compute_dapo_normalizer(mask, num_items_in_batch=num_items_in_batch) + elif loss_type == "luspo": + return (per_token_loss * mask.sum(-1, keepdim=True)).mean() + raise ValueError(f"Unknown loss_type: {loss_type}. Expected one of: grpo, bnpo, dr_grpo, dapo, cispo, sapo, luspo") + + +def _grpo_loss_forward_seq_ct( + logits, + old_logp, + ref_logp, + completion_ids, + advantages, + completion_mask, + temperature, + beta, + eps_low, + eps_high, + delta_val, + use_bias_correction_kl, + vllm_is_ratio, + vllm_is_ratio_stride, +): + B, L_ADD_1, N = logits.shape + L = L_ADD_1 - 1 + BLOCK_N = min(4096, _next_power_of_2(N)) + device = logits.device + mask = completion_mask.float() if completion_mask is not None else torch.ones(B, L, device=device) + + # Per-token log-probs (host) → per-sequence importance weights (GSPO). + lg = logits[:, :L, :].float() / temperature + lse_pt = torch.logsumexp(lg, dim=-1) + tgt = torch.gather(lg, -1, completion_ids.long().unsqueeze(-1)).squeeze(-1) + per_token_logps = tgt - lse_pt + log_ratio = torch.zeros_like(per_token_logps) if old_logp is None else (per_token_logps - old_logp) + seq_lens = mask.sum(-1).clamp(min=1.0) # (B,) + coef_1 = torch.exp((log_ratio * mask).sum(-1) / seq_lens) # (B,) + coef_2 = torch.clamp(coef_1, 1.0 - eps_low, 1.0 + eps_high) + is_clipped_seq = ( + ((coef_1 < 1.0 - eps_low) & (advantages < 0)) | ((coef_1 > 1.0 + eps_high) & (advantages > 0)) + ).float() + coef_1_for_loss = torch.clamp(coef_1, max=delta_val) if delta_val != 0.0 else coef_1 + + logits_2d = logits.reshape(B * L_ADD_1, N).contiguous() + loss = torch.zeros(B, L, device=device, dtype=torch.float32) + lse = torch.zeros(B, L, device=device, dtype=torch.float32) + is_clipped = torch.zeros(B, L, device=device, dtype=torch.float32) + has_beta = float(beta) != 0.0 + kl = torch.zeros(B, L, device=device, dtype=torch.float32) if has_beta else None + + dummy_f = torch.zeros(1, device=device, dtype=torch.float32) + dummy_i = torch.zeros(1, device=device, dtype=torch.int32) + ref_arg = ref_logp.contiguous() if ref_logp is not None else dummy_f + mask_arg = completion_mask.to(torch.int32).contiguous() if completion_mask is not None else dummy_i + kl_arg = kl if kl is not None else dummy_f + # Flatten to 1-D: the kernel gathers with a flat row*stride+col index, so the + # array must be rank-1 (strict tileiras rejects a rank-2 array with a 1-tuple index). + vllm_arg = vllm_is_ratio.contiguous().view(-1) if vllm_is_ratio is not None else dummy_f + has_mask = int(completion_mask is not None) + has_vllm = int(vllm_is_ratio is not None) + + ct.launch( + torch.cuda.current_stream(), + (B, L, 1), + _grpo_loss_fwd_seq_ct, + ( + logits_2d, + ref_arg, + completion_ids.contiguous(), + mask_arg, + advantages.contiguous(), + coef_1_for_loss.contiguous(), + coef_1.contiguous(), + coef_2.contiguous(), + is_clipped_seq.contiguous(), + vllm_arg, + loss, + lse, + kl_arg, + is_clipped, + int(B), + int(L), + int(N), + int(BLOCK_N), + float(temperature), + float(beta), + int(use_bias_correction_kl), + has_mask, + has_vllm, + int(vllm_is_ratio_stride), + ), + ) + return loss, lse, is_clipped, kl, coef_1, seq_lens + + +def _grpo_loss_backward_seq_ct( + dloss, + logits, + ref_logp, + completion_ids, + advantages, + completion_mask, + lse, + coef_1, + seq_lens, + temperature, + beta, + eps_low, + eps_high, + delta_val, + use_bias_correction_kl, + inplace, + vllm_is_ratio, +): + B, L_ADD_1, N = logits.shape + L = L_ADD_1 - 1 + BLOCK_N = min(4096, _next_power_of_2(N)) + if vllm_is_ratio is None: + dloss_sum = dloss.sum(-1).contiguous() + else: + ratio = vllm_is_ratio.unsqueeze(-1) if vllm_is_ratio.dim() == 1 else vllm_is_ratio + dloss_sum = (dloss * ratio).sum(-1).contiguous() + + logits_2d = logits.reshape(B * L_ADD_1, N).contiguous() + dlogits_2d = logits.data.reshape(B * L_ADD_1, N) if inplace else torch.empty_like(logits_2d) + dummy_f = torch.zeros(1, device=logits.device, dtype=torch.float32) + dummy_i = torch.zeros(1, device=logits.device, dtype=torch.int32) + ref_arg = ref_logp.contiguous() if ref_logp is not None else dummy_f + mask_arg = completion_mask.to(torch.int32).contiguous() if completion_mask is not None else dummy_i + has_mask = int(completion_mask is not None) + + bwd_kernel = _bwd_seq_occ_small if B <= 2 else _bwd_seq_occ_large + ct.launch( + torch.cuda.current_stream(), + (B, L, 1), + bwd_kernel, + ( + dloss.contiguous(), + dloss_sum, + dlogits_2d, + logits_2d, + ref_arg, + completion_ids.contiguous(), + advantages.contiguous(), + mask_arg, + lse, + coef_1.contiguous(), + seq_lens.contiguous(), + int(B), + int(L), + int(N), + int(BLOCK_N), + float(temperature), + float(beta), + int(use_bias_correction_kl), + float(eps_low), + float(eps_high), + float(delta_val), + has_mask, + ), + ) + dlogits = dlogits_2d.reshape(B, L_ADD_1, N) + dlogits[:, -1, :] = 0 + return dlogits + + +class GrpoLossFunction(torch.autograd.Function): + """CuTile autograd wrapper for GRPO loss (token-level). + + Full feature parity with the Triton GrpoLossFunction (all loss types, token/sequence-level IS). + """ + + @staticmethod + def forward( + ctx, + logits, + old_logp, + ref_logp, + completion_ids, + advantages, + completion_mask, + temperature, + beta, + eps_low, + eps_high, + inplace, + loss_type="grpo", + max_completion_length=None, + reduce=True, + importance_sampling_level="token", + sapo_temperature_pos=1.0, + sapo_temperature_neg=1.05, + vllm_is_ratio=None, + delta=None, + use_bias_correction_kl=False, + num_items_in_batch=None, + phi_seq=None, + ): + assert logits.is_contiguous() and completion_ids.is_contiguous() + if loss_type not in _str_to_loss_type: + raise ValueError(f"Unknown loss_type '{loss_type}'. Supported: {list(_str_to_loss_type.keys())}") + assert importance_sampling_level in ("token", "sequence"), ( + f"importance_sampling_level must be 'token' or 'sequence', got {importance_sampling_level}" + ) + if importance_sampling_level == "sequence" and loss_type in ("cispo", "sapo", "vespo"): + raise ValueError( + f"Sequence-level importance sampling is not supported for loss_type='{loss_type}'. " + f"Use importance_sampling_level='token' instead." + ) + if delta is not None and loss_type in ("cispo", "sapo", "vespo"): + raise ValueError(f"delta (two-sided clipping) is not supported for loss_type='{loss_type}'.") + if loss_type == "sapo": + if sapo_temperature_pos <= 0 or sapo_temperature_neg <= 0: + raise ValueError("sapo_temperature_pos/neg must be positive.") + + loss_type_int = _str_to_loss_type[loss_type] + delta_val = 0.0 if delta is None else float(delta) + + B, L_ADD_1, N = logits.shape + L = L_ADD_1 - 1 + + # VESPO requires a caller-precomputed per-sequence gamma weight phi_seq (B,). + if loss_type == "vespo": + if phi_seq is None: + raise ValueError("loss_type='vespo' requires phi_seq precomputed by the caller (B,) or (B, 1).") + assert phi_seq.shape in ((B,), (B, 1)), f"phi_seq must be (B,) or (B, 1), got {tuple(phi_seq.shape)}" + phi_seq = phi_seq.reshape(-1).contiguous() + else: + phi_seq = None + + vllm_is_ratio_stride = L + if vllm_is_ratio is not None: + assert vllm_is_ratio.dim() in (1, 2) + if vllm_is_ratio.dim() == 2: + assert vllm_is_ratio.shape[0] == B and vllm_is_ratio.shape[1] in (1, L) + else: + assert vllm_is_ratio.shape[0] == B + vllm_is_ratio = vllm_is_ratio.contiguous() + vllm_is_ratio_stride = vllm_is_ratio.shape[1] if vllm_is_ratio.dim() > 1 else 1 + + if importance_sampling_level == "sequence": + loss, lse, is_clipped, kl, coef_1, seq_lens = _grpo_loss_forward_seq_ct( + logits, + old_logp, + ref_logp, + completion_ids, + advantages, + completion_mask, + temperature, + beta, + eps_low, + eps_high, + delta_val, + int(use_bias_correction_kl), + vllm_is_ratio, + vllm_is_ratio_stride, + ) + ctx.save_for_backward(logits, ref_logp, completion_ids, advantages, completion_mask, lse, coef_1, seq_lens) + else: + loss, lse, is_clipped, kl = _grpo_loss_forward_ct( + logits, + old_logp, + ref_logp, + completion_ids, + advantages, + completion_mask, + temperature, + beta, + eps_low, + eps_high, + loss_type_int, + sapo_temperature_pos, + sapo_temperature_neg, + delta_val, + int(use_bias_correction_kl), + vllm_is_ratio, + vllm_is_ratio_stride, + phi_seq, + ) + ctx.save_for_backward(logits, old_logp, ref_logp, completion_ids, advantages, completion_mask, lse) + + ctx.importance_sampling_level = importance_sampling_level + ctx.vllm_is_ratio = vllm_is_ratio + ctx.phi_seq = phi_seq + ctx.infos = ( + temperature, + beta, + eps_low, + eps_high, + inplace, + loss_type, + loss_type_int, + sapo_temperature_pos, + sapo_temperature_neg, + max_completion_length, + reduce, + delta_val, + use_bias_correction_kl, + vllm_is_ratio_stride, + num_items_in_batch, + ) + + mask = completion_mask.float() if completion_mask is not None else torch.ones(B, L, device=logits.device) + mask_sum = mask.sum().clamp(min=1.0) + kl_mean = (kl * mask).sum() / mask_sum if kl is not None else None + clip_ratio = (is_clipped.float() * mask).sum() / mask_sum + + if not reduce: + loss_out = loss * mask + kl_out = kl * mask if kl is not None else None + is_clipped_out = is_clipped * mask + return loss_out, kl_out, is_clipped_out + + reduced_loss = _reduce_loss( + loss, mask, loss_type, max_completion_length, B, L, num_items_in_batch=num_items_in_batch + ) + return reduced_loss, kl_mean, clip_ratio + + @staticmethod + def backward(ctx, *args): + dloss_input = args[0] + level = ctx.importance_sampling_level + if level == "sequence": + logits, ref_logp, completion_ids, advantages, completion_mask, lse, coef_1, seq_lens = ctx.saved_tensors + old_logp = None + else: + logits, old_logp, ref_logp, completion_ids, advantages, completion_mask, lse = ctx.saved_tensors + ( + temperature, + beta, + eps_low, + eps_high, + inplace, + loss_type, + loss_type_int, + sapo_temp_pos, + sapo_temp_neg, + max_completion_length, + reduce, + delta_val, + use_bias_correction_kl, + vllm_is_ratio_stride, + num_items_in_batch, + ) = ctx.infos + vllm_is_ratio = ctx.vllm_is_ratio + phi_seq = ctx.phi_seq + + B, L_ADD_1, N = logits.shape + L = L_ADD_1 - 1 + mask = completion_mask.float() if completion_mask is not None else torch.ones(B, L, device=logits.device) + + if not reduce: + dloss = dloss_input + elif loss_type == "grpo" or loss_type == "sapo": + seq_lens_bwd = mask.sum(-1, keepdim=True).clamp(min=1.0) + dloss = dloss_input * mask / (seq_lens_bwd * B) + elif loss_type == "bnpo": + dloss = dloss_input * mask / mask.sum().clamp(min=1.0) + elif loss_type == "dr_grpo": + max_len = max_completion_length if max_completion_length is not None else L + dloss = dloss_input * mask / (B * max_len) + elif loss_type == "dapo" or loss_type == "cispo" or loss_type == "vespo": + dloss = dloss_input * mask / _compute_dapo_normalizer(mask, num_items_in_batch=num_items_in_batch) + elif loss_type == "luspo": + seq_lens_bwd = mask.sum(-1, keepdim=True).clamp(min=1.0) + # d(loss)/d(per_token_loss[b,l]) = seq_len[b] / (B*L), constant within a sequence. + # Broadcast the (B, 1) scale to (B, L). NOTE: this intentionally diverges from the + # Triton path, which passes the (B, 1) tensor with stride (1, 1) and reads + # dloss[off_b + off_l] — out of bounds (wrong) for B < L. The cuTile result is the + # mathematically correct per-sequence gradient. + dloss = (dloss_input * seq_lens_bwd / (B * L)).expand(B, L) + else: + raise ValueError(f"Unknown loss_type: {loss_type}") + + if level == "sequence": + dlogits = _grpo_loss_backward_seq_ct( + dloss, + logits, + ref_logp, + completion_ids, + advantages, + completion_mask, + lse, + coef_1, + seq_lens, + temperature, + beta, + eps_low, + eps_high, + delta_val, + use_bias_correction_kl, + inplace, + vllm_is_ratio, + ) + else: + dlogits = _grpo_loss_backward_ct( + dloss, + logits, + old_logp, + ref_logp, + completion_ids, + advantages, + completion_mask, + lse, + temperature, + beta, + eps_low, + eps_high, + inplace, + loss_type_int, + sapo_temp_pos, + sapo_temp_neg, + delta_val, + use_bias_correction_kl, + vllm_is_ratio, + vllm_is_ratio_stride, + phi_seq, + ) + # 22 forward inputs -> dlogits + 21 None + return (dlogits, None, None, None, None, None, None, None, None, None, None, None, + None, None, None, None, None, None, None, None, None, None) # fmt: skip diff --git a/src/liger_kernel/ops/cutile/ops/poly_norm.py b/src/liger_kernel/ops/cutile/ops/poly_norm.py new file mode 100644 index 000000000..97bbc29c7 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/poly_norm.py @@ -0,0 +1,412 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +PolyNorm normalization kernel (cuTile backend). + +Formula: y = w0*norm(x^3) + w1*norm(x^2) + w2*norm(x) + b, norm(u) = u / sqrt(mean(u^2) + eps) + +Forward is a 2-pass row-parallel kernel: pass 1 accumulates sum-of-squares per power (using +x^6 = x^4 * x^2 inline) and caches rstd; pass 2 applies the output in Horner form +(y = x2*(w0r3*x + w1r2) + w2r1*x + b) to avoid materialising x^3. Backward computes the +closed-form gradient and atomically reduces dW/dB into a 4-element buffer (no host .sum pass). +Aligned (power-of-2 n_cols) paths use check_bounds=False. +""" + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +MAX_FUSED_SIZE = 65536 + + +@ct.kernel(occupancy=4) +def _poly_norm_fwd_kernel( + x_input, # (n_rows, n_cols) input + y_output, # (n_rows, n_cols) output + weights, # (3,) weights [w0, w1, w2] + bias, # (1,) scalar bias + rstd3, # (n_rows,) cached rstd for x^3 power + rstd2, # (n_rows,) cached rstd for x^2 power + rstd1, # (n_rows,) cached rstd for x^1 power + N_COLS: ct.Constant[int], + eps, + BLOCK_SIZE: ct.Constant[int], + ALIGNED: ct.Constant[bool], +): + """ + PolyNorm forward kernel (row-parallel). + + Two passes per row: + 1. Accumulate sum-of-squares for each power, compute rstd, cache it. + 2. Compute output y = w0*norm(x^3) + w1*norm(x^2) + w2*norm(x) + b. + + ALIGNED=True: n_cols is a power of 2, so BLOCK_SIZE==n_cols and no partial + chunk exists. Uses check_bounds=False (hardware TMA path, ~10% faster). + ALIGNED=False: general case with software bounds checking. + """ + row_idx = ct.bid(0) + + # Load scalar weights and bias + w0 = ct.astype(ct.load(weights, 0, shape=()), ct.float32) + w1 = ct.astype(ct.load(weights, 1, shape=()), ct.float32) + w2 = ct.astype(ct.load(weights, 2, shape=()), ct.float32) + b = ct.astype(ct.load(bias, 0, shape=()), ct.float32) + + # Pass 1: accumulate sum-of-squares for each power using "fold" trick + # sum_sq_p[i] accumulates sum over col-chunks at position i within BLOCK_SIZE + sum_sq_3 = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + sum_sq_2 = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + sum_sq_1 = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + + num_chunks = (N_COLS + BLOCK_SIZE - 1) // BLOCK_SIZE + for ci in range(num_chunks): + col_indices = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + if ALIGNED: + x_tile = ct.gather(x_input, (row_idx, col_indices), check_bounds=False) + else: + x_tile = ct.gather(x_input, (row_idx, col_indices), check_bounds=True, padding_value=0.0) + x_f32 = ct.astype(x_tile, ct.float32) + # x4-path: reuse x2 for sum_sq_1 and x4 for sum_sq_2; x^6 = x4*x2 inline. + # Saves 2 FMUL/element vs computing x3=x^3 separately. + x2 = x_f32 * x_f32 # x^2 + x4 = x2 * x2 # x^4 + sum_sq_1 = sum_sq_1 + x2 # sum x^2 + sum_sq_2 = sum_sq_2 + x4 # sum x^4 + sum_sq_3 = sum_sq_3 + x4 * x2 # sum x^6 = x^4 * x^2 + + # Compute rstd = rsqrt(mean(u^2) + eps) for each power + rstd_3 = ct.rsqrt(ct.sum(sum_sq_3, axis=0, keepdims=False) / N_COLS + eps) + rstd_2 = ct.rsqrt(ct.sum(sum_sq_2, axis=0, keepdims=False) / N_COLS + eps) + rstd_1 = ct.rsqrt(ct.sum(sum_sq_1, axis=0, keepdims=False) / N_COLS + eps) + + # Cache rstd values for backward pass + ct.scatter(rstd3, row_idx, rstd_3) + ct.scatter(rstd2, row_idx, rstd_2) + ct.scatter(rstd1, row_idx, rstd_1) + + # Precompute loop-invariant scalar products (w × rstd) before pass 2. + # Horner form: y = x2*(w0r3*x + w1r2) + w2r1*x + b — avoids materialising x3 tile. + w0r3 = w0 * rstd_3 + w1r2 = w1 * rstd_2 + w2r1 = w2 * rstd_1 + + # Pass 2: compute output + for ci in range(num_chunks): + col_indices = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + if ALIGNED: + x_tile = ct.gather(x_input, (row_idx, col_indices), check_bounds=False) + else: + x_tile = ct.gather(x_input, (row_idx, col_indices), check_bounds=True, padding_value=0.0) + x_f32 = ct.astype(x_tile, ct.float32) + x2 = x_f32 * x_f32 + # Horner: y = x2*(w0r3*x + w1r2) + w2r1*x + b + inner = w0r3 * x_f32 + w1r2 + y_f32 = x2 * inner + w2r1 * x_f32 + b + if ALIGNED: + ct.scatter(y_output, (row_idx, col_indices), ct.astype(y_f32, x_tile.dtype), check_bounds=False) + else: + ct.scatter(y_output, (row_idx, col_indices), ct.astype(y_f32, x_tile.dtype), check_bounds=True) + + +_poly_norm_fwd_kernel_occ8 = _poly_norm_fwd_kernel.replace_hints(occupancy=8) + + +@ct.kernel(occupancy=4) +def _poly_norm_fwd_kernel_sc_large( + x_input, + y_output, + weights, + bias, + rstd3, + rstd2, + rstd1, + N_COLS: ct.Constant[int], + eps, + BLOCK_SIZE: ct.Constant[int], + ALIGNED: ct.Constant[bool], +): + """Single-chunk: re-gather x in pass2, latency=2 on both gathers.""" + row_idx = ct.bid(0) + + w0 = ct.astype(ct.load(weights, 0, shape=()), ct.float32) + w1 = ct.astype(ct.load(weights, 1, shape=()), ct.float32) + w2 = ct.astype(ct.load(weights, 2, shape=()), ct.float32) + b = ct.astype(ct.load(bias, 0, shape=()), ct.float32) + + col_indices = ct.arange(BLOCK_SIZE, dtype=ct.int32) + if ALIGNED: + x_tile = ct.gather(x_input, (row_idx, col_indices), check_bounds=False, latency=2) + else: + x_tile = ct.gather(x_input, (row_idx, col_indices), check_bounds=True, padding_value=0.0, latency=2) + x_f32 = ct.astype(x_tile, ct.float32) + x2 = x_f32 * x_f32 + x4 = x2 * x2 + + rstd_3 = ct.rsqrt(ct.sum(x4 * x2, axis=0, keepdims=False) / N_COLS + eps) + rstd_2 = ct.rsqrt(ct.sum(x4, axis=0, keepdims=False) / N_COLS + eps) + rstd_1 = ct.rsqrt(ct.sum(x2, axis=0, keepdims=False) / N_COLS + eps) + + ct.scatter(rstd3, row_idx, rstd_3) + ct.scatter(rstd2, row_idx, rstd_2) + ct.scatter(rstd1, row_idx, rstd_1) + + w0r3 = w0 * rstd_3 + w1r2 = w1 * rstd_2 + w2r1 = w2 * rstd_1 + + if ALIGNED: + x_tile2 = ct.gather(x_input, (row_idx, col_indices), check_bounds=False, latency=2) + else: + x_tile2 = ct.gather(x_input, (row_idx, col_indices), check_bounds=True, padding_value=0.0, latency=2) + x2_f32 = ct.astype(x_tile2, ct.float32) + x2sq = x2_f32 * x2_f32 + inner = w0r3 * x2_f32 + w1r2 + y_f32 = x2sq * inner + w2r1 * x2_f32 + b + if ALIGNED: + ct.scatter(y_output, (row_idx, col_indices), ct.astype(y_f32, x_tile2.dtype), check_bounds=False) + else: + ct.scatter(y_output, (row_idx, col_indices), ct.astype(y_f32, x_tile2.dtype), check_bounds=True) + + +_poly_norm_fwd_kernel_sc_large_occ4 = _poly_norm_fwd_kernel_sc_large.replace_hints(occupancy=4) +_poly_norm_fwd_kernel_sc_large_occ8 = _poly_norm_fwd_kernel_sc_large.replace_hints(occupancy=8) +_poly_norm_fwd_kernel_sc_large_occ16 = _poly_norm_fwd_kernel_sc_large.replace_hints(occupancy=16) + + +@ct.kernel(occupancy=2) +def _poly_norm_bwd_kernel( + dy, # (n_rows, n_cols) output gradient + dx, # (n_rows, n_cols) input gradient (output) + x_input, # (n_rows, n_cols) saved input + weights, # (3,) weights [w0, w1, w2] + rstd3, # (n_rows,) cached rstd for x^3 power + rstd2, # (n_rows,) cached rstd for x^2 power + rstd1, # (n_rows,) cached rstd for x^1 power + dwdb_output, # (4,) global atomic reduction target [dW0, dW1, dW2, dB] + N_COLS: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], + ALIGNED: ct.Constant[bool], +): + """ + PolyNorm backward kernel (row-parallel). + + Two passes per row: + 1. Compute S_p = sum(dy * x^p) for each power p, and dB = sum(dy). + 2. Compute dx using closed-form gradient formula. + + dW/dB contributions are atomically reduced into a single (4,) output tensor + directly inside the kernel — eliminates a separate host-side .sum(dim=1) + + .to(W.dtype) launch chain. + + ALIGNED=True uses check_bounds=False (hardware TMA, ~10% faster) when + n_cols is a power of 2 and BLOCK_SIZE==n_cols. + """ + row_idx = ct.bid(0) + + # Load weights and cached rstd values + w0 = ct.astype(ct.load(weights, 0, shape=()), ct.float32) + w1 = ct.astype(ct.load(weights, 1, shape=()), ct.float32) + w2 = ct.astype(ct.load(weights, 2, shape=()), ct.float32) + + rstd_3 = ct.load(rstd3, row_idx, shape=()) + rstd_2 = ct.load(rstd2, row_idx, shape=()) + rstd_1 = ct.load(rstd1, row_idx, shape=()) + + S_3_acc = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + S_2_acc = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + S_1_acc = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + dB_acc = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + + num_chunks = (N_COLS + BLOCK_SIZE - 1) // BLOCK_SIZE + for ci in range(num_chunks): + col_indices = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + if ALIGNED: + dy_t = ct.astype(ct.gather(dy, (row_idx, col_indices), check_bounds=False), ct.float32) + x = ct.astype(ct.gather(x_input, (row_idx, col_indices), check_bounds=False), ct.float32) + else: + dy_t = ct.astype(ct.gather(dy, (row_idx, col_indices), check_bounds=True, padding_value=0.0), ct.float32) + x = ct.astype(ct.gather(x_input, (row_idx, col_indices), check_bounds=True, padding_value=0.0), ct.float32) + S_3_acc = S_3_acc + dy_t * (x * x * x) + S_2_acc = S_2_acc + dy_t * (x * x) + S_1_acc = S_1_acc + dy_t * x + dB_acc = dB_acc + dy_t + + S_3 = ct.sum(S_3_acc, axis=0, keepdims=False) + S_2 = ct.sum(S_2_acc, axis=0, keepdims=False) + S_1 = ct.sum(S_1_acc, axis=0, keepdims=False) + dB_row = ct.sum(dB_acc, axis=0, keepdims=False) + + ct.atomic_add(dwdb_output, 0, rstd_3 * S_3, check_bounds=False) + ct.atomic_add(dwdb_output, 1, rstd_2 * S_2, check_bounds=False) + ct.atomic_add(dwdb_output, 2, rstd_1 * S_1, check_bounds=False) + ct.atomic_add(dwdb_output, 3, dB_row, check_bounds=False) + + for ci in range(num_chunks): + col_indices = ct.arange(BLOCK_SIZE, dtype=ct.int32) + ci * BLOCK_SIZE + if ALIGNED: + dy_t = ct.astype(ct.gather(dy, (row_idx, col_indices), check_bounds=False), ct.float32) + x = ct.astype(ct.gather(x_input, (row_idx, col_indices), check_bounds=False), ct.float32) + else: + dy_t = ct.astype(ct.gather(dy, (row_idx, col_indices), check_bounds=True, padding_value=0.0), ct.float32) + x = ct.astype(ct.gather(x_input, (row_idx, col_indices), check_bounds=True, padding_value=0.0), ct.float32) + x2 = x * x + x3 = x2 * x + + rstd3_cu = rstd_3 * rstd_3 * rstd_3 + grad_3 = w0 * (3.0 * x2 * rstd_3 * dy_t - (3.0 / N_COLS) * x2 * x3 * rstd3_cu * S_3) + + rstd2_cu = rstd_2 * rstd_2 * rstd_2 + grad_2 = w1 * (2.0 * x * rstd_2 * dy_t - (2.0 / N_COLS) * x3 * rstd2_cu * S_2) + + rstd1_cu = rstd_1 * rstd_1 * rstd_1 + grad_1 = w2 * (rstd_1 * dy_t - (1.0 / N_COLS) * x * rstd1_cu * S_1) + + dx_t = grad_3 + grad_2 + grad_1 + if ALIGNED: + ct.scatter(dx, (row_idx, col_indices), ct.astype(dx_t, dx.dtype), check_bounds=False) + else: + ct.scatter(dx, (row_idx, col_indices), ct.astype(dx_t, dx.dtype), check_bounds=True) + + +# Large-N variant (n_cols >= 8192): higher occupancy + 8 worker warps to widen +# parallelism on latency-bound 2-pass multi-chunk path. +_poly_norm_bwd_kernel_large = _poly_norm_bwd_kernel.replace_hints(occupancy=4, num_worker_warps=8) + + +class LigerPolyNormFunction(torch.autograd.Function): + """ + PolyNorm autograd function with CuTile forward and backward kernels. + + Formula: y = w0·norm(x^3) + w1·norm(x^2) + w2·norm(x) + b, norm(u) = u / sqrt(mean(u^2) + eps). + in_place is accepted for signature parity with the Triton implementation but ignored (the + cuTile forward always writes a fresh output). + """ + + @staticmethod + def forward(ctx, X, W, B, eps=1e-6, in_place=True): + shape = X.shape + dim = shape[-1] + X_2d = X.contiguous().view(-1, dim) + n_rows, n_cols = X_2d.shape + + # B is a scalar bias — accept 0-dim (torch.tensor(1.0)) or (1,); the kernel reads + # bias[0], so flatten to a 1-element tensor and restore B's shape for the dB gradient. + bias_shape = B.shape + B = B.reshape(1) + + # Per-shape BLOCK_SIZE & kernel variant chosen to match OAIT/NVT autotune + # picks. Single-chunk uses the SC kernel (re-gather, lower live-tile + # pressure); multi-chunk uses the fold-accumulator kernel. + # n_cols=2048 keeps BLOCK=1024 (multi-chunk) because BLOCK=2048 single-chunk + # spills the 3 fp32 accumulators (sum_sq_3/2/1). + if n_cols <= 1024: + BLOCK_SIZE = _next_power_of_2(n_cols) + elif n_cols == 2048: + BLOCK_SIZE = 1024 + elif n_cols <= 8192: + BLOCK_SIZE = 4096 + else: + BLOCK_SIZE = min(MAX_FUSED_SIZE, _next_power_of_2(n_cols)) + aligned = (n_cols % BLOCK_SIZE) == 0 + single_chunk = n_cols <= BLOCK_SIZE + + if n_cols <= 1024: + fwd_kernel = _poly_norm_fwd_kernel_sc_large_occ16 + elif n_cols == 2048: + fwd_kernel = _poly_norm_fwd_kernel + elif n_cols == 4096: + fwd_kernel = _poly_norm_fwd_kernel_sc_large_occ8 + elif n_cols <= 8192: + fwd_kernel = _poly_norm_fwd_kernel_occ8 + elif single_chunk: + fwd_kernel = _poly_norm_fwd_kernel_sc_large_occ4 + else: + fwd_kernel = _poly_norm_fwd_kernel + + Y = torch.empty_like(X_2d) + RSTD3 = torch.empty(n_rows, dtype=torch.float32, device=X.device) + RSTD2 = torch.empty(n_rows, dtype=torch.float32, device=X.device) + RSTD1 = torch.empty(n_rows, dtype=torch.float32, device=X.device) + + grid = (n_rows, 1, 1) + ct.launch( + torch.cuda.current_stream(), + grid, + fwd_kernel, + ( + X_2d, + Y, + W.contiguous(), + B.contiguous(), + RSTD3, + RSTD2, + RSTD1, + int(n_cols), + float(eps), + int(BLOCK_SIZE), + bool(aligned), + ), + ) + + ctx.save_for_backward(X_2d, W, RSTD3, RSTD2, RSTD1) + ctx.BLOCK_SIZE = BLOCK_SIZE + ctx.aligned = aligned + ctx.shape = shape + ctx.bias_shape = bias_shape + + return Y.view(*shape) + + @staticmethod + def backward(ctx, dy): + X_2d, W, RSTD3, RSTD2, RSTD1 = ctx.saved_tensors + shape = ctx.shape + + dim = shape[-1] + dY_2d = dy.contiguous().view(-1, dim) + n_rows, n_cols = dY_2d.shape + + # Bwd has 4 fp32 accumulators (S_3, S_2, S_1, dB) of BLOCK_SIZE each. + # Cap BLOCK_SIZE at 4096 to keep accumulators in registers (16KB per tile) + # and avoid spills observed at BLOCK_SIZE >= 16384 on norm-like bwd kernels. + BWD_MAX_BLOCK = 4096 + BLOCK_SIZE = min(BWD_MAX_BLOCK, _next_power_of_2(n_cols)) + aligned = (n_cols % BLOCK_SIZE) == 0 + + dx = torch.empty_like(dY_2d) + # The kernel atomically accumulates dW/dB contributions across all rows + # directly into this 4-element fp32 buffer — eliminates the host-side + # .sum(dim=1) + .to(W.dtype) launch chain. + dwdb_output = torch.zeros(4, dtype=torch.float32, device=W.device) + + # Medium/large-N (n_cols>=8192) is latency-bound on the 2-pass multi-chunk + # IR; the occ=4/nww=8 variant closes the gap vs OAIT. + kernel_choice = _poly_norm_bwd_kernel_large if n_cols >= 8192 else _poly_norm_bwd_kernel + + grid = (n_rows, 1, 1) + ct.launch( + torch.cuda.current_stream(), + grid, + kernel_choice, + ( + dY_2d, + dx, + X_2d, + W, + RSTD3, + RSTD2, + RSTD1, + dwdb_output, + int(n_cols), + int(BLOCK_SIZE), + bool(aligned), + ), + ) + + sums = dwdb_output.to(W.dtype) # (4,) cast to W.dtype + dW = sums[:3] + dB = sums[3:4].reshape(ctx.bias_shape) # match B's original shape (scalar or (1,)) + + return dx.view(*shape), dW, dB, None, None diff --git a/src/liger_kernel/ops/cutile/ops/rms_norm.py b/src/liger_kernel/ops/cutile/ops/rms_norm.py new file mode 100644 index 000000000..0bffebac6 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/rms_norm.py @@ -0,0 +1,422 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +RMS Normalization kernel (CuTile backend). + +Y = X / RMS(X) * (W + offset), RMS = sqrt(mean(x^2) + eps). + +Forward kernel: row-parallel (one block per row), single pass. + col_idx = arange(BLOCK_SIZE); gather X (check_bounds=True pads OOB to 0), + compute rstd, scatter Y. + +Backward kernel: SM-count partitioned, single DRAM pass per row (all BLOCK_SIZE). + - W loaded once per block; dW accumulated in registers, scattered once at end. + - dW_partial shape: (sm_count, n_cols) instead of (n_rows, n_cols). + +Casting modes: + - "llama" (0): X cast to fp32 for RMS; X*rstd cast BACK to X.dtype before W multiply. + RSTD stored as fp32. + - "gemma" (1): Both X and W cast to fp32; Y cast back to X.dtype. + RSTD stored as fp32. + - "none" (-1): No casting. Everything in X.dtype. RSTD stored in X.dtype. + +Uses gather/scatter with check_bounds=True for arbitrary n_cols. + +row_mode is accepted for signature parity with the Triton LigerRMSNormFunction but +is ignored: this backend always uses the row-parallel path. +""" + +import math + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +_CASTING_MODE_NONE = -1 +_CASTING_MODE_LLAMA = 0 +_CASTING_MODE_GEMMA = 1 + +_str_to_casting_mode = { + "llama": _CASTING_MODE_LLAMA, + "gemma": _CASTING_MODE_GEMMA, + "none": _CASTING_MODE_NONE, +} + + +def _calculate_settings(n_cols): + BLOCK_SIZE = _next_power_of_2(n_cols) + if BLOCK_SIZE > 65536: + raise RuntimeError(f"Feature dimension {n_cols} exceeds maximum supported (65536)") + return BLOCK_SIZE + + +# --------------------------------------------------------------------------- +# Forward kernel (row-parallel) +# --------------------------------------------------------------------------- + + +@ct.kernel +def _rms_norm_fwd_ct( + Y, # (n_rows, n_cols) output + X, # (n_rows, n_cols) input + W, # (n_cols,) affine weight (dummy 1-element tensor when elementwise_affine=False) + RSTD, # (n_rows,) cached rstd + n_cols, + eps, # runtime float + offset, # runtime float + BLOCK_SIZE: ct.Constant[int], + casting_mode: ct.Constant[int], + elementwise_affine: ct.Constant[bool], +): + """ + RMS norm forward (unified, single pass). + + Row-parallel forward pass: + col_idx = arange(BLOCK_SIZE) # BLOCK_SIZE = next_power_of_2(n_cols) + load X (check_bounds=True → OOB elements zero-padded, harmless for RMS sum) + compute rstd, store RSTD + scale X; optionally multiply by (W + offset) + store Y + + elementwise_affine is a compile-time constant — the W gather/multiply is + dead-code-eliminated when False, so the no-weight path has zero overhead. + + casting_mode: + llama (0): X cast to fp32 for RMS; X*rstd cast BACK to X.dtype before W multiply. + gemma (1): Both X and W cast to fp32; Y cast back to X.dtype. + none (-1): Compute in X.dtype (no upcast). x*x accumulated in X.dtype; + division by n_cols promotes to fp32. eps/offset rounded to X.dtype before + arithmetic. + """ + row_idx = ct.bid(0) + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + + if casting_mode == _CASTING_MODE_NONE: + x_val = ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=0.0) + if elementwise_affine: + w_val = ct.gather(W, col_idx, check_bounds=True, padding_value=0.0) + mean_sq = ct.astype(ct.sum(x_val * x_val, 0, keepdims=False), ct.float32) / n_cols + eps_rounded = ct.astype(ct.astype(eps, x_val.dtype), ct.float32) + rstd = ct.rsqrt(mean_sq + eps_rounded) # fp32 + ct.scatter(RSTD, row_idx, ct.astype(rstd, x_val.dtype), check_bounds=False) + x_scaled = ct.astype(x_val, ct.float32) * rstd # fp32 (upcast x for RMS computation) + if elementwise_affine: + offset_native = ct.astype(offset, x_val.dtype) # round offset to X.dtype precision + w_plus_offset_f32 = ct.astype(w_val + offset_native, ct.float32) + ct.scatter(Y, (row_idx, col_idx), ct.astype(x_scaled * w_plus_offset_f32, x_val.dtype), check_bounds=True) + else: + ct.scatter(Y, (row_idx, col_idx), ct.astype(x_scaled, x_val.dtype), check_bounds=True) + + elif casting_mode == _CASTING_MODE_LLAMA: + x_val = ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=0.0) + if elementwise_affine: + w_val = ct.gather(W, col_idx, check_bounds=True, padding_value=0.0) + x_f32 = ct.astype(x_val, ct.float32) + mean_sq = ct.sum(ct.mul(x_f32, x_f32, flush_to_zero=True), 0, keepdims=False) / n_cols + rstd = ct.rsqrt(mean_sq + eps) + ct.scatter(RSTD, row_idx, rstd, check_bounds=False) + # Cast X*rstd back to X.dtype before W multiply (llama behaviour) + x_scaled = ct.astype(x_f32 * rstd, X.dtype) + if elementwise_affine: + ct.scatter(Y, (row_idx, col_idx), ct.astype(x_scaled * (w_val + offset), Y.dtype), check_bounds=True) + else: + ct.scatter(Y, (row_idx, col_idx), ct.astype(x_scaled, Y.dtype), check_bounds=True) + + else: + # gemma: both X and W to fp32, Y cast back to X.dtype + x_f32 = ct.astype(ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + if elementwise_affine: + w_f32 = ct.astype(ct.gather(W, col_idx, check_bounds=True, padding_value=0.0), ct.float32) + mean_sq = ct.sum(x_f32 * x_f32, 0, keepdims=False) / n_cols + rstd = ct.rsqrt(mean_sq + eps) + ct.scatter(RSTD, row_idx, rstd, check_bounds=False) + x_scaled = x_f32 * rstd + if elementwise_affine: + ct.scatter(Y, (row_idx, col_idx), ct.astype(x_scaled * (w_f32 + offset), Y.dtype), check_bounds=True) + else: + ct.scatter(Y, (row_idx, col_idx), ct.astype(x_scaled, Y.dtype), check_bounds=True) + + +_rms_norm_fwd_large_ct = _rms_norm_fwd_ct.replace_hints(num_worker_warps=8) + + +# --------------------------------------------------------------------------- +# Backward kernels — SM-count grid, single DRAM pass +# --------------------------------------------------------------------------- + + +@ct.kernel +def _rms_norm_bwd_large_ct( + dY, # (n_rows, n_cols) upstream gradient + dX, # (n_rows, n_cols) output gradient + X, # (n_rows, n_cols) saved input + RSTD, # (n_rows,) cached rstd; OOB-safe via bounds-checked gather + n_cols: ct.Constant[int], + rows_per_program: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], + casting_mode: ct.Constant[int], +): + """ + RMS norm backward without affine weight. SM-count partitioned, single DRAM pass. + + Grid: (sm_count,). Block b processes rows [b*rpp, (b+1)*rpp). + Single pass: load dY and X once; sum_mX via register reduction; no re-read. + OOB rows return 0 via check_bounds; RSTD zero-padded for safe scalar load. + """ + block_id = ct.bid(0) + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + inv_n_cols = 1.0 / n_cols + + for ri in range(rows_per_program): + row_idx = block_id * rows_per_program + ri + + rstd = ct.astype(ct.gather(RSTD, (row_idx,), padding_value=0.0).item(), ct.float32) + dy_f32 = ct.astype( + ct.gather(dY, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32 + ) + x_f32 = ct.astype(ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32) + + sum_mX = ct.sum(dy_f32 * x_f32, 0, keepdims=False) + coeff = rstd * rstd * rstd * inv_n_cols * sum_mX + dx_f32 = rstd * dy_f32 - coeff * x_f32 + ct.scatter(dX, (row_idx, col_idx), ct.astype(dx_f32, dX.dtype), check_bounds=True) + + +@ct.kernel +def _rms_norm_bwd_w_large_ct( + dY, # (n_rows, n_cols) upstream gradient + dX, # (n_rows, n_cols) output gradient + X, # (n_rows, n_cols) saved input + W, # (n_cols,) affine weight + RSTD, # (n_rows,) cached rstd; OOB-safe via bounds-checked gather + dW_partial, # (sm_count, n_cols) per-block dW accumulation (host reduces) + n_cols: ct.Constant[int], + offset: ct.Constant[float], + rows_per_program: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], + casting_mode: ct.Constant[int], +): + """ + RMS norm backward with affine weight. SM-count partitioned, single DRAM pass. + + Grid: (sm_count,). Block b processes rows [b*rpp, (b+1)*rpp). + W loaded once per block and reused across all rows. + dW accumulated in registers throughout the row loop; scattered once at the end. + dW_partial shape: (sm_count, n_cols) — vastly smaller than (n_rows, n_cols). + OOB rows return 0 via check_bounds; RSTD zero-padded. + + casting_mode: + llama (0): load W in original dtype once; per row: dY in orig dtype, + m = (dY*(W+offset)) cast to fp32; dW += dy_orig*(X*rstd cast to X.dtype). + gemma (1): W loaded in fp32; per row: dY in fp32, m = dy_f32*(w_f32+offset); + dW += dy_f32 * x_f32 * rstd. + none (-1): load W in original dtype once; per row: dY in orig dtype, + m = dy_orig*(w_orig+offset) without cast to fp32 (cast for sum); + dW += dy_orig * (x_orig * rstd) without extra fp32 cast. + """ + block_id = ct.bid(0) + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + + # Per-block dW accumulator in registers; scattered to dW_partial once at end + dW_acc = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + inv_n_cols = 1.0 / n_cols + + # Load W once; dtype depends on mode (gemma keeps fp32; llama/none keep original). + if casting_mode == _CASTING_MODE_GEMMA: + w_f32 = ct.astype(ct.gather(W, col_idx, check_bounds=True, padding_value=0.0), ct.float32) + else: + w_orig = ct.gather(W, col_idx, check_bounds=True, padding_value=0.0) + + for ri in range(rows_per_program): + row_idx = block_id * rows_per_program + ri + + # Bounds-checked scalar read on RSTD (avoids host-side cat-padding). + rstd = ct.astype(ct.gather(RSTD, (row_idx,), padding_value=0.0).item(), ct.float32) + x_f32 = ct.astype(ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32) + + if casting_mode == _CASTING_MODE_GEMMA: + dy_f32 = ct.astype( + ct.gather(dY, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3), ct.float32 + ) + m_f32 = dy_f32 * (w_f32 + offset) + dW_term_f32 = dy_f32 * x_f32 * rstd + else: + dy_orig = ct.gather(dY, (row_idx, col_idx), check_bounds=True, padding_value=0.0, latency=3) + m_f32 = ct.astype(dy_orig * (w_orig + offset), ct.float32) + if casting_mode == _CASTING_MODE_LLAMA: + # x*rstd computed in fp32, then downcast for the dy multiply + dW_term_f32 = ct.astype(dy_orig * ct.astype(x_f32 * rstd, dY.dtype), ct.float32) + else: + # none: downcast x first, then multiply by rstd (mixed-precision) + x_orig = ct.astype(x_f32, dY.dtype) + dW_term_f32 = ct.astype(dy_orig * (x_orig * rstd), ct.float32) + + sum_mX = ct.sum(m_f32 * x_f32, 0, keepdims=False) + coeff = rstd * rstd * rstd * inv_n_cols * sum_mX + dx_f32 = rstd * m_f32 - coeff * x_f32 + ct.scatter(dX, (row_idx, col_idx), ct.astype(dx_f32, dX.dtype), check_bounds=True) + + # OOB rows contribute 0 (dy=0, x=0 via check_bounds) + dW_acc = ct.add(dW_acc, dW_term_f32) + + # Write this block's partial dW once (block_id < sm_count, always in-bounds) + ct.scatter(dW_partial, (block_id, col_idx), dW_acc, check_bounds=True) + + +_rms_norm_bwd_large_ct_nww8 = _rms_norm_bwd_large_ct.replace_hints(num_worker_warps=8) +_rms_norm_bwd_w_large_ct_nww8 = _rms_norm_bwd_w_large_ct.replace_hints(num_worker_warps=8) + + +# --------------------------------------------------------------------------- +# Python launch wrappers +# --------------------------------------------------------------------------- + + +def _rms_norm_forward_ct(X, W, eps, offset, casting_mode_int): + shape = X.shape + dim = shape[-1] + X2d = X.view(-1, dim).contiguous() + n_rows, n_cols = X2d.shape + BLOCK_SIZE = _calculate_settings(n_cols) + + Y = torch.empty_like(X2d) + # RSTD dtype: fp32 for llama/gemma, X.dtype for none + rstd_dtype = torch.float32 if casting_mode_int in (_CASTING_MODE_LLAMA, _CASTING_MODE_GEMMA) else X.dtype + RSTD = torch.empty(n_rows, dtype=rstd_dtype, device=X.device) + elementwise_affine = W is not None + + grid = (n_rows, 1, 1) + # When no weight, pass a 1-element dummy tensor; elementwise_affine=False causes the compiler + # to dead-code-eliminate every ct.gather(W, ...) so the dummy is never accessed. + W_tensor = W.contiguous() if elementwise_affine else X2d.new_empty(1) + fwd_kernel = _rms_norm_fwd_large_ct if BLOCK_SIZE >= 16384 else _rms_norm_fwd_ct + ct.launch( + torch.cuda.current_stream(), + grid, + fwd_kernel, + ( + Y, + X2d, + W_tensor, + RSTD, + int(n_cols), + float(eps), + float(offset) if elementwise_affine else 0.0, + int(BLOCK_SIZE), + int(casting_mode_int), + bool(elementwise_affine), + ), + ) + + return Y.view(*shape), X2d, RSTD, int(BLOCK_SIZE) + + +def _rms_norm_backward_ct(dY, X, W, RSTD, offset, BLOCK_SIZE, casting_mode_int, in_place): + shape = dY.shape + dim = shape[-1] + dY2d = dY.view(-1, dim).contiguous() + n_rows, n_cols = dY2d.shape + elementwise_affine = W is not None + + sm_count = torch.cuda.get_device_properties(X.device).multi_processor_count + rows_per_program = math.ceil(n_rows / sm_count) + grid = (sm_count, 1, 1) + + # When in_place=True, reuse dY2d buffer in-place for dX (safe: each row is processed + # independently, and within a row the load precedes the store in every kernel pass). + dX = dY2d if in_place else torch.zeros_like(dY2d) + + if elementwise_affine: + # Every (block_id, col) in dW_partial is written exactly once by the scatter at + # kernel end, so zero-init is unnecessary. + dW_partial = torch.empty(sm_count, n_cols, dtype=torch.float32, device=W.device) + # Larger hidden dims spill registers under the default nww=4; nww=8 fixes that. + bwd_w_kernel = _rms_norm_bwd_w_large_ct_nww8 if BLOCK_SIZE >= 8192 else _rms_norm_bwd_w_large_ct + ct.launch( + torch.cuda.current_stream(), + grid, + bwd_w_kernel, + ( + dY2d, + dX, + X.contiguous(), + W.contiguous(), + RSTD, + dW_partial, + int(n_cols), + float(offset), + int(rows_per_program), + int(BLOCK_SIZE), + int(casting_mode_int), + ), + ) + dW = dW_partial.sum(dim=0).to(W.dtype) + else: + bwd_kernel = _rms_norm_bwd_large_ct_nww8 if BLOCK_SIZE >= 8192 else _rms_norm_bwd_large_ct + ct.launch( + torch.cuda.current_stream(), + grid, + bwd_kernel, + ( + dY2d, + dX, + X.contiguous(), + RSTD, + int(n_cols), + int(rows_per_program), + int(BLOCK_SIZE), + int(casting_mode_int), + ), + ) + dW = None + + return dX.view(*shape), dW + + +class LigerRMSNormFunction(torch.autograd.Function): + """CuTile autograd wrapper for RMS normalization. + + Signature-compatible with the Triton ``LigerRMSNormFunction`` so the cuTile backend + swaps in transparently. ``row_mode`` is accepted but ignored (row-parallel only). + """ + + @staticmethod + def forward(ctx, X, W, eps, offset=0.0, casting_mode="llama", in_place=True, row_mode=None): + X = X.contiguous() + if W is not None: + W = W.contiguous() + + # Resolve casting_mode string → int + if isinstance(casting_mode, int): + casting_mode_int = casting_mode + else: + assert casting_mode in _str_to_casting_mode, f"Invalid casting_mode: {casting_mode}" + casting_mode_int = _str_to_casting_mode[casting_mode] + + Y, X_saved, RSTD, BLOCK_SIZE = _rms_norm_forward_ct(X, W, eps, offset, casting_mode_int) + + ctx.offset = offset + ctx.casting_mode = casting_mode_int + ctx.in_place = in_place + ctx.BLOCK_SIZE = BLOCK_SIZE + ctx.elementwise_affine = W is not None + if W is not None: + ctx.save_for_backward(X_saved, W, RSTD) + else: + ctx.save_for_backward(X_saved, RSTD) + return Y + + @staticmethod + def backward(ctx, dY): + dY = dY.contiguous() + if ctx.elementwise_affine: + X, W, RSTD = ctx.saved_tensors + else: + X, RSTD = ctx.saved_tensors + W = None + + dX, dW = _rms_norm_backward_ct(dY, X, W, RSTD, ctx.offset, ctx.BLOCK_SIZE, ctx.casting_mode, ctx.in_place) + return dX, dW, None, None, None, None, None diff --git a/src/liger_kernel/ops/cutile/ops/softmax.py b/src/liger_kernel/ops/cutile/ops/softmax.py new file mode 100644 index 000000000..7347d54ca --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/softmax.py @@ -0,0 +1,353 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +Softmax kernel (cuTile backend). + +Row-wise softmax with an exp2-based online formulation. Two regimes, all autotuned +(occupancy + num_worker_warps) and cached per shape: + - single-chunk: whole row in one tile. f32 uses a TMA load; bf16/fp16 use gather (aligned + fast path when n_cols is a power of 2). + - multi-chunk: 2-pass online softmax that re-reads the row; block size chosen for L2 reuse. + +The single-chunk threshold and multi-chunk block size both differ by direction. Forward (light: +max/sum/exp/div) scales to a 32768 tile; backward (holds y+dy+dot+dx) spills past 16384, so larger +rows use the multi-chunk path. The multi-chunk backward re-read is L2-bound, so it uses a larger +block (8192 vs the forward's 4096) to shrink the pass1->pass2 gap. Backward: dx = y*(dy - dot), +dot = sum(y*dy). +""" + +import math + +from types import SimpleNamespace + +import cuda.tile as ct +import torch + +from cuda.tile.tune import exhaustive_search + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +INV_LOG2 = 1.0 / math.log(2) + +# Single-chunk handles the whole row in one tile. Forward is light (max/sum/exp/div over one +# tile) and scales to a 32768 tile; backward holds y+dy+dot+dx and spills past 16384, so it +# uses a lower threshold and routes larger rows to the (autotuned) multi-chunk online loop. +_SINGLE_CHUNK_MAX_N_FWD = 32768 +_SINGLE_CHUNK_MAX_N_BWD = 16384 + +# Forward autotune: exp2+APPROX kernel has different register profile; nww=8/occ>=5 +# cause register spills at large N. Safe space: occ=[2,3,4], nww=4 only. +_SOFTMAX_FWD_TUNE_CONFIGS = [SimpleNamespace(occ=o, nww=4) for o in [2, 3, 4]] + +# Backward autotune: no exp, lighter register pressure — nww=8 beneficial at large N. +_SOFTMAX_BWD_TUNE_CONFIGS = [SimpleNamespace(occ=o, nww=n) for o in [2, 3, 4, 5, 6, 8] for n in [4, 8]] + +# Multi-chunk block sizes differ by direction. The online loop re-reads the row in pass 2, so +# its cost is dominated by L2 reuse on that re-read. For backward, BLOCK=8192 (vs 4096) halves +# the pass1->pass2 gap and reaches ~parity with the Triton single-block backward across large N. +# Forward is lighter and keeps BLOCK=4096 (8192 regresses the largest rows). occupancy+nww are +# autotuned per shape on top. +_MULTI_CHUNK_BLOCK_SIZE_FWD = 4096 +_MULTI_CHUNK_BLOCK_SIZE_BWD = 8192 +_SOFTMAX_MULTI_TUNE_CONFIGS = [SimpleNamespace(occ=o, nww=n) for o in [2, 3, 4, 6, 8] for n in [4, 8]] + +# Per-process cache: (path, n_cols, BLOCK_SIZE[, aligned]) -> tuned kernel +_SOFTMAX_FWD_TUNE_CACHE: dict = {} +_SOFTMAX_BWD_TUNE_CACHE: dict = {} +_SOFTMAX_MULTI_TUNE_CACHE: dict = {} + + +# --------------------------------------------------------------------------- +# Single-chunk forward kernels (plain functions, wrapped by ct.kernel below). +# No hardcoded occupancy — autotuned at runtime. +# --------------------------------------------------------------------------- + + +@ct.kernel +def _softmax_single_tma(Y, X, n_cols: ct.Constant[int], BLOCK_SIZE: ct.Constant[int]): + """f32, TMA load with NEG_INF padding. Scatter for write (check_bounds=True handles tail).""" + row_idx = ct.bid(0) + x_tile = ct.astype( + ct.load(X, index=(row_idx, 0), shape=(1, BLOCK_SIZE), padding_mode=ct.PaddingMode.NEG_INF).reshape( + (BLOCK_SIZE,) + ), + ct.float32, + ) + global_max = ct.max(x_tile, 0, keepdims=False) + exp_tile = ct.exp2(ct.mul(x_tile - global_max, INV_LOG2, flush_to_zero=True), flush_to_zero=True) + y_tile = ct.truediv( + exp_tile, ct.sum(exp_tile, 0, keepdims=False), rounding_mode=ct.RoundingMode.APPROX, flush_to_zero=True + ) + ct.scatter(Y, (row_idx, ct.arange(BLOCK_SIZE, dtype=ct.int32)), ct.astype(y_tile, Y.dtype), check_bounds=True) + + +@ct.kernel +def _softmax_single_gather(Y, X, n_cols: ct.Constant[int], BLOCK_SIZE: ct.Constant[int], ALIGNED: ct.Constant[bool]): + """bf16/fp16. ALIGNED=True: check_bounds=False (power-of-2 n_cols). ALIGNED=False: padded gather.""" + row_idx = ct.bid(0) + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + if ALIGNED: + x_tile = ct.astype(ct.gather(X, (row_idx, col_idx), check_bounds=False), ct.float32) + else: + x_tile = ct.astype(ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=-math.inf), ct.float32) + global_max = ct.max(x_tile, 0, keepdims=False) + exp_tile = ct.exp2(ct.mul(x_tile - global_max, INV_LOG2, flush_to_zero=True), flush_to_zero=True) + y_tile = ct.truediv( + exp_tile, ct.sum(exp_tile, 0, keepdims=False), rounding_mode=ct.RoundingMode.APPROX, flush_to_zero=True + ) + if ALIGNED: + ct.scatter(Y, (row_idx, col_idx), ct.astype(y_tile, Y.dtype), check_bounds=False) + else: + ct.scatter(Y, (row_idx, col_idx), ct.astype(y_tile, Y.dtype), check_bounds=True) + + +def _get_tuned_single_kernel(n_cols: int, BLOCK_SIZE: int, n_rows: int, dtype: torch.dtype, stream, y2d, x2d): + """Autotune occupancy+nww on first call; return cached kernel on subsequent calls.""" + is_tma = dtype == torch.float32 + is_aligned = n_cols == BLOCK_SIZE + + if is_tma: + key = ("tma", n_cols, BLOCK_SIZE) + base, args_fn = _softmax_single_tma, lambda cfg: (y2d, x2d, int(n_cols), int(BLOCK_SIZE)) + else: + key = ("gather", n_cols, BLOCK_SIZE, is_aligned) + base = _softmax_single_gather + args_fn = lambda cfg: (y2d, x2d, int(n_cols), int(BLOCK_SIZE), is_aligned) + + if key in _SOFTMAX_FWD_TUNE_CACHE: + return _SOFTMAX_FWD_TUNE_CACHE[key] + + result = exhaustive_search( + _SOFTMAX_FWD_TUNE_CONFIGS, + stream, + lambda cfg: (n_rows, 1, 1), + base, + args_fn, + lambda cfg: {"occupancy": cfg.occ, "num_worker_warps": cfg.nww}, + quiet=True, + ) + best = result.best.config + _SOFTMAX_FWD_TUNE_CACHE[key] = base.replace_hints(occupancy=best.occ, num_worker_warps=best.nww) + return _SOFTMAX_FWD_TUNE_CACHE[key] + + +# --------------------------------------------------------------------------- +# Multi-chunk forward kernel (n_cols > _SINGLE_CHUNK_MAX_N) +# --------------------------------------------------------------------------- + + +@ct.kernel +def _softmax_fwd_ct(Y, X, n_cols: ct.Constant[int], BLOCK_SIZE: ct.Constant[int]): + """2-pass online softmax. BLOCK_SIZE=4096 caps register pressure regardless of n_cols.""" + row_idx = ct.bid(0) + n_chunks = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE + + running_max = ct.full((1,), -math.inf, dtype=ct.float32) + running_sum = ct.full((1,), 0.0, dtype=ct.float32) + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + x_tile = ct.astype(ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=-math.inf), ct.float32) + blk_max = ct.max(x_tile, 0, keepdims=True) + new_max = ct.maximum(running_max, blk_max) + running_sum = running_sum * ct.exp2((running_max - new_max) * INV_LOG2) + ct.sum( + ct.exp2((x_tile - new_max) * INV_LOG2), 0, keepdims=True + ) + running_max = new_max + + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + x_tile = ct.astype(ct.gather(X, (row_idx, col_idx), check_bounds=True, padding_value=-math.inf), ct.float32) + y_tile = ct.exp2((x_tile - running_max) * INV_LOG2) / running_sum + ct.scatter(Y, (row_idx, col_idx), ct.astype(y_tile, Y.dtype), check_bounds=True) + + +# --------------------------------------------------------------------------- +# Backward kernels +# --------------------------------------------------------------------------- + + +@ct.kernel +def _softmax_bwd_ct(DX, DY, Y, n_cols: ct.Constant[int], BLOCK_SIZE: ct.Constant[int], ALIGNED: ct.Constant[bool]): + """Multi-chunk backward. dx = y * (dy - dot), dot = sum(y*dy). 2-pass fold.""" + row_idx = ct.bid(0) + n_chunks = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE + + dot_tile = ct.full((BLOCK_SIZE,), 0.0, dtype=ct.float32) + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + if ALIGNED: + y_tile = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=False), ct.float32) + dy_tile = ct.astype(ct.gather(DY, (row_idx, col_idx), check_bounds=False), ct.float32) + else: + y_tile = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + dy_tile = ct.astype(ct.gather(DY, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + dot_tile = ct.add(dot_tile, y_tile * dy_tile) + + dot = ct.sum(dot_tile, 0, keepdims=False) + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + if ALIGNED: + y_tile = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=False), ct.float32) + dy_tile = ct.astype(ct.gather(DY, (row_idx, col_idx), check_bounds=False), ct.float32) + ct.scatter(DX, (row_idx, col_idx), ct.astype(y_tile * (dy_tile - dot), DX.dtype), check_bounds=False) + else: + y_tile = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + dy_tile = ct.astype(ct.gather(DY, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + ct.scatter(DX, (row_idx, col_idx), ct.astype(y_tile * (dy_tile - dot), DX.dtype), check_bounds=True) + + +@ct.kernel +def _softmax_bwd_fused(DX, DY, Y, n_cols: ct.Constant[int], BLOCK_SIZE: ct.Constant[int], ALIGNED: ct.Constant[bool]): + """Single-chunk fused backward: loads Y+DY once, computes dot in registers, then dx. + + ALIGNED=True (power-of-2 n_cols): check_bounds=False eliminates predicate overhead. + """ + row_idx = ct.bid(0) + col_idx = ct.arange(BLOCK_SIZE, dtype=ct.int32) + if ALIGNED: + y_tile = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=False), ct.float32) + dy_tile = ct.astype(ct.gather(DY, (row_idx, col_idx), check_bounds=False), ct.float32) + dot = ct.sum(y_tile * dy_tile, 0, keepdims=False) + ct.scatter(DX, (row_idx, col_idx), ct.astype(y_tile * (dy_tile - dot), DX.dtype), check_bounds=False) + else: + y_tile = ct.astype(ct.gather(Y, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + dy_tile = ct.astype(ct.gather(DY, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + dot = ct.sum(y_tile * dy_tile, 0, keepdims=False) + ct.scatter(DX, (row_idx, col_idx), ct.astype(y_tile * (dy_tile - dot), DX.dtype), check_bounds=True) + + +def _get_tuned_bwd_kernel(n_cols: int, BLOCK_SIZE: int, n_rows: int, dtype: torch.dtype, device): + """Autotune occupancy+nww for single-chunk backward; cache by (n_cols, BLOCK_SIZE, aligned).""" + is_aligned = n_cols == BLOCK_SIZE + key = (n_cols, BLOCK_SIZE, is_aligned) + if key in _SOFTMAX_BWD_TUNE_CACHE: + return _SOFTMAX_BWD_TUNE_CACHE[key] + + # Use a fresh stream for timing — autograd backward stream is not safe for exhaustive_search. + tune_stream = torch.cuda.Stream(device=device) + dx = torch.empty(n_rows, BLOCK_SIZE, dtype=dtype, device=device) + dy = torch.empty(n_rows, BLOCK_SIZE, dtype=dtype, device=device) + y = torch.empty(n_rows, BLOCK_SIZE, dtype=dtype, device=device) + + result = exhaustive_search( + _SOFTMAX_BWD_TUNE_CONFIGS, + tune_stream, + lambda cfg: (n_rows, 1, 1), + _softmax_bwd_fused, + lambda cfg: (dx, dy, y, int(n_cols), int(BLOCK_SIZE), is_aligned), + lambda cfg: {"occupancy": cfg.occ, "num_worker_warps": cfg.nww}, + quiet=True, + ) + best = result.best.config + _SOFTMAX_BWD_TUNE_CACHE[key] = _softmax_bwd_fused.replace_hints(occupancy=best.occ, num_worker_warps=best.nww) + return _SOFTMAX_BWD_TUNE_CACHE[key] + + +def _get_tuned_multi_fwd_kernel(n_cols: int, BLOCK_SIZE: int, n_rows: int, stream, y2d, x2d): + """Autotune occupancy+nww for the multi-chunk forward; cache by (n_cols, BLOCK_SIZE).""" + key = ("multi_fwd", n_cols, BLOCK_SIZE) + if key in _SOFTMAX_MULTI_TUNE_CACHE: + return _SOFTMAX_MULTI_TUNE_CACHE[key] + result = exhaustive_search( + _SOFTMAX_MULTI_TUNE_CONFIGS, + stream, + lambda cfg: (n_rows, 1, 1), + _softmax_fwd_ct, + lambda cfg: (y2d, x2d, int(n_cols), int(BLOCK_SIZE)), + lambda cfg: {"occupancy": cfg.occ, "num_worker_warps": cfg.nww}, + quiet=True, + ) + best = result.best.config + _SOFTMAX_MULTI_TUNE_CACHE[key] = _softmax_fwd_ct.replace_hints(occupancy=best.occ, num_worker_warps=best.nww) + return _SOFTMAX_MULTI_TUNE_CACHE[key] + + +def _get_tuned_multi_bwd_kernel(n_cols: int, BLOCK_SIZE: int, n_rows: int, aligned: bool, dtype, device): + """Autotune occupancy+nww for the multi-chunk backward; cache by (n_cols, BLOCK_SIZE, aligned).""" + key = ("multi_bwd", n_cols, BLOCK_SIZE, aligned) + if key in _SOFTMAX_MULTI_TUNE_CACHE: + return _SOFTMAX_MULTI_TUNE_CACHE[key] + tune_stream = torch.cuda.Stream(device=device) + dx = torch.empty(n_rows, n_cols, dtype=dtype, device=device) + dy = torch.empty(n_rows, n_cols, dtype=dtype, device=device) + y = torch.empty(n_rows, n_cols, dtype=dtype, device=device) + result = exhaustive_search( + _SOFTMAX_MULTI_TUNE_CONFIGS, + tune_stream, + lambda cfg: (n_rows, 1, 1), + _softmax_bwd_ct, + lambda cfg: (dx, dy, y, int(n_cols), int(BLOCK_SIZE), aligned), + lambda cfg: {"occupancy": cfg.occ, "num_worker_warps": cfg.nww}, + quiet=True, + ) + best = result.best.config + _SOFTMAX_MULTI_TUNE_CACHE[key] = _softmax_bwd_ct.replace_hints(occupancy=best.occ, num_worker_warps=best.nww) + return _SOFTMAX_MULTI_TUNE_CACHE[key] + + +# --------------------------------------------------------------------------- +# Host-side dispatch +# --------------------------------------------------------------------------- + + +def _softmax_forward_ct(x: torch.Tensor): + *batch, n_cols = x.shape + x2d = x.contiguous().view(-1, n_cols) + n_rows = x2d.shape[0] + y2d = torch.empty_like(x2d) + stream = torch.cuda.current_stream() + + if n_cols <= _SINGLE_CHUNK_MAX_N_FWD: + BLOCK_SIZE = min(_next_power_of_2(n_cols), 65536) + is_tma = x2d.dtype == torch.float32 + is_aligned = n_cols == BLOCK_SIZE + kernel = _get_tuned_single_kernel(n_cols, BLOCK_SIZE, n_rows, x2d.dtype, stream, y2d, x2d) + if is_tma: + ct.launch(stream, (n_rows, 1, 1), kernel, (y2d, x2d, int(n_cols), int(BLOCK_SIZE))) + else: + ct.launch(stream, (n_rows, 1, 1), kernel, (y2d, x2d, int(n_cols), int(BLOCK_SIZE), is_aligned)) + else: + kernel = _get_tuned_multi_fwd_kernel(n_cols, _MULTI_CHUNK_BLOCK_SIZE_FWD, n_rows, stream, y2d, x2d) + ct.launch(stream, (n_rows, 1, 1), kernel, (y2d, x2d, int(n_cols), _MULTI_CHUNK_BLOCK_SIZE_FWD)) + + return y2d.view(*batch, n_cols) + + +def _softmax_backward_ct(dy: torch.Tensor, y: torch.Tensor): + *batch, n_cols = dy.shape + dy2d = dy.contiguous().view(-1, n_cols) + y2d = y.contiguous().view(-1, n_cols) + n_rows = dy2d.shape[0] + dx2d = torch.empty_like(dy2d) + stream = torch.cuda.current_stream() + + if n_cols <= _SINGLE_CHUNK_MAX_N_BWD: + BLOCK_SIZE = min(_next_power_of_2(n_cols), 65536) + aligned = n_cols == BLOCK_SIZE + kernel = _get_tuned_bwd_kernel(n_cols, BLOCK_SIZE, n_rows, dx2d.dtype, dx2d.device) + ct.launch(stream, (n_rows, 1, 1), kernel, (dx2d, dy2d, y2d, int(n_cols), int(BLOCK_SIZE), aligned)) + else: + aligned = (n_cols % _MULTI_CHUNK_BLOCK_SIZE_BWD) == 0 + kernel = _get_tuned_multi_bwd_kernel( + n_cols, _MULTI_CHUNK_BLOCK_SIZE_BWD, n_rows, aligned, dx2d.dtype, dx2d.device + ) + ct.launch(stream, (n_rows, 1, 1), kernel, (dx2d, dy2d, y2d, int(n_cols), _MULTI_CHUNK_BLOCK_SIZE_BWD, aligned)) + + return dx2d.view(*batch, n_cols) + + +class LigerSoftmaxFunction(torch.autograd.Function): + """CuTile autograd wrapper for row-wise softmax.""" + + @staticmethod + def forward(ctx, input_): + y = _softmax_forward_ct(input_) + ctx.save_for_backward(y) + return y + + @staticmethod + def backward(ctx, grad_output): + (y,) = ctx.saved_tensors + return _softmax_backward_ct(grad_output, y) diff --git a/src/liger_kernel/ops/cutile/ops/swiglu.py b/src/liger_kernel/ops/cutile/ops/swiglu.py new file mode 100644 index 000000000..1438c2664 --- /dev/null +++ b/src/liger_kernel/ops/cutile/ops/swiglu.py @@ -0,0 +1,283 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: MIT + +""" +SwiGLU activation kernel (CuTile backend). + +Computes: c = silu(a * gate_multiplier) * b where silu(x) = x * sigmoid(x) + +Row-parallel: grid = (n_rows, 1, 1). Each block handles one row. +Backward writes da into A and db into B in-place (memory optimization). + +gate_multiplier: applied inside the kernel as ct.Constant[float] (compile-time + constant; scales a before SiLU; chain rule applies extra factor in backward). +down_multiplier: applied at the Python wrapper level only (multiplied onto output + in forward; multiplied onto dc before backward kernel dispatch). Not in kernel. + +Two kernel variants per direction (fwd/bwd): + *_aligned: check_bounds=False — used when n_cols % BLOCK_SIZE == 0 + (all power-of-2 n_cols up to MAX_FUSED_SIZE, e.g. 4096, 8192), + ~17-20% faster vs check_bounds=True on B200. + *_ct: check_bounds=True — fallback for non-aligned n_cols. + +Forward uses @ct.kernel(occupancy=1) → 8 warps and the exp2 trick: + sigmoid via exp2(-a * LOG2E) → FMUL+EX2 on Blackwell. occupancy=1 is required + for the exp2→EX2 lowering. Backward does NOT set occupancy=1 (scatter inside a + backward loop risks hangs), so it uses exp(-a) instead of exp2. +""" + +import cuda.tile as ct +import torch + +from liger_kernel.ops.cutile.ops.utils import _next_power_of_2 + +MAX_FUSED_SIZE_FWD = 4096 # Forward: larger tile fits; forward is compute-bound, no register spill observed +MAX_FUSED_SIZE_BWD = 1024 # Backward: 14 chunks at n_cols=14336 (vs 28 at 512); stable without occupancy=1 + +# exp2 trick: sigmoid(x) = 1 / (1 + exp(-x)) = 1 / (1 + exp2(-x * LOG2E)) +# Using exp2(x * LOG2E) instead of exp(x) avoids Cody-Waite range reduction +# and maps to FMUL+EX2 on Blackwell (same as Triton's native sigmoid). +# CRITICAL: Only effective with @ct.kernel(occupancy=1) — without it, ct.exp2 calls exp internally. +LOG2E: float = 1.4426950408889634 # log2(e) = 1/ln(2) + + +@ct.kernel(occupancy=1, num_worker_warps=8) +def _swiglu_fwd_ct_aligned( + A, # (n_rows, n_cols) input a + B, # (n_rows, n_cols) input b + C, # (n_rows, n_cols) output c + n_cols: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], + gate_multiplier: ct.Constant[float], +): + """ + SwiGLU forward — aligned fast path (check_bounds=False). + + Safe only when n_cols % BLOCK_SIZE == 0 (no out-of-bounds accesses). + ~17-20% faster than the bounds-checked variant on B200. + Computes: c = silu(a * gate_multiplier) * b + """ + row_idx = ct.bid(0) + n_chunks = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE + + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + + a = ct.astype(ct.gather(A, (row_idx, col_idx), check_bounds=False, padding_value=0.0), ct.float32) + b = ct.gather(B, (row_idx, col_idx), check_bounds=False, padding_value=0.0) + + # Apply gate_multiplier before SiLU (Liger convention) + a_scaled = a * gate_multiplier + + # exp2 trick + flush_to_zero: sigmoid via exp2(-a*LOG2E) — FMUL+EX2 (avoids Cody-Waite range reduction). + # flush_to_zero=True skips denormal handling; sigmoid range is well above the denormal threshold. + # Requires occupancy=1 for correct exp2→EX2 lowering. + sig_a = ct.truediv( + 1.0, + 1.0 + ct.exp2(ct.mul(-a_scaled, LOG2E), flush_to_zero=True), + flush_to_zero=True, + rounding_mode=ct.RoundingMode.APPROX, + ) + silu_a = a_scaled * sig_a + + c = ct.astype(silu_a, b.dtype) * b + ct.scatter(C, (row_idx, col_idx), c, check_bounds=False) + + +@ct.kernel(occupancy=1, num_worker_warps=8) +def _swiglu_fwd_ct( + A, # (n_rows, n_cols) input a + B, # (n_rows, n_cols) input b + C, # (n_rows, n_cols) output c + n_cols: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], + gate_multiplier: ct.Constant[float], +): + """ + SwiGLU forward — general path (check_bounds=True). + + Handles arbitrary n_cols. Used as fallback when n_cols % BLOCK_SIZE != 0. + Computes: c = silu(a * gate_multiplier) * b + """ + row_idx = ct.bid(0) + n_chunks = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE + + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + + a = ct.astype(ct.gather(A, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + b = ct.gather(B, (row_idx, col_idx), check_bounds=True, padding_value=0.0) + + # Apply gate_multiplier before SiLU (Liger convention) + a_scaled = a * gate_multiplier + + # exp2 trick + flush_to_zero: sigmoid via exp2(-a*LOG2E) — FMUL+EX2 (avoids Cody-Waite range reduction). + # flush_to_zero=True skips denormal handling; sigmoid range is well above the denormal threshold. + # Requires occupancy=1 for correct exp2→EX2 lowering. + sig_a = ct.truediv( + 1.0, + 1.0 + ct.exp2(ct.mul(-a_scaled, LOG2E), flush_to_zero=True), + flush_to_zero=True, + rounding_mode=ct.RoundingMode.APPROX, + ) + silu_a = a_scaled * sig_a + + c = ct.astype(silu_a, b.dtype) * b + ct.scatter(C, (row_idx, col_idx), c, check_bounds=True) + + +@ct.kernel +def _swiglu_bwd_ct_aligned( + DC, # (n_rows, n_cols) upstream gradient + A, # (n_rows, n_cols) saved input a — DA written in-place + B, # (n_rows, n_cols) saved input b — DB written in-place + n_cols: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], + gate_multiplier: ct.Constant[float], +): + """ + SwiGLU backward — aligned fast path (check_bounds=False). + + Safe only when n_cols % BLOCK_SIZE == 0. da/db written in-place to A/B. + NOTE: No occupancy=1 — scatter inside a backward loop risks hangs. + + Chain rule: fwd computes c = silu(a * gm) * b + db = dc * silu(a * gm) + da = dc * d_silu(a*gm)/d(a*gm) * gm * b + = dc * (silu(a*gm) * (1 - sig(a*gm)) + sig(a*gm)) * gm * b + """ + row_idx = ct.bid(0) + n_chunks = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE + + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + + dc = ct.astype(ct.gather(DC, (row_idx, col_idx), check_bounds=False, padding_value=0.0), ct.float32) + # A holds original a (forward did not write back); reapply gate_multiplier + a = ct.astype(ct.gather(A, (row_idx, col_idx), check_bounds=False, padding_value=0.0), ct.float32) + b = ct.astype(ct.gather(B, (row_idx, col_idx), check_bounds=False, padding_value=0.0), ct.float32) + + a_scaled = a * gate_multiplier + sig_a = ct.truediv(1.0, 1.0 + ct.exp(0.0 - a_scaled), rounding_mode=ct.RoundingMode.APPROX) + silu_a = a_scaled * sig_a + + db = dc * silu_a + da = dc * (silu_a * (1.0 - sig_a) + sig_a) * b * gate_multiplier + + ct.scatter(A, (row_idx, col_idx), ct.astype(da, A.dtype), check_bounds=False) + ct.scatter(B, (row_idx, col_idx), ct.astype(db, B.dtype), check_bounds=False) + + +@ct.kernel +def _swiglu_bwd_ct( + DC, # (n_rows, n_cols) upstream gradient + A, # (n_rows, n_cols) saved input a — DA written in-place + B, # (n_rows, n_cols) saved input b — DB written in-place + n_cols: ct.Constant[int], + BLOCK_SIZE: ct.Constant[int], + gate_multiplier: ct.Constant[float], +): + """ + SwiGLU backward — general path (check_bounds=True). + + Recomputes sigmoid for memory efficiency (no saved activations). + da/db written in-place to A/B. Grid: (n_rows, 1, 1). + NOTE: No occupancy=1 — scatter inside a backward loop risks hangs. + """ + row_idx = ct.bid(0) + n_chunks = (n_cols + BLOCK_SIZE - 1) // BLOCK_SIZE + + for ci in range(n_chunks): + col_idx = ct.add(ct.arange(BLOCK_SIZE, dtype=ct.int32), ci * BLOCK_SIZE) + + dc = ct.astype(ct.gather(DC, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + # A holds original a (forward did not write back); reapply gate_multiplier + a = ct.astype(ct.gather(A, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + b = ct.astype(ct.gather(B, (row_idx, col_idx), check_bounds=True, padding_value=0.0), ct.float32) + + a_scaled = a * gate_multiplier + # Recompute sigmoid (APPROX division: no FCHK + software-fallback CALL) + sig_a = ct.truediv(1.0, 1.0 + ct.exp(0.0 - a_scaled), rounding_mode=ct.RoundingMode.APPROX) + silu_a = a_scaled * sig_a + + db = dc * silu_a + da = dc * (silu_a * (1.0 - sig_a) + sig_a) * b * gate_multiplier + + ct.scatter(A, (row_idx, col_idx), ct.astype(da, A.dtype), check_bounds=True) + ct.scatter(B, (row_idx, col_idx), ct.astype(db, B.dtype), check_bounds=True) + + +def _calculate_block_size(n_cols, max_fused_size): + # Cap the tile at max_fused_size (or next_pow2(n_cols) if smaller). + block = max(min(_next_power_of_2(n_cols), max_fused_size), 128) + # Largest power-of-2 tile <= block that evenly divides n_cols — this enables the + # check_bounds=False aligned fast path (which dispatch selects when block % n_cols == 0). + aligned = block + while aligned > 128 and n_cols % aligned != 0: + aligned //= 2 + # Prefer the aligned block only when it stays large (>= half the cap). For sizes with small + # odd factors (e.g. 11008 = 256*43, 13824 = 512*27) the largest aligned block collapses to a + # tiny tile with dozens of chunks; there, keep the full block and let the masked + # (check_bounds=True) kernel cover the remainder in far fewer chunks (~10% faster). + # 14336 = 2048*7 keeps a large aligned block (2048) and stays on the fast path. + if n_cols % aligned == 0 and aligned >= block // 2: + return aligned + return block + + +class LigerSiLUMulFunction(torch.autograd.Function): + """CuTile autograd wrapper for SwiGLU (silu(a * gate_multiplier) * b * down_multiplier). + + gate_multiplier is applied inside the kernel (consistent with Liger-Kernel). + down_multiplier is applied at the Python wrapper level. + """ + + @staticmethod + def forward(ctx, a, b, gate_multiplier: float = 1.0, down_multiplier: float = 1.0): + gate_multiplier = float(gate_multiplier) + down_multiplier = float(down_multiplier) + ori_shape = a.shape + n_cols = ori_shape[-1] + a = a.view(-1, n_cols).contiguous() + b = b.view(-1, n_cols).contiguous() + n_rows = a.shape[0] + + c = torch.empty_like(a) + BLOCK_SIZE = _calculate_block_size(n_cols, MAX_FUSED_SIZE_FWD) + fwd_kernel = _swiglu_fwd_ct_aligned if n_cols % BLOCK_SIZE == 0 else _swiglu_fwd_ct + + ct.launch( + torch.cuda.current_stream(), + (n_rows, 1, 1), + fwd_kernel, + (a, b, c, int(n_cols), int(BLOCK_SIZE), gate_multiplier), + ) + c_out = c.view(*ori_shape) + if down_multiplier != 1.0: + c_out = c_out * down_multiplier + ctx.save_for_backward(a, b) + ctx.ori_shape = ori_shape + ctx.gate_multiplier = gate_multiplier + ctx.down_multiplier = down_multiplier + return c_out + + @staticmethod + def backward(ctx, dc): + a, b = ctx.saved_tensors + ori_shape = ctx.ori_shape + n_cols = ori_shape[-1] + dc = dc.view(-1, n_cols).contiguous() + n_rows = dc.shape[0] + if ctx.down_multiplier != 1.0: + dc = dc * ctx.down_multiplier + BLOCK_SIZE = _calculate_block_size(n_cols, MAX_FUSED_SIZE_BWD) + bwd_kernel = _swiglu_bwd_ct_aligned if n_cols % BLOCK_SIZE == 0 else _swiglu_bwd_ct + + ct.launch( + torch.cuda.current_stream(), + (n_rows, 1, 1), + bwd_kernel, + (dc, a, b, int(n_cols), int(BLOCK_SIZE), ctx.gate_multiplier), + ) + return a.view(*ori_shape), b.view(*ori_shape), None, None diff --git a/test/transformers/test_cutile_backend.py b/test/transformers/test_cutile_backend.py index c64c6e930..53f572729 100644 --- a/test/transformers/test_cutile_backend.py +++ b/test/transformers/test_cutile_backend.py @@ -11,10 +11,18 @@ # Transformer modules bind liger_kernel.ops.* at import time (geglu != GELUMul name). TRANSFORMER_MODULES = { "LigerCrossEntropyFunction": "liger_kernel.transformers.cross_entropy", + "LigerDyTFunction": "liger_kernel.transformers.dyt", + "LigerFusedAddRMSNormFunction": "liger_kernel.transformers.fused_add_rms_norm", + "LigerFusedLinearCrossEntropyFunction": "liger_kernel.transformers.fused_linear_cross_entropy", "LigerFusedLinearJSDFunction": "liger_kernel.transformers.fused_linear_jsd", "LigerGELUMulFunction": "liger_kernel.transformers.geglu", + "GrpoLossFunction": "liger_kernel.transformers.grpo_loss", "LigerJSDFunction": "liger_kernel.transformers.jsd", "LigerLayerNormFunction": "liger_kernel.transformers.layer_norm", + "LigerPolyNormFunction": "liger_kernel.transformers.poly_norm", + "LigerRMSNormFunction": "liger_kernel.transformers.rms_norm", + "LigerSiLUMulFunction": "liger_kernel.transformers.swiglu", + "LigerSoftmaxFunction": "liger_kernel.transformers.softmax", } pytestmark = [ From ccf992f61da583630c0b850176ee64b70a10f709 Mon Sep 17 00:00:00 2001 From: jinmanx Date: Tue, 28 Jul 2026 03:13:21 +0000 Subject: [PATCH 2/2] [cuTile] Fix ruff formatting in fused_add_rms_norm Co-Authored-By: Claude Opus 4.8 (1M context) --- src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py b/src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py index e2144b5c1..648ff667c 100644 --- a/src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py +++ b/src/liger_kernel/ops/cutile/ops/fused_add_rms_norm.py @@ -101,7 +101,9 @@ def _fused_add_rms_norm_fwd_ct( S_tile = ct.astype(ct.mul(S_tile, rstd), X.dtype) # W may be higher precision than X (e.g. fp32 norm weight, bf16 activations); # the multiply promotes to W's dtype, so cast back to Y.dtype before storing. - ct.scatter(Y, (row_idx, col_idx), ct.astype(ct.mul(S_tile, ct.add(W_tile, offset)), Y.dtype), check_bounds=True) + ct.scatter( + Y, (row_idx, col_idx), ct.astype(ct.mul(S_tile, ct.add(W_tile, offset)), Y.dtype), check_bounds=True + ) elif casting_mode == _CASTING_MODE_GEMMA: S_tile = ct.astype(S_tile, ct.float32) rstd = ct.rsqrt(ct.sum(ct.mul(S_tile, S_tile), 0, keepdims=False) / n_cols + eps) @@ -155,7 +157,9 @@ def _fused_add_rms_norm_fwd_ct( if casting_mode == _CASTING_MODE_LLAMA: S_tile = ct.astype(S_tile, ct.float32) S_normed = ct.astype(ct.mul(S_tile, rstd), X.dtype) - ct.scatter(Y, (row_idx, col_idx), ct.astype(ct.mul(S_normed, ct.add(W_tile, offset)), Y.dtype), check_bounds=True) + ct.scatter( + Y, (row_idx, col_idx), ct.astype(ct.mul(S_normed, ct.add(W_tile, offset)), Y.dtype), check_bounds=True + ) elif casting_mode == _CASTING_MODE_GEMMA: S_tile = ct.astype(S_tile, ct.float32) W_shifted = ct.add(ct.astype(W_tile, ct.float32), offset)