megatron-lm-530b-a100-sxm-40gb-w16a16e16-tp8-pp35-dp2-sp1-fe0.5480769230769231-hbme1-summary.json

{
    "batch_size_per_gpu": 1,
    "max_batch_size_per_gpu": 1,
    "gradient_accumulation_steps": 1120,
    "global_batch_size": 2240,
    "dp_size": 2,
    "tp_size": 8,
    "pp_size": 35,
    "sp_size": 1,
    "ds_zero": "NONE",
    "total_num_gpus": 560,
    "seq_len": 2048,
    "total_num_tokens": 300000000000.0,
    "num_params_total": 529511874560,
    "activation_recomputation": "FULL",
    "achived_flops": 171.00000000000003,
    "flops_efficiency": 0.5480769230769231,
    "hbm_memory_efficiency": 1,
    "num_flops_total_per_micro_batch": 8819833453936640,
    "weight_memory_per_gpu": 4032266240.0,
    "gradient_memory_per_gpu": 3774873600.0,
    "optimizer_state_memory_per_gpu": 30198988800.0,
    "activation_memory_bs1": 1101004800.0,
    "activation_memory_per_gpu": 1101004800.0,
    "latency_per_micro_batch": 0.18420704791012193,
    "latency_fwd": 0.05444328305763335,
    "latency_fwd_attn": 0.01582356372210526,
    "latency_fwd_mlp": 0.030140121375438592,
    "latency_fwd_layernorm": 0.00064735238585209,
    "latency_fwd_tp_comm": 0.0029360128000000003,
    "latency_fwd_input_embedding": 0.001813541974705252,
    "latency_fwd_output_embedding_loss": 0.003082690799532163,
    "latency_per_iter": 206.31189365933656,
    "total_training_latency": 13491559.973958654,
    "gpu_hours": 2098687.107060235
}