megatron-lm-175b-a100-sxm-40gb-w16a16e16-tp8-pp12-dp8-sp1-fe0.4775641025641026-hbme1-summary.json

{
    "batch_size_per_gpu": 4,
    "max_batch_size_per_gpu": 5,
    "gradient_accumulation_steps": 48,
    "global_batch_size": 1536,
    "dp_size": 8,
    "tp_size": 8,
    "pp_size": 12,
    "sp_size": 1,
    "ds_zero": "NONE",
    "total_num_gpus": 768,
    "seq_len": 2048,
    "total_num_tokens": 300000000000.0,
    "num_params_total": 174563917824,
    "activation_recomputation": "FULL",
    "achived_flops": 149.0,
    "flops_efficiency": 0.4775641025641026,
    "hbm_memory_efficiency": 1,
    "num_flops_total_per_micro_batch": 11756880267313152,
    "weight_memory_per_gpu": 3778314240.0,
    "gradient_memory_per_gpu": 3623878656.0,
    "optimizer_state_memory_per_gpu": 28991029248.0,
    "activation_memory_bs1": 603979776.0,
    "activation_memory_per_gpu": 2415919104.0,
    "latency_per_micro_batch": 0.8219295488893423,
    "latency_fwd": 0.2381681304754309,
    "latency_fwd_attn": 0.07194790852896644,
    "latency_fwd_mlp": 0.13282690805347652,
    "latency_fwd_layernorm": 0.004143055269453376,
    "latency_fwd_tp_comm": 0.01879048192,
    "latency_fwd_input_embedding": 0.0019689290248231513,
    "latency_fwd_output_embedding_loss": 0.00849084767871141,
    "latency_per_iter": 39.45261834668843,
    "total_training_latency": 3762477.853868636,
    "gpu_hours": 802661.9421586423
}