megatron-lm-175b-a100-sxm-40gb-w16a16e16-tp8-pp12-dp16-sp1-fe0.4519230769230769-hbme1-summary.json

{
    "batch_size_per_gpu": 4,
    "max_batch_size_per_gpu": 5,
    "gradient_accumulation_steps": 24,
    "global_batch_size": 1536,
    "dp_size": 16,
    "tp_size": 8,
    "pp_size": 12,
    "sp_size": 1,
    "ds_zero": "NONE",
    "total_num_gpus": 1536,
    "seq_len": 2048,
    "total_num_tokens": 300000000000.0,
    "num_params_total": 174563917824,
    "activation_recomputation": "FULL",
    "achived_flops": 141.0,
    "flops_efficiency": 0.4519230769230769,
    "hbm_memory_efficiency": 1,
    "num_flops_total_per_micro_batch": 11756880267313152,
    "weight_memory_per_gpu": 3778314240.0,
    "gradient_memory_per_gpu": 3623878656.0,
    "optimizer_state_memory_per_gpu": 28991029248.0,
    "activation_memory_bs1": 603979776.0,
    "activation_memory_per_gpu": 2415919104.0,
    "latency_per_micro_batch": 0.8685638495355461,
    "latency_fwd": 0.25026831000797867,
    "latency_fwd_attn": 0.07603005936748936,
    "latency_fwd_mlp": 0.14036318652459576,
    "latency_fwd_layernorm": 0.004143055269453376,
    "latency_fwd_tp_comm": 0.01879048192,
    "latency_fwd_input_embedding": 0.0019689290248231513,
    "latency_fwd_output_embedding_loss": 0.008972597901617021,
    "latency_per_iter": 20.84553238885311,
    "total_training_latency": 1987975.8873277544,
    "gpu_hours": 848203.0452598418
}