megatron-lm-530b-a100-sxm-40gb-w16a16e16-tp8-pp35-dp2-sp1-fe0.5480769230769231-hbme1-summary-readable.txt

----------------------------------------------Summary-----------------------------------------------
batch_size_per_gpu: 1
max_batch_size_per_gpu: 1
gradient_accumulation_steps: 1120
global_batch_size: 2240
dp_size: 2
tp_size: 8
pp_size: 35
sp_size: 1
ds_zero: NONE
total_num_gpus: 560
seq_len: 2048
total_num_tokens: 300.0 G
num_params_total: 529.51 G
activation_recomputation: FULL
achived_flops: 171.00000000000003
flops_efficiency: 0.5480769230769231
hbm_memory_efficiency: 1
num_flops_total_per_micro_batch: 8819.83 T
weight_memory_per_gpu: 4.03 GB
gradient_memory_per_gpu: 3.77 GB
optimizer_state_memory_per_gpu: 30.2 GB
activation_memory_bs1: 1.1 GB
activation_memory_per_gpu: 1.1 GB
latency_per_micro_batch: 184.21 ms
latency_fwd: 54.44 ms
latency_fwd_attn: 15.82 ms
latency_fwd_mlp: 30.14 ms
latency_fwd_layernorm: 647.35 us
latency_fwd_tp_comm: 2.94 ms
latency_fwd_input_embedding: 1.81 ms
latency_fwd_output_embedding_loss: 3.08 ms
latency_per_iter: 3.44 minutes
total_training_latency: 156.15 days
gpu_hours: 2098687
----------------------------------------------------------------------------------------------------