config_phonelm_0.5b.yaml

name: PhoneLM-0.5B
wandb: True
resume: False
profile: False
training:
    use_bf16: True
    context_size: 2048
    num_train_epochs: 1
    learning_rate: 4e-4
    adam_beta1: 0.9
    adam_beta2: 0.95
    adam_epsilon: 1e-8
    weight_decay: 0.1
    deepspeed_config: "./ds_config_coslr.json"
    per_device_train_batch_size: 96
    per_device_eval_batch_size: 48
    gradient_accumulation_steps: 1
    set_logging_steps: 20
    bad_epochs_limit: 5
    warmup_steps: 500
    set_eval_steps: 2000
    set_save_steps: 2000
model:
    vocab_size: 49152
    hidden_size: 1024
    intermediate_size: 4864
    num_hidden_layers: 24
    hidden_act: "relu"
    num_attention_heads: 16
    num_key_value_heads: 16
    tokenizer_path: "./tokenizer"  
datasets:
    path: "./train_datasets"
    val_rate: 0.005