-
Notifications
You must be signed in to change notification settings - Fork 87
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'origin/main' into ksalahi/supervised-data
- Loading branch information
Showing
55 changed files
with
3,586 additions
and
2,052 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -117,3 +117,4 @@ dmypy.json | |
|
|
||
| # local execution commands | ||
| local_*.sh | ||
| .aider* | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -150,6 +150,9 @@ ledger.json | |
| /checkpoints | ||
| *.jaxpr | ||
|
|
||
| # local execution commands | ||
| local_*.sh | ||
|
|
||
| # aider | ||
| .aider* | ||
|
|
||
| .benchmarks | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| cache_dir: "gs://marin-us-central2/tokenized/gpt_neox/" | ||
| tokenizer: "EleutherAI/gpt-neox-20b" | ||
| cache_options: | ||
| batch_size: 256 | ||
| num_shard_groups: 1024 | ||
| stop_strategy: restart | ||
| shuffle: 100000 | ||
| configs: | ||
| "dclm": | ||
| train_urls: | ||
| - gs://marin-us-central2/raw/dclm/v2024-07-09-baseline-dedup/**/*.zstd | ||
| # these are just for eval | ||
| "paloma/4chan": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/4chan_meta_sep/val/val*.jsonl.gz | ||
| "paloma/c4_100_domains": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/c4_100_domains/val/val*.jsonl.gz | ||
| "paloma/c4_en": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/c4_en/val/val*.jsonl.gz | ||
| "paloma/dolma-v1_5": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/dolma-v1_5/val/val*.jsonl.gz | ||
| "paloma/dolma_100_programing_languages": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/dolma_100_programing_languages/val/val*.jsonl.gz | ||
| "paloma/dolma_100_subreddits": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/dolma_100_subreddits/val/val*.jsonl.gz | ||
| "paloma/falcon-refinedweb": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/falcon-refinedweb/val/val*.jsonl.gz | ||
| "paloma/gab": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/gab/val/val*.jsonl.gz | ||
| "paloma/m2d2_s2orc_unsplit": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/m2d2_s2orc_unsplit/val/val*.jsonl.gz | ||
| "paloma/m2d2_wikipedia_unsplit": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/m2d2_wikipedia_unsplit/val/val*.jsonl.gz | ||
| "paloma/manosphere_meta_sep": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/manosphere_meta_sep/val/val*.jsonl.gz | ||
| "paloma/mc4": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/mc4/val/val*.jsonl.gz | ||
| "paloma/ptb": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/ptb/val/val*.jsonl.gz | ||
| "paloma/redpajama": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/redpajama/val/val*.jsonl.gz | ||
| "paloma/twitterAAE_HELM_fixed": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/twitterAAE_HELM_fixed/val/val*.jsonl.gz | ||
| "paloma/wikitext_103": | ||
| validation_urls: | ||
| - gs://levanter-data/paloma/wikitext_103/val/val*.jsonl.gz | ||
| train_weights: | ||
| dclm: 1.0 | ||
| paloma/4chan: 0.0 | ||
| paloma/c4_100_domains: 0.0 | ||
| paloma/c4_en: 0.0 | ||
| paloma/dolma-v1_5: 0.0 | ||
| paloma/dolma_100_programing_languages: 0.0 | ||
| paloma/dolma_100_subreddits: 0.0 | ||
| paloma/falcon-refinedweb: 0.0 | ||
| paloma/gab: 0.0 | ||
| paloma/m2d2_s2orc_unsplit: 0.0 | ||
| paloma/m2d2_wikipedia_unsplit: 0.0 | ||
| paloma/manosphere_meta_sep: 0.0 | ||
| paloma/mc4: 0.0 | ||
| paloma/ptb: 0.0 | ||
| paloma/redpajama: 0.0 | ||
| paloma/twitterAAE_HELM_fixed: 0.0 | ||
| paloma/wikitext_103: 0.0 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| data: !include data/dclm_gpt_neo.yaml | ||
| model: # 7B class model | ||
| type: llama | ||
| seq_len: 2048 | ||
| hidden_dim: 4096 | ||
| intermediate_dim: 11008 | ||
| num_layers: 32 | ||
| num_heads: 32 | ||
| num_kv_heads: 32 | ||
| use_flash_attention: True | ||
| trainer: | ||
| tracker: | ||
| type: wandb | ||
| entity: "stanford-mercury" | ||
| project: "marin" | ||
| tags: ["dclm", "7B", "llama"] | ||
|
|
||
| mp: p=f32,c=bfloat16 | ||
| train_batch_size: 2048 | ||
| num_train_steps: 70000 # 280B / 4M | ||
| steps_per_eval: 1000 | ||
| tensor_parallel_axes: ["mlp", "heads"] | ||
| fsdp_axis: "embed" | ||
| batch_axis: "batch" | ||
| optimizer: | ||
| learning_rate: 4e-4 | ||
| weight_decay: 0.1 | ||
| min_lr_ratio: 0.1 | ||
| beta1: 0.9 | ||
| beta2: 0.95 | ||
| warmup: 5000 | ||
|
|
||
| z_loss_weight: 5e-6 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| # This dockerfile is used to build the docker image for using Ray to manage TPU slices. | ||
| ARG IMAGE=ghcr.io/stanford-crfm/levanter-base | ||
| ARG TAG=latest | ||
|
|
||
| FROM ${IMAGE}:${TAG} | ||
|
|
||
| # install docker in docker, but don't start it | ||
| RUN apt-get update && apt-get install -y docker.io | ||
|
|
||
| ENV TENSORSTORE_CURL_LOW_SPEED_TIME_SECONDS=60\ | ||
| TENSORSTORE_CURL_LOW_SPEED_LIMIT_BYTES=1024\ | ||
| RAY_USAGE_STATS_ENABLED=0\ | ||
| PATH=/opt/levanter/.venv/bin:$PATH\ | ||
| PYTHONPATH=/opt/levanter:/opt/levanter/src:/opt/levanter/examples:/opt/levanter/tests:src:.\ | ||
| HOME=/home/levanter | ||
| # Install dependencies | ||
|
|
||
| RUN apt-get install -y \ | ||
| sudo \ | ||
| git \ | ||
| libjemalloc-dev \ | ||
| wget \ | ||
| cmake \ | ||
| g++ \ | ||
| zlib1g-dev \ | ||
| tmux \ | ||
| screen \ | ||
| rsync \ | ||
| netbase \ | ||
| openssh-client \ | ||
| gnupg | ||
|
|
||
| RUN pip install --no-cache-dir \ | ||
| flatbuffers \ | ||
| cython==0.29.37 \ | ||
| # Necessary for Dataset to work properly. | ||
| numpy\>=1.20 \ | ||
| psutil \ | ||
| # Required a recent version of setuptools to be compatible with python 3.12+. | ||
| setuptools==71.1.0 \ | ||
| "google-api-python-client==1.7.8" \ | ||
| "google-oauth" | ||
|
|
||
|
|
||
| # Install gcloud so we can get secrets (maybe we should just curl?) | ||
| RUN curl https://dl.google.com/dl/cloudsdk/release/google-cloud-sdk.tar.gz > /tmp/google-cloud-sdk.tar.gz | ||
|
|
||
| RUN mkdir -p /usr/local/gcloud \ | ||
| && tar -C /usr/local/gcloud -xvf /tmp/google-cloud-sdk.tar.gz \ | ||
| && /usr/local/gcloud/google-cloud-sdk/install.sh \ | ||
| && rm -f /tmp/google-cloud-sdk.tar.gz | ||
|
|
||
| # Adding the package path to local | ||
| ENV PATH=$PATH:/usr/local/gcloud/google-cloud-sdk/bin | ||
|
|
||
| # GCP doesn't like it when root ssh's into a machine | ||
| RUN useradd -m -s /bin/bash levanter | ||
| RUN echo "levanter ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers | ||
| RUN usermod -aG docker levanter | ||
| RUN mkdir -p $HOME && touch $HOME/.bashrc && chown -R levanter $HOME | ||
| RUN echo "export PATH=$PATH" >> $HOME/.bashrc | ||
| RUN adduser levanter docker | ||
|
|
||
| RUN chown -R levanter /opt/levanter | ||
|
|
||
| USER levanter | ||
|
|
||
| # HACK until https://github.com/ray-project/ray/issues/47769 is resolved | ||
| RUN pip install 'ray[default,gcp]==2.34.0' | ||
| RUN git clone https://github.com/dlwh/ray.git ~/ray --branch tpu_docker_2.34 --depth 1 | ||
| RUN cp ~/ray/python/ray/autoscaler/_private/gcp/tpu_command_runner.py /opt/levanter/.venv/lib/python3.10/site-packages/ray/autoscaler/_private/gcp/tpu_command_runner.py | ||
|
|
||
|
|
||
| WORKDIR /opt/levanter |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.