diff --git a/scripts/launch_gpt2_small_fast_supervised_tpu.sh b/scripts/launch_gpt2_small_fast_supervised_tpu.sh new file mode 100644 index 000000000..df38aec99 --- /dev/null +++ b/scripts/launch_gpt2_small_fast_supervised_tpu.sh @@ -0,0 +1,6 @@ +# Launches the "gpt_small_fast" model on a TPU node + +python infra/launch.py --foreground --tpu_name $(whoami)-levanter-itest-32 --zone us-central2-b --tpu_type v4-32 --preemptible -- \ + python -m levanter.main.train_lm \ + --config_path config/gpt2_small_fast_supervised.yaml \ + --trainer.checkpointer.base_path gs://levanter-checkpoints/gpt-itest/ --trainer.checkpointer.save_interval 30m $* diff --git a/scripts/launch_gpt2_small_fast_tpu.sh b/scripts/launch_gpt2_small_fast_tpu.sh index df38aec99..0c09cdcfa 100644 --- a/scripts/launch_gpt2_small_fast_tpu.sh +++ b/scripts/launch_gpt2_small_fast_tpu.sh @@ -2,5 +2,5 @@ python infra/launch.py --foreground --tpu_name $(whoami)-levanter-itest-32 --zone us-central2-b --tpu_type v4-32 --preemptible -- \ python -m levanter.main.train_lm \ - --config_path config/gpt2_small_fast_supervised.yaml \ + --config_path config/gpt2_small_fast.yaml \ --trainer.checkpointer.base_path gs://levanter-checkpoints/gpt-itest/ --trainer.checkpointer.save_interval 30m $* diff --git a/src/levanter/data/text.py b/src/levanter/data/text.py index fdd935d82..dfd16f844 100644 --- a/src/levanter/data/text.py +++ b/src/levanter/data/text.py @@ -379,8 +379,6 @@ def num_gpus(self) -> int: return 0 - - def concatenate_and_group_texts( encoding: BatchEncoding, seq_len: int, diff --git a/src/levanter/utils/fsspec_utils.py b/src/levanter/utils/fsspec_utils.py index 6a1341bff..452ab3d84 100644 --- a/src/levanter/utils/fsspec_utils.py +++ b/src/levanter/utils/fsspec_utils.py @@ -1,6 +1,7 @@ import fsspec import braceexpand + def exists(url, **kwargs) -> bool: """Check if a file exists on a remote filesystem.""" fs, path = fsspec.core.url_to_fs(url, **kwargs) @@ -12,6 +13,7 @@ def mkdirs(path): fs, path = fsspec.core.url_to_fs(path) fs.makedirs(path, exist_ok=True) + def fsspec_expand_glob(url): expanded_urls = braceexpand.braceexpand(url) for expanded_url in expanded_urls: @@ -25,4 +27,4 @@ def fsspec_expand_glob(url): else: yield from [f"{protocol}://{path}" for path in globbed] else: - yield expanded_url \ No newline at end of file + yield expanded_url