forked from vllm-project/vllm
-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
439 changed files
with
29,472 additions
and
5,849 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,36 +1,43 @@ | ||
| import os | ||
| import sys | ||
| import zipfile | ||
|
|
||
| MAX_SIZE_MB = 250 | ||
| # Read the VLLM_MAX_SIZE_MB environment variable, defaulting to 250 MB | ||
| VLLM_MAX_SIZE_MB = int(os.environ.get('VLLM_MAX_SIZE_MB', 250)) | ||
|
|
||
|
|
||
| def print_top_10_largest_files(zip_file): | ||
| """Print the top 10 largest files in the given zip file.""" | ||
| with zipfile.ZipFile(zip_file, 'r') as z: | ||
| file_sizes = [(f, z.getinfo(f).file_size) for f in z.namelist()] | ||
| file_sizes.sort(key=lambda x: x[1], reverse=True) | ||
| for f, size in file_sizes[:10]: | ||
| print(f"{f}: {size/(1024*1024)} MBs uncompressed.") | ||
| print(f"{f}: {size / (1024 * 1024):.2f} MBs uncompressed.") | ||
|
|
||
|
|
||
| def check_wheel_size(directory): | ||
| """Check the size of .whl files in the given directory.""" | ||
| for root, _, files in os.walk(directory): | ||
| for f in files: | ||
| if f.endswith(".whl"): | ||
| wheel_path = os.path.join(root, f) | ||
| wheel_size = os.path.getsize(wheel_path) | ||
| wheel_size_mb = wheel_size / (1024 * 1024) | ||
| if wheel_size_mb > MAX_SIZE_MB: | ||
| print( | ||
| f"Wheel {wheel_path} is too large ({wheel_size_mb} MB) " | ||
| f"compare to the allowed size ({MAX_SIZE_MB} MB).") | ||
| for file_name in files: | ||
| if file_name.endswith(".whl"): | ||
| wheel_path = os.path.join(root, file_name) | ||
| wheel_size_mb = os.path.getsize(wheel_path) / (1024 * 1024) | ||
| if wheel_size_mb > VLLM_MAX_SIZE_MB: | ||
| print(f"Not allowed: Wheel {wheel_path} is larger " | ||
| f"({wheel_size_mb:.2f} MB) than the limit " | ||
| f"({VLLM_MAX_SIZE_MB} MB).") | ||
| print_top_10_largest_files(wheel_path) | ||
| return 1 | ||
| else: | ||
| print(f"Wheel {wheel_path} is within the allowed size " | ||
| f"({wheel_size_mb} MB).") | ||
| f"({wheel_size_mb:.2f} MB).") | ||
| return 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| import sys | ||
| sys.exit(check_wheel_size(sys.argv[1])) | ||
| if len(sys.argv) < 2: | ||
| print("Usage: python check-wheel-size.py <directory>") | ||
| sys.exit(1) | ||
|
|
||
| directory = sys.argv[1] | ||
| sys.exit(check_wheel_size(directory)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| # This script build the CPU docker image and run the offline inference inside the container. | ||
| # It serves a sanity check for compilation and basic model usage. | ||
| set -ex | ||
|
|
||
| # Try building the docker image | ||
| docker build -t cpu-test -f Dockerfile.ppc64le . | ||
|
|
||
| # Setup cleanup | ||
| remove_docker_container() { docker rm -f cpu-test || true; } | ||
| trap remove_docker_container EXIT | ||
| remove_docker_container | ||
|
|
||
| # Run the image, setting --shm-size=4g for tensor parallel. | ||
| source /etc/environment | ||
| #docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test cpu-test | ||
| docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true --network host -e HF_TOKEN=$HF_TOKEN --name cpu-test cpu-test | ||
|
|
||
| # Run basic model test | ||
| docker exec cpu-test bash -c " | ||
| pip install pytest matplotlib einops transformers_stream_generator | ||
| pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_oot_registration.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported | ||
|
|
||
| # online inference | ||
| docker exec cpu-test bash -c " | ||
| python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m & | ||
| timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1 | ||
| python3 benchmarks/benchmark_serving.py \ | ||
| --backend vllm \ | ||
| --dataset-name random \ | ||
| --model facebook/opt-125m \ | ||
| --num-prompts 20 \ | ||
| --endpoint /v1/completions \ | ||
| --tokenizer facebook/opt-125m" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.