From 55bdfa79b766eb7a31cc3e92f3565ddfa873a9e2 Mon Sep 17 00:00:00 2001 From: Xiaozhe Yao Date: Wed, 3 Jul 2024 15:14:30 +0200 Subject: [PATCH] benchmark infrastructure --- .github/FUNDING.yml | 12 ----- .github/init.sh | 68 ---------------------------- .github/release_message.sh | 3 -- .github/rename_project.sh | 36 --------------- .github/workflows/main.yml | 25 ---------- .github/workflows/release.yml | 50 -------------------- .github/workflows/rename_project.yml | 42 ----------------- benchmarks/bench_matmul.py | 27 +++++++++++ requirements.txt | 7 +-- triteia/python/configs/gpus/specs.py | 19 +++++++- triteia/python/utils/__init__.py | 3 ++ triteia/python/utils/benchmark.py | 54 ++++++++++++++++++++++ 12 files changed, 103 insertions(+), 243 deletions(-) delete mode 100644 .github/FUNDING.yml delete mode 100755 .github/init.sh delete mode 100755 .github/release_message.sh delete mode 100755 .github/rename_project.sh delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/rename_project.yml create mode 100644 benchmarks/bench_matmul.py diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 1e9d2a1..0000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,12 +0,0 @@ -# These are supported funding model platforms - -github: [rochacbruno] -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -otechie: # Replace with a single Otechie username -custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/init.sh b/.github/init.sh deleted file mode 100755 index 8a32ee3..0000000 --- a/.github/init.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -overwrite_template_dir=0 - -while getopts t:o flag -do - case "${flag}" in - t) template=${OPTARG};; - o) overwrite_template_dir=1;; - esac -done - -if [ -z "${template}" ]; then - echo "Available templates: flask" - read -p "Enter template name: " template -fi - -repo_urlname=$(basename -s .git `git config --get remote.origin.url`) -repo_name=$(basename -s .git `git config --get remote.origin.url` | tr '-' '_' | tr '[:upper:]' '[:lower:]') -repo_owner=$(git config --get remote.origin.url | awk -F ':' '{print $2}' | awk -F '/' '{print $1}') -echo "Repo name: ${repo_name}" -echo "Repo owner: ${repo_owner}" -echo "Repo urlname: ${repo_urlname}" - -if [ -f ".github/workflows/rename_project.yml" ]; then - .github/rename_project.sh -a "${repo_owner}" -n "${repo_name}" -u "${repo_urlname}" -d "Awesome ${repo_name} created by ${repo_owner}" -fi - -function download_template { - rm -rf "${template_dir}" - mkdir -p .github/templates - git clone "${template_url}" "${template_dir}" -} - -echo "Using template:${template}" -template_url="https://github.com/rochacbruno/${template}-project-template" -template_dir=".github/templates/${template}" -if [ -d "${template_dir}" ]; then - # Template directory already exists - if [ "${overwrite_template_dir}" -eq 1 ]; then - # user passed -o flag, delete and re-download - echo "Overwriting ${template_dir}" - download_template - else - # Ask user if they want to overwrite - echo "Directory ${template_dir} already exists." - read -p "Do you want to overwrite it? [y/N] " -n 1 -r - echo - if [[ $REPLY =~ ^[Yy]$ ]]; then - echo "Overwriting ${template_dir}" - download_template - else - # User decided not to overwrite - echo "Using existing ${template_dir}" - fi - fi -else - # Template directory does not exist, download it - echo "Downloading ${template_url}" - download_template -fi - -echo "Applying ${template} template to this project"} -./.github/templates/${template}/apply.sh -a "${repo_owner}" -n "${repo_name}" -u "${repo_urlname}" -d "Awesome ${repo_name} created by ${repo_owner}" - -# echo "Removing temporary template files" -# rm -rf .github/templates/${template} - -echo "Done! review, commit and push the changes" diff --git a/.github/release_message.sh b/.github/release_message.sh deleted file mode 100755 index f5a9062..0000000 --- a/.github/release_message.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -previous_tag=$(git tag --sort=-creatordate | sed -n 2p) -git shortlog "${previous_tag}.." | sed 's/^./ &/' diff --git a/.github/rename_project.sh b/.github/rename_project.sh deleted file mode 100755 index d4db9a1..0000000 --- a/.github/rename_project.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -while getopts a:n:u:d: flag -do - case "${flag}" in - a) author=${OPTARG};; - n) name=${OPTARG};; - u) urlname=${OPTARG};; - d) description=${OPTARG};; - esac -done - -echo "Author: $author"; -echo "Project Name: $name"; -echo "Project URL name: $urlname"; -echo "Description: $description"; - -echo "Renaming project..." - -original_author="FMStack" -original_name="triteia" -original_urlname="triteia" -original_description="Awesome triteia created by FMStack" -# for filename in $(find . -name "*.*") -for filename in $(git ls-files) -do - sed -i "s/$original_author/$author/g" $filename - sed -i "s/$original_name/$name/g" $filename - sed -i "s/$original_urlname/$urlname/g" $filename - sed -i "s/$original_description/$description/g" $filename - echo "Renamed $filename" -done - -mv triteia $name - -# This command runs only once on GHA! -rm -rf .github/template.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b30adc7..e69de29 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,25 +0,0 @@ -name: CI - -on: - push: - branches: [ version/v2 ] - pull_request: - branches: [ version/v2 ] - workflow_dispatch: - -jobs: - tests_linux: - strategy: - fail-fast: false - runs-on: [self-hosted, linux] - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install dependencies - run: make install - - name: Run tests - run: make test - - name: "Upload coverage to Codecov" - uses: codecov/codecov-action@v3 \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index c42c994..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Upload Python Package - -on: - push: - # Sequence of patterns matched against refs/tags - tags: - - '*' # Push events to matching v*, i.e. v1.0, v20.15.10 - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - release: - name: Create Release - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v3 - with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit - fetch-depth: 0 - - name: Generate Changelog - run: .github/release_message.sh > release_message.md - - name: Release - uses: softprops/action-gh-release@v1 - with: - body_path: release_message.md - - deploy: - needs: release - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.github/workflows/rename_project.yml b/.github/workflows/rename_project.yml deleted file mode 100644 index ae6d2eb..0000000 --- a/.github/workflows/rename_project.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Rename the project from template - -on: [push] - -permissions: write-all - -jobs: - rename-project: - if: ${{ !contains (github.repository, '/python-project-template') }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - # by default, it uses a depth of 1 - # this fetches all history so that we can read each commit - fetch-depth: 0 - ref: ${{ github.head_ref }} - - - run: echo "REPOSITORY_NAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}' | tr '-' '_' | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - shell: bash - - - run: echo "REPOSITORY_URLNAME=$(echo '${{ github.repository }}' | awk -F '/' '{print $2}')" >> $GITHUB_ENV - shell: bash - - - run: echo "REPOSITORY_OWNER=$(echo '${{ github.repository }}' | awk -F '/' '{print $1}')" >> $GITHUB_ENV - shell: bash - - - name: Is this still a template - id: is_template - run: echo "::set-output name=is_template::$(ls .github/template.yml &> /dev/null && echo true || echo false)" - - - name: Rename the project - if: steps.is_template.outputs.is_template == 'true' - run: | - echo "Renaming the project with -a(author) ${{ env.REPOSITORY_OWNER }} -n(name) ${{ env.REPOSITORY_NAME }} -u(urlname) ${{ env.REPOSITORY_URLNAME }}" - .github/rename_project.sh -a ${{ env.REPOSITORY_OWNER }} -n ${{ env.REPOSITORY_NAME }} -u ${{ env.REPOSITORY_URLNAME }} -d "Awesome ${{ env.REPOSITORY_NAME }} created by ${{ env.REPOSITORY_OWNER }}" - - - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: "✅ Ready to clone and code." - # commit_options: '--amend --no-edit' - push_options: --force diff --git a/benchmarks/bench_matmul.py b/benchmarks/bench_matmul.py new file mode 100644 index 0000000..1f1e4b0 --- /dev/null +++ b/benchmarks/bench_matmul.py @@ -0,0 +1,27 @@ +import torch +from triteia.python.ops import matmul_4bit_2_4, gen_sparse_quant4_NT +from triteia.python.utils import timing_function, print_results_table + +flops_func = lambda m, n, k: 2 * m * n * k + +def benchmark(m,n,k, dev="cuda", groupsize=-1): + x = torch.randn((n, k), dtype=torch.float16, device=dev) + weight_ref, qweight, scale, meta = gen_sparse_quant4_NT( + m, k, groupsize=groupsize, device=dev + ) + def fp16_func(x, weight_ref): + return torch.matmul(x, weight_ref) + def w4_2_4_func(qweight, x, meta, scale): + return matmul_4bit_2_4(qweight, x, meta, scale) + fp16_result = timing_function( + fp16_func, flops_func, kwargs={"m": m, "n": n, "k": k, "x": x, "weight_ref": weight_ref} + ) + w4_2_4_result = timing_function( + w4_2_4_func, flops_func, kwargs={"m": m, "n": n, "k": k, "qweight": qweight, "x": x, "meta": meta, "scale": scale} + ) + results = [fp16_result, w4_2_4_result] + + print_results_table("matmul_4bit_2_4", results) + +if __name__ == "__main__": + benchmark(4096*2, 32, 4096*2) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b05f2a6..bcd7cbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,2 @@ -# This template is a low-dependency template. -# By default there is no requirements added here. -# Add the requirements you need to this file. -# or run `make init` to create this file automatically based on the template. -# You can also run `make switch-to-poetry` to use the poetry package manager. +pynvml +rich \ No newline at end of file diff --git a/triteia/python/configs/gpus/specs.py b/triteia/python/configs/gpus/specs.py index c752068..8af49c8 100644 --- a/triteia/python/configs/gpus/specs.py +++ b/triteia/python/configs/gpus/specs.py @@ -1,5 +1,8 @@ +from pynvml import * +nvmlInit() + nvidia_rtx_3090 = { - 'name': 'NVIDIA RTX 3090', + 'name': 'NVIDIA GeForce RTX 3090', 'compute_capability': '8.6', 'memory': 24, # in GB 'bandwidth': 936.2, @@ -7,4 +10,16 @@ 'fp32_tflops': 35.58, } -nvidia_gpus = [nvidia_rtx_3090] \ No newline at end of file +nvidia_gpus = [nvidia_rtx_3090] + +def get_gpu_device_info(): + deviceCount = nvmlDeviceGetCount() + name = None + for i in range(deviceCount): + handle = nvmlDeviceGetHandleByIndex(i) + name = nvmlDeviceGetName(handle) + break + for gpu in nvidia_gpus: + if gpu['name'] == name: + return gpu + return None \ No newline at end of file diff --git a/triteia/python/utils/__init__.py b/triteia/python/utils/__init__.py index e69de29..cc942dc 100644 --- a/triteia/python/utils/__init__.py +++ b/triteia/python/utils/__init__.py @@ -0,0 +1,3 @@ +from .benchmark import timing_function, print_results_table + +__all__ = ["timing_function", "print_results_table"] \ No newline at end of file diff --git a/triteia/python/utils/benchmark.py b/triteia/python/utils/benchmark.py index e69de29..cedcb3d 100644 --- a/triteia/python/utils/benchmark.py +++ b/triteia/python/utils/benchmark.py @@ -0,0 +1,54 @@ +import inspect +import torch +from rich.console import Console +from rich.table import Table +from triteia.python.configs.gpus.specs import get_gpu_device_info + +def timing_function(func, flops_func, kwargs): + func_args_names = inspect.getfullargspec(func).args + func_args = {arg: kwargs[arg] for arg in func_args_names if arg in kwargs} + gpu_info = get_gpu_device_info() + if flops_func: + flops_func_args_names = inspect.getfullargspec(flops_func).args + flops_func_args = {arg: kwargs[arg] for arg in flops_func_args_names if arg in kwargs} + + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + output = func(**func_args) + end.record() + torch.cuda.synchronize() + elapsed = start.elapsed_time(end) + + if flops_func: + total_flops = flops_func(**flops_func_args) # FLOPS + perf_flops = total_flops/elapsed # FlOPS/ms + if gpu_info: + mfu = 100 * perf_flops/1e9/gpu_info["fp16_tflops"] + return { + "output": output, + "elapsed": elapsed, # ms + "func_name": func.__name__, + "total_flops": total_flops/1e9 if flops_func else None, # GFLOPS + "perf_flops": perf_flops/1e6 if flops_func else None, # GFLOPS/s + "mfu": mfu if flops_func and gpu_info else None, + "args": kwargs + } + +def print_results_table(title, results): + table = Table(title = title) + table.add_column("Func Name") + table.add_column("Elapsed (ms)") + table.add_column("Total FLOPS (GFLOPS)") + table.add_column("Perf FLOPS (GFLOPS/s)") + table.add_column("MFU (%)") + for result in results: + table.add_row( + result["func_name"], + f"{result['elapsed']:.2f}", + f"{result['total_flops']:.2f}" if result["total_flops"] else None, + f"{result['perf_flops']:.2f}" if result["perf_flops"] else None, + f"{result['mfu']:.2f}" if result["mfu"] else None + ) + console = Console() + console.print(table) \ No newline at end of file