From 77ad81a8aa96eeaee0eb5880ecca31ac7b77b722 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 15:01:31 +0200 Subject: [PATCH 01/12] add makefile --- Makefile | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..bd8fab32 --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SHELL := /bin/bash +CURRENT_DIR = $(shell pwd) +DEFAULT_CLONE_URL := https://github.com/huggingface/optimum-benchmark.git +# If CLONE_URL is empty, revert to DEFAULT_CLONE_URL +REAL_CLONE_URL = $(if $(CLONE_URL),$(CLONE_URL),$(DEFAULT_CLONE_URL)) + +# Install the library in development mode +.PHONY: style test + +# Run code quality checks +style_check: + black --check . + ruff . + +# Format the code +style: + black . + ruff . --fix + +# Run tests for the library +test: + python -m pytest tests \ No newline at end of file From d073d8be7f2c8990bb79a7972ee35cdcb9fc2b81 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 15:03:17 +0200 Subject: [PATCH 02/12] update setup with packages --- setup.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index cd83b6f7..9f8ed185 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,45 @@ from setuptools import find_packages, setup +INSTALL_REQUIRES = [ + # Mandatory HF dependencies + "optimum>=1.13.0", # backends, tasks and input generation + "transformers>=4.20.0", # pytorch, models, configs, hub, etc. + "accelerate>=0.22.0", # distributed inference and no weights init + # Hydra + "omegaconf==2.3.0", + "hydra-core==1.3.2", + "hydra_colorlog==1.2.0", + "hydra-joblib-launcher==1.2.0", + # Other + "codecarbon==2.3.1", + "psutil==5.9.0", + "pandas>=2.0.0", +] + + +EXTRAS_REQUIRE = { + "test": ["pytest"], + "quality": ["black", "ruff"], + "report": ["flatten_dict", "matplotlib", "seaborn", "rich"], + # cpu backends + "openvino": ["optimum[openvino,nncf]>=1.13.0"], + "onnxruntime": ["optimum[onnxruntime]>=1.13.0"], + "neural-compressor": ["optimum[neural-compressor]>=1.13.0"], + # cuda backends + "onnxruntime-gpu": ["py3nvml", "optimum[onnxruntime-gpu]>=1.13.0"], + "text-generation-inference": ["py3nvml-0.2.7", "docker==6.1.3"], + # specific settings + "peft": ["peft"], + "diffusers": ["diffusers"], +} + + setup( name="optimum-benchmark", version="0.0.1", packages=find_packages(), - extras_require={ - "test": ["pytest"], - }, + install_requires=INSTALL_REQUIRES, + extras_require=EXTRAS_REQUIRE, entry_points={ "console_scripts": [ "optimum-benchmark=optimum_benchmark.experiment:run_experiment", From 69ac286cec203008d1f0e60aaa3a161c5cf3e8f1 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 15:03:39 +0200 Subject: [PATCH 03/12] added quality checks --- .github/workflows/check_quality.yaml | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/check_quality.yaml diff --git a/.github/workflows/check_quality.yaml b/.github/workflows/check_quality.yaml new file mode 100644 index 00000000..5f550dc3 --- /dev/null +++ b/.github/workflows/check_quality.yaml @@ -0,0 +1,36 @@ +name: Quality checks + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + run_cpu_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install quality requirements + run: | + pip install --upgrade pip + pip install -e .[quality] + + - name: Check style with black + run: | + black --check . + + - name: Check style with ruff + run: | + ruff . From ce54ea1248ceafd52ed7c7891e8fb216514d7d45 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 15:03:56 +0200 Subject: [PATCH 04/12] update and seperate workflows --- .../workflows/test_cpu_neural_compressor.yaml | 32 ++++++++++++++++ .github/workflows/test_cpu_onnxruntime.yaml | 32 ++++++++++++++++ .github/workflows/test_cpu_openvino.yaml | 32 ++++++++++++++++ .../{test_cpu.yaml => test_cpu_pytorch.yaml} | 12 +++--- ...l => test_cuda_onnxruntime_inference.yaml} | 10 +++-- ...ml => test_cuda_onnxruntime_training.yaml} | 12 ++++-- .github/workflows/test_cuda_pytorch.yaml | 37 +++++++++++++++++++ docker/gpu.dockerfile | 8 ---- ...erfile => onnxruntime_training.dockerfile} | 7 ---- docker/scripts/build_gpu.sh | 1 - docker/scripts/build_ort_training.sh | 1 - 11 files changed, 154 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/test_cpu_neural_compressor.yaml create mode 100644 .github/workflows/test_cpu_onnxruntime.yaml create mode 100644 .github/workflows/test_cpu_openvino.yaml rename .github/workflows/{test_cpu.yaml => test_cpu_pytorch.yaml} (72%) rename .github/workflows/{test_gpu.yaml => test_cuda_onnxruntime_inference.yaml} (79%) rename .github/workflows/{test_ort_training.yaml => test_cuda_onnxruntime_training.yaml} (75%) create mode 100644 .github/workflows/test_cuda_pytorch.yaml rename docker/{ort_training.dockerfile => onnxruntime_training.dockerfile} (92%) delete mode 100644 docker/scripts/build_gpu.sh delete mode 100644 docker/scripts/build_ort_training.sh diff --git a/.github/workflows/test_cpu_neural_compressor.yaml b/.github/workflows/test_cpu_neural_compressor.yaml new file mode 100644 index 00000000..d6b868b5 --- /dev/null +++ b/.github/workflows/test_cpu_neural_compressor.yaml @@ -0,0 +1,32 @@ +name: Intel Neural Compressor CPU Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + run_cpu_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install Intel Neural Compressor CPU requirements + run: | + pip install --upgrade pip + pip install -e .[neural-compressor,test] + + - name: Run Intel Neural Compressor CPU tests + run: | + pytest -k "cpu_neural_compressor" diff --git a/.github/workflows/test_cpu_onnxruntime.yaml b/.github/workflows/test_cpu_onnxruntime.yaml new file mode 100644 index 00000000..4f7e7c73 --- /dev/null +++ b/.github/workflows/test_cpu_onnxruntime.yaml @@ -0,0 +1,32 @@ +name: OnnxRuntime CPU Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + run_cpu_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install requirements + run: | + pip install --upgrade pip + pip install -e .[onnxruntime,test] + + - name: Run tests + run: | + pytest -k "cpu_onnxruntime" diff --git a/.github/workflows/test_cpu_openvino.yaml b/.github/workflows/test_cpu_openvino.yaml new file mode 100644 index 00000000..54ed817e --- /dev/null +++ b/.github/workflows/test_cpu_openvino.yaml @@ -0,0 +1,32 @@ +name: OpenVINO CPU Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + run_cpu_tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install requirements + run: | + pip install --upgrade pip + pip install -e .[openvino,test] + + - name: Run tests + run: | + pytest -k "cpu_openvino" diff --git a/.github/workflows/test_cpu.yaml b/.github/workflows/test_cpu_pytorch.yaml similarity index 72% rename from .github/workflows/test_cpu.yaml rename to .github/workflows/test_cpu_pytorch.yaml index 90c8d3e9..464b5269 100644 --- a/.github/workflows/test_cpu.yaml +++ b/.github/workflows/test_cpu_pytorch.yaml @@ -1,4 +1,4 @@ -name: CPU Unit Tests +name: Pytorch CPU tests on: push: @@ -14,7 +14,7 @@ jobs: run_cpu_tests: runs-on: ubuntu-latest steps: - - name: Checkout code + - name: Checkout uses: actions/checkout@v2 - name: Set up Python 3.8 @@ -22,11 +22,11 @@ jobs: with: python-version: 3.8 - - name: Install CPU requirements + - name: Install requirements run: | pip install --upgrade pip - pip install -r cpu_requirements.txt pip install -e .[test] - - name: Run CPU tests - run: pytest -k "cpu" + - name: Run tests + run: | + pytest -k "cpu_pytorch" diff --git a/.github/workflows/test_gpu.yaml b/.github/workflows/test_cuda_onnxruntime_inference.yaml similarity index 79% rename from .github/workflows/test_gpu.yaml rename to .github/workflows/test_cuda_onnxruntime_inference.yaml index d76f763f..f4622205 100644 --- a/.github/workflows/test_gpu.yaml +++ b/.github/workflows/test_cuda_onnxruntime_inference.yaml @@ -1,4 +1,4 @@ -name: GPU Unit Tests +name: OnnxRuntime CUDA Inference Tests on: pull_request: @@ -12,7 +12,7 @@ jobs: build-and-test: runs-on: self-hosted steps: - - name: Restore docker ownership + - name: Restore files ownership run: docker run --rm --entrypoint /bin/bash @@ -23,13 +23,17 @@ jobs: ubuntu -c 'chown -R ${HOST_UID}:${HOST_GID} /workspace/optimum-benchmark' - - name: Checkout code + - name: Checkout uses: actions/checkout@v2 +<<<<<<< HEAD:.github/workflows/test_gpu.yaml - name: Build GPU Docker image run: docker build --no-cache --build-arg CACHEBUST=$(date +%s) -f docker/gpu.dockerfile -t optimum-benchmark-gpu . - name: Run GPU tests +======= + - name: Run tests +>>>>>>> seperated backend-device tests and update setup.py:.github/workflows/test_cuda_onnxruntime_inference.yaml run: docker run --rm --entrypoint /bin/bash diff --git a/.github/workflows/test_ort_training.yaml b/.github/workflows/test_cuda_onnxruntime_training.yaml similarity index 75% rename from .github/workflows/test_ort_training.yaml rename to .github/workflows/test_cuda_onnxruntime_training.yaml index 6540628e..07a0972c 100644 --- a/.github/workflows/test_ort_training.yaml +++ b/.github/workflows/test_cuda_onnxruntime_training.yaml @@ -12,7 +12,7 @@ jobs: build-and-test: runs-on: self-hosted steps: - - name: Restore docker ownership + - name: Restore files ownership run: docker run --rm --entrypoint /bin/bash @@ -23,18 +23,22 @@ jobs: ubuntu -c 'chown -R ${HOST_UID}:${HOST_GID} /workspace/optimum-benchmark' - - name: Checkout code + - name: Checkout uses: actions/checkout@v2 +<<<<<<< HEAD:.github/workflows/test_ort_training.yaml - name: Build OnnxRuntime Training Docker image run: docker build --no-cache --build-arg CACHEBUST=$(date +%s) -f docker/ort_training.dockerfile -t optimum-benchmark-ort-training . - name: Run OnnxRuntime Training tests +======= + - name: Run tests +>>>>>>> seperated backend-device tests and update setup.py:.github/workflows/test_cuda_onnxruntime_training.yaml run: docker run --rm --entrypoint /bin/bash --gpus '"device=0,1"' --volume $(pwd):/workspace/optimum-benchmark --workdir /workspace/optimum-benchmark - optimum-benchmark-ort-training - -c "pip install -e .[test] && pytest -k 'cuda_onnxruntime_training' -x" + onnxruntime-training + -c "pip install -e .[test] && pytest -k 'onnxruntime_training' -x" diff --git a/.github/workflows/test_cuda_pytorch.yaml b/.github/workflows/test_cuda_pytorch.yaml new file mode 100644 index 00000000..4f15e2ef --- /dev/null +++ b/.github/workflows/test_cuda_pytorch.yaml @@ -0,0 +1,37 @@ +name: Pytorch CUDA Tests + +on: + pull_request: + types: [opened, reopened, synchronize] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-and-test: + runs-on: self-hosted + steps: + - name: Restore files ownership + run: docker run + --rm + --entrypoint /bin/bash + --env HOST_UID=`id -u` + --env HOST_GID=`id -g` + --volume $(pwd):/workspace/optimum-benchmark + --workdir /workspace/optimum-benchmark + ubuntu + -c 'chown -R ${HOST_UID}:${HOST_GID} /workspace/optimum-benchmark' + + - name: Checkout + uses: actions/checkout@v2 + + - name: Run tests + run: docker run + --rm + --entrypoint /bin/bash + --gpus '"device=0,1"' + --volume $(pwd):/workspace/optimum-benchmark + --workdir /workspace/optimum-benchmark + optimum-benchmark-gpu + -c "pip install -e .[test] && pytest -k '(cuda or tensorrt) and not onnxruntime_training' -x" diff --git a/docker/gpu.dockerfile b/docker/gpu.dockerfile index f01d2bbe..42dfa69f 100644 --- a/docker/gpu.dockerfile +++ b/docker/gpu.dockerfile @@ -27,11 +27,3 @@ RUN apt-get install -y software-properties-common wget apt-utils patchelf git li apt-get clean RUN unattended-upgrade RUN apt-get autoremove -y -RUN pip install --upgrade pip - -# this line forces the docker build to rebuild from this point on -ARG CACHEBUST=1 - -# Install optimum-benchmark dependencies -COPY gpu_requirements.txt /tmp/gpu_requirements.txt -RUN pip install -r /tmp/gpu_requirements.txt \ No newline at end of file diff --git a/docker/ort_training.dockerfile b/docker/onnxruntime_training.dockerfile similarity index 92% rename from docker/ort_training.dockerfile rename to docker/onnxruntime_training.dockerfile index c40654d8..3a469a0a 100644 --- a/docker/ort_training.dockerfile +++ b/docker/onnxruntime_training.dockerfile @@ -63,10 +63,3 @@ RUN $PYTHON_EXE -m pip install torch-ort ENV TORCH_CUDA_ARCH_LIST="5.2 6.0 6.1 7.0 7.5 8.0 8.6+PTX" RUN $PYTHON_EXE -m pip install --upgrade protobuf==3.20.2 RUN $PYTHON_EXE -m torch_ort.configure - -# this line forces the docker build to rebuild from this point on -ARG CACHEBUST=1 - -# Install optimum-benchmark dependencies -COPY requirements.txt /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt \ No newline at end of file diff --git a/docker/scripts/build_gpu.sh b/docker/scripts/build_gpu.sh deleted file mode 100644 index 5ecfdb6a..00000000 --- a/docker/scripts/build_gpu.sh +++ /dev/null @@ -1 +0,0 @@ -docker build --no-cache --build-arg CACHEBUST=$(date +%s) -f docker/gpu.dockerfile -t optimum-benchmark-gpu . \ No newline at end of file diff --git a/docker/scripts/build_ort_training.sh b/docker/scripts/build_ort_training.sh deleted file mode 100644 index d2efd477..00000000 --- a/docker/scripts/build_ort_training.sh +++ /dev/null @@ -1 +0,0 @@ -docker build --no-cache --build-arg CACHEBUST=$(date +%s) -f docker/ort_training.dockerfile -t optimum-benchmark-ort-training . \ No newline at end of file From 18b5d11ad180af4469d7379d304dc9b219f47feb Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 15:12:12 +0200 Subject: [PATCH 05/12] added diffusers to worflows --- .github/workflows/test_cpu_neural_compressor.yaml | 2 +- .github/workflows/test_cpu_onnxruntime.yaml | 2 +- .github/workflows/test_cpu_openvino.yaml | 2 +- .github/workflows/test_cpu_pytorch.yaml | 2 +- .github/workflows/test_cuda_onnxruntime_inference.yaml | 9 +-------- .github/workflows/test_cuda_onnxruntime_training.yaml | 7 ------- 6 files changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_cpu_neural_compressor.yaml b/.github/workflows/test_cpu_neural_compressor.yaml index d6b868b5..a4c547b4 100644 --- a/.github/workflows/test_cpu_neural_compressor.yaml +++ b/.github/workflows/test_cpu_neural_compressor.yaml @@ -25,7 +25,7 @@ jobs: - name: Install Intel Neural Compressor CPU requirements run: | pip install --upgrade pip - pip install -e .[neural-compressor,test] + pip install -e .[test,neural-compressor,diffusers] - name: Run Intel Neural Compressor CPU tests run: | diff --git a/.github/workflows/test_cpu_onnxruntime.yaml b/.github/workflows/test_cpu_onnxruntime.yaml index 4f7e7c73..2d239171 100644 --- a/.github/workflows/test_cpu_onnxruntime.yaml +++ b/.github/workflows/test_cpu_onnxruntime.yaml @@ -25,7 +25,7 @@ jobs: - name: Install requirements run: | pip install --upgrade pip - pip install -e .[onnxruntime,test] + pip install -e .[test,onnxruntime,diffusers] - name: Run tests run: | diff --git a/.github/workflows/test_cpu_openvino.yaml b/.github/workflows/test_cpu_openvino.yaml index 54ed817e..515e3a4b 100644 --- a/.github/workflows/test_cpu_openvino.yaml +++ b/.github/workflows/test_cpu_openvino.yaml @@ -25,7 +25,7 @@ jobs: - name: Install requirements run: | pip install --upgrade pip - pip install -e .[openvino,test] + pip install -e .[test,openvino,diffusers] - name: Run tests run: | diff --git a/.github/workflows/test_cpu_pytorch.yaml b/.github/workflows/test_cpu_pytorch.yaml index 464b5269..a23e84cd 100644 --- a/.github/workflows/test_cpu_pytorch.yaml +++ b/.github/workflows/test_cpu_pytorch.yaml @@ -25,7 +25,7 @@ jobs: - name: Install requirements run: | pip install --upgrade pip - pip install -e .[test] + pip install -e .[test,diffusers] - name: Run tests run: | diff --git a/.github/workflows/test_cuda_onnxruntime_inference.yaml b/.github/workflows/test_cuda_onnxruntime_inference.yaml index f4622205..b7da2543 100644 --- a/.github/workflows/test_cuda_onnxruntime_inference.yaml +++ b/.github/workflows/test_cuda_onnxruntime_inference.yaml @@ -26,14 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 -<<<<<<< HEAD:.github/workflows/test_gpu.yaml - - name: Build GPU Docker image - run: docker build --no-cache --build-arg CACHEBUST=$(date +%s) -f docker/gpu.dockerfile -t optimum-benchmark-gpu . - - - name: Run GPU tests -======= - name: Run tests ->>>>>>> seperated backend-device tests and update setup.py:.github/workflows/test_cuda_onnxruntime_inference.yaml run: docker run --rm --entrypoint /bin/bash @@ -41,4 +34,4 @@ jobs: --volume $(pwd):/workspace/optimum-benchmark --workdir /workspace/optimum-benchmark optimum-benchmark-gpu - -c "pip install -e .[test] && pytest -k '(cuda or tensorrt) and not onnxruntime_training' -x" + -c "pip install -e .[test,diffusers] && pytest -k '(cuda or tensorrt) and not onnxruntime_training' -x" diff --git a/.github/workflows/test_cuda_onnxruntime_training.yaml b/.github/workflows/test_cuda_onnxruntime_training.yaml index 07a0972c..8b8f9023 100644 --- a/.github/workflows/test_cuda_onnxruntime_training.yaml +++ b/.github/workflows/test_cuda_onnxruntime_training.yaml @@ -26,14 +26,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 -<<<<<<< HEAD:.github/workflows/test_ort_training.yaml - - name: Build OnnxRuntime Training Docker image - run: docker build --no-cache --build-arg CACHEBUST=$(date +%s) -f docker/ort_training.dockerfile -t optimum-benchmark-ort-training . - - - name: Run OnnxRuntime Training tests -======= - name: Run tests ->>>>>>> seperated backend-device tests and update setup.py:.github/workflows/test_cuda_onnxruntime_training.yaml run: docker run --rm --entrypoint /bin/bash From 14af2d30974d544291904cc1aee4375e0d487699 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 16:02:08 +0200 Subject: [PATCH 06/12] set optimum version --- setup.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 9f8ed185..7ee4b095 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,11 @@ from setuptools import find_packages, setup +OPTIMUM_VERSION = "1.13.0" + INSTALL_REQUIRES = [ # Mandatory HF dependencies - "optimum>=1.13.0", # backends, tasks and input generation - "transformers>=4.20.0", # pytorch, models, configs, hub, etc. - "accelerate>=0.22.0", # distributed inference and no weights init + f"optimum=={OPTIMUM_VERSION}", # backends, tasks and input generation + "accelerate", # distributed inference and no weights init # Hydra "omegaconf==2.3.0", "hydra-core==1.3.2", @@ -13,24 +14,33 @@ # Other "codecarbon==2.3.1", "psutil==5.9.0", - "pandas>=2.0.0", + "pandas==2.1.0", ] +# add py3nvml if nvidia driver is installed +try: + import subprocess + + subprocess.run(["nvidia-smi"], stdout=subprocess.DEVNULL) + INSTALL_REQUIRES.append("py3nvml==0.2.7") +except FileNotFoundError: + pass + EXTRAS_REQUIRE = { "test": ["pytest"], "quality": ["black", "ruff"], "report": ["flatten_dict", "matplotlib", "seaborn", "rich"], # cpu backends - "openvino": ["optimum[openvino,nncf]>=1.13.0"], - "onnxruntime": ["optimum[onnxruntime]>=1.13.0"], - "neural-compressor": ["optimum[neural-compressor]>=1.13.0"], - # cuda backends - "onnxruntime-gpu": ["py3nvml", "optimum[onnxruntime-gpu]>=1.13.0"], - "text-generation-inference": ["py3nvml-0.2.7", "docker==6.1.3"], + "openvino": [f"optimum[openvino,nncf]=={OPTIMUM_VERSION}"], + "onnxruntime": [f"optimum[onnxruntime]=={OPTIMUM_VERSION}"], + "onnxruntime-gpu": [f"optimum[onnxruntime-gpu]=={OPTIMUM_VERSION}"], + "neural-compressor": [f"optimum[neural-compressor]=={OPTIMUM_VERSION}"], + # server-like backend + "text-generation-inference": ["docker==6.1.3"], # specific settings - "peft": ["peft"], "diffusers": ["diffusers"], + "peft": ["peft"], } From 092f8327b018aa428bbce4684be128d0be939845 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 16:02:31 +0200 Subject: [PATCH 07/12] corrected pytorch workflow --- .github/workflows/test_cuda_pytorch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cuda_pytorch.yaml b/.github/workflows/test_cuda_pytorch.yaml index 4f15e2ef..bada7928 100644 --- a/.github/workflows/test_cuda_pytorch.yaml +++ b/.github/workflows/test_cuda_pytorch.yaml @@ -34,4 +34,4 @@ jobs: --volume $(pwd):/workspace/optimum-benchmark --workdir /workspace/optimum-benchmark optimum-benchmark-gpu - -c "pip install -e .[test] && pytest -k '(cuda or tensorrt) and not onnxruntime_training' -x" + -c "pip install -e .[test,diffusers] && pytest -k 'cuda_pytorch' -x" From 5e023e7875d505a75b0cab0f618567488a7d8ac4 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 16:02:57 +0200 Subject: [PATCH 08/12] changes to TasksManager from latest optimum release --- optimum_benchmark/backends/base.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/optimum_benchmark/backends/base.py b/optimum_benchmark/backends/base.py index 774223e0..6727f855 100644 --- a/optimum_benchmark/backends/base.py +++ b/optimum_benchmark/backends/base.py @@ -64,30 +64,30 @@ def __init__(self, model: str, task: str, device: str, hub_kwargs: Dict[str, Any if self.is_diffusion_pipeline(): # for pipelines + self.library = "diffusers" + self.model_type = self.task self.pretrained_config = None self.pretrained_processor = None - self.model_type = self.task else: # for models + self.library = "transformers" self.pretrained_config = AutoConfig.from_pretrained( pretrained_model_name_or_path=self.model, **self.hub_kwargs ) self.model_type = self.pretrained_config.model_type try: - # the processor sometimes contains information about the model's - # input shapes that's not available in the config + # the processor sometimes contains information about the model's input shapes that's not available in the config self.pretrained_processor = AutoProcessor.from_pretrained( pretrained_model_name_or_path=self.model, **self.hub_kwargs ) except ValueError: + # sometimes the processor is not available or can't be determined/detected LOGGER.warning("Could not find the model's preprocessor") self.pretrained_processor = None self.automodel_class = TasksManager.get_model_class_for_task( - task=self.task, - framework="pt", - model_type=self.model_type, + task=self.task, library=self.library, model_type=self.model_type ) def is_text_generation_model(self) -> bool: From 49cbe2ef24551c5d1cf2164cccb6efadae4ac19a Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 16:05:40 +0200 Subject: [PATCH 09/12] fix pandas --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7ee4b095..5471e59f 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ # Other "codecarbon==2.3.1", "psutil==5.9.0", - "pandas==2.1.0", + "pandas>=2.0.0", ] # add py3nvml if nvidia driver is installed From 60fbe6bf16498ad0b5393e64b09c0010bce2e8e5 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 16:07:58 +0200 Subject: [PATCH 10/12] pytest key for ort training --- .github/workflows/test_cpu_neural_compressor.yaml | 2 +- .github/workflows/test_cuda_onnxruntime_inference.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cpu_neural_compressor.yaml b/.github/workflows/test_cpu_neural_compressor.yaml index a4c547b4..46b66c11 100644 --- a/.github/workflows/test_cpu_neural_compressor.yaml +++ b/.github/workflows/test_cpu_neural_compressor.yaml @@ -25,7 +25,7 @@ jobs: - name: Install Intel Neural Compressor CPU requirements run: | pip install --upgrade pip - pip install -e .[test,neural-compressor,diffusers] + pip install -e .[test,neural-compressor] - name: Run Intel Neural Compressor CPU tests run: | diff --git a/.github/workflows/test_cuda_onnxruntime_inference.yaml b/.github/workflows/test_cuda_onnxruntime_inference.yaml index b7da2543..161a7b6f 100644 --- a/.github/workflows/test_cuda_onnxruntime_inference.yaml +++ b/.github/workflows/test_cuda_onnxruntime_inference.yaml @@ -34,4 +34,4 @@ jobs: --volume $(pwd):/workspace/optimum-benchmark --workdir /workspace/optimum-benchmark optimum-benchmark-gpu - -c "pip install -e .[test,diffusers] && pytest -k '(cuda or tensorrt) and not onnxruntime_training' -x" + -c "pip install -e .[test,diffusers] && pytest -k '(cuda or tensorrt) and onnxruntime and not training' -x" From 74cb94691608e814df7b48fb5b715d72648e5c30 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 16:20:16 +0200 Subject: [PATCH 11/12] added peft to training workflows --- .github/workflows/test_cuda_onnxruntime_training.yaml | 2 +- .github/workflows/test_cuda_pytorch.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_cuda_onnxruntime_training.yaml b/.github/workflows/test_cuda_onnxruntime_training.yaml index 8b8f9023..f56f74c9 100644 --- a/.github/workflows/test_cuda_onnxruntime_training.yaml +++ b/.github/workflows/test_cuda_onnxruntime_training.yaml @@ -34,4 +34,4 @@ jobs: --volume $(pwd):/workspace/optimum-benchmark --workdir /workspace/optimum-benchmark onnxruntime-training - -c "pip install -e .[test] && pytest -k 'onnxruntime_training' -x" + -c "pip install -e .[test,peft] && pytest -k 'onnxruntime_training' -x" diff --git a/.github/workflows/test_cuda_pytorch.yaml b/.github/workflows/test_cuda_pytorch.yaml index bada7928..40c4d5dc 100644 --- a/.github/workflows/test_cuda_pytorch.yaml +++ b/.github/workflows/test_cuda_pytorch.yaml @@ -34,4 +34,4 @@ jobs: --volume $(pwd):/workspace/optimum-benchmark --workdir /workspace/optimum-benchmark optimum-benchmark-gpu - -c "pip install -e .[test,diffusers] && pytest -k 'cuda_pytorch' -x" + -c "pip install -e .[test,peft,diffusers] && pytest -k 'cuda_pytorch' -x" From a39cf0abc1f00030a761f40960b06045430ab596 Mon Sep 17 00:00:00 2001 From: IlyasMoutawwakil Date: Fri, 8 Sep 2023 16:21:41 +0200 Subject: [PATCH 12/12] naming --- .github/workflows/test_cuda_onnxruntime_training.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_cuda_onnxruntime_training.yaml b/.github/workflows/test_cuda_onnxruntime_training.yaml index f56f74c9..3aa34330 100644 --- a/.github/workflows/test_cuda_onnxruntime_training.yaml +++ b/.github/workflows/test_cuda_onnxruntime_training.yaml @@ -1,4 +1,4 @@ -name: OnnxRuntime Training Unit Tests +name: OnnxRuntime CUDA Training Tests on: pull_request: