add testing for example folder

.github/workflows/test_cli_cpu_ipex.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cpu') ||
       contains( github.event.pull_request.labels.*.name, 'ipex') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cpu_ipex')
@@ -47,5 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,ipex,diffusers,timm]
 
-      - name: Run tests
-        run: pytest tests/test_cli.py -s -k "cli and cpu and ipex"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and ipex"
+
+      - name: Run tests from example folder
+        run: |
+          pytest tests/test_examples.py -s -k "cpu and ipex"

.github/workflows/test_cli_cpu_llama_cpp.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cpu') ||
       contains( github.event.pull_request.labels.*.name, 'llama_cpp') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cpu_llama_cpp')
@@ -47,5 +48,6 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,llama-cpp]
 
-      - name: Run tests
-        run: pytest tests/test_cli.py -s -k "llama_cpp"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "llama_cpp"

.github/workflows/test_cli_cpu_neural_compressor.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cpu') ||
       contains( github.event.pull_request.labels.*.name, 'neural_compressor') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cpu_neural_compressor')
@@ -47,5 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,neural-compressor,diffusers,timm]
 
-      - name: Run tests
-        run: pytest tests/test_cli.py -s -k "cli and cpu and neural_compressor"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and neural_compressor"
+
+      - name: Run tests from example folder
+        run: |
+          pytest tests/test_examples.py -s -k "cpu and neural_compressor"

.github/workflows/test_cli_cpu_onnxruntime.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cpu') ||
       contains( github.event.pull_request.labels.*.name, 'onnxruntime') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cpu_onnxruntime')
@@ -47,5 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,onnxruntime,diffusers,timm]
 
-      - name: Run tests
-        run: pytest tests/test_cli.py -s -k "cli and cpu and onnxruntime"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and onnxruntime"
+
+      - name: Run tests from example folder
+        run: |
+          pytest tests/test_examples.py -s -k "cpu and onnxruntime"

.github/workflows/test_cli_cpu_openvino.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cpu') ||
       contains( github.event.pull_request.labels.*.name, 'openvino') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cpu_openvino')
@@ -47,5 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,openvino,diffusers,timm]
 
-      - name: Run tests
-        run: pytest tests/test_cli.py -s -k "cli and cpu and openvino"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and openvino"
+
+      - name: Run tests from example folder
+        run: |
+          pytest tests/test_examples.py -s -k "cpu and openvino"

.github/workflows/test_cli_cpu_py_txi.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cpu') ||
       contains( github.event.pull_request.labels.*.name, 'py_txi') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cpu_py_txi')
@@ -47,5 +48,10 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,py-txi]
 
-      - name: Run tests
-        run: pytest tests/test_cli.py -s -k "cli and cpu and py_txi"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and py_txi"
+
+      - name: Run tests from example folder
+        run: |
+          pytest tests/test_examples.py -s -k "examples and cpu and py_txi"

.github/workflows/test_cli_cpu_pytorch.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cpu') ||
       contains( github.event.pull_request.labels.*.name, 'pytorch') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cpu_pytorch')
@@ -47,5 +48,6 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing,diffusers,timm,peft]
 
-      - name: Run tests
-        run: pytest tests/test_cli.py -s -k "cli and cpu and pytorch"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -s -k "cli and cpu and pytorch"

.github/workflows/test_cli_cuda_onnxruntime.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cuda') ||
       contains( github.event.pull_request.labels.*.name, 'onnxruntime') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cuda_onnxruntime')
@@ -45,6 +46,6 @@ jobs:
         run: |
           pip install -e .[testing,onnxruntime-gpu,diffusers,timm]
 
-      - name: Run tests
-        run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and onnxruntime"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and onnxruntime"

.github/workflows/test_cli_cuda_py_txi.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cuda') ||
       contains( github.event.pull_request.labels.*.name, 'py_txi') ||
       contains( github.event.pull_request.labels.*.name, 'cli_cuda_py_txi')
@@ -47,5 +48,10 @@ jobs:
           pip install --upgrade pip
           pip install -e .[testing,py-txi]
 
-      - name: Run tests
-        run: pytest tests/test_cli.py -x -s -k "cli and cuda and py_txi"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and py_txi"
+
+      - name: Run tests from example folder
+        run: |
+          pytest tests/test_examples.py -x -s -k "cuda and py_txi"

.github/workflows/test_cli_cuda_pytorch.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cuda') ||
       contains( github.event.pull_request.labels.*.name, 'pytorch') ||
       contains( github.event.pull_request.labels.*.name, 'single_gpu') ||
@@ -44,11 +45,15 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install -e .[testing,diffusers,timm,peft,bitsandbytes,autoawq,auto-gptq]
+          pip install -e .[testing,diffusers,timm,peft,bitsandbytes,autoawq,auto-gptq,codecarbon]
 
-      - name: Run tests
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and not (dp or ddp or device_map or deepspeed)"
+
+      - name: Run tests from example folder
         run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and not (dp or ddp or device_map or deepspeed)"
+          pytest tests/test_examples.py -x -s -k "cuda and pytorch"
 
   run_cli_cuda_pytorch_multi_gpu_tests:
     if: ${{
@@ -76,10 +81,11 @@ jobs:
         run: |
           pip install -e .[testing,diffusers,timm,peft,deepspeed]
 
-      - name: Run tests (parallel)
+      - name: Run cli tests (parallel)
         run: |
           pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (dp or ddp or device_map)"
 
-      - name: Run tests (sequential)
+      - name: Run cli tests (sequential)
         run: |
           FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (deepspeed_inference)"
+

.github/workflows/test_cli_cuda_tensorrt_llm.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cuda') ||
       contains( github.event.pull_request.labels.*.name, 'tensorrt_llm') ||
       contains( github.event.pull_request.labels.*.name, 'single_gpu') ||
@@ -46,9 +47,13 @@ jobs:
         run: |
           pip install -e .[testing]
 
-      - name: Run tests
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and tensorrt_llm and not (tp or pp)"
+
+      - name: Run tests from example folder
         run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and tensorrt_llm and not (tp or pp)"
+          pytest tests/test_examples.py -x -s -k "cuda and tensorrt_llm"
 
   cli_cuda_tensorrt_llm_multi_gpu_tests:
     if: ${{
@@ -76,6 +81,7 @@ jobs:
         run: |
           pip install -e .[testing]
 
-      - name: Run tests (sequential)
+      - name: Run cli tests (sequential)
         run: |
           FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and tensorrt_llm and (tp or pp)"
+

.github/workflows/test_cli_cuda_torch_ort.yaml

@@ -56,6 +56,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cuda') ||
       contains( github.event.pull_request.labels.*.name, 'torch_ort') ||
       contains( github.event.pull_request.labels.*.name, 'multi_gpu') ||
@@ -78,6 +79,6 @@ jobs:
           pip install -e .[testing,torch-ort,peft] 
           pip install optimum@git+https://github.com/huggingface/optimum.git
 
-      - name: Run tests
-        run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and torch_ort and (dp or ddp or device_map)"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and torch_ort and (dp or ddp or device_map)"

.github/workflows/test_cli_cuda_vllm.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cuda') ||
       contains( github.event.pull_request.labels.*.name, 'vllm') ||
       contains( github.event.pull_request.labels.*.name, 'single_gpu') ||
@@ -55,6 +56,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'cuda') ||
       contains( github.event.pull_request.labels.*.name, 'vllm') ||
       contains( github.event.pull_request.labels.*.name, 'multi_gpu') ||
@@ -76,6 +78,10 @@ jobs:
         run: |
           pip install -e .[testing]
 
-      - name: Run tests (sequential)
+      # - name: Run cli tests (sequential)
+      #   run: |
+      #     FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and vllm and (tp or pp)"
+
+      - name: Run tests from example folder
         run: |
-          FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and vllm and (tp or pp)"
+          pytest tests/test_examples.py -x -s -k "cuda and vllm"

.github/workflows/test_cli_misc.yaml

@@ -60,6 +60,6 @@ jobs:
           pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
           pip install -e .[testing]
 
-      - name: Run tests
+      - name: Run cli tests
         run: |
-          pytest tests/test_cli.py -s -k "cli and not (cpu or cuda or rocm or mps)"
+          pytest tests/test_cli.py -s -k "cli and not (cpu or cuda or rocm or mps)"

.github/workflows/test_cli_rocm_pytorch.yaml

@@ -25,6 +25,7 @@ jobs:
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'rocm') ||
       contains( github.event.pull_request.labels.*.name, 'pytorch') ||
       contains( github.event.pull_request.labels.*.name, 'single_gpu') ||
@@ -49,15 +50,16 @@ jobs:
         run: |
           pip install -e .[testing,diffusers,timm,peft,autoawq,auto-gptq]
 
-      - name: Run tests
-        run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and not (dp or ddp or device_map or deepspeed) and not bnb"
+      # - name: Run cli tests
+      #   run: |
+      #     pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and not (dp or ddp or device_map or deepspeed) and not bnb"
 
   run_cli_rocm_pytorch_multi_gpu_tests:
     if: ${{
       (github.event_name == 'push') ||
       (github.event_name == 'workflow_dispatch') ||
       contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'examples') ||
       contains( github.event.pull_request.labels.*.name, 'rocm') ||
       contains( github.event.pull_request.labels.*.name, 'pytorch') ||
       contains( github.event.pull_request.labels.*.name, 'multi_gpu') ||
@@ -82,10 +84,11 @@ jobs:
         run: |
           pip install -e .[testing,diffusers,timm,deepspeed,peft,autoawq,auto-gptq] "deepspeed<0.15"
 
-      - name: Run tests (parallel)
+      - name: Run cli tests (parallel)
         run: |
           pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (dp or ddp or device_map)"
 
-      - name: Run tests (sequential)
+      - name: Run cli tests (sequential)
         run: |
           FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and (deepspeed_inference)"
+

Makefile

@@ -1,5 +1,5 @@
 # List of targets that are not associated with files
-.PHONY: quality style install build_cpu_image build_cuda_image build_cuda_ort_image build_rocm_image run_cpu_container run_cuda_container run_cuda_ort_container run_rocm_container install_api_misc install_api_cpu install_api_cuda install_api_rocm install_cli_misc install_cli_cpu_pytorch install_cli_cpu_openvino install_cli_cpu_onnxruntime install_cli_cpu_neural_compressor install_cli_cuda_pytorch install_cli_rocm_pytorch install_cli_cuda_torch_ort install_cli_cuda_onnxruntime test_api_misc test_api_cpu test_api_cuda test_api_rocm test_cli_misc test_cli_cpu_pytorch test_cli_cpu_openvino test_cli_cpu_onnxruntime test_cli_cpu_neural_compressor test_cli_cuda_onnxruntime test_cli_cuda_vllm test_cli_cuda_pytorch_multi_gpu test_cli_cuda_pytorch_single_gpu test_cli_cuda_torch_ort_multi_gpu test_cli_cuda_torch_ort_single_gpu test_cli_rocm_pytorch_multi_gpu test_cli_rocm_pytorch_single_gpu install_llm_perf_cuda_pytorch run_llm_perf_cuda_pytorch_unquantized run_llm_perf_cuda_pytorch_bnb run_llm_perf_cuda_pytorch_gptq run_llm_perf_cuda_pytorch_awq
+.PHONY: quality style install build_cpu_image build_cuda_image build_cuda_ort_image build_rocm_image run_cpu_container run_cuda_container run_cuda_ort_container run_rocm_container run_trt_container install_api_misc install_api_cpu install_api_cuda install_api_rocm install_cli_misc install_cli_cpu_pytorch install_cli_cpu_openvino install_cli_cpu_onnxruntime install_cli_cpu_neural_compressor install_cli_cuda_pytorch install_cli_rocm_pytorch install_cli_cuda_torch_ort install_cli_cuda_onnxruntime test_api_misc test_api_cpu test_api_cuda test_api_rocm test_cli_misc test_cli_cpu_pytorch test_cli_cpu_openvino test_cli_cpu_onnxruntime test_cli_cpu_neural_compressor test_cli_cuda_onnxruntime test_cli_cuda_vllm test_cli_cuda_pytorch_multi_gpu test_cli_cuda_pytorch_single_gpu test_cli_cuda_torch_ort_multi_gpu test_cli_cuda_torch_ort_single_gpu test_cli_rocm_pytorch_multi_gpu test_cli_rocm_pytorch_single_gpu install_llm_perf_cuda_pytorch run_llm_perf_cuda_pytorch_unquantized run_llm_perf_cuda_pytorch_bnb run_llm_perf_cuda_pytorch_gptq run_llm_perf_cuda_pytorch_awq
 
 PWD := $(shell pwd)
 USER_ID := $(shell id -u)
@@ -78,6 +78,18 @@ run_rocm_container:
 	--workdir /optimum-benchmark \
 	optimum-benchmark:latest-rocm
 
+run_trt_container:
+	docker run \
+	-it \
+	--rm \
+	--pid host \
+	--ipc host \
+	--gpus all \
+	--shm-size 64G \
+	--volume $(PWD):/optimum-benchmark \
+	--workdir /optimum-benchmark \
+	huggingface/optimum-nvidia:latest
+
 ## Install extras
 
 install_api_misc:

examples/ipex_bert.yaml

@@ -25,5 +25,4 @@ backend:
   device: cpu
   no_weights: false
   export: true
-  torch_dtype: bfloat16
   model: bert-base-uncased

examples/ipex_llama.yaml

@@ -33,5 +33,4 @@ backend:
   device: cpu
   export: true
   no_weights: false
-  torch_dtype: bfloat16
   model: TinyLlama/TinyLlama-1.1B-Chat-v1.0