Add the logic for Energy Star

.github/workflows/test_cli_energy_star.yaml

@@ -0,0 +1,49 @@
+name: CLI CUDA Energy Star Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - labeled
+      - unlabeled
+
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+jobs:
+  run_cli_energy_star_tests:
+    if: ${{
+      (github.event_name == 'push') ||
+      (github.event_name == 'workflow_dispatch') ||
+      contains( github.event.pull_request.labels.*.name, 'cli') ||
+      contains( github.event.pull_request.labels.*.name, 'energy_star') ||
+      contains( github.event.pull_request.labels.*.name, 'cli_energy_star')
+      }}
+
+    runs-on:
+      group: aws-g5-4xlarge-plus
+
+    container:
+      image: ghcr.io/huggingface/optimum-benchmark:latest-cuda
+      options: --ipc host --gpus all
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          pip install -e .[testing,diffusers,timm,codecarbon] librosa
+
+      - name: Run tests
+        run: |
+          pytest tests/test_energy_star.py -x -s

examples/energy_star/_base_.yaml

@@ -0,0 +1,20 @@
+log_report: true
+print_report: true
+
+# hydra/cli specific settings
+hydra:
+  run:
+    # define run directory
+    dir: runs/${name}
+  sweep:
+    # define sweep directory
+    dir: sweeps/${name}
+    subdir: ${hydra.job.override_dirname}
+  job:
+    # change working directory to the job directory
+    # so that artifacts are stored there
+    chdir: true
+    env_set:
+      # set environment variable OVERRIDE_BENCHMARKS to 1
+      # to not skip benchmarks that have been run before
+      OVERRIDE_BENCHMARKS: 1

examples/energy_star/automatic_speech_recognition.yaml

@@ -0,0 +1,28 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: automatic_speech_recognition
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  model: openai/whisper-large-v3
+  task: automatic-speech-recognition
+
+scenario:
+  dataset_name: EnergyStarAI/ASR
+  audio_column_name: audio
+  num_samples: 1000
+
+  input_shapes:
+    batch_size: 1

examples/energy_star/image_classification.yaml

@@ -0,0 +1,28 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: image_classification
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  task: image-classification
+  model: google/vit-base-patch16-224
+
+scenario:
+  dataset_name: EnergyStarAI/image_classification
+  image_column_name: image
+  num_samples: 1000
+
+  input_shapes:
+    batch_size: 1

examples/energy_star/image_to_text.yaml

@@ -0,0 +1,28 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: image_to_text
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  task: image-to-text
+  model: sashakunitsyn/vlrm-blip2-opt-2.7b
+
+scenario:
+  dataset_name: EnergyStarAI/image_captioning
+  image_column_name: image
+  num_samples: 1000
+
+  input_shapes:
+    batch_size: 1

examples/energy_star.yaml → examples/energy_star/object_detection.yaml

@@ -6,21 +6,23 @@ defaults:
   - _base_
   - _self_
 
-name: energy_star
+name: object_detection
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
 
 backend:
-  model: gpt2
   device: cuda
   device_ids: 0
   no_weights: true
-  task: feature-extraction
-
-launcher:
-  device_isolation: true
+  task: object-detection
+  model: facebook/detr-resnet-50
 
 scenario:
-  dataset_name: wikitext
-  dataset_config: wikitext-2-raw-v1
-  num_samples: 10
+  dataset_name: EnergyStarAI/object_detection
+  image_column_name: image
+  num_samples: 1000
+
   input_shapes:
     batch_size: 1

examples/energy_star/question_answering.yaml

@@ -0,0 +1,29 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: question_answering
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  task: question-answering
+  model: deepset/electra-base-squad2
+
+scenario:
+  dataset_name: EnergyStarAI/extractive_qa
+  question_column_name: question
+  context_column_name: context
+  num_samples: 1000
+
+  input_shapes:
+    batch_size: 1

examples/energy_star/sentence_similarity.yaml

@@ -0,0 +1,30 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: sentence_similarity_udever-bloom-7b1
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  library: transformers
+  task: sentence-similarity
+  model: sentence-transformers/all-MiniLM-L6-v2
+
+scenario:
+  dataset_name: EnergyStarAI/sentence_similarity
+  sentence1_column_name: sentence1
+  sentence2_column_name: sentence2
+  num_samples: 1000
+
+  input_shapes:
+    batch_size: 1

examples/energy_star/summarization.yaml

@@ -0,0 +1,33 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: summarization
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  task: summarization
+  model: sshleifer/distilbart-cnn-12-6
+
+scenario:
+  dataset_name: EnergyStarAI/summarization
+  text_column_name: text
+  num_samples: 1000
+  truncation: True
+
+  input_shapes:
+    batch_size: 1
+
+  generate_kwargs:
+    max_length: 10
+    min_new_tokens: 10

examples/energy_star/t5_question_answering.yaml

@@ -0,0 +1,32 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: question_answering_t5
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  model: google-t5/t5-large
+  task: text2text-generation
+
+scenario:
+  dataset_name: EnergyStarAI/extractive_qa
+  question_column_name: question
+  context_column_name: context
+  dataset_prefix1: "question: "
+  dataset_prefix2: " context: "
+  t5_task: question_answering
+  num_samples: 1000
+
+  input_shapes:
+    batch_size: 1

examples/energy_star/t5_summarization.yaml

@@ -0,0 +1,35 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: summarization_t5
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  model: google-t5/t5-large
+  task: text2text-generation
+
+scenario:
+  dataset_name: EnergyStarAI/summarization
+  dataset_prefix1: "summarize: "
+  text_column_name: text
+  t5_task: summarization
+  num_samples: 1000
+  truncation: True
+
+  input_shapes:
+    batch_size: 1
+
+  generate_kwargs:
+    max_new_tokens: 10
+    min_new_tokens: 10

examples/energy_star/t5_text_classification.yaml

@@ -0,0 +1,36 @@
+defaults:
+  - benchmark
+  - backend: pytorch
+  - launcher: process
+  - scenario: energy_star
+  - _base_
+  - _self_
+
+name: text_classification_t5
+
+launcher:
+  device_isolation: true
+  device_isolation_action: warn
+
+backend:
+  device: cuda
+  device_ids: 0
+  no_weights: true
+  model: google-t5/t5-large
+  task: text2text-generation
+
+scenario:
+  dataset_name: EnergyStarAI/text_classification
+  dataset_prefix1: "sst2 sentence: "
+  t5_task: text_classification
+  text_column_name: text
+
+  num_samples: 1000
+  truncation: True
+
+  input_shapes:
+    batch_size: 1
+
+  generate_kwargs:
+    max_new_tokens: 10
+    min_new_tokens: 10