action.yml

name: 'HoneyHive Evaluation'
description: 'Evaluations for LLM applications'
author: 'HoneyHive'

branding:
  icon: 'check-square'
  color: 'yellow'

inputs:
  root:
    description: 'The root directory of the evaluation'
    required: false
    default: '.'
  runtime:
    description: 'The runtime to use for the evaluation'
    required: true
  runId:
    description: 'The ID of the run to evaluate'
    required: true
  project:
    description: 'The project associated with the evaluation'
    required: true
  aggregateFunction:
    description:
      'The function used for aggregating the evaluation results (e.g., average,
      min, max, etc.)'
    required: false
    default: 'average'
  apiUrl:
    description: 'The base URL of the Honeyhive API'
    required: false
    default: 'https://api.honeyhive.ai'
  apiKey:
    description: 'The API key for the Honeyhive API'
    required: true
  openaiApiKey:
    description: 'The API key for the OpenAI API'
    required: true
  github_token:
    description: 'Your GitHub token'
    required: true
    default: ${{ github.token }}
  step_key:
    description:
      "A unique key to identify this step. Do not change this unless you know
      what you're doing."
    required: true
    default: ${{ github.workflow_ref }}-${{ github.action }}

outputs:
  status:
    description: 'The status of the evaluation run (e.g., pending or completed)'
  success:
    description: 'Whether all datapoints have passed (true or false)'
  passed:
    description: 'List of passed datapoint_ids or session_ids'
  failed:
    description: 'List of failed datapoint_ids or session_ids'
  metrics:
    description:
      'Metric level aggregates and details of passed/failed status for each
      metric'
  datapoints:
    description:
      'Details of each datapoint with associated session_id, pass status, and
      metric-level pass/fail status'

runs:
  using: node20
  main: dist/index.js