continuous-benchmarking-runtimes.yml

name: Continuous benchmarking - Runtime experiments

on:
  workflow_dispatch:
  schedule:
    - cron: "0 0 * * *"

jobs:
  build_client:
    name: Build framework
    runs-on: ubuntu-latest
    env:
      working-directory: ./src
    steps:
      - name: Check out code into the Go module directory
        uses: actions/checkout@v4
        with:
          ref: continuous-benchmarking

      - name: Set up Go 1.21
        uses: actions/setup-go@v3
        with:
          go-version: 1.21

      - name: Build client binary
        working-directory: ${{ env.working-directory }}
        run: env GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o stellar main.go

      - name: Package client artifact
        working-directory: ${{ env.working-directory }}
        run: tar -czvf build.tar ./stellar

      - name: Upload client artifact
        uses: actions/upload-artifact@v3
        with:
          name: STeLLAR-build
          path: ${{ env.working-directory }}/build.tar
          retention-days: 1

  cold-aws-experiments:
    name: AWS Cold Runtime Experiments
    needs: build_client
    runs-on: [ self-hosted, aws ]
    timeout-minutes: 1200
    strategy:
      fail-fast: false
      matrix:
        runtime:
          [
            go,
            py,
            node,
            java,
          ]
    env:
      working-directory: src
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
    steps:
      - name: "Cleanup build folder"
        run: |
          sudo rm -rf ./* || true
          sudo rm -rf ./.??* || true

      - name: Check out code into the Go module directory
        uses: actions/checkout@v4
        with:
          ref: continuous-benchmarking

      - name: Set up Node.js 16.16.0
        uses: actions/setup-node@v3
        with:
          node-version: 16.16.0

      - name: Set up Go 1.21
        if: ${{ matrix.runtime == 'go'}}
        uses: actions/setup-go@v3
        with:
          go-version: 1.21

      - name: Set up Java11
        if: ${{ matrix.runtime == 'java' }}
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up Gradle
        if: ${{ matrix.runtime == 'java'}}
        uses: gradle/gradle-build-action@v2

      - name: Configure AWS credentials using EASE lab account
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
          aws-region: us-west-1

      - name: Download client artifact
        uses: actions/download-artifact@v3
        with:
          name: STeLLAR-build

      - name: Untar client build
        working-directory: ${{ env.working-directory }}
        run: tar --strip-components=1 -xvf ../build.tar -C .

      - name: AWS Runtime ${{ matrix.runtime }}
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 1440
          max_attempts: 5
          retry_wait_seconds: 60
          command: cd src && ./stellar -o latency-samples-aws -l debug -c ../continuous-benchmarking/experiments/cold-function-invocations/language-runtime-deployment-method/aws/cold-hello${{ matrix.runtime }}-zip-aws.json -db

      - uses: actions/upload-artifact@v3
        with:
          name: cold-hello${{ matrix.runtime }}-zip-aws
          path: ${{ env.working-directory }}/latency-samples-aws

      - name: Add zero'ed data (Failed experiment)
        env:
          name: cold-hello${{ matrix.runtime }}-zip-aws
          DATA_INSERT_URL: ${{ secrets.DATA_INSERT_URL}}
        if: ${{ failure() }}
        run: |
          curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"'$name'","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"aws"}' $DATA_INSERT_URL

      - name: Send Slack message using Incoming Webhooks
        if: ${{ failure() }}
        uses: slackapi/slack-github-action@v1.24.0
        with:
          payload: |
            {
              "blocks": [
                {
                  "type": "header",
                  "text": {
                    "type": "plain_text",
                    "text": "Scheduled experiment (Job status)"
                  }
                },
                {
                  "type": "section",
                  "fields": [
                    {
                      "type": "mrkdwn",
                      "text": "*Workflow Name*: ${{ github.workflow }}"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Job Name*: ${{ github.job }}"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Job Status*: ${{ job.status }} ❌"
                    }
                  ]
                },
                {
                  "type": "section",
                  "fields": [
                    {
                      "type": "mrkdwn",
                      "text": "<!channel>"
                    }
                  ]
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

  cold-gcr-experiments:
    name: GCR Cold Runtime Experiments
    needs: build_client
    runs-on: [ self-hosted, gcr ]
    timeout-minutes: 1200
    strategy:
      fail-fast: false
      matrix:
        runtime:
          [
            go,
            py,
            node,
            java,
          ]
    env:
      working-directory: ./src
      DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
      DOCKER_HUB_ACCESS_TOKEN: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
    steps:
      - name: "Cleanup build folder"
        run: |
          sudo rm -rf ./* || true
          sudo rm -rf ./.??* || true

      - name: Check out code into the Go module directory
        uses: actions/checkout@v4
        with:
          ref: continuous-benchmarking

      - name: Set up Go 1.21
        if: ${{ matrix.runtime == 'go'}}
        uses: actions/setup-go@v3
        with:
          go-version: 1.21

      - name: Set up Java11
        if: ${{ matrix.runtime == 'java' }}
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: '11'

      - name: Set up Gradle
        if: ${{ matrix.runtime == 'java'}}
        uses: gradle/gradle-build-action@v2

      - id: auth
        name: Configure GCR credentials
        uses: google-github-actions/auth@v1
        with:
          credentials_json: ${{ secrets.GCR_CREDENTIALS }}

      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@v1
        with:
          version: ">= 363.0.0"

      - name: Download client artifact
        uses: actions/download-artifact@v3
        with:
          name: STeLLAR-build

      - name: Untar client build
        working-directory: ${{ env.working-directory }}
        run: tar --strip-components=1 -xvf ../build.tar -C .

      - name: GCR Runtime ${{ matrix.runtime }}
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 1440
          max_attempts: 5
          retry_wait_seconds: 60
          command: cd src && ./stellar -o latency-samples-gcr -l debug -c ../continuous-benchmarking/experiments/cold-function-invocations/language-runtime-deployment-method/gcr/cold-hello${{ matrix.runtime }}-img-gcr.json -db

      - uses: actions/upload-artifact@v3
        with:
          name: cold-hello${{ matrix.runtime }}-img-gcr
          path: ${{ env.working-directory }}/latency-samples-gcr

      - name: Add zero'ed data (Failed experiment)
        env:
          name: cold-hello${{ matrix.runtime }}-img-gcr
        if: ${{ failure() }}
        run: |
          curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"'$name'","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"gcr"}' $DATA_INSERT_URL

      - name: Send Slack message using Incoming Webhooks
        if: ${{ failure() }}
        uses: slackapi/slack-github-action@v1.24.0
        with:
          payload: |
            {
              "blocks": [
                {
                  "type": "header",
                  "text": {
                    "type": "plain_text",
                    "text": "Scheduled experiment (Job status)"
                  }
                },
                {
                  "type": "section",
                  "fields": [
                    {
                      "type": "mrkdwn",
                      "text": "*Workflow Name*: ${{ github.workflow }}"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Job Name*: ${{ github.job }}"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Job Status*: ${{ job.status }} ❌"
                    }
                  ]
                },
                {
                  "type": "section",
                  "fields": [
                    {
                      "type": "mrkdwn",
                      "text": "<!channel>"
                    }
                  ]
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

  cold-azure-experiments:
    name: Azure Cold Runtime Experiments
    needs: build_client
    runs-on: [ self-hosted, azure ]
    timeout-minutes: 1200
    strategy:
      fail-fast: false
      matrix:
        runtime:
          [
            py,
            node,
          ]
    env:
      working-directory: src
      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
      AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
    steps:
      - name: "Cleanup build folder"
        run: |
          sudo rm -rf ./* || true
          sudo rm -rf ./.??* || true

      - name: Check out code into the Go module directory
        uses: actions/checkout@v4
        with:
          ref: continuous-benchmarking

      - name: Download client artifact
        uses: actions/download-artifact@v3
        with:
          name: STeLLAR-build

      - name: Untar client build
        working-directory: ${{env.working-directory}}
        run: tar --strip-components=1 -xvf ../build.tar -C .

      - name: Azure Runtime ${{ matrix.runtime }}
        uses: nick-fields/retry@v2
        with:
          timeout_minutes: 1440
          max_attempts: 5
          retry_wait_seconds: 60
          command: cd src && ./stellar -o latency-samples-azure -l debug -c ../continuous-benchmarking/experiments/cold-function-invocations/language-runtime-deployment-method/azure/cold-hello${{matrix.runtime}}-zip-azure.json -db

      - uses: actions/upload-artifact@v3
        with:
          name: cold-hello${{ matrix.runtime }}-zip-azure
          path: ${{env.working-directory}}/latency-samples-azure

      - name: Add zero'ed data (Failed experiment)
        env:
          name: cold-hello${{ matrix.runtime }}-zip-azure
        if: ${{ failure() }}
        run: |
          curl -XPOST -H "Content-type: application/json" -d '{"experiment_type":"'$name'","date":"'$(date +%F)'","min":"0","max":"0","median":"0","tail_latency":"0","first_quartile":"0","third_quartile":"0","standard_deviation":"0","payload_size":"0","burst_size":"0","IATType":"0","count":"0","provider":"azure"}' $DATA_INSERT_URL

      - name: Send Slack message using Incoming Webhooks
        if: ${{ failure() }}
        uses: slackapi/slack-github-action@v1.24.0
        with:
          payload: |
            {
              "blocks": [
                {
                  "type": "header",
                  "text": {
                    "type": "plain_text",
                    "text": "Scheduled experiment (Job status)"
                  }
                },
                {
                  "type": "section",
                  "fields": [
                    {
                      "type": "mrkdwn",
                      "text": "*Workflow Name*: ${{ github.workflow }}"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Job Name*: ${{ github.job }}"
                    },
                    {
                      "type": "mrkdwn",
                      "text": "*Job Status*: ${{ job.status }} ❌"
                    }
                  ]
                },
                {
                  "type": "section",
                  "fields": [
                    {
                      "type": "mrkdwn",
                      "text": "<!channel>"
                    }
                  ]
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK