Skip to content
Cluster Run - Minimega

Cluster Run - Minimega #424

Sign in to view logs

Cluster Run - Minimega

Cluster Run - Minimega #424

Workflow file for this run

.github/workflows/cluster.yml at 1debf82
name: Cluster Run - Minimega
on:
pull_request:
branches:
- '*'
workflow_dispatch:
inputs:
azure_region:
description: 'Azure region to deploy resources'
required: true
default: 'centralus'
type: choice
options:
- centralus
- eastus
- eastus2
- westus
- westus2
- westus3
- northcentralus
- southcentralus
- canadacentral
- canadaeast
- uksouth
- ukwest
- northeurope
- westeurope
jobs:
build-and-test-cluster:
runs-on: self-hosted
env:
UNIQUE_ID: ${{ github.run_id }}-${{ github.run_number }}
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
IP_ADDRESS: ""
LS1_IP: ""
elastic: ""
AZURE_IP: ""
MINIMEGA_IP: ""
ENROLLMENT_TOKEN: ""
ES_PASSWORD: ""
KIBANA_PASSWORD: ""
steps:
- name: Checkout repository
uses: actions/[email protected]
- name: Setup environment variables
run: |
PUBLIC_IP=$(curl -s https://api.ipify.org)
echo "IP_ADDRESS=$PUBLIC_IP" >> $GITHUB_ENV
- name: Get branch name
shell: bash
run: |
if [ "${{ github.event_name }}" == "pull_request" ]; then
echo "BRANCH_NAME=${{ github.head_ref }}" >> $GITHUB_ENV
else
echo "BRANCH_NAME=${GITHUB_REF##*/}" >> $GITHUB_ENV
fi
- name: Set the environment for docker-compose
run: |
cd testing/v2/development
# Get the UID and GID of the current user
echo "HOST_UID=$(id -u)" > .env
echo "HOST_GID=$(id -g)" >> .env
cat .env
- name: Build pipeline container
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} build pipeline --no-cache
- name: Start pipeline container
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} up -d pipeline
- name: Install Python requirements
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
cd /home/lme-user/LME/testing/v2/installers/azure && \
pip install -r requirements.txt
"
- name: Build an Azure instance
env:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T \
-e AZURE_CLIENT_ID \
-e AZURE_CLIENT_SECRET \
-e AZURE_TENANT_ID \
-e AZURE_SUBSCRIPTION_ID \
pipeline bash -c "
cd /home/lme-user/LME/testing/v2/installers && \
python3 ./azure/build_azure_linux_network.py \
-g pipe-${{ env.UNIQUE_ID }} \
-s ${{ env.IP_ADDRESS }}/32 \
-vs Standard_D8_v4 \
-l ${{ inputs.azure_region || 'centralus' }} \
-ast 23:00 \
-y
"
- name: Install minimega on Azure instance
run: |
cd testing/v2/development
sleep 30
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
cd /home/lme-user/LME/testing/v2/installers && \
IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
./minimega/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt"
"
- name: Install windows minimega on Azure instance
env:
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }}
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
run: |
cd testing/v2/development
sleep 30
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
cd /home/lme-user/LME/testing/v2/installers && \
IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
AZURE_CLIENT_ID=${{ env.AZURE_CLIENT_ID }} && \
AZURE_CLIENT_SECRET=${{ env.AZURE_CLIENT_SECRET }} && \
AZURE_TENANT_ID=${{ env.AZURE_TENANT_ID }} && \
AZURE_SUBSCRIPTION_ID=${{ env.AZURE_SUBSCRIPTION_ID }} && \
./windows_qcow/pipeline_install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt"
"
- name: Install Linux in minimega
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
cd /home/lme-user/LME/testing/v2/installers && \
IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
./ubuntu_qcow_maker/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt"
"
- name: Check if linux is running in minimega
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
sleep 120 && \
cd /home/lme-user/LME/testing/v2/installers && \
IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
ssh lme-user@\$IP_ADDRESS 'sudo /opt/minimega/bin/minimega -e vm info'
"
- name: Get Azure and Minimega IP addresses
run: |
cd testing/v2/development
AZURE_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.ip.txt")
echo "AZURE_IP=$AZURE_IP" >> $GITHUB_ENV
echo "Azure IP:$AZURE_IP"
MINIMEGA_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh lme-user@$AZURE_IP 'sudo /opt/minimega/bin/minimega -e .json true .filter name=\"linux-runner\" vm info | jq -r \".[].Data[].Networks[].IP4\"'
" )
EXIT_CODE=$?
if [ $EXIT_CODE -ne 0 ]; then
echo "Failed to get Minimega IP. Exit code: $EXIT_CODE" >&2
exit 1
fi
if [ -z "$MINIMEGA_IP" ]; then
echo "Minimega IP is empty" >&2
exit 1
fi
echo "MINIMEGA_IP=$MINIMEGA_IP" >> $GITHUB_ENV
echo "Azure IP:$AZURE_IP Minimega IP:$MINIMEGA_IP"
- name: Run a command in Minimega
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ls -la'
"
- name: Install LME on Azure instance
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
cd /home/lme-user/LME/testing/v2/installers && \
IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
./install_v2/install.sh lme-user \$IP_ADDRESS "pipe-${{ env.UNIQUE_ID }}.password.txt" ${{ env.BRANCH_NAME }}
"
- name: Retrieve Elastic password
env:
AZURE_IP: ${{ env.AZURE_IP }}
run: |
cd testing/v2/development
ES_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$elastic'" | tail -n 1 | tr -d '\n')
echo "::add-mask::$ES_PASSWORD"
echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV
echo "Elastic password retrieved successfully: $ES_PASSWORD"
KIBANA_PASSWORD=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "ssh lme-user@$AZURE_IP '. /home/lme-user/LME/scripts/extract_secrets.sh -q && echo \$kibana_system'" | tail -n 1 | tr -d '\n')
echo "::add-mask::$KIBANA_PASSWORD"
echo "KIBANA_PASSWORD=$KIBANA_PASSWORD" >> $GITHUB_ENV
echo "Kibana password retrieved successfully: $KIBANA_PASSWORD"
- name: Install test requirements on Azure instance
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
cd /home/lme-user/LME/testing/v2/installers && \
IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
ssh lme-user@\$IP_ADDRESS 'whoami && hostname && \
wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \
sudo apt install -y ./google-chrome-stable_current_amd64.deb && \
cd /home/lme-user/LME/testing/tests && \
python3 -m venv venv && \
source venv/bin/activate && \
pip install -r requirements.txt '
"
- name: Retrieve Elastic policy ID and enrollment token
env:
KIBANA_URL: "https://localhost:5601"
ES_USERNAME: "elastic"
ES_PASSWORD: ${{ env.ES_PASSWORD }}
run: |
cd testing/v2/development
# Retrieve policy ID
POLICY_ID=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh lme-user@${{ env.AZURE_IP }} '
curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X GET \"$KIBANA_URL/api/fleet/agent_policies\" \
-H \"kbn-xsrf: true\" \
-H \"Content-Type: application/json\" |
jq -r '.items[0].id'
'
")
echo "Retrieved Policy ID: $POLICY_ID"
# Retrieve enrollment token using the policy ID
ENROLLMENT_TOKEN=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh lme-user@${{ env.AZURE_IP }} '
curl -kL -s -u \"$ES_USERNAME:$ES_PASSWORD\" -X POST \"$KIBANA_URL/api/fleet/enrollment-api-keys\" \
-H \"kbn-xsrf: true\" \
-H \"Content-Type: application/json\" \
-d \"{\\\"policy_id\\\":\\\"$POLICY_ID\\\"}\" |
jq -r .item.api_key
'
")
echo "Retrieved enrollment token: $ENROLLMENT_TOKEN"
# Mask the enrollment token in logs and set it as an environment variable
echo "::add-mask::$ENROLLMENT_TOKEN"
echo "ENROLLMENT_TOKEN=$ENROLLMENT_TOKEN" >> $GITHUB_ENV
echo "Policy ID and Enrollment Token retrieved successfully"
- name: Copy the Elastic Agent installer to Minimega
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
'sudo scp -p -o StrictHostKeyChecking=no /home/lme-user/LME/testing/v2/installers/lib/install_agent_linux.sh vmuser@${{ env.MINIMEGA_IP }}:~'
"
- name: Run a command in Minimega
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ls -la'
"
- name: Install the Elastic Agent in Minimega
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} chmod +x ./install_agent_linux.sh ' && \
ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ./install_agent_linux.sh --token ${{ env.ENROLLMENT_TOKEN }}'
"
- name: Check if the Elastic agent is reporting
env:
ES_PASSWORD: ${{ env.ES_PASSWORD }}
run: |
sleep 360
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
'export ES_PASSWORD=\"$ES_PASSWORD\" && /home/lme-user/LME/testing/v2/installers/lib/check_agent_reporting.sh'
"
- name: Run api tests on Azure instance
env:
ES_PASSWORD: ${{ env.ES_PASSWORD }}
KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }}
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \
echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \
echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \
echo elastic=\"$ES_PASSWORD\" >> .env && \
source venv/bin/activate && \
pytest -v api_tests/'
"
- name: Run selenium tests on Azure instance
env:
ES_PASSWORD: ${{ env.ES_PASSWORD }}
KIBANA_PASSWORD: ${{ env.KIBANA_PASSWORD }}
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
ssh lme-user@${{ env.AZURE_IP }} 'cd /home/lme-user/LME/testing/tests && \
echo ELASTIC_PASSWORD=\"$ES_PASSWORD\" >> .env && \
echo KIBANA_PASSWORD=\"$KIBANA_PASSWORD\" >> .env && \
echo elastic=\"$ES_PASSWORD\" >> .env && \
source venv/bin/activate && \
pytest -v selenium_tests/'
"
# - name: Cleanup Azure resources
# if: always()
# env:
# AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
# AZURE_SECRET: ${{ secrets.AZURE_SECRET }}
# AZURE_TENANT: ${{ secrets.AZURE_TENANT }}
# AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
# run: |
# cd testing/v2/development
# docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
# az login --service-principal -u $AZURE_CLIENT_ID -p $AZURE_SECRET --tenant $AZURE_TENANT
# az group delete --name pipe-${{ env.UNIQUE_ID }} --yes --no-wait
# "
- name: Stop and remove containers
if: always()
run: |
cd testing/v2/development
docker compose -p ${{ env.UNIQUE_ID }} down
docker system prune -af