EdenGCP #2
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: EdenGCP | |
| on: # yamllint disable-line rule:truthy | |
| workflow_run: | |
| workflows: | |
| - Publish | |
| types: | |
| - completed | |
| # yamllint disable rule:line-length | |
| jobs: | |
| integration: | |
| name: Integration test (${{ matrix.backend }};${{ matrix.hv }};${{ matrix.fs }}) | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| backend: ["gcp", "rol"] | |
| hv: ["kvm", "xen"] | |
| fs: ["zfs", "ext4"] | |
| steps: | |
| - name: Check | |
| run: | | |
| for addr in $(ip addr list|sed -En -e 's/.*inet ([0-9.]+).*/\1/p') | |
| do | |
| if echo "$addr" | grep -q -E "10.11.(12|13).[0-9]+"; then | |
| echo "$addr overlaps with test"; exit 1 | |
| fi | |
| if echo "$addr" | grep -q -E "10.8.0.[0-9]+"; then | |
| echo "$addr overlaps with vpn"; exit 1 | |
| fi | |
| done | |
| sudo df -h | |
| sudo swapoff -a | |
| sudo free | |
| - name: Check RoL secrets | |
| if: matrix.backend == 'rol' | |
| run: | | |
| if [ -z "$ROL_API_URL" ]; then echo "::error::ROL_API_URL is empty" && exit 1; fi | |
| if [ -z "$ROL_API_KEY" ]; then echo "::error::ROL_API_KEY is empty" && exit 1; fi | |
| if [ -z "$ROL_PROJECT" ]; then echo "::error::ROL_PROJECT is empty" && exit 1; fi | |
| if [ -z "$ROL_OVPN_CONF_BASE64" ]; then echo "::error::ROL_OVPN_CONF_BASE64 is empty" && exit 1; fi | |
| env: | |
| ROL_API_URL: ${{ secrets.ROL_API_URL }} | |
| ROL_API_KEY: ${{ secrets.ROL_API_KEY }} | |
| ROL_PROJECT: ${{ secrets.ROL_PROJECT }} | |
| ROL_OVPN_CONF_BASE64: ${{ secrets.ROL_OVPN_CONF_BASE64 }} | |
| - name: Setup packages | |
| run: | | |
| sudo apt update | |
| sudo apt install -y qemu binfmt-support qemu-user-static qemu-system-x86 qemu-system-aarch64 qemu-utils openvpn curl jq | |
| - name: GCP Set up Google Cloud SDK | |
| if: matrix.backend == 'gcp' | |
| uses: google-github-actions/[email protected] | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| - id: 'gcpauth' | |
| if: matrix.backend == 'gcp' | |
| name: GCP Auth to Google Cloud SDK | |
| uses: google-github-actions/[email protected] | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| credentials_json: ${{ secrets.GCP_SA_KEY }} | |
| create_credentials_file: true | |
| - name: GCP Clean | |
| if: matrix.backend == 'gcp' | |
| run: | | |
| gcloud compute instances delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -q --zone=us-west1-a || echo "Does not exist" | |
| gcloud compute images delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -q || echo "Does not exist" | |
| - name: Connect VPN | |
| id: connect_vpn | |
| timeout-minutes: 1 | |
| run: | | |
| echo "$OVPN_GCP_FILE" | base64 -d > ./config_gcp.ovpn | |
| echo "$OVPN_ROL_FILE" | base64 -d > ./config_rol.ovpn | |
| sudo openvpn --config ./config_${{ matrix.backend }}.ovpn --daemon | |
| until ip -f inet addr show tun0; do sleep 5; ip a; done | |
| echo ::set-output name=tunnel_ip::$(ip -f inet addr show tun0 | sed -En -e 's/.*inet ([0-9.]+).*/\1/p') | |
| env: | |
| OVPN_GCP_FILE: ${{ secrets.OVPN_FILE }} | |
| OVPN_ROL_FILE: ${{ secrets.ROL_OVPN_CONF_BASE64 }} | |
| - name: Get EVE | |
| uses: actions/checkout@v2 | |
| with: | |
| path: 'eve' | |
| # ROL introduced from Eden 0.4.0 | |
| # Conditions for backwards compatibility on old branches | |
| - name: Prepare eden | |
| run: | | |
| if [ "${{ matrix.backend }}" == "rol" ] || ! [ -f ${{ github.workspace }}/eve/tests/eden/eden-version ]; then | |
| EDEN_VERSION=lfedge/eden:0.5.0 | |
| else | |
| EDEN_VERSION=$(cat ${{ github.workspace }}/eve/tests/eden/eden-version) | |
| fi | |
| docker run -v $PWD:/out $EDEN_VERSION cp -a /eden/. /out/ | |
| sudo chown -R $(whoami) . | |
| - name: Eden setup | |
| run: | | |
| if [ "${{ matrix.backend }}" == "rol" ]; then | |
| devmodel='general' | |
| arch='arm64' | |
| netboot='true' | |
| tpm='false' | |
| else | |
| devmodel='GCP' | |
| arch='amd64' | |
| netboot='false' | |
| tpm='true' | |
| fi | |
| ./eden config add default --devmodel="${devmodel}" --arch="${arch}" | |
| ./eden config set default --key eve.tag --value="snapshot" | |
| ./eden config set default --key eve.hv --value ${{ matrix.hv }} | |
| ./eden config set default --key eve.tpm --value "${tpm}" | |
| ./eden config set default --key adam.eve-ip --value ${{ steps.connect_vpn.outputs.tunnel_ip }} | |
| ./eden config set default --key registry.ip --value ${{ steps.connect_vpn.outputs.tunnel_ip }} | |
| ./eden config set default --key=eden.tests --value=${{ github.workspace }}/eve/tests/eden | |
| if [ "${{ matrix.fs }}" == "zfs" ]; then | |
| if [ "${{ matrix.backend }}" == "gcp" ]; then | |
| ./eden config set default --key=eve.disks --value=4 | |
| fi | |
| grub_options='set_global dom0_extra_args "$dom0_extra_args eve_install_zfs_with_raid_level "' | |
| else | |
| grub_options='' | |
| fi | |
| ./eden setup -v debug --grub-options="${grub_options}" --netboot="${netboot}" | |
| ./eden start | |
| - name: GCP Create VM | |
| if: matrix.backend == 'gcp' | |
| run: | | |
| export ipv4=`curl https://api.ipify.org/?format=json | jq ".ip" -r` | |
| ./eden utils gcp firewall --source-range $ipv4/32 --name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}" | |
| ./eden utils gcp image --image-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}" upload | |
| ./eden utils gcp vm --image-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} --vm-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}" run | |
| sleep 100 | |
| BWD=$(./eden utils gcp vm get-ip --vm-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}") || { echo "no IP, probably VM does not exist"; exit 1; } | |
| echo "the IP is $BWD" | |
| ./eden utils gcp firewall -k "${{ steps.gcpauth.outputs.credentials_file_path }}" --source-range $BWD --name edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} || { echo "cannot set firewall"; exit 1; } | |
| - name: RoL rent RPi4 | |
| id: rol | |
| if: matrix.backend == 'rol' | |
| timeout-minutes: 60 | |
| run: | | |
| rent_id=$(./eden rol rent create -p "$ROL_PROJECT" -m raspberry --model pi_4_model_b_8gb -n GHAction-${{ github.run_number }}-snapshot-${{ matrix.hv }}-${{ matrix.fs }}) | |
| echo ::set-output name=id::$(echo $rent_id) | |
| until [[ "$(./eden rol rent get -p "$ROL_PROJECT" -i $rent_id | jq -r .machineState)" == "Ready" ]]; do sleep 10; echo "Waiting for EVE to load"; done | |
| env: | |
| ROL_API_URL: ${{ secrets.ROL_API_URL }} | |
| ROL_API_KEY: ${{ secrets.ROL_API_KEY }} | |
| ROL_PROJECT: ${{ secrets.ROL_PROJECT }} | |
| - name: Eden EVE onboard | |
| run: | | |
| ./eden eve onboard | |
| - name: Test | |
| run: | | |
| if [ "${{ matrix.backend }}" == "gcp" ]; then export EDEN_TEST=gcp; fi | |
| EDEN_TEST_STOP=n ./eden test ${{ github.workspace }}/eve/tests/eden/workflow -v debug | |
| - name: Collect logs | |
| if: ${{ always() }} | |
| run: | | |
| ./eden log --format json > trace.log || echo "no log" | |
| ./eden info > info.log || echo "no info" | |
| ./eden metric > metric.log || echo "no metric" | |
| ./eden netstat > netstat.log || echo "no netstat" | |
| docker logs eden_adam > adam.log 2>&1 || echo "no adam log" | |
| ./eden utils gcp vm log --vm-name eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -k "${{ steps.gcpauth.outputs.credentials_file_path }}" > console.log || echo "no device log" | |
| - name: Log counting | |
| if: ${{ always() }} | |
| run: | | |
| echo "::group::Total errors" | |
| echo "$(jq '.severity' trace.log|grep err|wc -l)" | |
| echo "::endgroup::" | |
| echo "::group::Errors by source" | |
| echo "errors by source: $(jq -s 'map(select(.severity|contains("err")))|group_by(.source)|map({"source": .[0].source, "total":length})|sort_by(.total)|reverse[]' trace.log)" | |
| echo "::endgroup::" | |
| echo "::group::Error log content duplicates" | |
| echo "$(jq -s 'map(select(.severity | contains("err")))|group_by(.content)|map(select(length>1))' trace.log)" | |
| echo "::endgroup::" | |
| echo "::group::Error log function filename duplicates" | |
| echo "$(jq -s 'map(select(.severity | contains("err")))|group_by(.filename)|map(select(length>10))|map({"source": .[0].source, "filename": .[0].filename, "function": .[0].function, "content": [.[].content], "total":length})|sort_by(.total)| reverse[]' trace.log)" | |
| echo "::endgroup::" | |
| echo "::group::Segfaults" | |
| echo "$(jq -s 'map(select(.content | contains("segfault at")))' trace.log)"|tee segfaults.log | |
| [ "$(jq length segfaults.log)" -gt 0 ] && echo "::warning::segfaults found, you can see them in Log counting->Segfaults section" | |
| echo "::endgroup::" | |
| - name: GCP Clean | |
| if: ${{ always() }} | |
| run: | | |
| gcloud compute firewall-rules delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} || echo "Does not exist" | |
| gcloud compute firewall-rules delete edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} || echo "Does not exist" | |
| gcloud compute instances delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -q --zone=us-west1-a || echo "Does not exist" | |
| gcloud compute images delete eve-edengcp-actions-${{ matrix.hv }}-${{ matrix.fs }}-${{github.run_number}} -q || echo "Does not exist" | |
| - name: RoL Clean | |
| if: ${{ always() }} | |
| run: | | |
| if [ -z "${{ steps.rol.outputs.id }}" ]; then exit 0; fi | |
| ./eden rol rent close -p $ROL_PROJECT -i ${{ steps.rol.outputs.id }} | |
| env: | |
| ROL_API_URL: ${{ secrets.ROL_API_URL }} | |
| ROL_API_KEY: ${{ secrets.ROL_API_KEY }} | |
| ROL_PROJECT: ${{ secrets.ROL_PROJECT }} | |
| - name: Store raw test results | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: eden-report-${{ matrix.backend }}-${{ matrix.hv }}-${{ matrix.fs }} | |
| path: | | |
| ${{ github.workspace }}/trace.log | |
| ${{ github.workspace }}/info.log | |
| ${{ github.workspace }}/adam.log | |
| ${{ github.workspace }}/netstat.log | |
| ${{ github.workspace }}/metric.log | |
| ${{ github.workspace }}/console.log |