From 181d5ed0c6d50e0bd876192f6c1febbdb266eb5e Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Mon, 3 Jun 2024 12:01:11 +0100 Subject: [PATCH] Build as part of GitHub action --- .github/workflows/build.yml | 217 ++++++++++++++++++++++++++++++++++++ builder/SCAI.tfvars | 8 ++ builder/build-image.sh | 29 +++++ builder/cloud-init.yaml | 70 ++++++++++++ builder/clouds.yaml | 30 +++++ builder/versions.tf | 14 +++ builder/vm.tf | 36 ++++++ tools/build.sh | 7 +- ubuntu/ubuntu-22.04.json | 2 +- 9 files changed, 409 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 builder/SCAI.tfvars create mode 100755 builder/build-image.sh create mode 100644 builder/cloud-init.yaml create mode 100644 builder/clouds.yaml create mode 100644 builder/versions.tf create mode 100644 builder/vm.tf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..619c846 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,217 @@ +--- +name: Build images that changed + +on: + - push + +jobs: + image-list: + name: build images + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.change-list.outputs.all_changed_files }} + + steps: + - name: checkout code + uses: actions/checkout@v4 + with: + # full git history needed to get proper list of changed files + fetch-depth: 0 + - name: Get list of changes + id: change-list + uses: tj-actions/changed-files@v44 + with: + files: | + **/*.json + + build-images: + name: Image builder + needs: image-list + runs-on: ubuntu-latest + strategy: + #matrix: ${{ fromJson(needs.image-list.outputs.matrix) }} + matrix: + images: ["ubuntu/ubuntu-22.04.json"] + + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup python + uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Install environment + run: | + curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 > jq + chmod +x jq + pip install yq git+https://github.com/tdviet/fedcloudclient.git + - name: Configure providers access + env: + REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }} + run: | + # using parametric scopes to only have access to the right VO + SCOPE="openid%20email%20profile%20voperson_id" + SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=vm_operator#aai.egi.eu" + SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=member#aai.egi.eu" + CLOUD_OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \ + -d "grant_type=refresh_token&client_id=token-portal&scope=$SCOPE&refresh_token=$REFRESH_TOKEN" \ + | jq -r ".access_token") + echo "::add-mask::$CLOUD_OIDC_TOKEN" + + + cd builder + BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)" + BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)" + BACKEND_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$CLOUD_OIDC_TOKEN" \ + --site "$BACKEND_SITE" --vo "$BACKEND_VO" -j | jq -r '.[0].Result.id')" + echo "::add-mask::$BACKEND_OS_TOKEN" + sed -i -e "s/backend_secret/$BACKEND_OS_TOKEN/" clouds.yaml + + SCOPE="openid%20email%20profile%20voperson_id" + SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:vo.access.egi.eu:role=vm_operator#aai.egi.eu" + SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:vo.access.egi.eu:role=member#aai.egi.eu" + ACCESS_OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \ + -d "grant_type=refresh_token&client_id=token-portal&scope=$SCOPE&refresh_token=$REFRESH_TOKEN" \ + | jq -r ".access_token") + echo "::add-mask::$ACCESS_OIDC_TOKEN" + DEPLOY_SITE="$(yq -r .clouds.deploy.site clouds.yaml)" + DEPLOY_VO="$(yq -r .clouds.deploy.vo clouds.yaml)" + echo "DEPLOY_SITE=$DEPLOY_SITE" >> "$GITHUB_ENV" + DEPLOY_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$ACCESS_OIDC_TOKEN" \ + --site "$DEPLOY_SITE" --vo "$DEPLOY_VO" -j | jq -r '.[0].Result.id')" + echo "::add-mask::$DEPLOY_OS_TOKEN" + sed -i -e "s/deploy_secret/$DEPLOY_OS_TOKEN/" clouds.yaml + + # Another one for images + IMAGES_SITE="$(yq -r .clouds.deploy.site clouds.yaml)" + IMAGES_VO="$(yq -r .clouds.deploy.vo clouds.yaml)" + echo "IMAGES_SITE=$IMAGES_SITE" >> "$GITHUB_ENV" + IMAGES_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$ACCESS_OIDC_TOKEN" \ + --site "$IMAGES_SITE" --vo "$IMAGES_VO" -j | jq -r '.[0].Result.id')" + echo "::add-mask::$IMAGES_OS_TOKEN" + sed -i -e "s/images_secret/$IMAGES_OS_TOKEN/" clouds.yaml + + mkdir -p ~/.config/openstack + touch ~/.config/openstack/secure.yaml + FEDCLOUD_LOCKER_TOKEN="$(fedcloud secret locker create \ + --oidc-access-token "$CLOUD_OIDC_TOKEN" \ + --ttl 1h --num-uses 2)" + echo "::add-mask::$FEDCLOUD_LOCKER_TOKEN" + fedcloud secret put --locker-token "$FEDCLOUD_LOCKER_TOKEN" deploy "data=@clouds.yaml" + echo "FEDCLOUD_LOCKER_TOKEN=$FEDCLOUD_LOCKER_TOKEN" >> "$GITHUB_ENV" + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.2.9 + - name: Terraform Format + id: fmt + run: | + cd builder + terraform fmt -check + - name: Terraform init + id: init + run: | + cd builder + terraform init + - name: Build the thing + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + cd builder + sed -i -e "s#%IMAGE%#${{ matrix.image }}#" cloud-init.yaml + sed -i -e "s/%TOKEN%/${{ secrets.GITHUB_TOKEN }}/" cloud-init.yaml + sed -i -e "s/%REF%/${{ github.sha }}/" cloud-init.yaml + sed -i -e "s/%SHORT_REF%/$(git rev-parse --short HEAD)/" cloud-init.yaml + sed -i -e "s/%FEDCLOUD_LOCKER_TOKEN%/$FEDCLOUD_LOCKER_TOKEN/" cloud-init.yaml + - name: terraform plan + id: plan + if: github.event_name == 'pull_request' + run: | + cd builder + terraform plan -no-color -var-file="$DEPLOY_SITE.tfvars" + continue-on-error: true + - name: Update Pull Request + uses: actions/github-script@v7 + if: github.event_name == 'pull_request' + env: + PLAN: "terraform\n${{ steps.plan.outputs.stdout }}" + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const output = `#### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\` + #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\` + #### Terraform Plan 📖\`${{ steps.plan.outcome }}\` +
Show Plan + + \`\`\` + ${process.env.PLAN} + \`\`\` + +
+ + *Pusher: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }) + - name: Terraform Plan Status + if: steps.plan.outcome == 'failure' + run: exit 1 + - name: Terraform Apply + id: terraform-apply +# if: github.ref == 'refs/heads/main' && github.event_name == 'push' + run: | + cd builder + terraform apply -auto-approve -var-file="$DEPLOY_SITE.tfvars" + - name: Get VM ID + id: terraform-vm-id +# if: github.ref == 'refs/heads/main' && github.event_name == 'push' + run: | + cd builder + terraform output -raw instance-id + - name: Re-configure providers access + env: + REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }} + run: | + # using parametric scopes to only have access to cloud.egi.eu VO + SCOPE="openid%20email%20profile%20voperson_id" + SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=vm_operator#aai.egi.eu" + SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:cloud.egi.eu:role=member#aai.egi.eu" + OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \ + -d "grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=token-portal&scope=$SCOPE" \ + | jq -r ".access_token") + echo "::add-mask::$OIDC_TOKEN" + cd builder + git checkout -- clouds.yaml + BACKEND_SITE="$(yq -r .clouds.backend.site clouds.yaml)" + BACKEND_VO="$(yq -r .clouds.backend.vo clouds.yaml)" + BACKEND_OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \ + --site "$BACKEND_SITE" --vo "$BACKEND_VO" -j | jq -r '.[0].Result.id')" + echo "::add-mask::$BACKEND_OS_TOKEN" + echo "BACKEND_OS_TOKEN=$BACKEND_OS_TOKEN" >> "$GITHUB_ENV" + sed -i -e "s/backend_secret/$BACKEND_OS_TOKEN/" clouds.yaml + mkdir -p ~/.config/openstack + touch ~/.config/openstack/secure.yaml + - name: Get the status file from swift +# if: github.ref == 'refs/heads/main' && github.event_name == 'push' + uses: nick-fields/retry@v3 + with: + timeout_minutes: 10 + max_attempts: 20 + retry_wait_seconds: 40 + command: > + pushd builder && + which openstack && + openstack --version && + openstack --help && + openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object save fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}" && + openstack --os-cloud backend --os-token "$BACKEND_OS_TOKEN" object delete fedcloud-catchall "${{ steps.terraform-vm-id.outputs.stdout }}" + - name: Look for errors + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + run: | + cd builder + # show the status in the build log + cat "${{ steps.terraform-vm-id.outputs.stdout }}" + grep -v "error" "${{ steps.terraform-vm-id.outputs.stdout }}" diff --git a/builder/SCAI.tfvars b/builder/SCAI.tfvars new file mode 100644 index 0000000..093f2c3 --- /dev/null +++ b/builder/SCAI.tfvars @@ -0,0 +1,8 @@ +# Network +net_id = "c38b0272-637e-4665-a79c-1a37222b572c" + +# Flavor: medium +flavor_id = "73302761-40dc-4ced-91d6-7dd18524a82e" + +# Image: ubuntu 22.04 +image_id = "adeba762-ad84-406a-b082-dd055a27afb4" diff --git a/builder/build-image.sh b/builder/build-image.sh new file mode 100755 index 0000000..a631597 --- /dev/null +++ b/builder/build-image.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -e +set -x + +IMAGE="$1" +FEDCLOUD_SECRET_LOCKER="$2" + +# create a virtual env for fedcloudclient +python3 -m venv "$PWD/.venv" +"$PWD/.venv/bin/pip" install fedcloudclient + +mkdir -p /etc/openstack/ +TMP_SECRETS="$(mktemp)" +"$PWD/.venv/bin/fedcloud" secret get --locker-token "$FEDCLOUD_SECRET_LOCKER" \ + deploy data >"$TMP_SECRETS" && mv "$TMP_SECRETS" /etc/openstack/clouds.yaml + +systemctl start notify + +# get packer +curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add - +apt-add-repository -y "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" +apt-get update && apt-get install -y packer +packer plugins install github.com/hashicorp/qemu +packer plugins install github.com/hashicorp/ansible + +if tools/build.sh "$IMAGE" >/var/log/image-build.log 2>&1; then + # upload the image is missing + echo "UPLOAD" +fi diff --git a/builder/cloud-init.yaml b/builder/cloud-init.yaml new file mode 100644 index 0000000..2be30e4 --- /dev/null +++ b/builder/cloud-init.yaml @@ -0,0 +1,70 @@ +#cloud-config +--- +users: + - name: egi + gecos: EGI + primary_group: egi + groups: users + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_import_id: + - gh:enolfc + - gh:gwarf + - gh:sebastian-luna-valero + +packages: + - git + - jq + - retry + - qemu-system-x86 + - qemu-utils + - virtualbox + - python3-venv + - python3-dev + - ansible + +write_files: + - content: | + #!/bin/sh + set -e + + mkdir -p /var/tmp/egi + cd /var/tmp/egi || exit + + # Valid GitHub token to access the repo + OAUTH_TOKEN="%TOKEN%" + COMMIT_SHA="%REF%" + SHORT_COMMIT_SHA="%SHORT_REF%" + IMAGE="%IMAGE%" + + + # get the repo code and untar at cwd + curl -L -H "Accept: application/vnd.github.v3+raw" \ + "https://api.github.com/repos/EGI-Federation/fedcloud-vmi-templates/tarball/$COMMIT_SHA" | \ + tar xz --strip=1 + builder/build-image.sh "$IMAGE" + path: /var/lib/cloud/scripts/per-boot/build.sh + permissions: '0755' + - content: | + #!/bin/sh + mkdir -p /var/tmp/egi + VM_ID="$(cloud-init query instance_id)" + + cloud-init status --wait >"/var/tmp/egi/$VM_ID" + [ -f /var/log/image-build.log ] && \ + cat /var/log/image-build.log >>"/var/tmp/egi/$VM_ID" + # try 10 times, otherwise just die + retry -t 25 -d 200 -- openstack --os-cloud backend object create \ + --name "$VM_ID" fedcloud-vmi-templates "/var/tmp/egi/$VM_ID" + path: /usr/local/bin/notify.sh + permissions: '0755' + - content: | + [Unit] + Description=Notify the github action + + [Service] + ExecStart=/usr/local/bin/notify.sh + + [Install] + WantedBy=multi-user.target + path: /etc/systemd/system/notify.service diff --git a/builder/clouds.yaml b/builder/clouds.yaml new file mode 100644 index 0000000..fd39be7 --- /dev/null +++ b/builder/clouds.yaml @@ -0,0 +1,30 @@ +--- +backend_token: &bt "backend_secret" +deploy_token: &dt "deploy_secret" +image_token: &it "images_secret" + +clouds: + backend: + site: NCG-INGRID-PT + vo: cloud.egi.eu + auth_type: token + auth: + auth_url: https://stratus.ncg.ingrid.pt:5000/v3 + token: *bt + project_id: 6b042927bcfa466cb9eb56d3ea679987 + deploy: + site: SCAI + vo: vo.access.egi.eu + auth_type: token + auth: + auth_url: https://cloud.scai.fraunhofer.de:5000/v3 + token: *dt + project_id: d12c055894f245b28c75c95e4ca78407 + images: + site: IFCA-LCG2 + vo: vo.access.egi.eu + auth_type: token + auth: + auth_url: https://api.cloud.ifca.es:5000/v3 + token: *it + project_id: 999f045cb1ff4684a15ebb338af69460 diff --git a/builder/versions.tf b/builder/versions.tf new file mode 100644 index 0000000..cba3cec --- /dev/null +++ b/builder/versions.tf @@ -0,0 +1,14 @@ +# This is where the info about the deployment is to be stored +terraform { + required_providers { + openstack = { + source = "terraform-provider-openstack/openstack" + version = "~> 1.48" + } + } + required_version = ">= 0.13" + backend "swift" { + container = "terraform-image-sync" + cloud = "backend" + } +} diff --git a/builder/vm.tf b/builder/vm.tf new file mode 100644 index 0000000..89e6729 --- /dev/null +++ b/builder/vm.tf @@ -0,0 +1,36 @@ +# The provider where the deployment is actually performed +provider "openstack" { + cloud = "deploy" +} + +# Configurable stuff +variable "net_id" { + type = string + description = "The id of the network" +} + +variable "image_id" { + type = string + description = "VM image id" +} + +variable "flavor_id" { + type = string + description = "VM flavor id" +} + + +resource "openstack_compute_instance_v2" "builder" { + name = "builder" + image_id = var.image_id + flavor_id = var.flavor_id + security_groups = ["default"] + user_data = file("cloud-init.yaml") + network { + uuid = var.net_id + } +} + +output "instance-id" { + value = openstack_compute_instance_v2.builder.id +} diff --git a/tools/build.sh b/tools/build.sh index e33d3ad..a5e7554 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -1,10 +1,11 @@ #!/bin/bash - # Takes as argument the json file describing the build for packer # e.g. build.sh centos-7.json - set -e +template_dir="$(dirname "$1")" +pushd "$template_dir" + # Create a temp ssh key that will be used to login to the VMs SSH_KEY_DIR=$(mktemp -d) ssh-keygen -f "$SSH_KEY_DIR/key" -N "" -t ed25519 @@ -18,7 +19,7 @@ done # build with this key packer build -var "SSH_PRIVATE_KEY_FILE=$SSH_KEY_DIR/key" \ -var "SSH_PUB_KEY=$(cat "$SSH_KEY_DIR/key.pub")" \ - "$1" + "$(basename "$1")" rm -rf "$SSH_KEY_DIR" diff --git a/ubuntu/ubuntu-22.04.json b/ubuntu/ubuntu-22.04.json index ce59da4..11dc376 100644 --- a/ubuntu/ubuntu-22.04.json +++ b/ubuntu/ubuntu-22.04.json @@ -34,7 +34,7 @@ "qemuargs": [ [ "-cpu", "host" ] ], - "vm_name": "Ubuntu.22.04-2023.10.24" + "vm_name": "Ubuntu.22.04-2024.07.31" } ], "provisioners": [