From dde6606a96374b618c7c7efd328220d342e683ed Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Tue, 27 Aug 2024 15:23:01 +0100 Subject: [PATCH 01/11] Improve output of the build --- builder/build-image.sh | 38 +++++++++++++++++++++--------------- builder/refresh.sh | 5 +++-- ubuntu/ubuntu-24.04.pkr.json | 4 ++-- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/builder/build-image.sh b/builder/build-image.sh index ddc40f8..f64427d 100755 --- a/builder/build-image.sh +++ b/builder/build-image.sh @@ -38,21 +38,27 @@ packer plugins install github.com/hashicorp/qemu packer plugins install github.com/hashicorp/ansible # do the build -if tools/build.sh "$IMAGE" >/var/log/image-build.log 2>&1; then - # compress the resulting image - QEMU_SOURCE_ID=$(hcl2tojson "$IMAGE" | jq -r '.source[0].qemu | keys[]') - VM_NAME=$(hcl2tojson "$IMAGE" | jq -r '.source[0].qemu.'"$QEMU_SOURCE_ID"'.vm_name') - QCOW_FILE="$VM_NAME.qcow2" - builder/refresh.sh vo.access.egi.eu "$(cat /var/tmp/egi/.refresh_token)" images - OS_TOKEN="$(yq -r '.clouds.images.auth.token' /etc/openstack/clouds.yaml)" - OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" - cd "$OUTPUT_DIR" - qemu-img convert -O qcow2 -c "$VM_NAME" "$QCOW_FILE" - openstack --os-cloud images --os-token "$OS_TOKEN" \ - object create egi_endorsed_vas "$QCOW_FILE" - ls -lh "$QCOW_FILE" - SHA="$(sha512sum -z "$QCOW_FILE" | cut -f1 -d" ")" - echo "SUCCESSFUL BUILD - $QCOW_FILE - $SHA" >>/var/log/image-build.log -fi +QEMU_SOURCE_ID=$(hcl2tojson "$IMAGE" | jq -r '.source[0].qemu | keys[]') +VM_NAME=$(hcl2tojson "$IMAGE" \ + | jq -r '.source[0].qemu.'"$QEMU_SOURCE_ID"'.vm_name') +QCOW_FILE="$VM_NAME.qcow2" + +{ + if tools/build.sh "$IMAGE"; then + # compress the resulting image + builder/refresh.sh vo.access.egi.eu \ + "$(cat /var/tmp/egi/.refresh_token)" images + OS_TOKEN="$(yq -r '.clouds.images.auth.token' \ + /etc/openstack/clouds.yaml)" + OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" + cd "$OUTPUT_DIR" + qemu-img convert -O qcow2 -c "$VM_NAME" "$QCOW_FILE" + openstack --os-cloud images --os-token "$OS_TOKEN" \ + object create egi_endorsed_vas "$QCOW_FILE" + ls -lh "$QCOW_FILE" + SHA="$(sha512sum -z "$QCOW_FILE" | cut -f1 -d" ")" + echo "SUCCESSFUL BUILD - $QCOW_FILE - $SHA" + fi +} >>/var/log/image-build.log 2>&1 echo "BUILD ENDED" >>/var/log/image-build.log diff --git a/builder/refresh.sh b/builder/refresh.sh index 24c7767..73a9793 100755 --- a/builder/refresh.sh +++ b/builder/refresh.sh @@ -5,7 +5,7 @@ # 2 --> the refresh token # 3 --> the list of clouds to update # -# Will throw the OIDC TOKEN to output! +# Will throw the OIDC TOKEN to output if $GITHUB_ACTION is defined! set -e @@ -27,12 +27,13 @@ SCOPE="$SCOPE%20eduperson_entitlement:urn:mace:egi.eu:group:$VO:role=member#aai. OIDC_TOKEN=$(curl -X POST "https://aai.egi.eu/auth/realms/egi/protocol/openid-connect/token" \ -d "grant_type=refresh_token&client_id=token-portal&scope=$SCOPE&refresh_token=$REFRESH_TOKEN" \ | jq -r ".access_token") -echo "::add-mask::$OIDC_TOKEN" +[ -n "$GITHUB_ACTION" ] && echo "::add-mask::$OIDC_TOKEN" for cloud in "$@" ; do SITE="$(yq -r ".clouds.$cloud.site" $CLOUDS_YAML)" VO="$(yq -r ".clouds.$cloud.vo" $CLOUDS_YAML)" OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \ --site "$SITE" --vo "$VO" -j | jq -r '.[0].Result.id')" + [ -n "$GITHUB_ACTION" ] && echo "::add-mask::$OIDC_TOKEN" echo "::add-mask::$OS_TOKEN" yq -y -i '.clouds.'"$cloud"'.auth.token="'"$OS_TOKEN"'"' $CLOUDS_YAML done diff --git a/ubuntu/ubuntu-24.04.pkr.json b/ubuntu/ubuntu-24.04.pkr.json index a38e197..ae96b2f 100644 --- a/ubuntu/ubuntu-24.04.pkr.json +++ b/ubuntu/ubuntu-24.04.pkr.json @@ -15,7 +15,7 @@ "boot", "" ], - "boot_wait": "5s", + "boot_wait": "6s", "disk_size": 8000, "format": "qcow2", "headless": true, @@ -34,7 +34,7 @@ "qemuargs": [ [ "-cpu", "host" ] ], - "vm_name": "Ubuntu.24.04-2024.04.30" + "vm_name": "Ubuntu.24.04-2024.08.27" } ], "provisioners": [ From 2210196ec35483e403d1a40173143c5b0fcc68c6 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 29 Aug 2024 10:18:41 +0100 Subject: [PATCH 02/11] Do not build if image is there --- builder/build-image.sh | 47 ++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/builder/build-image.sh b/builder/build-image.sh index f64427d..22eaf84 100755 --- a/builder/build-image.sh +++ b/builder/build-image.sh @@ -37,28 +37,39 @@ apt-get update && apt-get install -y packer packer plugins install github.com/hashicorp/qemu packer plugins install github.com/hashicorp/ansible -# do the build QEMU_SOURCE_ID=$(hcl2tojson "$IMAGE" | jq -r '.source[0].qemu | keys[]') VM_NAME=$(hcl2tojson "$IMAGE" \ | jq -r '.source[0].qemu.'"$QEMU_SOURCE_ID"'.vm_name') QCOW_FILE="$VM_NAME.qcow2" -{ - if tools/build.sh "$IMAGE"; then - # compress the resulting image - builder/refresh.sh vo.access.egi.eu \ - "$(cat /var/tmp/egi/.refresh_token)" images - OS_TOKEN="$(yq -r '.clouds.images.auth.token' \ - /etc/openstack/clouds.yaml)" - OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" - cd "$OUTPUT_DIR" - qemu-img convert -O qcow2 -c "$VM_NAME" "$QCOW_FILE" - openstack --os-cloud images --os-token "$OS_TOKEN" \ - object create egi_endorsed_vas "$QCOW_FILE" - ls -lh "$QCOW_FILE" - SHA="$(sha512sum -z "$QCOW_FILE" | cut -f1 -d" ")" - echo "SUCCESSFUL BUILD - $QCOW_FILE - $SHA" - fi -} >>/var/log/image-build.log 2>&1 +# Check if the image is already there +builder/refresh.sh vo.access.egi.eu "$(cat /var/tmp/egi/.refresh_token)" images +OS_TOKEN="$(yq -r '.clouds.images.auth.token' /etc/openstack/clouds.yaml)" +if openstack --os-cloud images --os-token "$OS_TOKEN" \ + object show egi_endorsed_vas \ + "$QCOW_FILE" > /dev/null ; then + # skip + echo "Skipped build as image is already uploaded" >>/var/log/image-build.log +else + # do the build + { + if tools/build.sh "$IMAGE"; then + # refresh the token, it may have expired + OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" + cd "$OUTPUT_DIR" + # compress the resulting image + qemu-img convert -O qcow2 -c "$VM_NAME" "$QCOW_FILE" + builder/refresh.sh vo.access.egi.eu \ + "$(cat /var/tmp/egi/.refresh_token)" images + OS_TOKEN="$(yq -r '.clouds.images.auth.token' \ + /etc/openstack/clouds.yaml)" + openstack --os-cloud images --os-token "$OS_TOKEN" \ + object create egi_endorsed_vas \ + "$QCOW_FILE" + SHA="$(sha512sum -z "$QCOW_FILE" | cut -f1 -d" ")" + echo "SUCCESSFUL BUILD - $QCOW_FILE - $SHA" + fi + } >>/var/log/image-build.log 2>&1 +fi echo "BUILD ENDED" >>/var/log/image-build.log From 20dc109520c7a042cb498baa0ec66ee196d67b23 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 29 Aug 2024 10:50:23 +0100 Subject: [PATCH 03/11] Fix indent --- builder/build-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/build-image.sh b/builder/build-image.sh index 22eaf84..7795346 100755 --- a/builder/build-image.sh +++ b/builder/build-image.sh @@ -55,8 +55,8 @@ else { if tools/build.sh "$IMAGE"; then # refresh the token, it may have expired - OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" - cd "$OUTPUT_DIR" + OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" + cd "$OUTPUT_DIR" # compress the resulting image qemu-img convert -O qcow2 -c "$VM_NAME" "$QCOW_FILE" builder/refresh.sh vo.access.egi.eu \ From c38285ee28cffd9aed083104be3d02113ae66f1a Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 29 Aug 2024 10:51:49 +0100 Subject: [PATCH 04/11] Force build --- ubuntu/ubuntu-24.04.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubuntu/ubuntu-24.04.pkr.hcl b/ubuntu/ubuntu-24.04.pkr.hcl index d3324f6..9761cb2 100644 --- a/ubuntu/ubuntu-24.04.pkr.hcl +++ b/ubuntu/ubuntu-24.04.pkr.hcl @@ -32,7 +32,7 @@ source "qemu" "ubuntu_24_04" { "boot", "" ] - boot_wait = "5s" + boot_wait = "6s" disk_size = 8000 format = "qcow2" headless = true From ae390595cabe68e8df79ce0522acbbec457b9dfe Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Thu, 29 Aug 2024 11:35:35 +0100 Subject: [PATCH 05/11] Remove unused files --- ubuntu/datahub-jupyter-ubuntu-22.04.pkr.json | 59 ------------------ ubuntu/docker-ubuntu-22.04.pkr.json | 61 ------------------ ubuntu/small-ubuntu-20.04.pkr.json | 62 ------------------- ubuntu/ubuntu-20.04.pkr.json | 62 ------------------- ubuntu/ubuntu-22.04.pkr.json | 59 ------------------ ubuntu/ubuntu-24.04.pkr.json | 65 -------------------- 6 files changed, 368 deletions(-) delete mode 100644 ubuntu/datahub-jupyter-ubuntu-22.04.pkr.json delete mode 100644 ubuntu/docker-ubuntu-22.04.pkr.json delete mode 100644 ubuntu/small-ubuntu-20.04.pkr.json delete mode 100644 ubuntu/ubuntu-20.04.pkr.json delete mode 100644 ubuntu/ubuntu-22.04.pkr.json delete mode 100644 ubuntu/ubuntu-24.04.pkr.json diff --git a/ubuntu/datahub-jupyter-ubuntu-22.04.pkr.json b/ubuntu/datahub-jupyter-ubuntu-22.04.pkr.json deleted file mode 100644 index 400f024..0000000 --- a/ubuntu/datahub-jupyter-ubuntu-22.04.pkr.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "variables": { - "SSH_PUB_KEY": "", - "SSH_PRIVATE_KEY_FILE": "" - }, - "builders": [ - { - "boot_command": [ - "c", - "linux /casper/vmlinuz --- autoinstall ds=\"nocloud-net;seedfrom=http://{{.HTTPIP}}:{{.HTTPPort}}/\"", - " PACKER_USER=ubuntu PACKER_AUTHORIZED_KEY={{ user `SSH_PUB_KEY` | urlquery }}", - "", - "initrd /casper/initrd", - "", - "boot", - "" - ], - "boot_wait": "5s", - "disk_size": 8000, - "format": "qcow2", - "headless": true, - "http_directory": "httpdir", - "http_port_max": 8550, - "http_port_min": 8500, - "iso_url": "https://releases.ubuntu.com/jammy/ubuntu-22.04.4-live-server-amd64.iso", - "iso_checksum": "sha256:45f873de9f8cb637345d6e66a583762730bbea30277ef7b32c9c3bd6700a32b2", - "memory": 1024, - "ssh_timeout": "20m", - "ssh_clear_authorized_keys": true, - "ssh_private_key_file": "{{ user `SSH_PRIVATE_KEY_FILE` }}", - "shutdown_command": "sudo -- sh -c 'rm /etc/sudoers.d/99-egi-installation && shutdown -h now'", - "ssh_username": "ubuntu", - "type": "qemu", - "qemuargs": [ - [ "-cpu", "host" ] - ], - "vm_name": "DataHub-Jupyter-Ubuntu.22.04-2024.03.21" - } - ], - "provisioners": [ - { - "playbook_file": "provisioners/init.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "pause_before": "30s", - "playbook_file": "provisioners/datahub-jupyter.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "type": "shell", - "script": "provisioners/cleanup.sh" - } - ] -} diff --git a/ubuntu/docker-ubuntu-22.04.pkr.json b/ubuntu/docker-ubuntu-22.04.pkr.json deleted file mode 100644 index 9ebfa84..0000000 --- a/ubuntu/docker-ubuntu-22.04.pkr.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "variables": { - "SSH_PUB_KEY": "", - "SSH_PRIVATE_KEY_FILE": "" - }, - "builders": [ - { - "boot_command": [ - "c", - "linux /casper/vmlinuz --- autoinstall ds=\"nocloud-net;seedfrom=http://{{.HTTPIP}}:{{.HTTPPort}}/\"", - " PACKER_USER=ubuntu PACKER_AUTHORIZED_KEY={{ user `SSH_PUB_KEY` | urlquery }}", - "", - "initrd /casper/initrd", - "", - "boot", - "" - ], - "boot_wait": "5s", - "disk_size": 8000, - "format": "qcow2", - "headless": true, - "http_directory": "httpdir", - "http_port_max": 8550, - "http_port_min": 8500, - "iso_url": "https://releases.ubuntu.com/22.04/ubuntu-22.04.3-live-server-amd64.iso", - "iso_checksum": "sha256:a4acfda10b18da50e2ec50ccaf860d7f20b389df8765611142305c0e911d16fd", - "memory": 1024, - "ssh_timeout": "20m", - "ssh_clear_authorized_keys": true, - "ssh_private_key_file": "{{ user `SSH_PRIVATE_KEY_FILE` }}", - "shutdown_command": "sudo -- sh -c 'rm /etc/sudoers.d/99-egi-installation && shutdown -h now'", - "ssh_username": "ubuntu", - "type": "qemu", - "vnc_port_min": 5934, - "vnc_port_max": 5934, - "qemuargs": [ - [ "-cpu", "host" ] - ], - "vm_name": "Docker.Ubuntu.22.04-2024.03.14" - } - ], - "provisioners": [ - { - "playbook_file": "provisioners/init.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "pause_before": "30s", - "playbook_file": "provisioners/docker.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "type": "shell", - "script": "provisioners/cleanup.sh" - } - ] -} diff --git a/ubuntu/small-ubuntu-20.04.pkr.json b/ubuntu/small-ubuntu-20.04.pkr.json deleted file mode 100644 index b3a6ffd..0000000 --- a/ubuntu/small-ubuntu-20.04.pkr.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "variables": { - "SSH_PUB_KEY": "", - "SSH_PRIVATE_KEY_FILE": "" - }, - "builders": [ - { - "boot_command": [ - "", - "", - "", - "", - "", - "", - "/casper/vmlinuz ", - "initrd=/casper/initrd ", - " autoinstall ds=nocloud-net;s=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ ", - " PACKER_USER=ubuntu PACKER_AUTHORIZED_KEY={{ user `SSH_PUB_KEY` | urlquery }}", - "" - ], - "boot_wait": "3s", - "disk_size": 3500, - "format": "qcow2", - "headless": true, - "http_directory": "httpdir", - "http_port_max": 8550, - "http_port_min": 8500, - "iso_url": "https://releases.ubuntu.com/20.04/ubuntu-20.04.6-live-server-amd64.iso", - "iso_checksum": "sha256:b8f31413336b9393ad5d8ef0282717b2ab19f007df2e9ed5196c13d8f9153c8b", - "memory": 1024, - "ssh_timeout": "20m", - "ssh_clear_authorized_keys": true, - "ssh_private_key_file": "{{ user `SSH_PRIVATE_KEY_FILE` }}", - "shutdown_command": "sudo -- sh -c 'rm /etc/sudoers.d/99-egi-installation && shutdown -h now'", - "ssh_username": "ubuntu", - "type": "qemu", - "qemuargs": [ - [ "-cpu", "host" ] - ], - "vm_name": "Small.Ubuntu.20.04-2024.06.03" - } - ], - "provisioners": [ - { - "playbook_file": "provisioners/init.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "pause_before": "30s", - "playbook_file": "provisioners/base.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "type": "shell", - "script": "provisioners/cleanup.sh" - } - ] -} diff --git a/ubuntu/ubuntu-20.04.pkr.json b/ubuntu/ubuntu-20.04.pkr.json deleted file mode 100644 index 3945cc0..0000000 --- a/ubuntu/ubuntu-20.04.pkr.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "variables": { - "SSH_PUB_KEY": "", - "SSH_PRIVATE_KEY_FILE": "" - }, - "builders": [ - { - "boot_command": [ - "", - "", - "", - "", - "", - "", - "/casper/vmlinuz ", - "initrd=/casper/initrd ", - " autoinstall ds=nocloud-net;s=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ ", - " PACKER_USER=ubuntu PACKER_AUTHORIZED_KEY={{ user `SSH_PUB_KEY` | urlquery }}", - "" - ], - "boot_wait": "3s", - "disk_size": 8000, - "format": "qcow2", - "headless": true, - "http_directory": "httpdir", - "http_port_max": 8550, - "http_port_min": 8500, - "iso_url": "https://releases.ubuntu.com/20.04/ubuntu-20.04.6-live-server-amd64.iso", - "iso_checksum": "sha256:b8f31413336b9393ad5d8ef0282717b2ab19f007df2e9ed5196c13d8f9153c8b", - "memory": 1024, - "ssh_timeout": "20m", - "ssh_clear_authorized_keys": true, - "ssh_private_key_file": "{{ user `SSH_PRIVATE_KEY_FILE` }}", - "shutdown_command": "sudo -- sh -c 'rm /etc/sudoers.d/99-egi-installation && shutdown -h now'", - "ssh_username": "ubuntu", - "type": "qemu", - "qemuargs": [ - [ "-cpu", "host" ] - ], - "vm_name": "Ubuntu.20.04-2024.04.22" - } - ], - "provisioners": [ - { - "playbook_file": "provisioners/init.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "pause_before": "30s", - "playbook_file": "provisioners/base.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "type": "shell", - "script": "provisioners/cleanup.sh" - } - ] -} diff --git a/ubuntu/ubuntu-22.04.pkr.json b/ubuntu/ubuntu-22.04.pkr.json deleted file mode 100644 index 548d501..0000000 --- a/ubuntu/ubuntu-22.04.pkr.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "variables": { - "SSH_PUB_KEY": "", - "SSH_PRIVATE_KEY_FILE": "" - }, - "builders": [ - { - "boot_command": [ - "c", - "linux /casper/vmlinuz --- autoinstall ds=\"nocloud-net;seedfrom=http://{{.HTTPIP}}:{{.HTTPPort}}/\"", - " PACKER_USER=ubuntu PACKER_AUTHORIZED_KEY={{ user `SSH_PUB_KEY` | urlquery }}", - "", - "initrd /casper/initrd", - "", - "boot", - "" - ], - "boot_wait": "4s", - "disk_size": 8000, - "format": "qcow2", - "headless": true, - "http_directory": "httpdir", - "http_port_max": 8550, - "http_port_min": 8500, - "iso_url": "https://releases.ubuntu.com/22.04/ubuntu-22.04.4-live-server-amd64.iso", - "iso_checksum": "sha256:45f873de9f8cb637345d6e66a583762730bbea30277ef7b32c9c3bd6700a32b2", - "memory": 1024, - "ssh_timeout": "20m", - "ssh_clear_authorized_keys": true, - "ssh_private_key_file": "{{ user `SSH_PRIVATE_KEY_FILE` }}", - "shutdown_command": "sudo -- sh -c 'rm /etc/sudoers.d/99-egi-installation && shutdown -h now'", - "ssh_username": "ubuntu", - "type": "qemu", - "qemuargs": [ - [ "-cpu", "host" ] - ], - "vm_name": "Ubuntu.22.04-2024.08.06" - } - ], - "provisioners": [ - { - "playbook_file": "provisioners/init.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "pause_before": "30s", - "playbook_file": "provisioners/base.yaml", - "type": "ansible", - "use_proxy": false, - "user": "ubuntu" - }, - { - "type": "shell", - "script": "provisioners/cleanup.sh" - } - ] -} diff --git a/ubuntu/ubuntu-24.04.pkr.json b/ubuntu/ubuntu-24.04.pkr.json deleted file mode 100644 index ae96b2f..0000000 --- a/ubuntu/ubuntu-24.04.pkr.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "variables": { - "SSH_PUB_KEY": "", - "SSH_PRIVATE_KEY_FILE": "" - }, - "builders": [ - { - "boot_command": [ - "c", - "linux /casper/vmlinuz --- autoinstall ds=\"nocloud-net;seedfrom=http://{{.HTTPIP}}:{{.HTTPPort}}/\"", - " PACKER_USER=ubuntu PACKER_AUTHORIZED_KEY={{ user `SSH_PUB_KEY` | urlquery }}", - "", - "initrd /casper/initrd", - "", - "boot", - "" - ], - "boot_wait": "6s", - "disk_size": 8000, - "format": "qcow2", - "headless": true, - "http_directory": "httpdir", - "http_port_max": 8550, - "http_port_min": 8500, - "iso_url": "https://releases.ubuntu.com/24.04/ubuntu-24.04-live-server-amd64.iso", - "iso_checksum": "sha256:8762f7e74e4d64d72fceb5f70682e6b069932deedb4949c6975d0f0fe0a91be3", - "memory": 1024, - "ssh_timeout": "20m", - "ssh_clear_authorized_keys": true, - "ssh_private_key_file": "{{ user `SSH_PRIVATE_KEY_FILE` }}", - "shutdown_command": "sudo -- sh -c 'rm /etc/sudoers.d/99-egi-installation && shutdown -h now'", - "ssh_username": "ubuntu", - "type": "qemu", - "qemuargs": [ - [ "-cpu", "host" ] - ], - "vm_name": "Ubuntu.24.04-2024.08.27" - } - ], - "provisioners": [ - { - "playbook_file": "provisioners/init.yaml", - "type": "ansible", - "extra_arguments": [ - "--extra-vars", "ansible_python_interpreter=/usr/bin/python3" - ], - "use_proxy": false, - "user": "ubuntu" - }, - { - "pause_before": "30s", - "playbook_file": "provisioners/base.yaml", - "type": "ansible", - "extra_arguments": [ - "--extra-vars", "ansible_python_interpreter=/usr/bin/python3" - ], - "use_proxy": false, - "user": "ubuntu" - }, - { - "type": "shell", - "script": "provisioners/cleanup.sh" - } - ] -} From 21ab1b534875c7988ffbd92bfad830c25118b033 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Mon, 7 Oct 2024 10:05:53 +0100 Subject: [PATCH 06/11] Rely on error trap --- builder/build-image.sh | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/builder/build-image.sh b/builder/build-image.sh index 7795346..2582747 100755 --- a/builder/build-image.sh +++ b/builder/build-image.sh @@ -51,25 +51,22 @@ if openstack --os-cloud images --os-token "$OS_TOKEN" \ # skip echo "Skipped build as image is already uploaded" >>/var/log/image-build.log else - # do the build - { - if tools/build.sh "$IMAGE"; then - # refresh the token, it may have expired - OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" - cd "$OUTPUT_DIR" - # compress the resulting image - qemu-img convert -O qcow2 -c "$VM_NAME" "$QCOW_FILE" - builder/refresh.sh vo.access.egi.eu \ - "$(cat /var/tmp/egi/.refresh_token)" images - OS_TOKEN="$(yq -r '.clouds.images.auth.token' \ - /etc/openstack/clouds.yaml)" - openstack --os-cloud images --os-token "$OS_TOKEN" \ - object create egi_endorsed_vas \ - "$QCOW_FILE" - SHA="$(sha512sum -z "$QCOW_FILE" | cut -f1 -d" ")" - echo "SUCCESSFUL BUILD - $QCOW_FILE - $SHA" - fi - } >>/var/log/image-build.log 2>&1 + if tools/build.sh "$IMAGE" >/var/log/image-build.log 2>&1; then + # compress the resulting image + QEMU_SOURCE_ID=$(hcl2tojson "$IMAGE" | jq -r '.source[0].qemu | keys[]') + VM_NAME=$(hcl2tojson "$IMAGE" | jq -r '.source[0].qemu.'"$QEMU_SOURCE_ID"'.vm_name') + QCOW_FILE="$VM_NAME.qcow2" + builder/refresh.sh vo.access.egi.eu "$(cat /var/tmp/egi/.refresh_token)" images + OS_TOKEN="$(yq -r '.clouds.images.auth.token' /etc/openstack/clouds.yaml)" + OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" + cd "$OUTPUT_DIR" + qemu-img convert -O qcow2 -c "$VM_NAME" "$QCOW_FILE" + openstack --os-cloud images --os-token "$OS_TOKEN" \ + object create egi_endorsed_vas "$QCOW_FILE" + ls -lh "$QCOW_FILE" + SHA="$(sha512sum -z "$QCOW_FILE" | cut -f1 -d" ")" + echo "SUCCESSFUL BUILD - $QCOW_FILE - $SHA" >>/var/log/image-build.log + fi fi echo "BUILD ENDED" >>/var/log/image-build.log From 871b3c6b7876db67f8a1019d4d557a9997289820 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Mon, 7 Oct 2024 11:34:12 +0100 Subject: [PATCH 07/11] Remove duplicated code --- builder/build-image.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/builder/build-image.sh b/builder/build-image.sh index 2582747..1796db2 100755 --- a/builder/build-image.sh +++ b/builder/build-image.sh @@ -53,14 +53,12 @@ if openstack --os-cloud images --os-token "$OS_TOKEN" \ else if tools/build.sh "$IMAGE" >/var/log/image-build.log 2>&1; then # compress the resulting image - QEMU_SOURCE_ID=$(hcl2tojson "$IMAGE" | jq -r '.source[0].qemu | keys[]') - VM_NAME=$(hcl2tojson "$IMAGE" | jq -r '.source[0].qemu.'"$QEMU_SOURCE_ID"'.vm_name') - QCOW_FILE="$VM_NAME.qcow2" - builder/refresh.sh vo.access.egi.eu "$(cat /var/tmp/egi/.refresh_token)" images - OS_TOKEN="$(yq -r '.clouds.images.auth.token' /etc/openstack/clouds.yaml)" OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" cd "$OUTPUT_DIR" qemu-img convert -O qcow2 -c "$VM_NAME" "$QCOW_FILE" + # upload the image + builder/refresh.sh vo.access.egi.eu "$(cat /var/tmp/egi/.refresh_token)" images + OS_TOKEN="$(yq -r '.clouds.images.auth.token' /etc/openstack/clouds.yaml)" openstack --os-cloud images --os-token "$OS_TOKEN" \ object create egi_endorsed_vas "$QCOW_FILE" ls -lh "$QCOW_FILE" From 26c995346776c3b62ebb8c2fe58f2e103d540178 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Mon, 7 Oct 2024 15:00:31 +0100 Subject: [PATCH 08/11] Improve usage of logs and build result --- .github/workflows/build.yml | 2 +- builder/build-image.sh | 14 +++++++------- builder/cloud-init.yaml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 482ab56..68f9429 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,7 +121,7 @@ jobs: cat "${{ steps.terraform-vm-id.outputs.stdout }}" echo "BUILDEOF" } >> "$GITHUB_OUTPUT" - if grep "SUCCESSFUL BUILD" "${{ steps.terraform-vm-id.outputs.stdout }}"; then + if egrep "^### BUILD-IMAGE: (SUCCESS)|(SKIP) -" "${{ steps.terraform-vm-id.outputs.stdout }}"; then echo "outcome=success" >> "$GITHUB_OUTPUT" else echo "outcome=failure" >> "$GITHUB_OUTPUT" diff --git a/builder/build-image.sh b/builder/build-image.sh index 1796db2..5ae5d10 100755 --- a/builder/build-image.sh +++ b/builder/build-image.sh @@ -2,11 +2,11 @@ set -e error_handler() { - echo " Error in line: $1 " >> /var/log/image-build.log 2>&1 + echo "### BUILD-IMAGE: ERROR - line $1" shift - echo " Exit status: $1 " >> /var/log/image-build.log 2>&1 + echo " Exit status: $1" shift - echo " Command: $* " >> /var/log/image-build.log 2>&1 + echo " Command: $*" } trap 'error_handler ${LINENO} $? ${BASH_COMMAND}' ERR INT TERM @@ -49,9 +49,9 @@ if openstack --os-cloud images --os-token "$OS_TOKEN" \ object show egi_endorsed_vas \ "$QCOW_FILE" > /dev/null ; then # skip - echo "Skipped build as image is already uploaded" >>/var/log/image-build.log + echo "### BUILD-IMAGE: SKIP - Image $QCOW_FILE is already uploaded" else - if tools/build.sh "$IMAGE" >/var/log/image-build.log 2>&1; then + if tools/build.sh "$IMAGE"; then # compress the resulting image OUTPUT_DIR="$(dirname "$IMAGE")/output-$QEMU_SOURCE_ID" cd "$OUTPUT_DIR" @@ -63,8 +63,8 @@ else object create egi_endorsed_vas "$QCOW_FILE" ls -lh "$QCOW_FILE" SHA="$(sha512sum -z "$QCOW_FILE" | cut -f1 -d" ")" - echo "SUCCESSFUL BUILD - $QCOW_FILE - $SHA" >>/var/log/image-build.log + echo "### BUILD-IMAGE: SUCCESS - qcow: $QCOW_FILE sha512sum: $SHA" fi fi -echo "BUILD ENDED" >>/var/log/image-build.log +echo "### BUILD ENDED" diff --git a/builder/cloud-init.yaml b/builder/cloud-init.yaml index 9bba4cb..41bda22 100644 --- a/builder/cloud-init.yaml +++ b/builder/cloud-init.yaml @@ -46,7 +46,7 @@ write_files: ansible-galaxy role install -p /var/tmp/egi/ubuntu/provisioners/roles/ grycap.docker # build image - builder/build-image.sh "$IMAGE" "$FEDCLOUD_LOCKER_TOKEN" + builder/build-image.sh "$IMAGE" "$FEDCLOUD_LOCKER_TOKEN" > /var/log/image-build.log 2>&1 path: /var/lib/cloud/scripts/per-boot/build.sh permissions: '0755' - content: | From a42272d912ad66d9b15ef71f1581073bfa1cc238 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Mon, 7 Oct 2024 15:17:54 +0100 Subject: [PATCH 09/11] Better status extraction --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68f9429..eb6f42d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,11 +121,11 @@ jobs: cat "${{ steps.terraform-vm-id.outputs.stdout }}" echo "BUILDEOF" } >> "$GITHUB_OUTPUT" - if egrep "^### BUILD-IMAGE: (SUCCESS)|(SKIP) -" "${{ steps.terraform-vm-id.outputs.stdout }}"; then - echo "outcome=success" >> "$GITHUB_OUTPUT" - else - echo "outcome=failure" >> "$GITHUB_OUTPUT" - fi + outcome=$((grep "^### BUILD-IMAGE: " \ + "${{ steps.terraform-vm-id.outputs.stdout }}" \ + || echo "ERROR") \ + | cut -f2 -d":" | cut -f1 -d"-" | tr -d " ") + echo "outcome=$outcome" >> "$GITHUB_OUTPUT" - name: Update PR with build status uses: actions/github-script@v7 with: From 350315a3cd644aec1ffbaaff329fc703fdcd6610 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Mon, 7 Oct 2024 15:27:09 +0100 Subject: [PATCH 10/11] Be a bit less verbose --- .github/workflows/build.yml | 6 +++--- builder/build-image.sh | 6 +++--- builder/refresh.sh | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb6f42d..690f30b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,9 +121,9 @@ jobs: cat "${{ steps.terraform-vm-id.outputs.stdout }}" echo "BUILDEOF" } >> "$GITHUB_OUTPUT" - outcome=$((grep "^### BUILD-IMAGE: " \ - "${{ steps.terraform-vm-id.outputs.stdout }}" \ - || echo "ERROR") \ + outcome=$( (grep "^### BUILD-IMAGE: " \ + "${{ steps.terraform-vm-id.outputs.stdout }}" \ + || echo "ERROR") \ | cut -f2 -d":" | cut -f1 -d"-" | tr -d " ") echo "outcome=$outcome" >> "$GITHUB_OUTPUT" - name: Update PR with build status diff --git a/builder/build-image.sh b/builder/build-image.sh index 5ae5d10..8515a7a 100755 --- a/builder/build-image.sh +++ b/builder/build-image.sh @@ -17,7 +17,7 @@ FEDCLOUD_SECRET_LOCKER="$2" # create a virtual env for fedcloudclient python3 -m venv "$PWD/.venv" export PATH="$PWD/.venv/bin:$PATH" -pip install fedcloudclient simplejson yq python-hcl2 +pip install -qqq fedcloudclient simplejson yq python-hcl2 # Get openstack ready mkdir -p /etc/openstack/ @@ -31,9 +31,9 @@ systemctl start notify # get packer export PACKER_CONFIG_DIR="$PWD" -curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo tee /etc/apt/trusted.gpg.d/hashicorp.asc +curl -fsSL https://apt.releases.hashicorp.com/gpg > /etc/apt/trusted.gpg.d/hashicorp.asc apt-add-repository -y "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -apt-get update && apt-get install -y packer +apt-get -q update && apt-get install -yq packer packer plugins install github.com/hashicorp/qemu packer plugins install github.com/hashicorp/ansible diff --git a/builder/refresh.sh b/builder/refresh.sh index 73a9793..0c308bb 100755 --- a/builder/refresh.sh +++ b/builder/refresh.sh @@ -34,7 +34,6 @@ for cloud in "$@" ; do OS_TOKEN="$(fedcloud openstack token issue --oidc-access-token "$OIDC_TOKEN" \ --site "$SITE" --vo "$VO" -j | jq -r '.[0].Result.id')" [ -n "$GITHUB_ACTION" ] && echo "::add-mask::$OIDC_TOKEN" - echo "::add-mask::$OS_TOKEN" yq -y -i '.clouds.'"$cloud"'.auth.token="'"$OS_TOKEN"'"' $CLOUDS_YAML done From 60741a09ce6fa93a591ca96d28c9f17cf9b21476 Mon Sep 17 00:00:00 2001 From: Enol Fernandez Date: Mon, 7 Oct 2024 15:35:56 +0100 Subject: [PATCH 11/11] And less verbose --- builder/build-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/build-image.sh b/builder/build-image.sh index 8515a7a..d3e39ea 100755 --- a/builder/build-image.sh +++ b/builder/build-image.sh @@ -33,7 +33,7 @@ systemctl start notify export PACKER_CONFIG_DIR="$PWD" curl -fsSL https://apt.releases.hashicorp.com/gpg > /etc/apt/trusted.gpg.d/hashicorp.asc apt-add-repository -y "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" -apt-get -q update && apt-get install -yq packer +apt-get -qq update && apt-get -qq install -y packer packer plugins install github.com/hashicorp/qemu packer plugins install github.com/hashicorp/ansible