fix ubuntu 22.10 builds and output_directory (chef#1465)

* fix ubuntu 22.10 builds, updated amz linux script and readme
* fix output_directory for windows workstations, updated readme, updated amz_readme
* added qemu provider to x86_64 build pipeline

.github/workflows/packer-builds.yml

@@ -44,14 +44,12 @@ jobs:
           - windows-2019
           - windows-2022
         provider:
-          # - qemu
+          - qemu
           - virtualbox-iso
         exclude:
           - os: centos-stream-9
             provider: virtualbox-iso
     steps:
-      # - name: HomeBrew
-      #   run: brew update && brew upgrade qemu && brew upgrade --cask virtualbox
       - name: Checkout
         uses: actions/checkout@main
       - name: Pull Change list
@@ -78,7 +76,7 @@ jobs:
         run: packer init -upgrade packer_templates
       - name: Packer build
         if: steps.verify-changed-files.outputs.files_changed == 'true'
-        run: packer build -only=${{ matrix.provider }}.vm -var-file=os_pkrvars/$(echo ${{ matrix.os }} | cut -d "-" -f 1)/"${{ matrix.os }}-x86_64.pkrvars.hcl" packer_templates
+        run: packer build -only=${{ matrix.provider }}.vm -var "qemu_accelerator=hvf" -var-file=os_pkrvars/$(echo ${{ matrix.os }} | cut -d "-" -f 1)/"${{ matrix.os }}-x86_64.pkrvars.hcl" packer_templates
       - name: Upload build artifact
         uses: actions/upload-artifact@main
         with:
@@ -116,8 +114,6 @@ jobs:
           # - virtualbox-iso
           # - vmware-iso
     steps:
-      # - name: Update HomeBrew
-      #   run: brew update && brew upgrade && brew upgrade --cask
       - name: Checkout
         uses: actions/checkout@main
       - name: Pull Change list
@@ -144,7 +140,7 @@ jobs:
         run: packer init -upgrade packer_templates
       - name: Packer build
         if: steps.verify-changed-files.outputs.files_changed == 'true'
-        run: packer build -only=${{ matrix.provider }}.vm -var-file=os_pkrvars/$(echo ${{ matrix.os }} | cut -d "-" -f 1)/"${{ matrix.os }}-aarch64.pkrvars.hcl" packer_templates
+        run: packer build -only=${{ matrix.provider }}.vm -var "qemu_accelerator=hvf" -var-file=os_pkrvars/$(echo ${{ matrix.os }} | cut -d "-" -f 1)/"${{ matrix.os }}-aarch64.pkrvars.hcl" packer_templates
       - name: Upload build artifact
         uses: actions/upload-artifact@v3
         with:
@@ -222,7 +218,7 @@ jobs:
 #        run: packer init -upgrade packer_templates
 #      - name: Packer build
 #        if: steps.verify-changed-files.outputs.files_changed == 'true'
-#        run: packer build -only=${{ matrix.provider }}.vm -var-file=os_pkrvars/$(echo ${{ matrix.os }} | cut -d "-" -f 1)/"${{ matrix.os }}-x86_64.pkrvars.hcl" packer_templates
+#        run: packer build -only=${{ matrix.provider }}.vm -var "qemu_accelerator=whpx" -var-file=os_pkrvars/$(echo ${{ matrix.os }} | cut -d "-" -f 1)/"${{ matrix.os }}-x86_64.pkrvars.hcl" packer_templates
 #      - name: Upload build artifact
 #        uses: actions/upload-artifact@main
 #        with:

.gitignore

@@ -1,4 +1,5 @@
 .bundle/
+**/builds/
 *.iso
 *.box
 *.ovf

AMZ_README_FIRST.md

@@ -2,7 +2,11 @@ This is not your normal Bento box. Instead of building a system from an ISO we'r
 
 # Building this box
 
-1. Download the VirtualBox .vdi file for Amazon Linux 2 and place it in the same directory as this readme file. Amazon hosts these at <https://cdn.amazonlinux.com/os-images/latest/virtualbox/>. Make sure to name it amazon.vdi instead of the version specific name that Amazon gives it on their site.
-1. Run the STEP1_build_ovf.sh script to prepare this VDI file for packer and export it as a OVF file.
-1. From the amazonlinux directory run packer: bento build amazon-2-x86_64.json
-1. OPTIONAL: Cleanup the leftover files in the directory
+Simply run the AMZ_build_virtualbox-ovf.sh script
+
+This script will:
+
+1. Download the VirtualBox .vdi file for Amazon Linux 2 and place it in the amz_working_files directory. Amazon hosts these at <https://cdn.amazonlinux.com/os-images/latest/virtualbox/>. It will name it amazon.vdi instead of the version specific name that Amazon gives it on their site
+1. It will prepare this VDI file for packer and export it as a OVF file
+1. It will run the packer build
+1. Lastly it will clean up the leftover files in the working directory

AMZ_build_virtualbox-ovf.sh

@@ -59,7 +59,7 @@ echo Deleting the VM
 vboxmanage unregistervm $VM --delete
 
 echo starting packer build of amazonlinux
-if packer build -only=virtualbox-ovf.amazonlinux -var-file="$AMZDIR/../../os_pkrvars/amazonlinux/amazonlinux-2-x86_64.pkrvars.hcl" "$AMZDIR"/..; then
+if packer build -only=virtualbox-ovf.amazonlinux -var-file="$SCRIPT_RELATIVE_DIR"/os_pkrvars/amazonlinux/amazonlinux-2-x86_64.pkrvars.hcl "$SCRIPT_RELATIVE_DIR"/packer_templates; then
   echo "Cleaning up files"
   rm "$AMZDIR"/*.ovf "$AMZDIR"/*.vmdk "$AMZDIR"/*.iso
 fi

CHANGELOG.md

@@ -1,5 +1,48 @@
 # Change Log
 
+## [v202301.19.0] (2023-01-19)
+
+### Notes
+
+- When running packer build command the output directory is relative to the working directory the command is currently running in. Suggest running packer build commands from bento root directory for build working files to be placed in bento/builds/(build_name) directory by default. If the output_directory variable isn't overwritten a directory called builds/(build_name) will be created in the current working directory that you are running the command from
+
+### Fixes and updates
+
+- fixed x86_64 Ubuntu 22.10 build
+- set qemu_accelerator to null
+- fixed bug in output_directory variable - the provisioner option doesn't take template variables only hcl variables
+- updated AMZ_README_FIRST.md with how to use the updated script and actions it'll perform
+- updated README.md with vbox 7 fix and added note on builds directory being relative to the current working directory
+
+### Known Issues
+
+#### Failing Builds
+
+- OpenSUSE Leap 15 - x86_64
+- SpringdaleLinux 7 - x86_64
+- almalinux 8 - aarch64
+- CentOS 7 - aarch64
+- CentOS-Stream 8 - aarch64
+- CentOS-Stream - aarch64
+- Fedora 36 - aarch64
+- Fedora 37 - aarch64
+- OracleLinux 7 - aarch64
+- OracleLinux 8 - aarch64
+- Ubuntu 22.04 - aarch64
+- Ubuntu 22.10 - aarch64
+
+### Todo
+
+- Fix failing builds
+- Add more Virtualization providers to build pipelines
+- Finish removal of deprecated chef-solo provider to powershell provider for windows
+- migrate from http directory for hosting files to cd_files in source templates
+  - This makes all builds compatable with hyper-v gen 2 which removes floppy disk capability
+  - This also makes things universal for Virtualbox 6.1 to 7.x due to latter requiring extra config for guests on NAT to be able to connect to host
+- Update pipelines to only run on updated pkrvars files
+- Create CD pipeline to upload vagrant boxes after PR is merged
+- Create CD pipeline to build and upload new versions of vagrant boxes once every 3 months with the latest patches
+
 ## [v202301.09.0] (2023-01-09)
 
 ### Notes

README.md

@@ -2,7 +2,10 @@
 
 Bento is a project that encapsulates [Packer](https://www.packer.io/) templates for building [Vagrant](https://www.vagrantup.com/) base boxes. A subset of templates are built and published to the [bento org](https://app.vagrantup.com/bento) on Vagrant Cloud. These published boxes serve as the default boxes for [kitchen-vagrant](https://github.com/test-kitchen/kitchen-vagrant/).
 
-***NOTE:** Virutalbox 7.x requires extra config to allow nat network to connect to the host. To use uncomment lines #79 and #80 in bento/packer_templates/pkr-sources.pkr.hcl
+***NOTE:**
+
+- Virutalbox 7.x requires extra config to allow nat network to connect to the host. To use uncomment lines #154 and #155 in bento/packer_templates/pkr-variables.pkr.hcl
+- When running packer build command the output directory is relative to the working directory the command is currently running in. Suggest running packer build commands from bento root directory for build working files to be placed in bento/builds/(build_name) directory by default. If the output_directory variable isn't overwritten a directory called builds/(build_name) will be created in the current working directory that you are running the command from
 
 ## Using Public Boxes
 

os_pkrvars/almalinux/almalinux-8-aarch64.pkrvars.hcl

@@ -1,10 +1,10 @@
 os_name                 = "almalinux"
 os_version              = "8.7"
 os_arch                 = "aarch64"
-iso_url                 = "https://seattle.crazyfast.us/almalinux/8.7/isos/aarch64/AlmaLinux-8.7-aarch64-dvd.iso"
-iso_checksum            = "c14fae79d5e26bc41e75f0a65063ab684199a7ea3a6bd0a8086e1ffd2a7afac8"
+iso_url                 = "https://repo.almalinux.org/almalinux/8/isos/aarch64/AlmaLinux-8.7-aarch64-dvd.iso"
+iso_checksum            = "file:https://repo.almalinux.org/almalinux/8/isos/aarch64/CHECKSUM"
 parallels_guest_os_type = "centos"
 vbox_guest_os_type      = "RedHat_64"
 vmware_guest_os_type    = "centos-64"
-boot_command            = ["<up>e<wait><down><down><end><wait> inst.text inst.ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/rhel/8ks.cfg<leftCtrlOn>x<leftCtrlOff>"]
+boot_command            = ["<up>e<wait><down><down><end><wait> inst.text inst.ks=http://{{ .HTTPIP }}:{{ .HTTPPort }}/rhel/8ks.cfg <leftCtrlOn>x<leftCtrlOff>"]
 boot_command_hyperv     = ["<wait5><up><wait5><tab> text ks=hd:fd0:/ks.cfg<enter><wait5><esc>"]

os_pkrvars/almalinux/almalinux-8-x86_64.pkrvars.hcl

@@ -1,7 +1,7 @@
 os_name                 = "almalinux"
 os_version              = "8.7"
 os_arch                 = "x86_64"
-iso_url                 = "https://repo.almalinux.org/almalinux/8.7/isos/x86_64/AlmaLinux-8.7-x86_64-dvd.iso"
+iso_url                 = "https://repo.almalinux.org/almalinux/8/isos/x86_64/AlmaLinux-8.7-x86_64-dvd.iso"
 iso_checksum            = "b95ddf9d56a849cc8eb4b95dd2321c13af637d3379b91f5d96c39e96fb4403b3"
 parallels_guest_os_type = "centos"
 vbox_guest_os_type      = "RedHat_64"

os_pkrvars/almalinux/almalinux-9-aarch64.pkrvars.hcl

@@ -1,7 +1,7 @@
 os_name                 = "almalinux"
 os_version              = "9.1"
 os_arch                 = "aarch64"
-iso_url                 = "https://repo.almalinux.org/almalinux/9.1/isos/aarch64/AlmaLinux-9.1-aarch64-dvd.iso"
+iso_url                 = "https://repo.almalinux.org/almalinux/9/isos/aarch64/AlmaLinux-9.1-aarch64-dvd.iso"
 iso_checksum            = "a6f0fb355b9c82f13a95f3f02e19b0f07906a7e0f27e3bca144338ebac9abf40"
 parallels_guest_os_type = "centos"
 vbox_guest_os_type      = "RedHat_64"

os_pkrvars/almalinux/almalinux-9-x86_64.pkrvars.hcl

@@ -1,7 +1,7 @@
 os_name                 = "almalinux"
 os_version              = "9.1"
 os_arch                 = "x86_64"
-iso_url                 = "https://repo.almalinux.org/almalinux/9.1/isos/x86_64/AlmaLinux-9.1-x86_64-dvd.iso"
+iso_url                 = "https://repo.almalinux.org/almalinux/9/isos/x86_64/AlmaLinux-9.1-x86_64-dvd.iso"
 iso_checksum            = "2a44e3f8a012c132da19b9aae2bf949e20b116f0a2a7ac3eca111972f4ac952f"
 parallels_guest_os_type = "centos"
 vbox_guest_os_type      = "RedHat_64"

os_pkrvars/ubuntu/ubuntu-22.04-x86_64.pkrvars.hcl

@@ -8,12 +8,4 @@ parallels_guest_os_type = "ubuntu"
 vbox_guest_os_type      = "Ubuntu_64"
 vmware_guest_os_type    = "ubuntu-64"
 boot_command            = ["c<wait>set gfxpayload=keep<enter><wait>linux /casper/vmlinuz quiet autoinstall ds=nocloud-net\\;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/ ---<enter><wait>initrd /casper/initrd<wait><enter><wait>boot<enter><wait>"]
-#boot_command = [
-#  "e<wait>",
-#  "<down><down><down>",
-#  "<end><bs><bs><bs><bs><wait>",
-#  "quiet autoinstall ds=nocloud-net\\;s=http://{{ .HTTPIP }}:{{ .HTTPPort }}/ubuntu/ ---<wait>",
-#  "<f10><wait>"
-#]
-#boot_command            = ["c<wait>set gfxpayload=keep<enter><wait>linux /casper/vmlinuz quiet autoinstall ds=nocloud-net;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/ ---<enter><wait>initrd /casper/initrd<wait><enter><wait>boot<enter><wait>"]
-#boot_command_hyperv     = ["c<wait>set gfxpayload=keep<enter><wait>linux /casper/vmlinuz quiet autoinstall ds=nocloud-net;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/ ---<enter><wait>initrd /casper/initrd<wait><enter><wait>boot<enter><wait>"]
+boot_command_hyperv     = ["c<wait>set gfxpayload=keep<enter><wait>linux /casper/vmlinuz quiet autoinstall ds=nocloud-net\\;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/ ---<enter><wait>initrd /casper/initrd<wait><enter><wait>boot<enter><wait>"]

os_pkrvars/ubuntu/ubuntu-22.10-x86_64.pkrvars.hcl

@@ -7,5 +7,5 @@ hyperv_generation       = 2
 parallels_guest_os_type = "ubuntu"
 vbox_guest_os_type      = "Ubuntu_64"
 vmware_guest_os_type    = "ubuntu-64"
-boot_command            = ["c<wait>set gfxpayload=keep<enter><wait>linux /casper/vmlinuz quiet autoinstall ds=nocloud-net\\;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/ ---<enter><wait>initrd /casper/initrd<wait><enter><wait>boot<enter><wait>"]
-boot_command_hyperv     = ["c<wait>set gfxpayload=keep<enter><wait>linux /casper/vmlinuz quiet autoinstall ds=nocloud-net\\;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/ ---<enter><wait>initrd /casper/initrd<wait><enter><wait>boot<enter><wait>"]
+boot_command            = ["<wait>e<wait><down><down><down><end> autoinstall ds=nocloud-net\\;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/<wait><f10><wait>"]
+boot_command_hyperv     = ["<wait>e<wait><down><down><down><end> autoinstall ds=nocloud-net\\;s=http://{{.HTTPIP}}:{{.HTTPPort}}/ubuntu/<wait><f10><wait>"]

packer_templates/http/ubuntu/user-data

@@ -7,7 +7,8 @@ autoinstall:
     password: '$6$rounds=4096$5CU3LEj/MQvbkfPb$LmKEF9pCfU8R.dA.GemgE/8GT6r9blge3grJvdsVTMFKyLEQwzEF3SGWqAzjawY/XHRpWj4fOiLBrRyxJhIRJ1'
   early-commands:
     # otherwise packer tries to connect and exceed max attempts:
-    - systemctl stop ssh
+    - systemctl stop ssh.service
+    - systemctl stop ssh.socket
   ssh:
     install-server: yes
     allow-pw: yes

packer_templates/pkr-sources.pkr.hcl

@@ -40,7 +40,7 @@ locals {
         ["-m", "${local.memory}"],
         ["-smp", "2"],
         ["-drive", "file=~/virtio-win.iso,media=cdrom,index=3"],
-        ["-drive", "file=${path.root}/../builds/packer-${var.os_name}-${var.os_version}-qemu/{{ .Name }},if=virtio,cache=writeback,discard=ignore,format=qcow2,index=1"],
+        ["-drive", "file=${path.root}/../builds/packer-${var.os_name}-${var.os_version}-${var.os_arch}-qemu/{{ .Name }},if=virtio,cache=writeback,discard=ignore,format=qcow2,index=1"],
         ["-display", "none"]
         ] : [
         ["-m", "${local.memory}"],
@@ -102,7 +102,7 @@ locals {
   ) : var.floppy_files
   http_directory   = var.http_directory == null ? "${path.root}/http" : var.http_directory
   memory           = var.memory == null ? (var.is_windows ? 4096 : 2048) : var.memory
-  output_directory = var.output_directory == null ? "${path.root}/../builds/packer-${var.os_name}-${var.os_version}-${var.os_arch}-${source.type}" : var.output_directory
+  output_directory = var.output_directory == null ? "builds/packer-${var.os_name}-${var.os_version}-${var.os_arch}" : var.output_directory
   shutdown_command = var.shutdown_command == null ? (
     var.is_windows ? "shutdown /s /t 10 /f /d p:4:1 /c \"Packer Shutdown\"" : (
       var.os_name == "freebsd" ? "echo 'vagrant' | su -m root -c 'shutdown -p now'" : "echo 'vagrant' | sudo -S /sbin/halt -h -p"
@@ -131,7 +131,7 @@ source "hyperv-iso" "vm" {
   iso_checksum          = var.iso_checksum
   iso_url               = var.iso_url
   memory                = local.memory
-  output_directory      = local.output_directory
+  output_directory      = "${local.output_directory}-hyperv"
   shutdown_command      = local.shutdown_command
   shutdown_timeout      = var.shutdown_timeout
   ssh_password          = var.ssh_password
@@ -159,7 +159,7 @@ source "parallels-iso" "vm" {
   iso_checksum           = var.iso_checksum
   iso_url                = var.iso_url
   memory                 = local.memory
-  output_directory       = local.output_directory
+  output_directory       = "${local.output_directory}-parallels"
   shutdown_command       = local.shutdown_command
   shutdown_timeout       = var.shutdown_timeout
   ssh_password           = var.ssh_password
@@ -186,7 +186,7 @@ source "qemu" "vm" {
   iso_checksum     = var.iso_checksum
   iso_url          = var.iso_url
   memory           = local.memory
-  output_directory = local.output_directory
+  output_directory = "${local.output_directory}-qemu"
   shutdown_command = local.shutdown_command
   shutdown_timeout = var.shutdown_timeout
   ssh_password     = var.ssh_password
@@ -220,7 +220,7 @@ source "virtualbox-iso" "vm" {
   iso_checksum              = var.iso_checksum
   iso_url                   = var.iso_url
   memory                    = local.memory
-  output_directory          = local.output_directory
+  output_directory          = "${local.output_directory}-virtualbox"
   shutdown_command          = local.shutdown_command
   shutdown_timeout          = var.shutdown_timeout
   ssh_password              = var.ssh_password
@@ -239,7 +239,7 @@ source "virtualbox-ovf" "amazonlinux" {
   virtualbox_version_file = var.virtualbox_version_file
   communicator            = local.communicator
   headless                = var.headless
-  output_directory        = local.output_directory
+  output_directory        = "${local.output_directory}-virtualbox-ovf"
   shutdown_command        = local.shutdown_command
   shutdown_timeout        = var.shutdown_timeout
   ssh_password            = var.ssh_password
@@ -267,7 +267,7 @@ source "vmware-iso" "vm" {
   iso_checksum                   = var.iso_checksum
   iso_url                        = var.iso_url
   memory                         = local.memory
-  output_directory               = local.output_directory
+  output_directory               = "${local.output_directory}-vmware"
   shutdown_command               = local.shutdown_command
   shutdown_timeout               = var.shutdown_timeout
   ssh_password                   = var.ssh_password

packer_templates/pkr-variables.pkr.hcl

@@ -102,7 +102,7 @@ variable "parallels_prlctl_version_file" {
 # qemu
 variable "qemu_accelerator" {
   type    = string
-  default = "kvm"
+  default = null
 }
 variable "qemuargs" {
   type    = list(list(string))

packer_templates/vagrantfile-windows-gen2.template

@@ -19,4 +19,4 @@ Vagrant.configure("2") do |config|
         hv.cpus = 2
         hv.enable_virtualization_extensions = true
     end
-end
+end