From 294ca161ce8aa199a942a8c625af704796f4db22 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 14 Jul 2022 14:47:43 -0400 Subject: [PATCH 1/5] CI: pin the versions of tp-qemu and tp-libvirt Up until now, the bootstrap process run on CI will fetch the HEAD of the repository URIs given. In testing, it's a good practice to only change one component at a time. By changing only one component at time, we are able to quickly point at compatibility regressions (say a tp-qemu test needs something that is being dropped or changing behavior in Avocado-VT), or, be required to update the reference on the test provider to a version that can make use of a new or improved Avocado-VT API. In order to expand the integration coverage of the test providers and Avocado-VT, it's my understanding that this locking of versions is needed so that maintainers can keep their sanity. Signed-off-by: Cleber Rosa --- .cirrus.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index f8224d3ccd..9c725aaafc 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -28,6 +28,11 @@ fedora_35_task: - test -z $AVOCADO_SRC || python3 -m pip install $AVOCADO_SRC - python3 $SETUP bootstrap_script: &bootstrap_scr + # Pin the versions of tp-qemu and tp-libvirt to known working versions. + # These versions can be bumped at any time, but provide better understanding + # of regressions or version update requirements on the test providers side. + - python3 -c "from virttest.asset import ConfigLoader; cl = ConfigLoader('virttest/test-providers.d/io-github-autotest-qemu.ini'); cl.set('provider', 'ref', '33ed9b0ac6e4255877ff109a1d91fa84c9097dc5'); cl.save()" + - python3 -c "from virttest.asset import ConfigLoader; cl = ConfigLoader('virttest/test-providers.d/io-github-autotest-libvirt.ini'); cl.set('provider', 'ref', 'e6af7700f879fe1547bfb1421439121d35c919bd'); cl.save()" - python3 -m avocado vt-bootstrap --vt-type=$VT_TYPE --vt-skip-verify-download-assets --yes-to-all list_script: &list_scr - python3 -m avocado list --vt-save-config=/tmp/config --vt-type=$VT_TYPE -- boot | tee /tmp/list From 7cd09e0de7bdd29e3f94c2a9ea7f4c87f39613db Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 14 Jul 2022 14:47:43 -0400 Subject: [PATCH 2/5] CI: increase integration with test providers coverage By expanding the number of tests from test providers that we run, we increase the level of confidence that Avocado-VT API changes are not breaking tests. The list of tests added was curated by attempting to add tests that are known to be stable on other testing scenarios, but adapted to the Cirrus CI environment (based on JeOS, different KVM host environment, etc). This list is not set in stone, and can and should be continually curated to represent a better sampling of tests and thus a better integration coverage. Signed-off-by: Cleber Rosa --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 9c725aaafc..549ba304a7 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -44,8 +44,8 @@ fedora_35_task: - python3 -m avocado vt-list-archs --vt-type=$VT_TYPE dry_run_script: &dry_run_scr - PATH=~/.local/bin:$PATH python3 -m avocado --show all run --vt-type=$VT_TYPE --dry-run -- io-github-autotest-qemu.boot - run_boot_script: &run_boot_scr - - test $VT_TYPE == "libvirt" || PATH=$HOME/.local/bin:$PATH python3 -m avocado --show all run --vt-type=$VT_TYPE --vt-extra-params nettype=user -- io-github-autotest-qemu.boot + run_tests_script: &run_tests_scr + - test $VT_TYPE == "libvirt" || PATH=$HOME/.local/bin:$PATH python3 -m avocado --show all run --vt-type=$VT_TYPE --vt-extra-params nettype=user -- io-github-autotest-qemu.boot io-github-autotest-qemu.qemu_disk_img.convert.base_to_raw io-github-autotest-qemu.commit_with_backing.default centos_8_1_task: container: From d5741fa0e0b42f0f58bbe02780a6a50b16ba887a Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 14 Jul 2022 14:47:43 -0400 Subject: [PATCH 3/5] CI: include the path of avocado-runner-avocado-vt on list This is a workaround for the fact that current Avocado will look for the runner script while doing resolution (creating Runnables) because it will populate the Runnable configuration with the "used configuration" for that kind of runner. If a change in Avocado will be needed is TBD, but this takes care of the broken resolution in the CI environment. Reference: https://github.com/avocado-framework/avocado-vt/issues/3457 Signed-off-by: Cleber Rosa --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 549ba304a7..efc9e84300 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -35,10 +35,10 @@ fedora_35_task: - python3 -c "from virttest.asset import ConfigLoader; cl = ConfigLoader('virttest/test-providers.d/io-github-autotest-libvirt.ini'); cl.set('provider', 'ref', 'e6af7700f879fe1547bfb1421439121d35c919bd'); cl.save()" - python3 -m avocado vt-bootstrap --vt-type=$VT_TYPE --vt-skip-verify-download-assets --yes-to-all list_script: &list_scr - - python3 -m avocado list --vt-save-config=/tmp/config --vt-type=$VT_TYPE -- boot | tee /tmp/list + - PATH=~/.local/bin:$PATH python3 -m avocado list --vt-save-config=/tmp/config --vt-type=$VT_TYPE -- boot | tee /tmp/list - test $VT_TYPE == "qemu" || test $(wc -l < /tmp/list) -gt 50 - test $VT_TYPE == "libvirt" || test $(wc -l < /tmp/list) -eq 1 - - python3 -m avocado list --vt-config=/tmp/config --vt-type=$VT_TYPE -- boot | tee /tmp/list_from_config + - PATH=~/.local/bin:$PATH python3 -m avocado list --vt-config=/tmp/config --vt-type=$VT_TYPE -- boot | tee /tmp/list_from_config - diff /tmp/list /tmp/list_from_config - python3 -m avocado vt-list-guests --vt-type=$VT_TYPE - python3 -m avocado vt-list-archs --vt-type=$VT_TYPE From ea320e1b18705842d2affa89c30ee2d2f22f17a3 Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 14 Jul 2022 14:47:43 -0400 Subject: [PATCH 4/5] CI: increase testing coverage of "avocado devel" tasks By actually running actual tests, and not just a dry-run, we can increase the compatibility coverage between latest Avocado and Avocado-VT. Signed-off-by: Cleber Rosa --- .cirrus.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index efc9e84300..f293c239df 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -98,3 +98,5 @@ avocado_devel_task: *list_scr dry_run_script: *dry_run_scr + run_tests_script: + *run_tests_scr From 53e8f1ea5ad60a830dea4b961e87fb276aeae9fb Mon Sep 17 00:00:00 2001 From: Cleber Rosa Date: Thu, 14 Jul 2022 14:47:43 -0400 Subject: [PATCH 5/5] CI: remove the waiver of failures with latest Avocado This marks the point where Avocado-VT and Avocado are compatible (to the extent that is being tested here). With this, an Avocado-VT release matching an Avocado release will be produced. In short, from the latest Avocado (tested with here) there will be a release 98.0, and from the latest Avocado-VT commit there will be produced a release 98.0 too. Then both Avocado and Avocado-VT 98.0 will be compatible releases between themselves. There are two ways to deal with the waiver after the releases are made: 1) Reinstante the waiver right away 2) Reinstante the waiver when a compatibility with the post Avocado 98.0 release (the development version of Avocado) is found. I'd suggest going with 2, and creating issues to track the compatibility problems found and that need to be addressed before the next compatible release. Signed-off-by: Cleber Rosa --- .cirrus.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index f293c239df..8db3aef335 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -72,7 +72,6 @@ avocado_devel_task: container: image: quay.io/avocado-framework/avocado-vt-ci-fedora-35 kvm: true - allow_failures: $AVOCADO_SRC == 'git+https://github.com/avocado-framework/avocado#egg=avocado_framework' env: matrix: # Latest Avocado