diff --git a/docs/Installing.asciidoc b/docs/Installing.asciidoc index a64038421cc..ebc2b9200c6 100644 --- a/docs/Installing.asciidoc +++ b/docs/Installing.asciidoc @@ -1049,6 +1049,27 @@ comment = openQA test distributions systemctl enable --now rsyncd -------------------------------------------------------------------------------- +=== Alternative caching implementations + +Caching described above works well for a single worker host, but in case of +several hosts in a single site (that is remote from the main openQA webui +instance) it results in downloading the same assets several times. In +such case, one can setup local cache on their own (without using +openqa-worker-cacheservice service) and share it with workers using +some network filesystem (see <> +section above). +Such setups can use `SYNC_ASSETS_HOOK` in `/etc/openqa/workers.ini` to ensure +the cache is up to date before starting the job (or resuming test in developer +mode). The setting takes a shell command that is executed just before +evaluating assets. It is up to the system administrator to decide what it +should do, but there are few suggestions: + +* Call rsync, possibly via ssh on the cache host +* Wait for a lock file signaling that cache download is in progress to disappear + +If the command exits with code 32, re-downloading needles in developer mode +will be skipped. + === Enable linking files referred by job settings Specific job settings might refer to files within the test distribution. diff --git a/lib/OpenQA/Worker/Engines/isotovideo.pm b/lib/OpenQA/Worker/Engines/isotovideo.pm index 035164ed7de..4dc21ea228d 100644 --- a/lib/OpenQA/Worker/Engines/isotovideo.pm +++ b/lib/OpenQA/Worker/Engines/isotovideo.pm @@ -326,6 +326,12 @@ sub engine_workit ($job, $callback) { log_debug "Job settings:\n" . format_settings(\%vars); + # start pre-job hook if any + if ($job_settings->{SYNC_ASSETS_HOOK}) { + log_debug 'Running SYNC_ASSETS_HOOK'; + system($job_settings->{SYNC_ASSETS_HOOK}); + } + # cache/locate assets, set ASSETDIR my $assetkeys = detect_asset_keys(\%vars); if (my $cache_dir = $global_settings->{CACHEDIRECTORY}) { diff --git a/t/24-worker-engine.t b/t/24-worker-engine.t index 78b9a90eaaf..393cc7d40be 100644 --- a/t/24-worker-engine.t +++ b/t/24-worker-engine.t @@ -27,6 +27,7 @@ use OpenQA::Test::FakeWorker; use Mojo::File qw(path tempdir); use Mojo::JSON 'decode_json'; use OpenQA::Utils qw(testcasedir productdir needledir locate_asset base_host); +use Cwd qw(getcwd); # define fake packages for testing asset caching { @@ -511,14 +512,18 @@ subtest 'behavior with ABSOLUTE_TEST_CONFIG_PATHS=1' => sub { }; subtest 'link asset' => sub { + my $cwd = getcwd; my $pool_directory = tempdir('poolXXXX'); my $worker = OpenQA::Test::FakeWorker->new(pool_directory => $pool_directory); my $client = Test::FakeClient->new; + # just in case cleanup the symlink to really check if it gets re-created + unlink 't/data/openqa/share/factory/hdd/symlink.qcow2' if -e 't/data/openqa/share/factory/hdd/symlink.qcow2'; my $settings = { JOBTOKEN => 'token000', ISO => 'openSUSE-13.1-DVD-x86_64-Build0091-Media.iso', HDD_1 => 'foo.qcow2', HDD_2 => 'symlink.qcow2', + SYNC_ASSETS_HOOK => "ln -s foo.qcow2 $cwd/t/data/openqa/share/factory/hdd/symlink.qcow2" }; my $job = OpenQA::Worker::Job->new($worker, $client, {id => 16, settings => $settings}); combined_like { my $result = _run_engine($job) } @@ -537,6 +542,7 @@ subtest 'link asset' => sub { is $vars_data->{ISO}, 'openSUSE-13.1-DVD-x86_64-Build0091-Media.iso', 'the value of ISO is basename when doing link'; is $vars_data->{HDD_1}, 'foo.qcow2', 'the value of HDD_1 is basename when doing link'; + unlink 't/data/openqa/share/factory/hdd/symlink.qcow2'; }; subtest 'using cgroupv2' => sub { diff --git a/t/data/openqa/share/factory/hdd/symlink.qcow2 b/t/data/openqa/share/factory/hdd/symlink.qcow2 deleted file mode 120000 index e54be38efb3..00000000000 --- a/t/data/openqa/share/factory/hdd/symlink.qcow2 +++ /dev/null @@ -1 +0,0 @@ -foo.qcow2 \ No newline at end of file