From 3608db725fb76090a934d06c7e923ae20fe9b936 Mon Sep 17 00:00:00 2001 From: Sebastian Riedel Date: Tue, 1 Oct 2024 15:30:15 +0200 Subject: [PATCH] Add embargo feature for IBS --- assets/vue/ClassifySnippets.vue | 7 +- assets/vue/ReportMetadata.vue | 10 + lib/Cavil/Command/learn.pm | 11 +- lib/Cavil/Controller/Queue.pm | 34 ++- lib/Cavil/Controller/Reviewer.pm | 1 + lib/Cavil/Controller/Snippet.pm | 2 +- lib/Cavil/FileIndexer.pm | 3 +- lib/Cavil/Model/Packages.pm | 15 +- lib/Cavil/Model/Reports.pm | 2 +- lib/Cavil/Model/Snippets.pm | 34 ++- lib/Cavil/OBS.pm | 16 ++ lib/Cavil/Task/Import.pm | 18 +- lib/Cavil/Util.pm | 8 +- migrations/cavil.sql | 7 +- staging/start.pl | 3 +- t/command_learn.t | 80 ++++++ t/embargo.t | 241 ++++++++++++++++++ .../harbor.changes | 1 + t/lib/Cavil/Test.pm | 8 + t/obs.t | 168 ++++++++++++ t/snippet.t | 2 +- t/util.t | 14 +- 22 files changed, 639 insertions(+), 46 deletions(-) create mode 100644 t/embargo.t diff --git a/assets/vue/ClassifySnippets.vue b/assets/vue/ClassifySnippets.vue index 040a077a21..4bf92db035 100644 --- a/assets/vue/ClassifySnippets.vue +++ b/assets/vue/ClassifySnippets.vue @@ -101,8 +101,9 @@ Unknown file , and 1 other file , and {{ snippet.files }} other files -
- +
+ +
@@ -214,7 +215,7 @@ export default { for (const snippet of snippets) { snippet.buttonPressed = null; - snippet.fileUrl = `/reviews/file_view/${snippet.package}/${snippet.filename}`; + snippet.fileUrl = `/reviews/file_view/${snippet.filepackage}/${snippet.filename}`; snippet.editUrl = `/snippet/edit/${snippet.id}`; let num = snippet.sline ?? 1; const lines = []; diff --git a/assets/vue/ReportMetadata.vue b/assets/vue/ReportMetadata.vue index 2bed2fbc39..12fd282ce3 100644 --- a/assets/vue/ReportMetadata.vue +++ b/assets/vue/ReportMetadata.vue @@ -35,6 +35,14 @@ (not SPDX) + + + + + Embargoed: + Yes + No + @@ -348,6 +356,7 @@ export default { history: [], notice: null, pkgChecksum: null, + pkgEmbargoed: false, pkgFiles: [], pkgLicense: null, pkgName: null, @@ -410,6 +419,7 @@ export default { this.pkgType = data.package_type; this.pkgUrl = data.package_url; this.pkgVersion = data.package_version; + this.pkgEmbargoed = data.embargoed; this.pkgChecksum = data.package_checksum; this.checkoutUrl = `/reviews/file_view/${this.pkgId}`; diff --git a/lib/Cavil/Command/learn.pm b/lib/Cavil/Command/learn.pm index 98fc21eeea..497f2be38d 100644 --- a/lib/Cavil/Command/learn.pm +++ b/lib/Cavil/Command/learn.pm @@ -120,14 +120,19 @@ sub _output_snippets ($self, $good, $bad) { my $count = my $last_id = 0; while (1) { - my $batch - = $db->query('SELECT * FROM snippets WHERE approved = true AND id > ? ORDER BY id ASC LIMIT 100', $last_id); + my $batch = $db->query( + 'SELECT s.*, bp.embargoed FROM snippets s LEFT JOIN bot_packages bp ON (bp.id = s.package) + WHERE approved = true AND s.id > ? ORDER BY s.id ASC LIMIT 100', $last_id + ); last if $batch->rows == 0; for my $hash ($batch->hashes->each) { - $count++; my $id = $hash->{id}; $last_id = $id if $id > $last_id; + + next if $hash->{embargoed}; + + $count++; my $dir = $hash->{license} ? $good : $bad; my $file = $dir->child("$hash->{hash}.txt"); next unless _spew($file, $hash->{text}); diff --git a/lib/Cavil/Controller/Queue.pm b/lib/Cavil/Controller/Queue.pm index b894d914d3..574237106b 100644 --- a/lib/Cavil/Controller/Queue.pm +++ b/lib/Cavil/Controller/Queue.pm @@ -78,20 +78,26 @@ sub create_package ($self) { $obj->{obsolete} = 0; $pkgs->update($obj); - $pkgs->obs_import( - $obj->{id}, - { - api => $api, - project => $project, - pkg => $pkg, - srcpkg => $srcpkg, - rev => $rev, - srcmd5 => $srcmd5, - verifymd5 => $verifymd5, - priority => $prio - }, - $prio + 10 - ) if $create; + if ($create) { + $pkgs->obs_import( + $obj->{id}, + { + api => $api, + project => $project, + pkg => $pkg, + srcpkg => $srcpkg, + rev => $rev, + srcmd5 => $srcmd5, + verifymd5 => $verifymd5, + external_link => $obj->{external_link}, + priority => $prio + }, + $prio + 10 + ); + } + else { + $pkgs->obs_embargo($obj->{id}, {api => $api, external_link => $obj->{external_link}}); + } $self->render(json => {saved => $obj}); } diff --git a/lib/Cavil/Controller/Reviewer.pm b/lib/Cavil/Controller/Reviewer.pm index 7818f26608..87a956a211 100644 --- a/lib/Cavil/Controller/Reviewer.pm +++ b/lib/Cavil/Controller/Reviewer.pm @@ -114,6 +114,7 @@ sub meta ($self) { actions => $actions, copied_files => {'%doc' => [sort keys %docs], '%license' => [sort keys %lics]}, created => $pkg->{created_epoch}, + embargoed => \!!$pkg->{embargoed}, errors => $spec->{errors} // [], external_link => $pkg->{external_link}, has_spdx_report => \!!$has_spdx_report, diff --git a/lib/Cavil/Controller/Snippet.pm b/lib/Cavil/Controller/Snippet.pm index dfc017946a..93afc19e04 100644 --- a/lib/Cavil/Controller/Snippet.pm +++ b/lib/Cavil/Controller/Snippet.pm @@ -148,7 +148,7 @@ sub list_meta ($self) { my $snippets = $unclassified->{snippets}; for my $snippet (@$snippets) { - $snippet->{$_} = $snippet->{$_} ? true : false for qw(license classified approved); + $snippet->{$_} = $snippet->{$_} ? true : false for qw(embargoed license classified approved); } $self->render(json => {snippets => $snippets, total => $unclassified->{total}}); diff --git a/lib/Cavil/FileIndexer.pm b/lib/Cavil/FileIndexer.pm index 5be4f6c65d..63abb2efbd 100644 --- a/lib/Cavil/FileIndexer.pm +++ b/lib/Cavil/FileIndexer.pm @@ -173,7 +173,8 @@ sub _snippet ($self, $file_id, $matches, $path, $first_line, $last_line) { return; } - my $snippet = $self->{snippets}->{$hash} ||= $self->{app}->snippets->find_or_create($hash, $text); + my $snippet = $self->{snippets}->{$hash} + ||= $self->{app}->snippets->find_or_create({hash => $hash, text => $text, package => $self->{package}}); $self->{db}->insert('file_snippets', {package => $self->{package}, snippet => $snippet, sline => $first_line, eline => $last_line, file => $file_id}); diff --git a/lib/Cavil/Model/Packages.pm b/lib/Cavil/Model/Packages.pm index c35c631589..40b952205e 100644 --- a/lib/Cavil/Model/Packages.pm +++ b/lib/Cavil/Model/Packages.pm @@ -36,7 +36,8 @@ sub add ($self, %args) { source => $source_id, requesting_user => $args{requesting_user}, priority => $args{priority}, - state => 'new' + state => 'new', + embargoed => $args{embargoed} ? 1 : 0 }; return $db->insert('bot_packages', $pkg, {returning => 'id'})->hash->{id}; } @@ -393,6 +394,16 @@ sub name_suggestions ($self, $partial) { )->arrays->flatten->to_array; } +sub obs_embargo ($self, $id, $data, $priority = 5) { + my $pkg = $self->find($id); + return $self->minion->enqueue( + obs_embargo => [$id, $data] => { + priority => $priority, + notes => {external_link => $pkg->{external_link}, package => $pkg->{name}, "pkg_$id" => 1} + } + ); +} + sub obs_import ($self, $id, $data, $priority = 5) { my $pkg = $self->find($id); return $self->minion->enqueue( @@ -481,7 +492,7 @@ sub unpacked ($self, $id) { sub update ($self, $pkg) { my %updates = map { exists $pkg->{$_} ? ($_ => $pkg->{$_}) : () } - (qw(created checksum priority state obsolete result notice reviewed reviewing_user external_link)); + (qw(created checksum priority state obsolete result notice reviewed reviewing_user external_link embargoed)); $updates{reviewed} = \'now()' if $pkg->{review_timestamp}; return $self->pg->db->update('bot_packages', \%updates, {id => $pkg->{id}}); } diff --git a/lib/Cavil/Model/Reports.pm b/lib/Cavil/Model/Reports.pm index 417d80d57b..40b1ad5195 100644 --- a/lib/Cavil/Model/Reports.pm +++ b/lib/Cavil/Model/Reports.pm @@ -295,7 +295,7 @@ sub _dig_report { } my $matches = $db->select('pattern_matches', [qw(id file pattern sline eline)], $query); - $query = {package => $pkg->{id}}; + $query = {'file_snippets.package' => $pkg->{id}}; if ($limit_to_file) { $query->{file} = $limit_to_file; } diff --git a/lib/Cavil/Model/Snippets.pm b/lib/Cavil/Model/Snippets.pm index f9672f18f7..ca8183c725 100644 --- a/lib/Cavil/Model/Snippets.pm +++ b/lib/Cavil/Model/Snippets.pm @@ -31,17 +31,25 @@ sub find ($self, $id) { return $self->pg->db->select('snippets', '*', {id => $id})->hash; } -sub find_or_create ($self, $hash, $text, $prefix = '') { +sub find_or_create ($self, $new) { + $new->{prefix} //= ''; my $db = $self->pg->db; - my $snip = $db->select('snippets', 'id', {hash => $hash})->hash; - return $snip->{id} if $snip; + my $old = $db->query( + 'SELECT s.id, bp.embargoed FROM snippets s LEFT JOIN bot_packages bp ON (bp.id = s.package) + WHERE hash = ?', $new->{hash} + )->hash; + + # Inherit embargo status until there is no embargo anymore (the value will tell us which package lifted the embargo) + if ($old) { + $db->query('UPDATE snippets SET package = ? WHERE id = ?', $new->{package}, $old->{id}) if $old->{embargoed}; + return $old->{id}; + } - $db->query( - 'insert into snippets (hash, text) values (?, ?) - on conflict do nothing', "$prefix$hash", $text - ); - return $db->select('snippets', 'id', {hash => "$prefix$hash"})->hash->{id}; + my $hash = "$new->{prefix}$new->{hash}"; + $db->query('INSERT INTO snippets (hash, text, package) VALUES (?, ?, ?) ON CONFLICT DO NOTHING', + $hash, $new->{text}, $new->{package}); + return $db->select('snippets', 'id', {hash => $hash})->hash->{id}; } sub from_file ($self, $file_id, $first_line, $last_line) { @@ -53,7 +61,8 @@ sub from_file ($self, $file_id, $first_line, $last_line) { my $path = path($self->checkout_dir, $package->{name}, $package->{checkout_dir}, '.unpacked', $file->{filename}); my ($text, $hash) = file_and_checksum($path, $first_line, $last_line); - my $snippet_id = $self->find_or_create($hash, $text, 'manual:'); + my $snippet_id + = $self->find_or_create({hash => $hash, text => $text, package => $package->{id}, prefix => 'manual:'}); $db->insert('file_snippets', {package => $package->{id}, snippet => $snippet_id, sline => $first_line, eline => $last_line, file => $file_id}); @@ -97,7 +106,8 @@ sub unclassified ($self, $options) { } my $snippets = $db->query( - "SELECT *, COUNT(*) OVER() AS total FROM snippets + "SELECT s.*, bp.embargoed, COUNT(*) OVER() AS total + FROM snippets s LEFT JOIN bot_packages bp ON (bp.id = s.package) WHERE $is_approved AND $is_classified $before $legal $confidence $timeframe ORDER BY id DESC LIMIT 10" )->hashes; @@ -106,13 +116,13 @@ sub unclassified ($self, $options) { $total = delete $snippet->{total}; $snippet->{likelyness} = int($snippet->{likelyness} * 100); my $files = $db->query( - 'SELECT fs.sline, mf.filename, mf.package + 'SELECT fs.sline, mf.filename, mf.package AS filepackage FROM file_snippets fs JOIN matched_files mf ON (fs.file = mf.id) WHERE fs.snippet = ? ORDER BY fs.id DESC LIMIT 1', $snippet->{id} )->hashes; $snippet->{files} = $files->size; my $file = $files->[0] || {}; - $snippet->{$_} = $file->{$_} for qw(filename sline package); + $snippet->{$_} = $file->{$_} for qw(filename sline filepackage); my $license = $db->query('SELECT license, risk FROM license_patterns WHERE id = ? AND license != ?', $snippet->{like_pattern} // 0, '')->hash // {}; diff --git a/lib/Cavil/OBS.pm b/lib/Cavil/OBS.pm index 7e4a94b6b6..82634573a9 100644 --- a/lib/Cavil/OBS.pm +++ b/lib/Cavil/OBS.pm @@ -41,6 +41,22 @@ has ua => sub { }; has user => sub { die 'Missing ssh user' }; +sub check_for_embargo ($self, $api, $request) { + my $url = _url($api, 'public', 'request', $request); + my $res = $self->_get($url); + croak "$url: " . $res->code unless $res->is_success; + + for my $project ($res->dom->find('action [project]')->map('attr', 'project')->uniq->each) { + my $url = _url($api, 'public', 'source', $project, '_attribute'); + my $res = $self->_get($url); + next if $res->code == 404; + croak "$url: " . $res->code unless $res->is_success; + return 1 if $res->dom->at('attributes attribute[name=EmbargoDate]'); + } + + return 0; +} + sub download_source ($self, $api, $project, $pkg, $dir, $options = {}) { $dir = path($dir)->make_path; diff --git a/lib/Cavil/Task/Import.pm b/lib/Cavil/Task/Import.pm index 52d12f8cbf..13adae9d03 100644 --- a/lib/Cavil/Task/Import.pm +++ b/lib/Cavil/Task/Import.pm @@ -17,10 +17,21 @@ package Cavil::Task::Import; use Mojo::Base 'Mojolicious::Plugin', -signatures; use Cavil::Checkout; -use Mojo::File 'path'; +use Mojo::File qw(path); +use Cavil::Util qw(request_id_from_external_link); sub register ($self, $app, $config) { - $app->minion->add_task(obs_import => \&_obs); + $app->minion->add_task(obs_embargo => \&_embargo); + $app->minion->add_task(obs_import => \&_obs); +} + +sub _embargo ($job, $id, $data) { + return unless my $link = $data->{external_link}; + return unless my $request_id = request_id_from_external_link($link); + + my $app = $job->app; + my $embargoed = $app->obs->check_for_embargo($data->{api}, $request_id); + $app->packages->update({id => $id, embargoed => $embargoed}); } sub _obs ($job, $id, $data) { @@ -33,6 +44,9 @@ sub _obs ($job, $id, $data) { return $job->finish("Package $id is already being processed") unless my $guard = $minion->guard("processing_pkg_$id", 172800); + # Check embargo status before checkout + _embargo($job, $id, $data); + my $checkout_dir = $app->config->{checkout_dir}; my ($srcpkg, $verifymd5, $api, $project, $pkg, $srcmd5, $priority) = @{$data}{qw(srcpkg verifymd5 api project pkg srcmd5 priority)}; diff --git a/lib/Cavil/Util.pm b/lib/Cavil/Util.pm index ba9f491ecb..2fb6b98aaf 100644 --- a/lib/Cavil/Util.pm +++ b/lib/Cavil/Util.pm @@ -29,7 +29,8 @@ $Text::Glob::strict_wildcard_slash = 0; our @EXPORT_OK = ( qw(buckets file_and_checksum slurp_and_decode load_ignored_files lines_context obs_ssh_auth paginate), - qw(parse_exclude_file pattern_checksum pattern_matches read_lines snippet_checksum ssh_sign) + qw(parse_exclude_file pattern_checksum pattern_matches read_lines request_id_from_external_link snippet_checksum), + qw(ssh_sign) ); my $MAX_FILE_SIZE = 30000; @@ -216,6 +217,11 @@ sub read_lines ($path, $start_line, $end_line) { return $text; } +sub request_id_from_external_link ($link) { + return $1 if $link =~ /^(?:obs|ibs)#(\d+)$/; + return undef; +} + # Based on https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/ sub ssh_sign ($key, $realm, $value) { diff --git a/migrations/cavil.sql b/migrations/cavil.sql index 437884ae94..c45a2817a5 100644 --- a/migrations/cavil.sql +++ b/migrations/cavil.sql @@ -227,6 +227,11 @@ ALTER TABLE ignored_lines ADD COLUMN contributor int REFERENCES bot_users(id); ALTER TABLE pattern_matches ADD COLUMN ignored_line int REFERENCES ignored_lines(id) ON DELETE SET NULL; CREATE INDEX ON pattern_matches(ignored_line); ---23 up +-- 23 up ALTER TYPE bot_state RENAME VALUE 'correct' TO 'acceptable_by_lawyer'; ALTER TABLE bot_packages ADD COLUMN notice text; + +-- 24 up +ALTER TABLE bot_packages ADD COLUMN embargoed boolean DEFAULT false NOT NULL; +CREATE INDEX ON bot_packages(embargoed); +ALTER TABLE snippets ADD COLUMN package int REFERENCES bot_packages(id) ON DELETE SET NULL; diff --git a/staging/start.pl b/staging/start.pl index 07603a5815..77c75b6b3b 100644 --- a/staging/start.pl +++ b/staging/start.pl @@ -145,7 +145,8 @@ project => 'just:a:test', package => 'harbor-helm', srcmd5 => 'abc1c36647a5d356883d490da2140def', - priority => 5 + priority => 5, + embargoed => 1 ); $pkgs->imported($pkg_id); my $harbor = $pkgs->find($pkg_id); diff --git a/t/command_learn.t b/t/command_learn.t index fbb6f4db37..0f0ebeda77 100644 --- a/t/command_learn.t +++ b/t/command_learn.t @@ -153,4 +153,84 @@ subtest 'Snippets added' => sub { }; }; +subtest 'Embargo handling' => sub { + my $dir = $tmp->child('embargo')->make_path; + + my $pkg_id = $app->packages->add( + name => 'some-security-package', + checkout_dir => 'f51a419bea8f272484680fb72e5e1234', + api_url => 'https://api.opensuse.org', + requesting_user => 1, + project => 'openSUSE:Factory', + priority => 3, + package => 'some-security-package', + srcmd5 => 'a51a419bea8f272484680fb72e5e123f', + embargoed => 1 + ); + my $snippets = $app->snippets; + my $snippet_one_id = $snippets->find_or_create( + {hash => 'b51a469b6a8f2624866806b7265e123c', text => 'This is an embargo test', package => $pkg_id}); + my $snippet_two_id = $snippets->find_or_create( + {hash => '751746976a8f7624766807b7265e1237', text => 'This is another embargo test', package => $pkg_id}); + my $snippet_three_id + = $snippets->find_or_create({hash => 'abcd46976a8f7624766807b7265e12cd', text => 'Abandoned unembargoed snippet'}); + $snippets->approve($snippet_one_id, 'true'); + $snippets->approve($snippet_two_id, 'false'); + $snippets->approve($snippet_three_id, 'true'); + + subtest 'Embargoed snippets are not exported' => sub { + my $buffer = ''; + { + open my $handle, '>', \$buffer; + local *STDOUT = $handle; + $app->start('learn', '-o', "$dir"); + } + like $buffer, qr/Exporting snippet 1/, 'first snippet'; + like $buffer, qr/Exporting snippet 2/, 'second snippet'; + like $buffer, qr/Exporting snippet 9/, 'third snippet'; + like $buffer, qr/Exported 3 snippets/, 'two snippets exported'; + + ok !-e $dir->child('bad', '751746976a8f7624766807b7265e1237.txt'), 'embargoed file does not exist'; + ok !-e $dir->child('good', 'b51a469b6a8f2624866806b7265e123c.txt'), 'embargoed file does not exist'; + + my $bad = $dir->child('bad')->list; + is $bad->size, 1, 'one file'; + like $bad->first->slurp, qr/Fixed copyright notice/, 'right content'; + my $good = $dir->child('good')->list; + is $good->size, 2, 'two files'; + like $good->first->slurp, qr/Copyright Holder/, 'right content'; + like $good->last->slurp, qr/Abandoned unembargoed snippet/, 'right content'; + }; + + subtest 'Snippets are exported once the embargo status has been lifted' => sub { + $snippets->find_or_create( + {hash => 'b51a469b6a8f2624866806b7265e123c', text => 'This is an embargo test', package => 1}); + $snippets->find_or_create( + {hash => '751746976a8f7624766807b7265e1237', text => 'This is another embargo test', package => 1}); + + + my $buffer = ''; + { + open my $handle, '>', \$buffer; + local *STDOUT = $handle; + $app->start('learn', '-o', "$dir"); + } + like $buffer, qr/Exporting snippet 7/, 'first unembargoed snippet'; + like $buffer, qr/Exporting snippet 8/, 'second unembargoed snippet'; + like $buffer, qr/Exported 5 snippets/, 'five snippets exported'; + + ok -e $dir->child('bad', '751746976a8f7624766807b7265e1237.txt'), 'unembargoed file does exist'; + ok -e $dir->child('good', 'b51a469b6a8f2624866806b7265e123c.txt'), 'unembargoed file does exist'; + + my $bad = $dir->child('bad')->list; + is $bad->size, 2, 'two files'; + my $good = $dir->child('good')->list; + is $good->size, 3, 'three files'; + like $dir->child('bad', '751746976a8f7624766807b7265e1237.txt')->slurp, qr/This is another embargo test/, + 'right content'; + like $dir->child('good', 'b51a469b6a8f2624866806b7265e123c.txt')->slurp, qr/This is an embargo test/, + 'right content'; + }; +}; + done_testing(); diff --git a/t/embargo.t b/t/embargo.t new file mode 100644 index 0000000000..e46ad2e891 --- /dev/null +++ b/t/embargo.t @@ -0,0 +1,241 @@ +# Copyright (C) 2024 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +use Mojo::Base -strict; + +use FindBin; +use lib "$FindBin::Bin/lib"; + +use Test::More; +use Test::Mojo; +use Cavil::OBS; +use Cavil::Test; +use Mojolicious::Lite; +use Mojo::File qw(path); + +plan skip_all => 'set TEST_ONLINE to enable this test' unless $ENV{TEST_ONLINE}; + +app->log->level('error'); + +app->routes->add_condition( + query => sub { + my ($route, $c, $captures, $hash) = @_; + + for my $key (keys %$hash) { + my $values = ref $hash->{$key} ? $hash->{$key} : [$hash->{$key}]; + my $param = $c->req->url->query->param($key); + return undef unless defined $param && grep { $param eq $_ } @$values; + } + + return 1; + } +); + +get '/public/source/:project/perl-Mojolicious.SUSE_SLE-15-SP2_Update' => [project => ['SUSE:Maintenance:4321']] => + (query => {view => 'info'}) => {text => <<'EOF'}; + + perl-Mojolicious.spec + +EOF + +get '/public/source/:project/perl-Mojolicious.SUSE_SLE-15-SP2_Update' => [project => ['SUSE:Maintenance:4321']] => + (query => {expand => 1}) => {text => <<'EOF'}; + + + + + + +EOF + +get '/public/source/:project/perl-Mojolicious.SUSE_SLE-15-SP2_Update/_meta' => + [project => ['SUSE:Maintenance:4321']] => {text => <<'EOF'}; + + Job Queue + Test package + + http://search.cpan.org/dist/Minion + +EOF + +my @files = qw(Mojolicious-7.25.tar.gz perl-Mojolicious.changes perl-Mojolicious.spec); +get("/public/source/:project/perl-Mojolicious.SUSE_SLE-15-SP2_Update//$_" => [project => ['SUSE:Maintenance:4321']] => + {data => path(__FILE__)->sibling('legal-bot', 'perl-Mojolicious', 'c7cfdab0e71b0bebfdf8b2dc3badfecd', $_)->slurp}) + for @files; + +get '/public/request/4321' => {text => <<'EOF'}; + + + + + + requesting release + +EOF + +get '/public/source/:project/_attribute' => [project => 'SUSE:Maintenance:4321'] => {text => <<'EOF'}; + + + 2024-03-27 07:00 UTC + + +EOF + +get '/*whatever' => {whatever => ''} => {text => '', status => 404}; + +my $cavil_test = Cavil::Test->new(online => $ENV{TEST_ONLINE}, schema => 'manual_review_test'); +my $t = Test::Mojo->new(Cavil => $cavil_test->default_config); +$cavil_test->embargo_fixtures($t->app); +my $dir = $cavil_test->checkout_dir; + +# Connect mock web service +my $mock_app = app; +my $api = 'http://127.0.0.1:' . $t->app->obs->ua->server->app($mock_app)->url->port; + +subtest 'Embargoed package does not existy yet' => sub { + $t->get_ok('/package/3' => {Authorization => 'Token test_token'})->status_is(404)->content_like(qr/No such package/); +}; + +subtest 'Existing packages are not embargoed' => sub { + $t->get_ok('/package/1' => {Authorization => 'Token test_token'})->status_is(200)->json_is('/embargoed', 0); + $t->get_ok('/package/2' => {Authorization => 'Token test_token'})->status_is(200)->json_is('/embargoed', 0); +}; + +subtest 'Embargoed packages' => sub { + my $form = { + api => $api, + package => 'perl-Mojolicious.SUSE_SLE-15-SP2_Update', + project => 'SUSE:Maintenance:4321', + external_link => 'ibs#4321' + }; + $t->app->minion->on( + worker => sub { + my ($minion, $worker) = @_; + $worker->on( + dequeue => sub { + my ($worker, $job) = @_; + $job->on( + start => sub { + my $job = shift; + my $task = $job->task; + return unless $task eq 'obs_import' || $task eq 'obs_embargo'; + $job->app->obs(Cavil::OBS->new); + my $api = 'http://127.0.0.1:' . $job->app->obs->ua->server->app($mock_app)->url->port; + $job->args->[1]{api} = $api; + } + ); + } + ); + } + ); + + subtest 'Create package with embargo (detected via OBS API)' => sub { + $t->post_ok('/packages' => {Authorization => 'Token test_token'} => form => $form)->status_is(200) + ->json_is('/saved/checkout_dir', '236d7b56886a0d2799c0d114eddbb7ff')->json_is('/saved/id', 3); + $t->get_ok('/package/3/report' => {Authorization => 'Token test_token'})->status_is(408) + ->content_like(qr/package being processed/); + $t->get_ok('/api/1.0/source' => form => $form)->status_is(200)->json_is('/review' => 3, '/history' => []); + $t->app->minion->perform_jobs; + my $checkout = $dir->child('perl-Mojolicious.SUSE_SLE-15-SP2_Update', '236d7b56886a0d2799c0d114eddbb7ff'); + ok -d $checkout, 'directory exists'; + ok -f $checkout->child('Mojolicious-7.25.tar.gz'), 'file exists'; + ok -f $checkout->child('perl-Mojolicious.changes'), 'file exists'; + ok -f $checkout->child('perl-Mojolicious.spec'), 'file exists'; + ok !-d $checkout->child('Mojolicious'), 'directory does not exist yet'; + }; + + subtest 'Embargoed package has been created' => sub { + $t->get_ok('/package/3' => {Authorization => 'Token test_token'})->status_is(200)->json_is('/state', 'new') + ->json_is('/priority', 5)->json_is('/embargoed', 1)->json_is('/external_link', 'ibs#4321'); + $t->get_ok('/package/3/report' => {Authorization => 'Token test_token'})->status_is(200) + ->content_type_like(qr/application\/json/)->json_is('/package/checkout_dir', '236d7b56886a0d2799c0d114eddbb7ff') + ->json_has('/report/risks'); + $t->get_ok('/source/3' => {Authorization => 'Token test_token'})->status_is(200) + ->content_type_like(qr/application\/json/)->json_has('/source/filename'); + }; + + subtest 'Check embargo status on re-import' => sub { + $t->app->packages->obsolete_if_not_in_product(3); + is $t->app->minion->jobs({tasks => ['obs_import']})->total, 1, 'one import job'; + is $t->app->minion->jobs({tasks => ['obs_embargo']})->total, 0, 'no embargo jobs'; + + $t->post_ok('/packages' => {Authorization => 'Token test_token'} => form => $form)->status_is(200) + ->json_is('/saved/checkout_dir', '236d7b56886a0d2799c0d114eddbb7ff')->json_is('/saved/id', 3); + $t->app->minion->perform_jobs; + $t->post_ok('/packages/import/3' => {Authorization => 'Token test_token'} => form => {state => 'new'}) + ->status_is(200)->json_is('/imported/id', 3)->json_is('/imported/state', 'new'); + $t->get_ok('/package/3' => {Authorization => 'Token test_token'})->status_is(200)->json_is('/state', 'new') + ->json_is('/priority', 5)->json_is('/embargoed', 1)->json_is('/external_link', 'ibs#4321'); + is $t->app->minion->jobs({tasks => ['obs_import']})->total, 1, 'one import job'; + is $t->app->minion->jobs({tasks => ['obs_embargo']})->total, 1, 'one embargo job'; + }; +}; + +subtest 'Embargoed snippets' => sub { + subtest 'Unembargoed snippet does not become embargoed again' => sub { + my $unembargoed_snippet = $t->app->snippets->find_or_create( + {hash => 'manual:236d7b56836a4d2759c061147dd8b7ab', text => 'This is an embargo test', package => 1}); + is $t->app->pg->db->select('snippets', '*', {id => $unembargoed_snippet})->hash->{package}, 1, + 'linked to unembargoed package'; + is $t->app->snippets->find_or_create( + {hash => 'manual:236d7b56836a4d2759c061147dd8b7ab', text => 'This is an embargo test', package => 3}), + $unembargoed_snippet, 'no new snippet created'; + is $t->app->pg->db->select('snippets', '*', {id => $unembargoed_snippet})->hash->{package}, 1, + 'still linked to unembargoed package'; + }; + + subtest 'Package specific snippets are embargoed' => sub { + $t->get_ok('/login')->status_is(302)->header_is(Location => '/'); + + $t->get_ok('/snippets/meta?confidence=100&isClassified=false&isApproved=false&isLegal=true¬Legal=true') + ->status_is(200)->json_is('/snippets/0/embargoed', 0)->json_like('/snippets/0/text', qr/This is an embargo test/) + ->json_is('/snippets/1/package', 3)->json_is('/snippets/1/embargoed', 1) + ->json_like('/snippets/1/text', qr/added EXPERIMENTAL support for IPv6/)->json_is('/snippets/2/package', 3) + ->json_is('/snippets/2/embargoed', 1)->json_like('/snippets/2/text', qr/added EXPERIMENTAL xml attribute/); + + $t->get_ok('/logout')->status_is(302)->header_is(Location => '/'); + }; + + subtest 'Embargoed snippets become unembargoed when they are referenced by unembargoed packages' => sub { + $t->get_ok('/login')->status_is(302)->header_is(Location => '/'); + + subtest 'Snippets are linked to first unembargoed package' => sub { + ok $t->app->packages->unpack(1), 'indexing first unembargoed package'; + ok $t->app->packages->unpack(2), 'indexing second unembargoed package'; + $t->app->minion->perform_jobs; + $t->get_ok('/snippets/meta?confidence=100&isClassified=false&isApproved=false&isLegal=true¬Legal=true') + ->status_is(200)->json_is('/snippets/0/embargoed', 0) + ->json_like('/snippets/0/text', qr/This is an embargo test/)->json_is('/snippets/1/package', 1) + ->json_is('/snippets/1/filepackage', 2)->json_is('/snippets/1/embargoed', 0) + ->json_like('/snippets/1/text', qr/added EXPERIMENTAL support for IPv6/)->json_is('/snippets/2/package', 1) + ->json_is('/snippets/2/filepackage', 2)->json_is('/snippets/2/embargoed', 0) + ->json_like('/snippets/2/text', qr/added EXPERIMENTAL xml attribute/); + }; + + $t->get_ok('/logout')->status_is(302)->header_is(Location => '/'); + }; +}; + +done_testing; diff --git a/t/legal-bot/harbor-helm/4fcfdab0e71b0bebfdf8b5cc3badfec4/harbor.changes b/t/legal-bot/harbor-helm/4fcfdab0e71b0bebfdf8b5cc3badfec4/harbor.changes index cd91cbd237..9454c81bb2 100644 --- a/t/legal-bot/harbor-helm/4fcfdab0e71b0bebfdf8b5cc3badfec4/harbor.changes +++ b/t/legal-bot/harbor-helm/4fcfdab0e71b0bebfdf8b5cc3badfec4/harbor.changes @@ -2,3 +2,4 @@ Tue May 21 17:38:38 CEST 2021 - sales@suse.com - Initial version +- Test comment with keyword: responsibility, Bureau of Industry and Security, just a test diff --git a/t/lib/Cavil/Test.pm b/t/lib/Cavil/Test.pm index fa661fe688..c898c7d875 100644 --- a/t/lib/Cavil/Test.pm +++ b/t/lib/Cavil/Test.pm @@ -57,6 +57,14 @@ sub default_config ($self) { }; } +sub embargo_fixtures ($self, $app) { + $self->mojo_fixtures($app); + my $patterns = $app->patterns; + + # A pattern that will create new snippets with embargo + $patterns->create(pattern => 'Added EXPERIMENTAL'); +} + sub just_patterns_fixtures ($self, $app) { $self->no_fixtures($app); my $patterns = $app->patterns; diff --git a/t/obs.t b/t/obs.t index b1f0e06091..b77a311088 100644 --- a/t/obs.t +++ b/t/obs.t @@ -393,6 +393,168 @@ get '/public/source/:project/postgresql96-plr/_meta' => [project => ['server:dat EOF +get '/public/request/1234' => {text => <<'EOF'}; + + + + + + requesting release + +EOF + +get '/public/source/:project/_attribute' => [project => 'SUSE:Maintenance:4321'] => {text => <<'EOF'}; + + + 2024-03-27 07:00 UTC + + +EOF + +get '/public/request/1235' => {text => <<'EOF'}; + + + + + + requesting release + +EOF + +get '/public/source/:project/_attribute' => [project => 'SUSE:Maintenance:5321'] => {text => <<'EOF'}; + +EOF + +get '/public/request/324874' => {text => <<'EOF'}; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Auto accept + + + reviewed_okay + + Review got accepted + reviewed_okay + + + + OK + + Review got accepted + OK + + + + reviewers added: qam-openqa + + Review got accepted + reviewers added: qam-openqa + + + + Request accepted for 'qam-openqa' based on data in http://dashboard.qam.suse.de/ + + Review got accepted + Request accepted for 'qam-openqa' based on data in http://dashboard.qam.suse.de/ + + + requesting release + +EOF + +get '/public/source/:project/_attribute' => [project => 'SUSE:Maintenance:33127'] => {text => <<'EOF'}; + +EOF + +get '/public/source/:project/_attribute' => [project => 'SUSE:Maintenance:34725'] => {text => <<'EOF'}; + + + 2024-07-16 12:00 UTC + + + 338592:admin + 341702:admin + + + 2024-09-24 12:00 UTC + + + +EOF + my $AUTHENTICATED = 0; get '/source/:project/kernel-default' => [project => ['openSUSE:Factory']] => (query => {view => 'info'}) => sub ($c) { if (($c->req->headers->authorization // '') =~ /^Signature keyId="legaldb",algorithm="ssh",.+,created="\d+"$/) { @@ -493,6 +655,12 @@ subtest 'Source download for missing packages' => sub { like $@, qr/Mojo-SQLite-1.004.tar.gz/, 'right error'; }; +subtest 'Embargo' => sub { + is $obs->check_for_embargo($api, 1234), 1, 'embargoed'; + is $obs->check_for_embargo($api, 1235), 0, 'not embargoed'; + is $obs->check_for_embargo($api, 324874), 1, 'embargoed'; +}; + subtest 'Bot API (with Minion background jobs)' => sub { my $cavil_test = Cavil::Test->new(online => $ENV{TEST_ONLINE}, schema => 'import_test'); my $config = $cavil_test->default_config; diff --git a/t/snippet.t b/t/snippet.t index fca0ecae71..e74521e518 100644 --- a/t/snippet.t +++ b/t/snippet.t @@ -35,7 +35,7 @@ subtest 'Snippet metadata' => sub { $t->get_ok('/login')->status_is(302)->header_is(Location => '/'); $t->get_ok('/snippets/meta?isClassified=false')->status_is(200)->json_hasnt('/snippets/0'); - my $id = $t->app->snippets->find_or_create('0000', 'Licenses are cool'); + my $id = $t->app->snippets->find_or_create({hash => '0000', text => 'Licenses are cool'}); $t->get_ok('/snippets/meta?isClassified=false')->status_is(200)->json_has('/snippets/0') ->json_is('/snippets/0/classified', false)->json_is('/snippets/0/approved', false); diff --git a/t/util.t b/t/util.t index ce677a0c2a..ff96399ba2 100644 --- a/t/util.t +++ b/t/util.t @@ -16,9 +16,10 @@ use Mojo::Base -strict; use Test::More; -use Mojo::File qw(curfile tempfile); -use Mojo::JSON qw(decode_json); -use Cavil::Util qw(buckets lines_context obs_ssh_auth parse_exclude_file pattern_matches ssh_sign); +use Mojo::File qw(curfile tempfile); +use Mojo::JSON qw(decode_json); +use Cavil::Util + qw(buckets lines_context obs_ssh_auth parse_exclude_file pattern_matches request_id_from_external_link ssh_sign); my $PRIVATE_KEY = tempfile->spew(<<'EOF'); -----BEGIN OPENSSH PRIVATE KEY----- @@ -80,6 +81,13 @@ subtest 'pattern_matches' => sub { ok !pattern_matches('foo $SKIP3 bar', 'foo ya da ya da bar foo'), 'no match'; }; +subtest 'request_id_from_external_link' => sub { + is request_id_from_external_link('obs#1234'), 1234, 'right id'; + is request_id_from_external_link('ibs#4321'), 4321, 'right id'; + is request_id_from_external_link('unknown#4321'), undef, 'no id'; + is request_id_from_external_link(''), undef, 'no id'; +}; + subtest 'ssh_sign' => sub { my $signature = ssh_sign($PRIVATE_KEY, 'realm', 'message'); like $signature, qr/^[-A-Za-z0-9+\/]+={0,3}$/, 'valid Base64 encoded signature';