Add embargo feature for IBS

assets/vue/ClassifySnippets.vue

@@ -101,8 +101,9 @@
             <a v-else>Unknown file</a>
             <span v-if="snippet.files === 2">, and 1 other file</span>
             <span v-else-if="snippet.files > 2">, and {{ snippet.files }} other files</span>
-            <div v-if="snippet.approved === true" class="float-end">
-              <i class="fas fa-check-circle"></i>
+            <div class="float-end">
+              <i v-if="snippet.embargoed === true" class="fas fa-lock" title="Embargoed"></i>
+              <i v-if="snippet.approved === true" class="fas fa-check-circle ms-2" title="Approved by a human"></i>
             </div>
           </div>
           <div class="snippet-source">
@@ -214,7 +215,7 @@ export default {
 
       for (const snippet of snippets) {
         snippet.buttonPressed = null;
-        snippet.fileUrl = `/reviews/file_view/${snippet.package}/${snippet.filename}`;
+        snippet.fileUrl = `/reviews/file_view/${snippet.filepackage}/${snippet.filename}`;
         snippet.editUrl = `/snippet/edit/${snippet.id}`;
         let num = snippet.sline ?? 1;
         const lines = [];

assets/vue/ReportMetadata.vue

@@ -35,6 +35,14 @@
             <small v-if="pkgLicense.spdx === false">(not SPDX)</small>
           </td>
         </tr>
+        <tr>
+          <th class="fit text-start noleftpad" scope="row">
+            <i class="fas fa-lock"></i>
+          </th>
+          <th class="fit text-start noleftpad" scope="row">Embargoed:</th>
+          <td v-if="pkgEmbargoed === true" id="pkg-embargoed">Yes</td>
+          <td v-else id="pkg-embargoed">No</td>
+        </tr>
         <tr v-if="pkgFiles.length > 0">
           <th class="fit text-start noleftpad" scope="row">
             <i class="fas fa-cubes"></i>
@@ -348,6 +356,7 @@ export default {
       history: [],
       notice: null,
       pkgChecksum: null,
+      pkgEmbargoed: false,
       pkgFiles: [],
       pkgLicense: null,
       pkgName: null,
@@ -410,6 +419,7 @@ export default {
       this.pkgType = data.package_type;
       this.pkgUrl = data.package_url;
       this.pkgVersion = data.package_version;
+      this.pkgEmbargoed = data.embargoed;
 
       this.pkgChecksum = data.package_checksum;
       this.checkoutUrl = `/reviews/file_view/${this.pkgId}`;

lib/Cavil/Command/learn.pm

@@ -120,14 +120,19 @@ sub _output_snippets ($self, $good, $bad) {
 
   my $count = my $last_id = 0;
   while (1) {
-    my $batch
-      = $db->query('SELECT * FROM snippets WHERE approved = true AND id > ? ORDER BY id ASC LIMIT 100', $last_id);
+    my $batch = $db->query(
+      'SELECT s.*, bp.embargoed FROM snippets s LEFT JOIN bot_packages bp ON (bp.id = s.package)
+       WHERE approved = true AND s.id > ? ORDER BY s.id ASC LIMIT 100', $last_id
+    );
     last if $batch->rows == 0;
 
     for my $hash ($batch->hashes->each) {
-      $count++;
       my $id = $hash->{id};
       $last_id = $id if $id > $last_id;
+
+      next if $hash->{embargoed};
+
+      $count++;
       my $dir  = $hash->{license} ? $good : $bad;
       my $file = $dir->child("$hash->{hash}.txt");
       next unless _spew($file, $hash->{text});

lib/Cavil/Controller/Queue.pm

@@ -78,20 +78,26 @@ sub create_package ($self) {
 
   $obj->{obsolete} = 0;
   $pkgs->update($obj);
-  $pkgs->obs_import(
-    $obj->{id},
-    {
-      api       => $api,
-      project   => $project,
-      pkg       => $pkg,
-      srcpkg    => $srcpkg,
-      rev       => $rev,
-      srcmd5    => $srcmd5,
-      verifymd5 => $verifymd5,
-      priority  => $prio
-    },
-    $prio + 10
-  ) if $create;
+  if ($create) {
+    $pkgs->obs_import(
+      $obj->{id},
+      {
+        api           => $api,
+        project       => $project,
+        pkg           => $pkg,
+        srcpkg        => $srcpkg,
+        rev           => $rev,
+        srcmd5        => $srcmd5,
+        verifymd5     => $verifymd5,
+        external_link => $obj->{external_link},
+        priority      => $prio
+      },
+      $prio + 10
+    );
+  }
+  else {
+    $pkgs->obs_embargo($obj->{id}, {api => $api, external_link => $obj->{external_link}});
+  }
 
   $self->render(json => {saved => $obj});
 }

lib/Cavil/Controller/Reviewer.pm

@@ -114,6 +114,7 @@ sub meta ($self) {
       actions           => $actions,
       copied_files      => {'%doc' => [sort keys %docs], '%license' => [sort keys %lics]},
       created           => $pkg->{created_epoch},
+      embargoed         => \!!$pkg->{embargoed},
       errors            => $spec->{errors} // [],
       external_link     => $pkg->{external_link},
       has_spdx_report   => \!!$has_spdx_report,

lib/Cavil/Controller/Snippet.pm

@@ -148,7 +148,7 @@ sub list_meta ($self) {
 
   my $snippets = $unclassified->{snippets};
   for my $snippet (@$snippets) {
-    $snippet->{$_} = $snippet->{$_} ? true : false for qw(license classified approved);
+    $snippet->{$_} = $snippet->{$_} ? true : false for qw(embargoed license classified approved);
   }
 
   $self->render(json => {snippets => $snippets, total => $unclassified->{total}});

lib/Cavil/FileIndexer.pm

@@ -173,7 +173,8 @@ sub _snippet ($self, $file_id, $matches, $path, $first_line, $last_line) {
     return;
   }
 
-  my $snippet = $self->{snippets}->{$hash} ||= $self->{app}->snippets->find_or_create($hash, $text);
+  my $snippet = $self->{snippets}->{$hash}
+    ||= $self->{app}->snippets->find_or_create({hash => $hash, text => $text, package => $self->{package}});
   $self->{db}->insert('file_snippets',
     {package => $self->{package}, snippet => $snippet, sline => $first_line, eline => $last_line, file => $file_id});
 

lib/Cavil/Model/Packages.pm

@@ -36,7 +36,8 @@ sub add ($self, %args) {
     source          => $source_id,
     requesting_user => $args{requesting_user},
     priority        => $args{priority},
-    state           => 'new'
+    state           => 'new',
+    embargoed       => $args{embargoed} ? 1 : 0
   };
   return $db->insert('bot_packages', $pkg, {returning => 'id'})->hash->{id};
 }
@@ -393,6 +394,16 @@ sub name_suggestions ($self, $partial) {
   )->arrays->flatten->to_array;
 }
 
+sub obs_embargo ($self, $id, $data, $priority = 5) {
+  my $pkg = $self->find($id);
+  return $self->minion->enqueue(
+    obs_embargo => [$id, $data] => {
+      priority => $priority,
+      notes    => {external_link => $pkg->{external_link}, package => $pkg->{name}, "pkg_$id" => 1}
+    }
+  );
+}
+
 sub obs_import ($self, $id, $data, $priority = 5) {
   my $pkg = $self->find($id);
   return $self->minion->enqueue(
@@ -481,7 +492,7 @@ sub unpacked ($self, $id) {
 
 sub update ($self, $pkg) {
   my %updates = map { exists $pkg->{$_} ? ($_ => $pkg->{$_}) : () }
-    (qw(created checksum priority state obsolete result notice reviewed reviewing_user external_link));
+    (qw(created checksum priority state obsolete result notice reviewed reviewing_user external_link embargoed));
   $updates{reviewed} = \'now()' if $pkg->{review_timestamp};
   return $self->pg->db->update('bot_packages', \%updates, {id => $pkg->{id}});
 }

lib/Cavil/Model/Reports.pm

@@ -295,7 +295,7 @@ sub _dig_report {
   }
   my $matches = $db->select('pattern_matches', [qw(id file pattern sline eline)], $query);
 
-  $query = {package => $pkg->{id}};
+  $query = {'file_snippets.package' => $pkg->{id}};
   if ($limit_to_file) {
     $query->{file} = $limit_to_file;
   }

lib/Cavil/Model/Snippets.pm

@@ -31,17 +31,25 @@ sub find ($self, $id) {
   return $self->pg->db->select('snippets', '*', {id => $id})->hash;
 }
 
-sub find_or_create ($self, $hash, $text, $prefix = '') {
+sub find_or_create ($self, $new) {
+  $new->{prefix} //= '';
   my $db = $self->pg->db;
 
-  my $snip = $db->select('snippets', 'id', {hash => $hash})->hash;
-  return $snip->{id} if $snip;
+  my $old = $db->query(
+    'SELECT s.id, bp.embargoed FROM snippets s LEFT JOIN bot_packages bp ON (bp.id = s.package)
+     WHERE hash = ?', $new->{hash}
+  )->hash;
+
+  # Inherit embargo status until there is no embargo anymore (the value will tell us which package lifted the embargo)
+  if ($old) {
+    $db->query('UPDATE snippets SET package = ? WHERE id = ?', $new->{package}, $old->{id}) if $old->{embargoed};
+    return $old->{id};
+  }
 
-  $db->query(
-    'insert into snippets (hash, text) values (?, ?)
-   on conflict do nothing', "$prefix$hash", $text
-  );
-  return $db->select('snippets', 'id', {hash => "$prefix$hash"})->hash->{id};
+  my $hash = "$new->{prefix}$new->{hash}";
+  $db->query('INSERT INTO snippets (hash, text, package) VALUES (?, ?, ?) ON CONFLICT DO NOTHING',
+    $hash, $new->{text}, $new->{package});
+  return $db->select('snippets', 'id', {hash => $hash})->hash->{id};
 }
 
 sub from_file ($self, $file_id, $first_line, $last_line) {
@@ -53,7 +61,8 @@ sub from_file ($self, $file_id, $first_line, $last_line) {
   my $path    = path($self->checkout_dir, $package->{name}, $package->{checkout_dir}, '.unpacked', $file->{filename});
 
   my ($text, $hash) = file_and_checksum($path, $first_line, $last_line);
-  my $snippet_id = $self->find_or_create($hash, $text, 'manual:');
+  my $snippet_id
+    = $self->find_or_create({hash => $hash, text => $text, package => $package->{id}, prefix => 'manual:'});
   $db->insert('file_snippets',
     {package => $package->{id}, snippet => $snippet_id, sline => $first_line, eline => $last_line, file => $file_id});
 
@@ -97,7 +106,8 @@ sub unclassified ($self, $options) {
   }
 
   my $snippets = $db->query(
-    "SELECT *, COUNT(*) OVER() AS total FROM snippets
+    "SELECT s.*, bp.embargoed, COUNT(*) OVER() AS total
+     FROM snippets s LEFT JOIN bot_packages bp ON (bp.id = s.package)
      WHERE $is_approved AND $is_classified $before $legal $confidence $timeframe ORDER BY id DESC LIMIT 10"
   )->hashes;
 
@@ -106,13 +116,13 @@ sub unclassified ($self, $options) {
     $total = delete $snippet->{total};
     $snippet->{likelyness} = int($snippet->{likelyness} * 100);
     my $files = $db->query(
-      'SELECT fs.sline, mf.filename, mf.package
+      'SELECT fs.sline, mf.filename, mf.package AS filepackage
        FROM file_snippets fs JOIN matched_files mf ON (fs.file = mf.id)
        WHERE fs.snippet = ? ORDER BY fs.id DESC LIMIT 1', $snippet->{id}
     )->hashes;
     $snippet->{files} = $files->size;
     my $file = $files->[0] || {};
-    $snippet->{$_} = $file->{$_} for qw(filename sline package);
+    $snippet->{$_} = $file->{$_} for qw(filename sline filepackage);
 
     my $license = $db->query('SELECT license, risk FROM license_patterns WHERE id = ? AND license != ?',
       $snippet->{like_pattern} // 0, '')->hash // {};

lib/Cavil/OBS.pm

@@ -41,6 +41,22 @@ has ua        => sub {
 };
 has user => sub { die 'Missing ssh user' };
 
+sub check_for_embargo ($self, $api, $request) {
+  my $url = _url($api, 'public', 'request', $request);
+  my $res = $self->_get($url);
+  croak "$url: " . $res->code unless $res->is_success;
+
+  for my $project ($res->dom->find('action [project]')->map('attr', 'project')->uniq->each) {
+    my $url = _url($api, 'public', 'source', $project, '_attribute');
+    my $res = $self->_get($url);
+    next if $res->code == 404;
+    croak "$url: " . $res->code unless $res->is_success;
+    return 1 if $res->dom->at('attributes attribute[name=EmbargoDate]');
+  }
+
+  return 0;
+}
+
 sub download_source ($self, $api, $project, $pkg, $dir, $options = {}) {
   $dir = path($dir)->make_path;
 

lib/Cavil/Task/Import.pm

@@ -17,10 +17,21 @@ package Cavil::Task::Import;
 use Mojo::Base 'Mojolicious::Plugin', -signatures;
 
 use Cavil::Checkout;
-use Mojo::File 'path';
+use Mojo::File  qw(path);
+use Cavil::Util qw(request_id_from_external_link);
 
 sub register ($self, $app, $config) {
-  $app->minion->add_task(obs_import => \&_obs);
+  $app->minion->add_task(obs_embargo => \&_embargo);
+  $app->minion->add_task(obs_import  => \&_obs);
+}
+
+sub _embargo ($job, $id, $data) {
+  return unless my $link       = $data->{external_link};
+  return unless my $request_id = request_id_from_external_link($link);
+
+  my $app       = $job->app;
+  my $embargoed = $app->obs->check_for_embargo($data->{api}, $request_id);
+  $app->packages->update({id => $id, embargoed => $embargoed});
 }
 
 sub _obs ($job, $id, $data) {
@@ -33,6 +44,9 @@ sub _obs ($job, $id, $data) {
   return $job->finish("Package $id is already being processed")
     unless my $guard = $minion->guard("processing_pkg_$id", 172800);
 
+  # Check embargo status before checkout
+  _embargo($job, $id, $data);
+
   my $checkout_dir = $app->config->{checkout_dir};
   my ($srcpkg, $verifymd5, $api, $project, $pkg, $srcmd5, $priority)
     = @{$data}{qw(srcpkg verifymd5 api project pkg srcmd5 priority)};

lib/Cavil/Util.pm

@@ -29,7 +29,8 @@ $Text::Glob::strict_wildcard_slash = 0;
 
 our @EXPORT_OK = (
   qw(buckets file_and_checksum slurp_and_decode load_ignored_files lines_context obs_ssh_auth paginate),
-  qw(parse_exclude_file pattern_checksum pattern_matches read_lines snippet_checksum ssh_sign)
+  qw(parse_exclude_file pattern_checksum pattern_matches read_lines request_id_from_external_link snippet_checksum),
+  qw(ssh_sign)
 );
 
 my $MAX_FILE_SIZE = 30000;
@@ -216,6 +217,11 @@ sub read_lines ($path, $start_line, $end_line) {
   return $text;
 }
 
+sub request_id_from_external_link ($link) {
+  return $1 if $link =~ /^(?:obs|ibs)#(\d+)$/;
+  return undef;
+}
+
 # Based on https://www.suse.com/c/multi-factor-authentication-on-suses-build-service/
 sub ssh_sign ($key, $realm, $value) {
 

migrations/cavil.sql

@@ -227,6 +227,11 @@ ALTER TABLE ignored_lines ADD COLUMN contributor int REFERENCES bot_users(id);
 ALTER TABLE pattern_matches ADD COLUMN ignored_line int REFERENCES ignored_lines(id) ON DELETE SET NULL;
 CREATE INDEX ON pattern_matches(ignored_line);
 
---23 up
+-- 23 up
 ALTER TYPE bot_state RENAME VALUE 'correct' TO 'acceptable_by_lawyer';
 ALTER TABLE bot_packages ADD COLUMN notice text;
+
+-- 24 up
+ALTER TABLE bot_packages ADD COLUMN embargoed boolean DEFAULT false NOT NULL;
+CREATE INDEX ON bot_packages(embargoed);
+ALTER TABLE snippets ADD COLUMN package int REFERENCES bot_packages(id) ON DELETE SET NULL;

staging/start.pl

@@ -145,7 +145,8 @@
   project         => 'just:a:test',
   package         => 'harbor-helm',
   srcmd5          => 'abc1c36647a5d356883d490da2140def',
-  priority        => 5
+  priority        => 5,
+  embargoed       => 1
 );
 $pkgs->imported($pkg_id);
 my $harbor = $pkgs->find($pkg_id);