Add authentication to remote data sources

Allow remote data ingestion to perform basic and bearer authentication.

README.md

@@ -298,12 +298,66 @@ To provide a set of fixed data, create a JSON file ending in `.`_format_`-input`
 To access data remotely, create a file ending in `.`_format_`-remote` as follows:
 
     {
+       "authentication": null,
        "url": "http://some.url/format/endpoint",
        "ttl": 10
     }
 
 where `url` is the URL to download the data and `ttl` is the number of minutes
-to cache the data for.
+to cache the data for. If no authentication is required, `null` can be used for
+`"authentication"`. Alternatively, it can be one of the following
+authentication methods.
+
+### Basic Authentication
+Basic authentication sends a user name and password to the remote server. The
+password can be stored in the configuration file like this:
+
+    {
+       "authentication": {
+         "type": "basic",
+         "username": "jrhacker",
+         "password": "s3cr3t"
+       },
+       "url": "http://some.url/format/endpoint",
+       "ttl": 10
+    }
+
+or it can be stored separately using `basic-file`:
+
+    {
+       "authentication": {
+         "type": "basic-file",
+         "username": "jrhacker",
+         "passwordFile": "/home/shesmu/secret-password"
+       },
+       "url": "http://some.url/format/endpoint",
+       "ttl": 10
+    }
+
+
+### Bearer Authentication
+Bearer authentication sends a single token to the remote server. The
+token can be stored in the configuration file like this:
+
+    {
+       "authentication": {
+         "type": "bearer",
+         "token": "01234567890ABCDEF"
+       },
+       "url": "http://some.url/format/endpoint",
+       "ttl": 10
+    }
+
+or it can be stored separately using `bearer-file`:
+
+    {
+       "authentication": {
+         "type": "bearer-file",
+         "tokenFile": "/home/shesmu/secret-token"
+       },
+       "url": "http://some.url/format/endpoint",
+       "ttl": 10
+    }
 
 ## Saved Searches
 Shesmu's _Actions_ dashboard provides a way to sift through the actions that

changes/add_json_authentication.md

@@ -0,0 +1 @@
+* Allow HTTP authentication for `-remote` input format sources

shesmu-pluginapi/src/main/java/ca/on/oicr/gsi/shesmu/plugin/authentication/AuthenticationConfiguration.java

@@ -0,0 +1,29 @@
+package ca.on.oicr.gsi.shesmu.plugin.authentication;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import java.io.IOException;
+import java.net.http.HttpRequest.Builder;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+@JsonSubTypes({
+  @JsonSubTypes.Type(value = AuthenticationConfigurationBasic.class, name = "basic"),
+  @JsonSubTypes.Type(value = AuthenticationConfigurationBearer.class, name = "bearer"),
+  @JsonSubTypes.Type(value = AuthenticationConfigurationFileBasic.class, name = "basic-file"),
+  @JsonSubTypes.Type(value = AuthenticationConfigurationFileBearer.class, name = "bearer-file")
+})
+public abstract sealed class AuthenticationConfiguration
+    permits AuthenticationConfigurationBasic,
+        AuthenticationConfigurationBearer,
+        AuthenticationConfigurationFileBasic,
+        AuthenticationConfigurationFileBearer {
+
+  public static void addAuthenticationHeader(
+      AuthenticationConfiguration authentication, Builder request) throws IOException {
+    if (authentication != null) {
+      request.header("Authorization", authentication.prepareAuthentication());
+    }
+  }
+
+  public abstract String prepareAuthentication() throws IOException;
+}

shesmu-pluginapi/src/main/java/ca/on/oicr/gsi/shesmu/plugin/authentication/AuthenticationConfigurationBasic.java

@@ -0,0 +1,36 @@
+package ca.on.oicr.gsi.shesmu.plugin.authentication;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+public final class AuthenticationConfigurationBasic extends AuthenticationConfiguration {
+  public static String makeHeader(String username, String password) {
+    return "Basic "
+        + Base64.getEncoder()
+            .encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8));
+  }
+
+  private String password;
+  private String username;
+
+  public String getPassword() {
+    return password;
+  }
+
+  public String getUsername() {
+    return username;
+  }
+
+  @Override
+  public String prepareAuthentication() {
+    return makeHeader(username, password);
+  }
+
+  public void setPassword(String password) {
+    this.password = password;
+  }
+
+  public void setUsername(String username) {
+    this.username = username;
+  }
+}

shesmu-pluginapi/src/main/java/ca/on/oicr/gsi/shesmu/plugin/authentication/AuthenticationConfigurationBearer.java

@@ -0,0 +1,20 @@
+package ca.on.oicr.gsi.shesmu.plugin.authentication;
+
+import java.io.IOException;
+
+public final class AuthenticationConfigurationBearer extends AuthenticationConfiguration {
+  private String token;
+
+  public String getToken() {
+    return token;
+  }
+
+  @Override
+  public String prepareAuthentication() throws IOException {
+    return "Bearer " + token;
+  }
+
+  public void setToken(String token) {
+    this.token = token;
+  }
+}

shesmu-pluginapi/src/main/java/ca/on/oicr/gsi/shesmu/plugin/authentication/AuthenticationConfigurationFileBasic.java

@@ -0,0 +1,32 @@
+package ca.on.oicr.gsi.shesmu.plugin.authentication;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+public final class AuthenticationConfigurationFileBasic extends AuthenticationConfiguration {
+  private String passwordFile;
+  private String username;
+
+  public String getPasswordFile() {
+    return passwordFile;
+  }
+
+  public String getUsername() {
+    return username;
+  }
+
+  @Override
+  public String prepareAuthentication() throws IOException {
+    return AuthenticationConfigurationBasic.makeHeader(
+        username, Files.readString(Paths.get(passwordFile)).trim());
+  }
+
+  public void setPasswordFile(String passwordFile) {
+    this.passwordFile = passwordFile;
+  }
+
+  public void setUsername(String username) {
+    this.username = username;
+  }
+}

shesmu-pluginapi/src/main/java/ca/on/oicr/gsi/shesmu/plugin/authentication/AuthenticationConfigurationFileBearer.java

@@ -0,0 +1,22 @@
+package ca.on.oicr.gsi.shesmu.plugin.authentication;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+public final class AuthenticationConfigurationFileBearer extends AuthenticationConfiguration {
+  private String tokenFile;
+
+  public String getTokenFile() {
+    return tokenFile;
+  }
+
+  @Override
+  public String prepareAuthentication() throws IOException {
+    return "Bearer " + Files.readString(Paths.get(tokenFile)).trim();
+  }
+
+  public void setTokenFile(String tokenFile) {
+    this.tokenFile = tokenFile;
+  }
+}

shesmu-pluginapi/src/main/java/ca/on/oicr/gsi/shesmu/plugin/authentication/package-info.java

@@ -0,0 +1,2 @@
+/** Utilities for configuring HTTP authentication in configuration files */
+package ca.on.oicr.gsi.shesmu.plugin.authentication;

shesmu-server/src/main/java/ca/on/oicr/gsi/shesmu/server/plugins/AnnotatedInputFormatDefinition.java

@@ -10,6 +10,7 @@
 import ca.on.oicr.gsi.shesmu.plugin.ErrorableStream;
 import ca.on.oicr.gsi.shesmu.plugin.PluginFile;
 import ca.on.oicr.gsi.shesmu.plugin.Tuple;
+import ca.on.oicr.gsi.shesmu.plugin.authentication.AuthenticationConfiguration;
 import ca.on.oicr.gsi.shesmu.plugin.cache.InitialCachePopulationException;
 import ca.on.oicr.gsi.shesmu.plugin.cache.InvalidatableRecord;
 import ca.on.oicr.gsi.shesmu.plugin.cache.LabelledKeyValueCache;
@@ -130,9 +131,14 @@ public Imyhat type() {
   }
 
   public static final class Configuration {
+    private AuthenticationConfiguration authentication;
     private int ttl;
     private String url;
 
+    public AuthenticationConfiguration getAuthentication() {
+      return authentication;
+    }
+
     public int getTtl() {
       return ttl;
     }
@@ -141,6 +147,10 @@ public String getUrl() {
       return url;
     }
 
+    public void setAuthentication(AuthenticationConfiguration authentication) {
+      this.authentication = authentication;
+    }
+
     public void setTtl(int ttl) {
       this.ttl = ttl;
     }
@@ -311,10 +321,10 @@ public RemoteReloader(Path fileName) {
       protected Stream<Object> fetch(Instant lastUpdated) throws Exception {
         if (config.isEmpty()) return new ErrorableStream<>(Stream.empty(), false);
         final var url = config.get().getUrl();
-        var response =
-            Server.HTTP_CLIENT.send(
-                HttpRequest.newBuilder(URI.create(url)).GET().version(Version.HTTP_1_1).build(),
-                BodyHandlers.ofInputStream());
+        final var request = HttpRequest.newBuilder(URI.create(url)).GET().version(Version.HTTP_1_1);
+        AuthenticationConfiguration.addAuthenticationHeader(
+            config.get().getAuthentication(), request);
+        var response = Server.HTTP_CLIENT.send(request.build(), BodyHandlers.ofInputStream());
         try (var parser = RuntimeSupport.MAPPER.getFactory().createParser(response.body())) {
           if (response.statusCode() != 200) return new ErrorableStream<>(Stream.empty(), false);
           final List<Object> results = new ArrayList<>();