Add tests for multi-repo setup

README.md

@@ -50,7 +50,7 @@ You can configure Monorobot to [unfurl GitHub links](https://api.slack.com/refer
 
 ### Documentation
 
-The bot expects two configuration files to be present.
+Add a configuration file to each repository you want to support, and a secrets file on the bot server itself. Read on for instructions to set up each file:
 
 * [Repository configuration](./documentation/config_docs.md)
 * [Secrets](./documentation/secret_docs.md)

documentation/secret_docs.md

@@ -17,6 +17,8 @@ A secrets file stores sensitive information. Unlike the repository configuration
 |-|-|-|-|
 | `gh_token` | specify to grant the bot access to private repositories; omit for public repositories | Yes | - |
 | `gh_hook_token` | specify to ensure the bot only receives GitHub notifications from pre-approved repositories | Yes | - |
+| `repos` | an object mapping repository URLs to repository-specific GitHub secrets | Yes | - |
+| `allowed_repos` | a whitelist of repository URLs to process payloads for | Yes | all incoming payloads are processed |
 | `slack_access_token` | slack bot access token to enable message posting to the workspace | Yes | try to use webhooks defined in `slack_hooks` instead |
 | `slack_hooks` | list of channel names and their corresponding webhook endpoint | Yes | try to use token defined in `slack_access_token` instead |
 | `slack_signing_secret` | specify to verify incoming slack requests | Yes | - |
@@ -27,10 +29,36 @@ Note that either `slack_access_token` or `slack_hooks` must be defined.
 
 Some operations, such as fetching a config file from a private repository, or the commit corresponding to a commit comment event, require a personal access token. Refer [here](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) for detailed instructions on token generation.
 
+*See `repos` if you need to support multiple repositories that use different tokens.*
+
 ## `gh_hook_token`
 
 Refer [here](https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/securing-your-webhooks) for more information on securing webhooks with a token.
 
+*See `repos` if you need to support multiple repositories that use different tokens.*
+
+## `repos`
+
+If you're using Monorobot for multiple repositories that need different secrets (e.g., one on github.com and another on GitHub Enterprise), you can provide them as an object. Secrets defined here will take precedence over those defined at the top level of the secrets file.
+
+Repository URLs should be fully qualified (include the protocol).
+
+```json
+{
+  "https://github.com/ahrefs/runner" : {
+    "gh_token": "XXX"
+  },
+  "https://git.ahrefs.com/ahrefs/coyote" : {
+    "gh_token": "XXX",
+    "gh_hook_token": "XXX"
+  }
+}
+```
+
+## `allowed_repos`
+
+Use this option to restrict incoming notifications from GitHub to approved repository URLs.
+
 ## `slack_access_token`
 
 Refer [here](https://api.slack.com/authentication/oauth-v2) for obtaining an access token via OAuth.

lib/action.ml

@@ -87,13 +87,14 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
     if List.is_empty matched_channel_names then default else matched_channel_names
 
   let partition_status (ctx : Context.t) (n : status_notification) =
-    let cfg = Context.get_config_exn ctx in
+    let repo = n.repository in
+    let cfg = Context.find_repo_config_exn ctx repo.url in
     let pipeline = n.context in
     let current_status = n.state in
     let rules = cfg.status_rules.rules in
     let action_on_match (branches : branch list) =
       let default = Option.to_list cfg.prefix_rules.default_channel in
-      let () = Context.refresh_pipeline_status ~pipeline ~branches ~status:current_status ctx in
+      State.set_repo_pipeline_status ctx.state repo.url ~pipeline ~branches ~status:current_status;
       match List.is_empty branches with
       | true -> Lwt.return []
       | false ->
@@ -105,18 +106,18 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
       | false -> Lwt.return default
       | true ->
         let sha = n.commit.sha in
-        let repo = n.repository in
         ( match%lwt Github_api.get_api_commit ~ctx ~repo ~sha with
         | Error e -> action_error e
         | Ok commit -> Lwt.return @@ partition_commit cfg commit.files
         )
     in
-    if Context.is_pipeline_allowed ctx ~pipeline then begin
+    if Context.is_pipeline_allowed ctx repo.url ~pipeline then begin
+      let repo_state = State.find_or_add_repo ctx.state repo.url in
       match Rule.Status.match_rules ~rules n with
       | Some Ignore | None -> Lwt.return []
       | Some Allow -> action_on_match n.branches
       | Some Allow_once ->
-      match Map.find ctx.state.pipeline_statuses pipeline with
+      match Map.find repo_state.pipeline_statuses pipeline with
       | Some branch_statuses ->
         let has_same_status_state_as_prev (branch : branch) =
           match Map.find branch_statuses branch.name with
@@ -130,7 +131,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
     else Lwt.return []
 
   let partition_commit_comment (ctx : Context.t) n =
-    let cfg = Context.get_config_exn ctx in
+    let cfg = Context.find_repo_config_exn ctx n.repository.url in
     match n.comment.commit_id with
     | None -> action_error "unable to find commit id for this commit comment event"
     | Some sha ->
@@ -149,7 +150,8 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
       )
 
   let generate_notifications (ctx : Context.t) req =
-    let cfg = Context.get_config_exn ctx in
+    let repo = Github.repo_of_notification req in
+    let cfg = Context.find_repo_config_exn ctx repo.url in
     match req with
     | Github.Push n ->
       partition_push cfg n |> List.map ~f:(fun (channel, n) -> generate_push_notification n channel) |> Lwt.return
@@ -178,20 +180,20 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
     in
     Lwt_list.iter_s notify notifications
 
-  (** `refresh_config_of_context ctx n` updates the current context if the configuration
-      hasn't been loaded yet, or if the incoming request `n` is a push
+  (** `refresh_repo_config ctx n` fetches the latest repo config if it's
+      uninitialized, or if the incoming request `n` is a push
       notification containing commits that touched the config file. *)
-  let refresh_config_of_context (ctx : Context.t) notification =
+  let refresh_repo_config (ctx : Context.t) notification =
+    let repo = Github.repo_of_notification notification in
     let fetch_config () =
-      let repo = Github.repo_of_notification notification in
       match%lwt Github_api.get_config ~ctx ~repo with
       | Ok config ->
-        ctx.config <- Some config;
-        Context.print_config ctx;
+        Context.set_repo_config ctx repo.url config;
+        Context.print_config ctx repo.url;
         Lwt.return @@ Ok ()
       | Error e -> action_error e
     in
-    match ctx.config with
+    match Context.find_repo_config ctx repo.url with
     | None -> fetch_config ()
     | Some _ ->
     match notification with
@@ -203,12 +205,28 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct
     | _ -> Lwt.return @@ Ok ()
 
   let process_github_notification (ctx : Context.t) headers body =
+    let validate_signature secrets payload =
+      let repo = Github.repo_of_notification payload in
+      let signing_key = Context.gh_hook_token_of_secrets secrets repo.url in
+      Github.validate_signature ?signing_key ~headers body
+    in
+    let repo_is_allowed secrets payload =
+      let repo = Github.repo_of_notification payload in
+      let allowed_repos = secrets.allowed_repos in
+      List.is_empty allowed_repos || List.exists allowed_repos ~f:(String.equal repo.url)
+    in
     try%lwt
       let secrets = Context.get_secrets_exn ctx in
-      match Github.parse_exn ~secret:secrets.gh_hook_token headers body with
+      match Github.parse_exn headers body with
       | exception exn -> Exn_lwt.fail ~exn "failed to parse payload"
       | payload ->
-        ( match%lwt refresh_config_of_context ctx payload with
+      match validate_signature secrets payload with
+      | Error e -> action_error e
+      | Ok () ->
+      match repo_is_allowed secrets payload with
+      | false -> action_error "unsupported repository"
+      | true ->
+        ( match%lwt refresh_repo_config ctx payload with
         | Error e -> action_error e
         | Ok () ->
           let%lwt notifications = generate_notifications ctx payload in

lib/api_local.ml

@@ -7,8 +7,10 @@ let cwd = Caml.Sys.getcwd ()
 let cache_dir = Caml.Filename.concat cwd "github-api-cache"
 
 module Github : Api.Github = struct
+  let mock_config_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_config"
+
   let get_config ~(ctx : Context.t) ~repo:_ =
-    let url = Caml.Filename.concat cwd ctx.config_filename in
+    let url = Caml.Filename.concat mock_config_dir ctx.config_filename in
     match get_local_file url with
     | Error e -> Lwt.return @@ fmt_error "error while getting local file: %s\nfailed to get config %s" e url
     | Ok file -> Lwt.return @@ Ok (Config_j.config_of_string file)

lib/api_remote.ml

@@ -23,7 +23,8 @@ module Github : Api.Github = struct
   let get_config ~(ctx : Context.t) ~repo =
     let secrets = Context.get_secrets_exn ctx in
     let url = contents_url ~repo ~path:ctx.config_filename in
-    let headers = build_headers ?token:secrets.gh_token () in
+    let token = Context.gh_token_of_secrets secrets repo.url in
+    let headers = build_headers ?token () in
     match%lwt http_request ~headers `GET url with
     | Error e -> Lwt.return @@ fmt_error "error while querying remote: %s\nfailed to get config from file %s" e url
     | Ok res ->
@@ -44,23 +45,23 @@ module Github : Api.Github = struct
         @@ fmt_error "unexpected encoding '%s' in Github response\nfailed to get config from file %s" encoding url
       )
 
-  let get_resource (ctx : Context.t) url =
-    let secrets = Context.get_secrets_exn ctx in
-    let headers = build_headers ?token:secrets.gh_token () in
+  let get_resource secrets repo_url url =
+    let token = Context.gh_token_of_secrets secrets repo_url in
+    let headers = build_headers ?token () in
     match%lwt http_request ~headers `GET url with
     | Ok res -> Lwt.return @@ Ok res
     | Error e -> Lwt.return @@ fmt_error "error while querying remote: %s\nfailed to get resource from %s" e url
 
-  let get_api_commit ~(ctx : Context.t) ~repo ~sha =
-    let%lwt res = commits_url ~repo ~sha |> get_resource ctx in
+  let get_api_commit ~(ctx : Context.t) ~(repo : Github_t.repository) ~sha =
+    let%lwt res = commits_url ~repo ~sha |> get_resource (Context.get_secrets_exn ctx) repo.url in
     Lwt.return @@ Result.map res ~f:Github_j.api_commit_of_string
 
-  let get_pull_request ~(ctx : Context.t) ~repo ~number =
-    let%lwt res = pulls_url ~repo ~number |> get_resource ctx in
+  let get_pull_request ~(ctx : Context.t) ~(repo : Github_t.repository) ~number =
+    let%lwt res = pulls_url ~repo ~number |> get_resource (Context.get_secrets_exn ctx) repo.url in
     Lwt.return @@ Result.map res ~f:Github_j.pull_request_of_string
 
-  let get_issue ~(ctx : Context.t) ~repo ~number =
-    let%lwt res = issues_url ~repo ~number |> get_resource ctx in
+  let get_issue ~(ctx : Context.t) ~(repo : Github_t.repository) ~number =
+    let%lwt res = issues_url ~repo ~number |> get_resource (Context.get_secrets_exn ctx) repo.url in
     Lwt.return @@ Result.map res ~f:Github_j.issue_of_string
 end
 

lib/common.atd

@@ -1,3 +1,7 @@
 type 'v map_as_object =
   (string * 'v) list <json repr="object">
     wrap <ocaml module="Common.StringMap" t="'v Common.StringMap.t">
+
+type 'v table_as_object =
+  (string * 'v) list <json repr="object">
+    wrap <ocaml module="Common.Table" t="'v Common.Table.t">

lib/common.ml

@@ -15,6 +15,20 @@ module StringMap = struct
   let unwrap = to_list
 end
 
+module Table = struct
+  type 'a t = (string, 'a) Hashtbl.t
+
+  let empty () = Hashtbl.create (module String)
+
+  let to_list (l : 'a t) : (string * 'a) list = Hashtbl.to_alist l
+
+  let of_list (m : (string * 'a) list) : 'a t = Hashtbl.of_alist_exn (module String) m
+
+  let wrap = of_list
+
+  let unwrap = to_list
+end
+
 let fmt_error fmt = Printf.ksprintf (fun s -> Error s) fmt
 
 let first_line s =

lib/config.atd

@@ -1,6 +1,7 @@
 type status_rule <ocaml from="Rule"> = abstract
 type prefix_rule <ocaml from="Rule"> = abstract
 type label_rule <ocaml from="Rule"> = abstract
+type 'v map_as_object <ocaml from="Common"> = abstract
 
 (* This type of rule is used for CI build notifications. *)
 type status_rules = {
@@ -36,9 +37,20 @@ type webhook = {
   channel : string; (* name of the Slack channel to post the message *)
 }
 
+type gh_repo_secrets = {
+  (* GitHub personal access token, if repo access requires it *)
+  ?gh_token : string nullable;
+  (* GitHub webhook token to secure the webhook *)
+  ?gh_hook_token : string nullable;
+}
+
 (* This is the structure of the secrets file which stores sensitive information, and
    shouldn't be checked into version control.  *)
 type secrets = {
+  (* repo-specific secrets; overrides global values if defined for a given repo *)
+  ~repos <ocaml default="Common.StringMap.empty"> : gh_repo_secrets map_as_object;
+  (* whitelist of repository URLs to handle notifications for *)
+  ~allowed_repos <ocaml default="[]"> : string list;
   (* GitHub personal access token, if repo access requires it *)
   ?gh_token : string nullable;
   (* GitHub webhook token to secure the webhook *)

lib/context.ml

@@ -4,62 +4,73 @@ open Devkit
 
 exception Context_error of string
 
-let context_error msg = raise (Context_error msg)
+let context_error fmt = Printf.ksprintf (fun msg -> raise (Context_error msg)) fmt
 
 type t = {
   config_filename : string;
   secrets_filepath : string;
   state_filepath : string option;
   mutable secrets : Config_t.secrets option;
-  mutable config : Config_t.config option;
+  config : Config_t.config Table.t;
   state : State_t.state;
 }
 
-let default : t =
+let default_config_filename = "monorobot.json"
+
+let default_secrets_filepath = "secrets.json"
+
+let make ?config_filename ?secrets_filepath ?state_filepath () =
   {
-    config_filename = "monorobot.json";
-    secrets_filepath = "secrets.json";
-    state_filepath = None;
+    config_filename = Option.value config_filename ~default:default_config_filename;
+    secrets_filepath = Option.value secrets_filepath ~default:default_secrets_filepath;
+    state_filepath;
     secrets = None;
-    config = None;
-    state = State.empty;
+    config = Table.empty ();
+    state = State.empty ();
   }
 
-let make ?config_filename ?secrets_filepath ?state_filepath () =
-  let config_filename = Option.value config_filename ~default:default.config_filename in
-  let secrets_filepath = Option.value secrets_filepath ~default:default.secrets_filepath in
-  { default with config_filename; secrets_filepath; state_filepath }
-
 let get_secrets_exn ctx =
   match ctx.secrets with
   | None -> context_error "secrets is uninitialized"
   | Some secrets -> secrets
 
-let get_config_exn ctx =
-  match ctx.config with
-  | None -> context_error "config is uninitialized"
+let find_repo_config ctx repo_url = Hashtbl.find ctx.config repo_url
+
+let find_repo_config_exn ctx repo_url =
+  match find_repo_config ctx repo_url with
+  | None -> context_error "config uninitialized for repo %s" repo_url
   | Some config -> config
 
+let set_repo_config ctx repo_url config = Hashtbl.set ctx.config ~key:repo_url ~data:config
+
+let gh_token_of_secrets (secrets : Config_t.secrets) repo_url =
+  match Map.find secrets.repos repo_url with
+  | None -> secrets.gh_token
+  | Some repo_secrets -> repo_secrets.gh_token
+
+let gh_hook_token_of_secrets (secrets : Config_t.secrets) repo_url =
+  match Map.find secrets.repos repo_url with
+  | None -> secrets.gh_hook_token
+  | Some repo_secrets -> repo_secrets.gh_hook_token
+
 let hook_of_channel ctx channel_name =
   let secrets = get_secrets_exn ctx in
   match List.find secrets.slack_hooks ~f:(fun webhook -> String.equal webhook.channel channel_name) with
   | Some hook -> Some hook.url
   | None -> None
 
-(** `is_pipeline_allowed ctx p` returns `true` if ctx.config.status_rules
-    doesn't define a whitelist of allowed pipelines, or if the list
+(** `is_pipeline_allowed s r p` returns `true` if
+    `status_rules` doesn't define a whitelist of allowed
+    pipelines in the config of repo `r`, or if the list
     contains pipeline `p`; returns `false` otherwise. *)
-let is_pipeline_allowed ctx ~pipeline =
-  match ctx.config with
+let is_pipeline_allowed ctx repo_url ~pipeline =
+  match find_repo_config ctx repo_url with
   | None -> false
   | Some config ->
   match config.status_rules.allowed_pipelines with
   | Some allowed_pipelines when not @@ List.exists allowed_pipelines ~f:(String.equal pipeline) -> false
   | _ -> true
 
-let refresh_pipeline_status ctx ~pipeline ~(branches : Github_t.branch list) ~status =
-  if is_pipeline_allowed ctx ~pipeline then State.refresh_pipeline_status ctx.state ~pipeline ~branches ~status else ()
-
 let log = Log.from "context"
 
 let refresh_secrets ctx =
@@ -90,11 +101,12 @@ let refresh_state ctx =
     end
     else Ok ctx
 
-let print_config ctx =
-  let cfg = get_config_exn ctx in
+let print_config ctx repo_url =
+  let cfg = find_repo_config_exn ctx repo_url in
   let secrets = get_secrets_exn ctx in
+  let token = gh_hook_token_of_secrets secrets repo_url in
   log#info "using prefix routing:";
   Rule.Prefix.print_prefix_routing cfg.prefix_rules.rules;
   log#info "using label routing:";
   Rule.Label.print_label_routing cfg.label_rules.rules;
-  log#info "signature checking %s" (if Option.is_some secrets.gh_hook_token then "enabled" else "disabled")
+  log#info "signature checking %s" (if Option.is_some token then "enabled" else "disabled")