From 0f95bfba14e07cfd20f1b935a2f2a4e9bb0e3e05 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 11:32:38 +0800 Subject: [PATCH 01/13] add atd wrapper for hash tables --- lib/common.atd | 4 ++++ lib/common.ml | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/lib/common.atd b/lib/common.atd index 7354e2d2..280a4c5d 100644 --- a/lib/common.atd +++ b/lib/common.atd @@ -1,3 +1,7 @@ type 'v map_as_object = (string * 'v) list wrap + +type 'v table_as_object = + (string * 'v) list + wrap diff --git a/lib/common.ml b/lib/common.ml index b5de531e..d2018069 100644 --- a/lib/common.ml +++ b/lib/common.ml @@ -15,6 +15,20 @@ module StringMap = struct let unwrap = to_list end +module Table = struct + type 'a t = (string, 'a) Hashtbl.t + + let empty () = Hashtbl.create (module String) + + let to_list (l : 'a t) : (string * 'a) list = Hashtbl.to_alist l + + let of_list (m : (string * 'a) list) : 'a t = Hashtbl.of_alist_exn (module String) m + + let wrap = of_list + + let unwrap = to_list +end + let fmt_error fmt = Printf.ksprintf (fun s -> Error s) fmt let first_line s = From 3ef41591c6778f073a322e2bfa0a441efbd66142 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 11:43:43 +0800 Subject: [PATCH 02/13] make config a hash table mapping urls to config object --- lib/action.ml | 26 ++++++++++++++------------ lib/context.ml | 32 +++++++++++++++++++------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/lib/action.ml b/lib/action.ml index 07a5fe73..ff0e0c8f 100644 --- a/lib/action.ml +++ b/lib/action.ml @@ -87,7 +87,8 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct if List.is_empty matched_channel_names then default else matched_channel_names let partition_status (ctx : Context.t) (n : status_notification) = - let cfg = Context.get_config_exn ctx in + let repo = n.repository in + let cfg = Context.find_repo_config_exn ctx repo.url in let pipeline = n.context in let current_status = n.state in let rules = cfg.status_rules.rules in @@ -111,7 +112,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct | Ok commit -> Lwt.return @@ partition_commit cfg commit.files ) in - if Context.is_pipeline_allowed ctx ~pipeline then begin + if Context.is_pipeline_allowed ctx repo.url ~pipeline then begin match Rule.Status.match_rules ~rules n with | Some Ignore | None -> Lwt.return [] | Some Allow -> action_on_match n.branches @@ -130,7 +131,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct else Lwt.return [] let partition_commit_comment (ctx : Context.t) n = - let cfg = Context.get_config_exn ctx in + let cfg = Context.find_repo_config_exn ctx n.repository.url in match n.comment.commit_id with | None -> action_error "unable to find commit id for this commit comment event" | Some sha -> @@ -149,7 +150,8 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct ) let generate_notifications (ctx : Context.t) req = - let cfg = Context.get_config_exn ctx in + let repo = Github.repo_of_notification req in + let cfg = Context.find_repo_config_exn ctx repo.url in match req with | Github.Push n -> partition_push cfg n |> List.map ~f:(fun (channel, n) -> generate_push_notification n channel) |> Lwt.return @@ -178,20 +180,20 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct in Lwt_list.iter_s notify notifications - (** `refresh_config_of_context ctx n` updates the current context if the configuration - hasn't been loaded yet, or if the incoming request `n` is a push + (** `refresh_repo_config ctx n` fetches the latest repo config if it's + uninitialized, or if the incoming request `n` is a push notification containing commits that touched the config file. *) - let refresh_config_of_context (ctx : Context.t) notification = + let refresh_repo_config (ctx : Context.t) notification = + let repo = Github.repo_of_notification notification in let fetch_config () = - let repo = Github.repo_of_notification notification in match%lwt Github_api.get_config ~ctx ~repo with | Ok config -> - ctx.config <- Some config; - Context.print_config ctx; + Context.set_repo_config ctx repo.url config; + Context.print_config ctx repo.url; Lwt.return @@ Ok () | Error e -> action_error e in - match ctx.config with + match Context.find_repo_config ctx repo.url with | None -> fetch_config () | Some _ -> match notification with @@ -208,7 +210,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct match Github.parse_exn ~secret:secrets.gh_hook_token headers body with | exception exn -> Exn_lwt.fail ~exn "failed to parse payload" | payload -> - ( match%lwt refresh_config_of_context ctx payload with + ( match%lwt refresh_repo_config ctx payload with | Error e -> action_error e | Ok () -> let%lwt notifications = generate_notifications ctx payload in diff --git a/lib/context.ml b/lib/context.ml index 0c44f68c..bed82e3f 100644 --- a/lib/context.ml +++ b/lib/context.ml @@ -4,14 +4,14 @@ open Devkit exception Context_error of string -let context_error msg = raise (Context_error msg) +let context_error fmt = Printf.ksprintf (fun msg -> raise (Context_error msg)) fmt type t = { config_filename : string; secrets_filepath : string; state_filepath : string option; mutable secrets : Config_t.secrets option; - mutable config : Config_t.config option; + config : Config_t.config Table.t; state : State_t.state; } @@ -21,7 +21,7 @@ let default : t = secrets_filepath = "secrets.json"; state_filepath = None; secrets = None; - config = None; + config = Table.empty (); state = State.empty; } @@ -35,22 +35,27 @@ let get_secrets_exn ctx = | None -> context_error "secrets is uninitialized" | Some secrets -> secrets -let get_config_exn ctx = - match ctx.config with - | None -> context_error "config is uninitialized" +let find_repo_config ctx repo_url = Hashtbl.find ctx.config repo_url + +let find_repo_config_exn ctx repo_url = + match find_repo_config ctx repo_url with + | None -> context_error "config uninitialized for repo %s" repo_url | Some config -> config +let set_repo_config ctx repo_url config = Hashtbl.set ctx.config ~key:repo_url ~data:config + let hook_of_channel ctx channel_name = let secrets = get_secrets_exn ctx in match List.find secrets.slack_hooks ~f:(fun webhook -> String.equal webhook.channel channel_name) with | Some hook -> Some hook.url | None -> None -(** `is_pipeline_allowed ctx p` returns `true` if ctx.config.status_rules - doesn't define a whitelist of allowed pipelines, or if the list +(** `is_pipeline_allowed s r p` returns `true` if + `status_rules` doesn't define a whitelist of allowed + pipelines in the config of repo `r`, or if the list contains pipeline `p`; returns `false` otherwise. *) -let is_pipeline_allowed ctx ~pipeline = - match ctx.config with +let is_pipeline_allowed ctx repo_url ~pipeline = + match find_repo_config ctx repo_url with | None -> false | Some config -> match config.status_rules.allowed_pipelines with @@ -58,7 +63,8 @@ let is_pipeline_allowed ctx ~pipeline = | _ -> true let refresh_pipeline_status ctx ~pipeline ~(branches : Github_t.branch list) ~status = - if is_pipeline_allowed ctx ~pipeline then State.refresh_pipeline_status ctx.state ~pipeline ~branches ~status else () + if is_pipeline_allowed ctx "" ~pipeline then State.refresh_pipeline_status ctx.state ~pipeline ~branches ~status + else () let log = Log.from "context" @@ -90,8 +96,8 @@ let refresh_state ctx = end else Ok ctx -let print_config ctx = - let cfg = get_config_exn ctx in +let print_config ctx repo_url = + let cfg = find_repo_config_exn ctx repo_url in let secrets = get_secrets_exn ctx in log#info "using prefix routing:"; Rule.Prefix.print_prefix_routing cfg.prefix_rules.rules; From cafc7c26f32662e4d39505625c60e4b8a54bf7e3 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 11:51:43 +0800 Subject: [PATCH 03/13] store runtime repo state in a hash table indexed by repo url --- lib/action.ml | 5 +++-- lib/context.ml | 6 +----- lib/state.atd | 10 ++++++++-- lib/state.ml | 14 ++++++++++---- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/action.ml b/lib/action.ml index ff0e0c8f..92c2c2f1 100644 --- a/lib/action.ml +++ b/lib/action.ml @@ -94,7 +94,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct let rules = cfg.status_rules.rules in let action_on_match (branches : branch list) = let default = Option.to_list cfg.prefix_rules.default_channel in - let () = Context.refresh_pipeline_status ~pipeline ~branches ~status:current_status ctx in + State.set_repo_pipeline_status ctx.state repo.url ~pipeline ~branches ~status:current_status; match List.is_empty branches with | true -> Lwt.return [] | false -> @@ -113,11 +113,12 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct ) in if Context.is_pipeline_allowed ctx repo.url ~pipeline then begin + let repo_state = State.find_or_add_repo ctx.state repo.url in match Rule.Status.match_rules ~rules n with | Some Ignore | None -> Lwt.return [] | Some Allow -> action_on_match n.branches | Some Allow_once -> - match Map.find ctx.state.pipeline_statuses pipeline with + match Map.find repo_state.pipeline_statuses pipeline with | Some branch_statuses -> let has_same_status_state_as_prev (branch : branch) = match Map.find branch_statuses branch.name with diff --git a/lib/context.ml b/lib/context.ml index bed82e3f..bb321fb3 100644 --- a/lib/context.ml +++ b/lib/context.ml @@ -22,7 +22,7 @@ let default : t = state_filepath = None; secrets = None; config = Table.empty (); - state = State.empty; + state = State.empty (); } let make ?config_filename ?secrets_filepath ?state_filepath () = @@ -62,10 +62,6 @@ let is_pipeline_allowed ctx repo_url ~pipeline = | Some allowed_pipelines when not @@ List.exists allowed_pipelines ~f:(String.equal pipeline) -> false | _ -> true -let refresh_pipeline_status ctx ~pipeline ~(branches : Github_t.branch list) ~status = - if is_pipeline_allowed ctx "" ~pipeline then State.refresh_pipeline_status ctx.state ~pipeline ~branches ~status - else () - let log = Log.from "context" let refresh_secrets ctx = diff --git a/lib/state.atd b/lib/state.atd index 514ea20b..85c5b420 100644 --- a/lib/state.atd +++ b/lib/state.atd @@ -1,5 +1,6 @@ type status_state = abstract type 'v map_as_object = abstract +type 'v table_as_object = abstract (* A map from branch names to build statuses *) type branch_statuses = status_state map_as_object @@ -9,7 +10,12 @@ type branch_statuses = status_state map_as_object branch *) type pipeline_statuses = branch_statuses map_as_object +(* The runtime state of a given GitHub repository *) +type repo_state = { + pipeline_statuses : pipeline_statuses +} + (* The serializable runtime state of the bot *) type state = { - pipeline_statuses : pipeline_statuses -} \ No newline at end of file + ~repos : repo_state table_as_object +} diff --git a/lib/state.ml b/lib/state.ml index dcc7afe6..e7d8dc9f 100644 --- a/lib/state.ml +++ b/lib/state.ml @@ -2,15 +2,21 @@ open Base open Common open Devkit -let empty : State_t.state = { pipeline_statuses = StringMap.empty } +let empty_repo_state () : State_t.repo_state = { pipeline_statuses = StringMap.empty } -let refresh_pipeline_status (state : State_t.state) ~pipeline ~(branches : Github_t.branch list) ~status = - let update_pipeline_status branch_statuses = +let empty () : State_t.state = { repos = Table.empty () } + +let find_or_add_repo (state : State_t.state) repo_url = + Hashtbl.find_or_add state.repos repo_url ~default:empty_repo_state + +let set_repo_pipeline_status (state : State_t.state) repo_url ~pipeline ~(branches : Github_t.branch list) ~status = + let set_branch_status branch_statuses = let new_statuses = List.map branches ~f:(fun b -> b.name, status) in let init = Option.value branch_statuses ~default:(Map.empty (module String)) in List.fold_left new_statuses ~init ~f:(fun m (key, data) -> Map.set m ~key ~data) in - state.pipeline_statuses <- Map.update state.pipeline_statuses pipeline ~f:update_pipeline_status + let repo_state = find_or_add_repo state repo_url in + repo_state.pipeline_statuses <- Map.update repo_state.pipeline_statuses pipeline ~f:set_branch_status let log = Log.from "state" From 5a8fc14035421220b29021dd3e7fd4c02cccaa36 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 11:54:45 +0800 Subject: [PATCH 04/13] allocate new hash table for every Context.make () --- lib/context.ml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/context.ml b/lib/context.ml index bb321fb3..0c6c336e 100644 --- a/lib/context.ml +++ b/lib/context.ml @@ -15,21 +15,20 @@ type t = { state : State_t.state; } -let default : t = +let default_config_filename = "monorobot.json" + +let default_secrets_filepath = "secrets.json" + +let make ?config_filename ?secrets_filepath ?state_filepath () = { - config_filename = "monorobot.json"; - secrets_filepath = "secrets.json"; - state_filepath = None; + config_filename = Option.value config_filename ~default:default_config_filename; + secrets_filepath = Option.value secrets_filepath ~default:default_secrets_filepath; + state_filepath; secrets = None; config = Table.empty (); state = State.empty (); } -let make ?config_filename ?secrets_filepath ?state_filepath () = - let config_filename = Option.value config_filename ~default:default.config_filename in - let secrets_filepath = Option.value secrets_filepath ~default:default.secrets_filepath in - { default with config_filename; secrets_filepath; state_filepath } - let get_secrets_exn ctx = match ctx.secrets with | None -> context_error "secrets is uninitialized" From 0a81d7f4bcff3b99889efe43d2d5054c1e765b13 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 11:59:26 +0800 Subject: [PATCH 05/13] move webhook signature checking logic out of parse_exn With repo-specific secrets, we'll need the repo name in order to obtain the webhook token used for signature validation. So parsing the request body for a GH payload needs to happen before, not after, the signature check. --- lib/action.ml | 9 ++++++++- lib/github.ml | 18 +++++++++--------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/lib/action.ml b/lib/action.ml index 92c2c2f1..95d55ce4 100644 --- a/lib/action.ml +++ b/lib/action.ml @@ -206,11 +206,18 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct | _ -> Lwt.return @@ Ok () let process_github_notification (ctx : Context.t) headers body = + let validate_signature secrets = + let signing_key = secrets.gh_hook_token in + Github.validate_signature ?signing_key ~headers body + in try%lwt let secrets = Context.get_secrets_exn ctx in - match Github.parse_exn ~secret:secrets.gh_hook_token headers body with + match Github.parse_exn headers body with | exception exn -> Exn_lwt.fail ~exn "failed to parse payload" | payload -> + match validate_signature secrets with + | Error e -> action_error e + | Ok () -> ( match%lwt refresh_repo_config ctx payload with | Error e -> action_error e | Ok () -> diff --git a/lib/github.ml b/lib/github.ml index 6914dae0..8f8c6209 100644 --- a/lib/github.ml +++ b/lib/github.ml @@ -64,16 +64,16 @@ let is_valid_signature ~secret headers_sig body = let (`Hex request_hash) = Hex.of_string request_hash in String.equal headers_sig (sprintf "sha1=%s" request_hash) +let validate_signature ?signing_key ~headers body = + match signing_key with + | None -> Ok () + | Some secret -> + match List.Assoc.find headers "x-hub-signature" ~equal:String.equal with + | None -> Error "unable to find header x-hub-signature" + | Some signature -> if is_valid_signature ~secret signature body then Ok () else Error "signatures don't match" + (* Parse a payload. The type of the payload is detected from the headers. *) -let parse_exn ~secret headers body = - begin - match secret with - | None -> () - | Some secret -> - match List.Assoc.find headers "x-hub-signature" ~equal:String.equal with - | None -> Exn.fail "unable to find header x-hub-signature" - | Some req_sig -> if not @@ is_valid_signature ~secret req_sig body then failwith "request signature invalid" - end; +let parse_exn headers body = match List.Assoc.find_exn headers "x-github-event" ~equal:String.equal with | exception exn -> Exn.fail ~exn "unable to read x-github-event" | "push" -> Push (commit_pushed_notification_of_string body) From cc32884e1a278c48fe85752bbe8b79d7abf4867b Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 12:09:34 +0800 Subject: [PATCH 06/13] allow repo-specific configuration of gh_token and gh_hook_token Define custom getters for retrieving GH secret values. As the getter for each token type defaults to looking for a global value if a repo-specific vaue isn't found, existing deployments don't need to change. --- lib/action.ml | 8 ++++---- lib/api_remote.ml | 21 +++++++++++---------- lib/config.atd | 10 ++++++++++ lib/context.ml | 13 ++++++++++++- 4 files changed, 37 insertions(+), 15 deletions(-) diff --git a/lib/action.ml b/lib/action.ml index 95d55ce4..1f8383a2 100644 --- a/lib/action.ml +++ b/lib/action.ml @@ -106,7 +106,6 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct | false -> Lwt.return default | true -> let sha = n.commit.sha in - let repo = n.repository in ( match%lwt Github_api.get_api_commit ~ctx ~repo ~sha with | Error e -> action_error e | Ok commit -> Lwt.return @@ partition_commit cfg commit.files @@ -206,8 +205,9 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct | _ -> Lwt.return @@ Ok () let process_github_notification (ctx : Context.t) headers body = - let validate_signature secrets = - let signing_key = secrets.gh_hook_token in + let validate_signature secrets payload = + let repo = Github.repo_of_notification payload in + let signing_key = Context.gh_hook_token_of_secrets secrets repo.url in Github.validate_signature ?signing_key ~headers body in try%lwt @@ -215,7 +215,7 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct match Github.parse_exn headers body with | exception exn -> Exn_lwt.fail ~exn "failed to parse payload" | payload -> - match validate_signature secrets with + match validate_signature secrets payload with | Error e -> action_error e | Ok () -> ( match%lwt refresh_repo_config ctx payload with diff --git a/lib/api_remote.ml b/lib/api_remote.ml index 13ca53b6..188a59ee 100644 --- a/lib/api_remote.ml +++ b/lib/api_remote.ml @@ -23,7 +23,8 @@ module Github : Api.Github = struct let get_config ~(ctx : Context.t) ~repo = let secrets = Context.get_secrets_exn ctx in let url = contents_url ~repo ~path:ctx.config_filename in - let headers = build_headers ?token:secrets.gh_token () in + let token = Context.gh_token_of_secrets secrets repo.url in + let headers = build_headers ?token () in match%lwt http_request ~headers `GET url with | Error e -> Lwt.return @@ fmt_error "error while querying remote: %s\nfailed to get config from file %s" e url | Ok res -> @@ -44,23 +45,23 @@ module Github : Api.Github = struct @@ fmt_error "unexpected encoding '%s' in Github response\nfailed to get config from file %s" encoding url ) - let get_resource (ctx : Context.t) url = - let secrets = Context.get_secrets_exn ctx in - let headers = build_headers ?token:secrets.gh_token () in + let get_resource secrets repo_url url = + let token = Context.gh_token_of_secrets secrets repo_url in + let headers = build_headers ?token () in match%lwt http_request ~headers `GET url with | Ok res -> Lwt.return @@ Ok res | Error e -> Lwt.return @@ fmt_error "error while querying remote: %s\nfailed to get resource from %s" e url - let get_api_commit ~(ctx : Context.t) ~repo ~sha = - let%lwt res = commits_url ~repo ~sha |> get_resource ctx in + let get_api_commit ~(ctx : Context.t) ~(repo : Github_t.repository) ~sha = + let%lwt res = commits_url ~repo ~sha |> get_resource (Context.get_secrets_exn ctx) repo.url in Lwt.return @@ Result.map res ~f:Github_j.api_commit_of_string - let get_pull_request ~(ctx : Context.t) ~repo ~number = - let%lwt res = pulls_url ~repo ~number |> get_resource ctx in + let get_pull_request ~(ctx : Context.t) ~(repo : Github_t.repository) ~number = + let%lwt res = pulls_url ~repo ~number |> get_resource (Context.get_secrets_exn ctx) repo.url in Lwt.return @@ Result.map res ~f:Github_j.pull_request_of_string - let get_issue ~(ctx : Context.t) ~repo ~number = - let%lwt res = issues_url ~repo ~number |> get_resource ctx in + let get_issue ~(ctx : Context.t) ~(repo : Github_t.repository) ~number = + let%lwt res = issues_url ~repo ~number |> get_resource (Context.get_secrets_exn ctx) repo.url in Lwt.return @@ Result.map res ~f:Github_j.issue_of_string end diff --git a/lib/config.atd b/lib/config.atd index d90b1828..4f21529f 100644 --- a/lib/config.atd +++ b/lib/config.atd @@ -1,6 +1,7 @@ type status_rule = abstract type prefix_rule = abstract type label_rule = abstract +type 'v map_as_object = abstract (* This type of rule is used for CI build notifications. *) type status_rules = { @@ -36,9 +37,18 @@ type webhook = { channel : string; (* name of the Slack channel to post the message *) } +type gh_repo_secrets = { + (* GitHub personal access token, if repo access requires it *) + ?gh_token : string nullable; + (* GitHub webhook token to secure the webhook *) + ?gh_hook_token : string nullable; +} + (* This is the structure of the secrets file which stores sensitive information, and shouldn't be checked into version control. *) type secrets = { + (* repo-specific secrets; overrides global values if defined for a given repo *) + ~repos : gh_repo_secrets map_as_object; (* GitHub personal access token, if repo access requires it *) ?gh_token : string nullable; (* GitHub webhook token to secure the webhook *) diff --git a/lib/context.ml b/lib/context.ml index 0c6c336e..986a78bb 100644 --- a/lib/context.ml +++ b/lib/context.ml @@ -43,6 +43,16 @@ let find_repo_config_exn ctx repo_url = let set_repo_config ctx repo_url config = Hashtbl.set ctx.config ~key:repo_url ~data:config +let gh_token_of_secrets (secrets : Config_t.secrets) repo_url = + match Map.find secrets.repos repo_url with + | None -> secrets.gh_token + | Some repo_secrets -> repo_secrets.gh_token + +let gh_hook_token_of_secrets (secrets : Config_t.secrets) repo_url = + match Map.find secrets.repos repo_url with + | None -> secrets.gh_hook_token + | Some repo_secrets -> repo_secrets.gh_hook_token + let hook_of_channel ctx channel_name = let secrets = get_secrets_exn ctx in match List.find secrets.slack_hooks ~f:(fun webhook -> String.equal webhook.channel channel_name) with @@ -94,8 +104,9 @@ let refresh_state ctx = let print_config ctx repo_url = let cfg = find_repo_config_exn ctx repo_url in let secrets = get_secrets_exn ctx in + let token = gh_hook_token_of_secrets secrets repo_url in log#info "using prefix routing:"; Rule.Prefix.print_prefix_routing cfg.prefix_rules.rules; log#info "using label routing:"; Rule.Label.print_label_routing cfg.label_rules.rules; - log#info "signature checking %s" (if Option.is_some secrets.gh_hook_token then "enabled" else "disabled") + log#info "signature checking %s" (if Option.is_some token then "enabled" else "disabled") From 419e6e18a3116d28575719ddd0e5a525bfb615fa Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 12:11:15 +0800 Subject: [PATCH 07/13] add option to restrict handling of GH payloads to certain repo urls --- lib/action.ml | 8 ++++++++ lib/config.atd | 2 ++ 2 files changed, 10 insertions(+) diff --git a/lib/action.ml b/lib/action.ml index 1f8383a2..4eab9b26 100644 --- a/lib/action.ml +++ b/lib/action.ml @@ -210,6 +210,11 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct let signing_key = Context.gh_hook_token_of_secrets secrets repo.url in Github.validate_signature ?signing_key ~headers body in + let repo_is_allowed secrets payload = + let repo = Github.repo_of_notification payload in + let allowed_repos = secrets.allowed_repos in + List.is_empty allowed_repos || List.exists allowed_repos ~f:(String.equal repo.url) + in try%lwt let secrets = Context.get_secrets_exn ctx in match Github.parse_exn headers body with @@ -218,6 +223,9 @@ module Action (Github_api : Api.Github) (Slack_api : Api.Slack) = struct match validate_signature secrets payload with | Error e -> action_error e | Ok () -> + match repo_is_allowed secrets payload with + | false -> action_error "unsupported repository" + | true -> ( match%lwt refresh_repo_config ctx payload with | Error e -> action_error e | Ok () -> diff --git a/lib/config.atd b/lib/config.atd index 4f21529f..13242b3e 100644 --- a/lib/config.atd +++ b/lib/config.atd @@ -49,6 +49,8 @@ type gh_repo_secrets = { type secrets = { (* repo-specific secrets; overrides global values if defined for a given repo *) ~repos : gh_repo_secrets map_as_object; + (* whitelist of repository URLs to handle notifications for *) + ~allowed_repos : string list; (* GitHub personal access token, if repo access requires it *) ?gh_token : string nullable; (* GitHub webhook token to secure the webhook *) From c39d7a55fc4e59fdd93a06fa613b02291c087172 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 12:12:18 +0800 Subject: [PATCH 08/13] modify tests to recreate context per test case For each test case, initializes a repo state from file if one exists. --- test/test.ml | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/test/test.ml b/test/test.ml index 008f74e7..ff824136 100644 --- a/test/test.ml +++ b/test/test.ml @@ -19,24 +19,34 @@ let get_mock_payloads () = let state_path = Caml.Filename.concat mock_state_dir fn in if Caml.Sys.file_exists state_path then kind, payload_path, Some state_path else kind, payload_path, None) -let process ~(ctx : Context.t) (kind, path, state_path) = - let%lwt ctx = +let process ~(secrets : Config_t.secrets) ~config (kind, path, state_path) = + let headers = [ "x-github-event", kind ] in + let make_test_context event = + let repo = Github.repo_of_notification @@ Github.parse_exn headers event in + let ctx = Context.make () in + ctx.secrets <- Some secrets; + ignore (State.find_or_add_repo ctx.state repo.url); match state_path with - | None -> Lwt.return ctx + | None -> + Context.set_repo_config ctx repo.url config; + Lwt.return ctx | Some state_path -> match Common.get_local_file state_path with | Error e -> log#error "failed to read %s: %s" state_path e; Lwt.return ctx | Ok file -> - let state = State_j.state_of_string file in - Lwt.return { ctx with state } + let repo_state = State_j.repo_state_of_string file in + Hashtbl.set ctx.state.repos ~key:repo.url ~data:repo_state; + Context.set_repo_config ctx repo.url config; + Lwt.return ctx in Stdio.printf "===== file %s =====\n" path; let headers = [ "x-github-event", kind ] in match Common.get_local_file path with | Error e -> Lwt.return @@ log#error "failed to read %s: %s" path e | Ok event -> + let%lwt ctx = make_test_context event in let%lwt _ctx = Action_local.process_github_notification ctx headers event in Lwt.return_unit @@ -52,12 +62,10 @@ let () = log#error "%s" e; Lwt.return_unit | Ok config -> - let ctx = { ctx with config = Some config } in - ( match Context.refresh_secrets ctx with - | Ok ctx -> Lwt_list.iter_s (process ~ctx) payloads - | Error e -> - log#error "failed to read secrets:"; - log#error "%s" e; - Lwt.return_unit - ) + match Context.refresh_secrets ctx with + | Ok ctx -> Lwt_list.iter_s (process ~secrets:(Option.value_exn ctx.secrets) ~config) payloads + | Error e -> + log#error "failed to read secrets:"; + log#error "%s" e; + Lwt.return_unit ) From 0fdd590c20b8a48771ac160e3b57a1d519d97414 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 12:13:45 +0800 Subject: [PATCH 09/13] update documentation with multi-repo info --- README.md | 2 +- documentation/secret_docs.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 53f8f276..052d0446 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ You can configure Monorobot to [unfurl GitHub links](https://api.slack.com/refer ### Documentation -The bot expects two configuration files to be present. +Add a configuration file to each repository you want to support, and a secrets file on the bot server itself. Read on for instructions to set up each file: * [Repository configuration](./documentation/config_docs.md) * [Secrets](./documentation/secret_docs.md) diff --git a/documentation/secret_docs.md b/documentation/secret_docs.md index 87389bce..bc84f77d 100644 --- a/documentation/secret_docs.md +++ b/documentation/secret_docs.md @@ -17,6 +17,8 @@ A secrets file stores sensitive information. Unlike the repository configuration |-|-|-|-| | `gh_token` | specify to grant the bot access to private repositories; omit for public repositories | Yes | - | | `gh_hook_token` | specify to ensure the bot only receives GitHub notifications from pre-approved repositories | Yes | - | +| `repos` | an object mapping repository URLs to repository-specific GitHub secrets | Yes | - | +| `allowed_repos` | a whitelist of repository URLs to process payloads for | Yes | all incoming payloads are processed | | `slack_access_token` | slack bot access token to enable message posting to the workspace | Yes | try to use webhooks defined in `slack_hooks` instead | | `slack_hooks` | list of channel names and their corresponding webhook endpoint | Yes | try to use token defined in `slack_access_token` instead | | `slack_signing_secret` | specify to verify incoming slack requests | Yes | - | @@ -27,10 +29,36 @@ Note that either `slack_access_token` or `slack_hooks` must be defined. Some operations, such as fetching a config file from a private repository, or the commit corresponding to a commit comment event, require a personal access token. Refer [here](https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token) for detailed instructions on token generation. +*See `repos` if you need to support multiple repositories that use different tokens.* + ## `gh_hook_token` Refer [here](https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/securing-your-webhooks) for more information on securing webhooks with a token. +*See `repos` if you need to support multiple repositories that use different tokens.* + +## `repos` + +If you're using Monorobot for multiple repositories that need different secrets (e.g., one on github.com and another on GitHub Enterprise), you can provide them as an object. Secrets defined here will take precedence over those defined at the top level of the secrets file. + +Repository URLs should be fully qualified (include the protocol). + +```json +{ + "https://github.com/ahrefs/runner" : { + "gh_token": "XXX" + }, + "https://git.ahrefs.com/ahrefs/coyote" : { + "gh_token": "XXX", + "gh_hook_token": "XXX" + } +} +``` + +## `allowed_repos` + +Use this option to restrict incoming notifications from GitHub to approved repository URLs. + ## `slack_access_token` Refer [here](https://api.slack.com/authentication/oauth-v2) for obtaining an access token via OAuth. From 5f7c02c7ab8b8e404c65eb975540b80ab84bea19 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 14:06:46 +0800 Subject: [PATCH 10/13] tests: load secrets and config files per test case Make secrets, config, state, and incoming payload configurable individually, so that it's easier to test multi-repo behavior. --- lib/api_local.ml | 4 +- {test => mock_config}/monorobot.json | 0 {test => mock_secrets}/secrets.json | 0 test/dune | 6 +-- test/test.ml | 79 +++++++++++++--------------- 5 files changed, 43 insertions(+), 46 deletions(-) rename {test => mock_config}/monorobot.json (100%) rename {test => mock_secrets}/secrets.json (100%) diff --git a/lib/api_local.ml b/lib/api_local.ml index 915faa6d..34e540d2 100644 --- a/lib/api_local.ml +++ b/lib/api_local.ml @@ -7,8 +7,10 @@ let cwd = Caml.Sys.getcwd () let cache_dir = Caml.Filename.concat cwd "github-api-cache" module Github : Api.Github = struct + let mock_config_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_config" + let get_config ~(ctx : Context.t) ~repo:_ = - let url = Caml.Filename.concat cwd ctx.config_filename in + let url = Caml.Filename.concat mock_config_dir ctx.config_filename in match get_local_file url with | Error e -> Lwt.return @@ fmt_error "error while getting local file: %s\nfailed to get config %s" e url | Ok file -> Lwt.return @@ Ok (Config_j.config_of_string file) diff --git a/test/monorobot.json b/mock_config/monorobot.json similarity index 100% rename from test/monorobot.json rename to mock_config/monorobot.json diff --git a/test/secrets.json b/mock_secrets/secrets.json similarity index 100% rename from test/secrets.json rename to mock_secrets/secrets.json diff --git a/test/dune b/test/dune index 71cdc617..e6ad603e 100644 --- a/test/dune +++ b/test/dune @@ -9,9 +9,9 @@ (deps (source_tree ../mock_states) (source_tree ../mock_payloads) - (source_tree github-api-cache) - monorobot.json - secrets.json) + (source_tree ../mock_config) + (source_tree ../mock_secrets) + (source_tree github-api-cache)) (action (with-stdout-to slack_payloads.out diff --git a/test/test.ml b/test/test.ml index ff824136..e5a5e375 100644 --- a/test/test.ml +++ b/test/test.ml @@ -1,12 +1,19 @@ open Base open Lib +open Common let log = Devkit.Log.from "test" +let () = Devkit.Log.set_loglevels "error" + let mock_payload_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_payloads" let mock_state_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_states" +let mock_secrets_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_secrets" + +let mock_config_dir = Caml.Filename.concat Caml.Filename.parent_dir_name "mock_config" + module Action_local = Action.Action (Api_local.Github) (Api_local.Slack) let get_mock_payloads () = @@ -16,56 +23,44 @@ let get_mock_payloads () = |> List.filter_map ~f:(fun fn -> Github.event_of_filename fn |> Option.map ~f:(fun kind -> kind, fn)) |> List.map ~f:(fun (kind, fn) -> let payload_path = Caml.Filename.concat mock_payload_dir fn in - let state_path = Caml.Filename.concat mock_state_dir fn in - if Caml.Sys.file_exists state_path then kind, payload_path, Some state_path else kind, payload_path, None) + let state_filepath = + let path = Caml.Filename.concat mock_state_dir fn in + if Caml.Sys.file_exists path then Some path else None + in + let secrets_filepath = + let path = Caml.Filename.concat mock_secrets_dir fn in + if Caml.Sys.file_exists path then Some path else None + in + let config_filename = + let path = Caml.Filename.concat mock_config_dir fn in + if Caml.Sys.file_exists path then Some fn else None + in + kind, payload_path, state_filepath, secrets_filepath, config_filename) -let process ~(secrets : Config_t.secrets) ~config (kind, path, state_path) = - let headers = [ "x-github-event", kind ] in - let make_test_context event = - let repo = Github.repo_of_notification @@ Github.parse_exn headers event in - let ctx = Context.make () in - ctx.secrets <- Some secrets; - ignore (State.find_or_add_repo ctx.state repo.url); - match state_path with - | None -> - Context.set_repo_config ctx repo.url config; - Lwt.return ctx - | Some state_path -> - match Common.get_local_file state_path with - | Error e -> - log#error "failed to read %s: %s" state_path e; - Lwt.return ctx - | Ok file -> - let repo_state = State_j.repo_state_of_string file in - Hashtbl.set ctx.state.repos ~key:repo.url ~data:repo_state; - Context.set_repo_config ctx repo.url config; - Lwt.return ctx +let process (kind, path, state_filepath, secrets_filepath, config_filename) = + let make_test_context () = + let secrets_filepath = + Option.value ~default:(Caml.Filename.concat mock_secrets_dir Context.default_secrets_filepath) secrets_filepath + in + let ctx = Context.make ?config_filename ~secrets_filepath ?state_filepath () in + match Context.refresh_state ctx with + | Error e -> fmt_error "failed to read state: %s" e + | Ok ctx -> + match Context.refresh_secrets ctx with + | Error e -> fmt_error "failed to read secrets: %s" e + | Ok ctx -> Ok ctx in Stdio.printf "===== file %s =====\n" path; let headers = [ "x-github-event", kind ] in - match Common.get_local_file path with + match get_local_file path with | Error e -> Lwt.return @@ log#error "failed to read %s: %s" path e | Ok event -> - let%lwt ctx = make_test_context event in + match make_test_context () with + | Error e -> Lwt.return @@ log#error "%s" e + | Ok ctx -> let%lwt _ctx = Action_local.process_github_notification ctx headers event in Lwt.return_unit let () = let payloads = get_mock_payloads () in - let repo : Github_t.repository = - { name = ""; full_name = ""; url = ""; commits_url = ""; contents_url = ""; pulls_url = ""; issues_url = "" } - in - let ctx = Context.make ~state_filepath:"state.json" () in - Lwt_main.run - ( match%lwt Api_local.Github.get_config ~ctx ~repo with - | Error e -> - log#error "%s" e; - Lwt.return_unit - | Ok config -> - match Context.refresh_secrets ctx with - | Ok ctx -> Lwt_list.iter_s (process ~secrets:(Option.value_exn ctx.secrets) ~config) payloads - | Error e -> - log#error "failed to read secrets:"; - log#error "%s" e; - Lwt.return_unit - ) + Lwt_main.run (Lwt_list.iter_s process payloads) From 59cb9ee16740279053c24f46318dae47acbc5bfa Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 14:08:31 +0800 Subject: [PATCH 11/13] fully use logger for config printing --- lib/rule.ml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/rule.ml b/lib/rule.ml index e1645af5..49995bf5 100644 --- a/lib/rule.ml +++ b/lib/rule.ml @@ -65,19 +65,20 @@ module Prefix = struct |> Option.map ~f:(fun (res : prefix_rule * int) -> (fst res).channel_name) let print_prefix_routing rules = + let log = Devkit.Log.from "context" in let show_match l = String.concat ~sep:" or " @@ List.map ~f:(fun s -> s ^ "*") l in rules |> List.iter ~f:(fun (rule : prefix_rule) -> begin match rule.allow, rule.ignore with - | None, None -> Stdio.printf " any" - | None, Some [] -> Stdio.printf " any" - | None, Some l -> Stdio.printf " not %s" (show_match l) - | Some l, None -> Stdio.printf " %s" (show_match l) - | Some l, Some [] -> Stdio.printf " %s" (show_match l) - | Some l, Some i -> Stdio.printf " %s and not %s" (show_match l) (show_match i) + | None, None -> log#info " any" + | None, Some [] -> log#info " any" + | None, Some l -> log#info " not %s" (show_match l) + | Some l, None -> log#info " %s" (show_match l) + | Some l, Some [] -> log#info " %s" (show_match l) + | Some l, Some i -> log#info " %s and not %s" (show_match l) (show_match i) end; - Stdio.printf " -> #%s\n%!" rule.channel_name) + log#info " -> #%s\n%!" rule.channel_name) end module Label = struct @@ -100,17 +101,18 @@ module Label = struct rules |> List.filter_map ~f:match_rule |> List.dedup_and_sort ~compare:String.compare let print_label_routing rules = + let log = Devkit.Log.from "context" in let show_match l = String.concat ~sep:" or " l in rules |> List.iter ~f:(fun (rule : label_rule) -> begin match rule.allow, rule.ignore with - | None, None -> Stdio.printf " any" - | None, Some [] -> Stdio.printf " any" - | None, Some l -> Stdio.printf " not %s" (show_match l) - | Some l, None -> Stdio.printf " %s" (show_match l) - | Some l, Some [] -> Stdio.printf " %s" (show_match l) - | Some l, Some i -> Stdio.printf " %s and not %s" (show_match l) (show_match i) + | None, None -> log#info " any" + | None, Some [] -> log#info " any" + | None, Some l -> log#info " not %s" (show_match l) + | Some l, None -> log#info " %s" (show_match l) + | Some l, Some [] -> log#info " %s" (show_match l) + | Some l, Some i -> log#info " %s and not %s" (show_match l) (show_match i) end; - Stdio.printf " -> #%s\n%!" rule.channel_name) + log#info " -> #%s\n%!" rule.channel_name) end From ad12cd2783f06fb05138c5c0a30448a8ca7764c8 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 14:09:30 +0800 Subject: [PATCH 12/13] tests: promote mock states to new state format --- mock_states/status.state_hide_success_test.json | 16 ++++++++++------ ...te_hide_success_test_disallowed_pipeline.json | 10 +++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/mock_states/status.state_hide_success_test.json b/mock_states/status.state_hide_success_test.json index 0a4b7dec..725acca5 100644 --- a/mock_states/status.state_hide_success_test.json +++ b/mock_states/status.state_hide_success_test.json @@ -1,11 +1,15 @@ { - "pipeline_statuses": { - "default": { - "master": "failure" - }, - "buildkite/pipeline2": { - "master": "success" + "repos": { + "https://github.com/ahrefs/monorepo": { + "pipeline_statuses": { + "default": { + "master": "failure" + }, + "buildkite/pipeline2": { + "master": "success" + } + } } } } \ No newline at end of file diff --git a/mock_states/status.state_hide_success_test_disallowed_pipeline.json b/mock_states/status.state_hide_success_test_disallowed_pipeline.json index 25b4592a..70a77e09 100644 --- a/mock_states/status.state_hide_success_test_disallowed_pipeline.json +++ b/mock_states/status.state_hide_success_test_disallowed_pipeline.json @@ -1,7 +1,11 @@ { - "pipeline_statuses": { - "buildkite/pipeline2": { - "master": "failure" + "repos": { + "https://github.com/ahrefs/monorepo": { + "pipeline_statuses": { + "buildkite/pipeline2": { + "master": "failure" + } + } } } } \ No newline at end of file From 34f4997a23ecd25444ad49ce831a3b31af810c39 Mon Sep 17 00:00:00 2001 From: Yasunari Watanabe Date: Tue, 5 Jan 2021 14:09:49 +0800 Subject: [PATCH 13/13] tests: add cases that check repo-specific behavior `status.multi_repo_disallowed_repo` If `allowed_repositories` excludes the payload repo, it should be ignored even if payload would generate notification otherwise. `status.multi_repo_independent_status_state` The status rules should be evaluated for the correct repo. In this test case, it should correctly detect that `master`'s last build status was a failure for the given payload repo, and issue a success notification, despite the last build status success in a different repo. --- .../status.multi_repo_disallowed_repo.json | 222 ++++++++++++++++++ ...s.multi_repo_independent_status_state.json | 222 ++++++++++++++++++ .../status.multi_repo_disallowed_repo.json | 4 + ...s.multi_repo_independent_status_state.json | 19 ++ test/slack_payloads.expected | 23 ++ 5 files changed, 490 insertions(+) create mode 100644 mock_payloads/status.multi_repo_disallowed_repo.json create mode 100644 mock_payloads/status.multi_repo_independent_status_state.json create mode 100644 mock_secrets/status.multi_repo_disallowed_repo.json create mode 100644 mock_secrets/status.multi_repo_independent_status_state.json diff --git a/mock_payloads/status.multi_repo_disallowed_repo.json b/mock_payloads/status.multi_repo_disallowed_repo.json new file mode 100644 index 00000000..71c7e867 --- /dev/null +++ b/mock_payloads/status.multi_repo_disallowed_repo.json @@ -0,0 +1,222 @@ +{ + "id": 0, + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "name": "ahrefs/monorobot_test", + "target_url": "https://buildkite.com/org/pipeline2/builds/2", + "avatar_url": "https://example.org/avatars/oa/0", + "context": "buildkite/pipeline2", + "description": "Build #2 passed (5 minutes, 19 seconds)", + "state": "success", + "commit": { + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "node_id": "00000000000000000000", + "commit": { + "author": { + "name": "Wile E. Coyote", + "email": "wile.e.coyote@example.org", + "date": "2020-06-02T03:14:51Z" + }, + "committer": { + "name": "GitHub Enterprise", + "email": "git@example.org", + "date": "2020-06-02T03:14:51Z" + }, + "message": "Update README.md", + "tree": { + "sha": "ee5c539cad37c77348ce7a55756acc542b41cfc7", + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/trees/ee5c539cad37c77348ce7a55756acc542b41cfc7" + }, + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "comment_count": 0, + "verification": { + "verified": false, + "reason": "unsigned", + "signature": null, + "payload": null + } + }, + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "html_url": "https://example.org/ahrefs/monorobot_test/commit/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "comments_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478/comments", + "author": { + "login": "wileecoyote", + "id": 0, + "node_id": "00000000000000000000", + "avatar_url": "https://example.org/avatars/u/0", + "gravatar_id": "", + "url": "https://example.org/api/v3/users/wileecoyote", + "html_url": "https://example.org/wileecoyote", + "followers_url": "https://example.org/api/v3/users/wileecoyote/followers", + "following_url": "https://example.org/api/v3/users/wileecoyote/following{/other_user}", + "gists_url": "https://example.org/api/v3/users/wileecoyote/gists{/gist_id}", + "starred_url": "https://example.org/api/v3/users/wileecoyote/starred{/owner}{/repo}", + "subscriptions_url": "https://example.org/api/v3/users/wileecoyote/subscriptions", + "organizations_url": "https://example.org/api/v3/users/wileecoyote/orgs", + "repos_url": "https://example.org/api/v3/users/wileecoyote/repos", + "events_url": "https://example.org/api/v3/users/wileecoyote/events{/privacy}", + "received_events_url": "https://example.org/api/v3/users/wileecoyote/received_events", + "type": "User", + "site_admin": false + }, + "committer": null, + "parents": [ + { + "sha": "04cb72d6dc8d92131282a7eff57f6caf632f0a39", + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits/04cb72d6dc8d92131282a7eff57f6caf632f0a39", + "html_url": "https://example.org/ahrefs/monorobot_test/commit/04cb72d6dc8d92131282a7eff57f6caf632f0a39" + } + ] + }, + "branches": [ + { + "name": "master", + "commit": { + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478" + }, + "protected": false + } + ], + "created_at": "2020-06-02T03:21:39+00:00", + "updated_at": "2020-06-02T03:21:39+00:00", + "repository": { + "id": 0, + "node_id": "00000000000000000000", + "name": "monorobot_test", + "full_name": "ahrefs/monorobot_test", + "private": true, + "owner": { + "login": "ahrefs", + "id": 0, + "node_id": "00000000000000000000", + "avatar_url": "https://example.org/avatars/u/0", + "gravatar_id": "", + "url": "https://example.org/api/v3/users/ahrefs", + "html_url": "https://example.org/ahrefs", + "followers_url": "https://example.org/api/v3/users/ahrefs/followers", + "following_url": "https://example.org/api/v3/users/ahrefs/following{/other_user}", + "gists_url": "https://example.org/api/v3/users/ahrefs/gists{/gist_id}", + "starred_url": "https://example.org/api/v3/users/ahrefs/starred{/owner}{/repo}", + "subscriptions_url": "https://example.org/api/v3/users/ahrefs/subscriptions", + "organizations_url": "https://example.org/api/v3/users/ahrefs/orgs", + "repos_url": "https://example.org/api/v3/users/ahrefs/repos", + "events_url": "https://example.org/api/v3/users/ahrefs/events{/privacy}", + "received_events_url": "https://example.org/api/v3/users/ahrefs/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://example.org/ahrefs/monorobot_test", + "description": null, + "fork": false, + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test", + "forks_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/forks", + "keys_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/keys{/key_id}", + "collaborators_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/collaborators{/collaborator}", + "teams_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/teams", + "hooks_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/hooks", + "issue_events_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/issues/events{/number}", + "events_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/events", + "assignees_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/assignees{/user}", + "branches_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/branches{/branch}", + "tags_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/tags", + "blobs_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/blobs{/sha}", + "git_tags_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/tags{/sha}", + "git_refs_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/refs{/sha}", + "trees_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/trees{/sha}", + "statuses_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/statuses/{sha}", + "languages_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/languages", + "stargazers_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/stargazers", + "contributors_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/contributors", + "subscribers_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/subscribers", + "subscription_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/subscription", + "commits_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits{/sha}", + "git_commits_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/commits{/sha}", + "comments_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/comments{/number}", + "issue_comment_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/issues/comments{/number}", + "contents_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/contents/{+path}", + "compare_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/compare/{base}...{head}", + "merges_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/merges", + "archive_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/{archive_format}{/ref}", + "downloads_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/downloads", + "issues_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/issues{/number}", + "pulls_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/pulls{/number}", + "milestones_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/milestones{/number}", + "notifications_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/notifications{?since,all,participating}", + "labels_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/labels{/name}", + "releases_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/releases{/id}", + "deployments_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/deployments", + "created_at": "2020-06-01T18:44:17Z", + "updated_at": "2020-06-02T03:14:53Z", + "pushed_at": "2020-06-02T03:14:51Z", + "git_url": "git://example.org/ahrefs/monorobot_test.git", + "ssh_url": "git@example.org:ahrefs/monorobot_test.git", + "clone_url": "https://example.org/ahrefs/monorobot_test.git", + "svn_url": "https://example.org/ahrefs/monorobot_test", + "homepage": null, + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Shell", + "has_issues": true, + "has_projects": false, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": null, + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master" + }, + "organization": { + "login": "ahrefs", + "id": 0, + "node_id": "00000000000000000000", + "url": "https://example.org/api/v3/orgs/ahrefs", + "repos_url": "https://example.org/api/v3/orgs/ahrefs/repos", + "events_url": "https://example.org/api/v3/orgs/ahrefs/events", + "hooks_url": "https://example.org/api/v3/orgs/ahrefs/hooks", + "issues_url": "https://example.org/api/v3/orgs/ahrefs/issues", + "members_url": "https://example.org/api/v3/orgs/ahrefs/members{/member}", + "public_members_url": "https://example.org/api/v3/orgs/ahrefs/public_members{/member}", + "avatar_url": "https://example.org/avatars/u/0", + "description": null + }, + "enterprise": { + "id": 0, + "slug": "ahrefs-corp", + "name": "Acme Corp", + "node_id": "00000000000000000000", + "avatar_url": "https://example.org/avatars/b/0", + "description": null, + "website_url": null, + "html_url": "https://example.org/enterprises/ahrefs-corp", + "created_at": "2019-01-09T18:50:55Z", + "updated_at": "2019-03-01T16:07:28Z" + }, + "sender": { + "login": "ygrek", + "id": 0, + "node_id": "00000000000000000000", + "avatar_url": "https://example.org/avatars/u/0", + "gravatar_id": "", + "url": "https://example.org/api/v3/users/ygrek", + "html_url": "https://example.org/ip", + "followers_url": "https://example.org/api/v3/users/ygrek/followers", + "following_url": "https://example.org/api/v3/users/ygrek/following{/other_user}", + "gists_url": "https://example.org/api/v3/users/ygrek/gists{/gist_id}", + "starred_url": "https://example.org/api/v3/users/ygrek/starred{/owner}{/repo}", + "subscriptions_url": "https://example.org/api/v3/users/ygrek/subscriptions", + "organizations_url": "https://example.org/api/v3/users/ygrek/orgs", + "repos_url": "https://example.org/api/v3/users/ygrek/repos", + "events_url": "https://example.org/api/v3/users/ygrek/events{/privacy}", + "received_events_url": "https://example.org/api/v3/users/ygrek/received_events", + "type": "User", + "site_admin": true + } +} diff --git a/mock_payloads/status.multi_repo_independent_status_state.json b/mock_payloads/status.multi_repo_independent_status_state.json new file mode 100644 index 00000000..71c7e867 --- /dev/null +++ b/mock_payloads/status.multi_repo_independent_status_state.json @@ -0,0 +1,222 @@ +{ + "id": 0, + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "name": "ahrefs/monorobot_test", + "target_url": "https://buildkite.com/org/pipeline2/builds/2", + "avatar_url": "https://example.org/avatars/oa/0", + "context": "buildkite/pipeline2", + "description": "Build #2 passed (5 minutes, 19 seconds)", + "state": "success", + "commit": { + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "node_id": "00000000000000000000", + "commit": { + "author": { + "name": "Wile E. Coyote", + "email": "wile.e.coyote@example.org", + "date": "2020-06-02T03:14:51Z" + }, + "committer": { + "name": "GitHub Enterprise", + "email": "git@example.org", + "date": "2020-06-02T03:14:51Z" + }, + "message": "Update README.md", + "tree": { + "sha": "ee5c539cad37c77348ce7a55756acc542b41cfc7", + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/trees/ee5c539cad37c77348ce7a55756acc542b41cfc7" + }, + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "comment_count": 0, + "verification": { + "verified": false, + "reason": "unsigned", + "signature": null, + "payload": null + } + }, + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "html_url": "https://example.org/ahrefs/monorobot_test/commit/0d95302addd66c1816bce1b1d495ed1c93ccd478", + "comments_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478/comments", + "author": { + "login": "wileecoyote", + "id": 0, + "node_id": "00000000000000000000", + "avatar_url": "https://example.org/avatars/u/0", + "gravatar_id": "", + "url": "https://example.org/api/v3/users/wileecoyote", + "html_url": "https://example.org/wileecoyote", + "followers_url": "https://example.org/api/v3/users/wileecoyote/followers", + "following_url": "https://example.org/api/v3/users/wileecoyote/following{/other_user}", + "gists_url": "https://example.org/api/v3/users/wileecoyote/gists{/gist_id}", + "starred_url": "https://example.org/api/v3/users/wileecoyote/starred{/owner}{/repo}", + "subscriptions_url": "https://example.org/api/v3/users/wileecoyote/subscriptions", + "organizations_url": "https://example.org/api/v3/users/wileecoyote/orgs", + "repos_url": "https://example.org/api/v3/users/wileecoyote/repos", + "events_url": "https://example.org/api/v3/users/wileecoyote/events{/privacy}", + "received_events_url": "https://example.org/api/v3/users/wileecoyote/received_events", + "type": "User", + "site_admin": false + }, + "committer": null, + "parents": [ + { + "sha": "04cb72d6dc8d92131282a7eff57f6caf632f0a39", + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits/04cb72d6dc8d92131282a7eff57f6caf632f0a39", + "html_url": "https://example.org/ahrefs/monorobot_test/commit/04cb72d6dc8d92131282a7eff57f6caf632f0a39" + } + ] + }, + "branches": [ + { + "name": "master", + "commit": { + "sha": "0d95302addd66c1816bce1b1d495ed1c93ccd478", + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits/0d95302addd66c1816bce1b1d495ed1c93ccd478" + }, + "protected": false + } + ], + "created_at": "2020-06-02T03:21:39+00:00", + "updated_at": "2020-06-02T03:21:39+00:00", + "repository": { + "id": 0, + "node_id": "00000000000000000000", + "name": "monorobot_test", + "full_name": "ahrefs/monorobot_test", + "private": true, + "owner": { + "login": "ahrefs", + "id": 0, + "node_id": "00000000000000000000", + "avatar_url": "https://example.org/avatars/u/0", + "gravatar_id": "", + "url": "https://example.org/api/v3/users/ahrefs", + "html_url": "https://example.org/ahrefs", + "followers_url": "https://example.org/api/v3/users/ahrefs/followers", + "following_url": "https://example.org/api/v3/users/ahrefs/following{/other_user}", + "gists_url": "https://example.org/api/v3/users/ahrefs/gists{/gist_id}", + "starred_url": "https://example.org/api/v3/users/ahrefs/starred{/owner}{/repo}", + "subscriptions_url": "https://example.org/api/v3/users/ahrefs/subscriptions", + "organizations_url": "https://example.org/api/v3/users/ahrefs/orgs", + "repos_url": "https://example.org/api/v3/users/ahrefs/repos", + "events_url": "https://example.org/api/v3/users/ahrefs/events{/privacy}", + "received_events_url": "https://example.org/api/v3/users/ahrefs/received_events", + "type": "Organization", + "site_admin": false + }, + "html_url": "https://example.org/ahrefs/monorobot_test", + "description": null, + "fork": false, + "url": "https://example.org/api/v3/repos/ahrefs/monorobot_test", + "forks_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/forks", + "keys_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/keys{/key_id}", + "collaborators_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/collaborators{/collaborator}", + "teams_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/teams", + "hooks_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/hooks", + "issue_events_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/issues/events{/number}", + "events_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/events", + "assignees_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/assignees{/user}", + "branches_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/branches{/branch}", + "tags_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/tags", + "blobs_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/blobs{/sha}", + "git_tags_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/tags{/sha}", + "git_refs_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/refs{/sha}", + "trees_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/trees{/sha}", + "statuses_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/statuses/{sha}", + "languages_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/languages", + "stargazers_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/stargazers", + "contributors_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/contributors", + "subscribers_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/subscribers", + "subscription_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/subscription", + "commits_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/commits{/sha}", + "git_commits_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/git/commits{/sha}", + "comments_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/comments{/number}", + "issue_comment_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/issues/comments{/number}", + "contents_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/contents/{+path}", + "compare_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/compare/{base}...{head}", + "merges_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/merges", + "archive_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/{archive_format}{/ref}", + "downloads_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/downloads", + "issues_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/issues{/number}", + "pulls_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/pulls{/number}", + "milestones_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/milestones{/number}", + "notifications_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/notifications{?since,all,participating}", + "labels_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/labels{/name}", + "releases_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/releases{/id}", + "deployments_url": "https://example.org/api/v3/repos/ahrefs/monorobot_test/deployments", + "created_at": "2020-06-01T18:44:17Z", + "updated_at": "2020-06-02T03:14:53Z", + "pushed_at": "2020-06-02T03:14:51Z", + "git_url": "git://example.org/ahrefs/monorobot_test.git", + "ssh_url": "git@example.org:ahrefs/monorobot_test.git", + "clone_url": "https://example.org/ahrefs/monorobot_test.git", + "svn_url": "https://example.org/ahrefs/monorobot_test", + "homepage": null, + "size": 0, + "stargazers_count": 0, + "watchers_count": 0, + "language": "Shell", + "has_issues": true, + "has_projects": false, + "has_downloads": true, + "has_wiki": true, + "has_pages": false, + "forks_count": 0, + "mirror_url": null, + "archived": false, + "disabled": false, + "open_issues_count": 0, + "license": null, + "forks": 0, + "open_issues": 0, + "watchers": 0, + "default_branch": "master" + }, + "organization": { + "login": "ahrefs", + "id": 0, + "node_id": "00000000000000000000", + "url": "https://example.org/api/v3/orgs/ahrefs", + "repos_url": "https://example.org/api/v3/orgs/ahrefs/repos", + "events_url": "https://example.org/api/v3/orgs/ahrefs/events", + "hooks_url": "https://example.org/api/v3/orgs/ahrefs/hooks", + "issues_url": "https://example.org/api/v3/orgs/ahrefs/issues", + "members_url": "https://example.org/api/v3/orgs/ahrefs/members{/member}", + "public_members_url": "https://example.org/api/v3/orgs/ahrefs/public_members{/member}", + "avatar_url": "https://example.org/avatars/u/0", + "description": null + }, + "enterprise": { + "id": 0, + "slug": "ahrefs-corp", + "name": "Acme Corp", + "node_id": "00000000000000000000", + "avatar_url": "https://example.org/avatars/b/0", + "description": null, + "website_url": null, + "html_url": "https://example.org/enterprises/ahrefs-corp", + "created_at": "2019-01-09T18:50:55Z", + "updated_at": "2019-03-01T16:07:28Z" + }, + "sender": { + "login": "ygrek", + "id": 0, + "node_id": "00000000000000000000", + "avatar_url": "https://example.org/avatars/u/0", + "gravatar_id": "", + "url": "https://example.org/api/v3/users/ygrek", + "html_url": "https://example.org/ip", + "followers_url": "https://example.org/api/v3/users/ygrek/followers", + "following_url": "https://example.org/api/v3/users/ygrek/following{/other_user}", + "gists_url": "https://example.org/api/v3/users/ygrek/gists{/gist_id}", + "starred_url": "https://example.org/api/v3/users/ygrek/starred{/owner}{/repo}", + "subscriptions_url": "https://example.org/api/v3/users/ygrek/subscriptions", + "organizations_url": "https://example.org/api/v3/users/ygrek/orgs", + "repos_url": "https://example.org/api/v3/users/ygrek/repos", + "events_url": "https://example.org/api/v3/users/ygrek/events{/privacy}", + "received_events_url": "https://example.org/api/v3/users/ygrek/received_events", + "type": "User", + "site_admin": true + } +} diff --git a/mock_secrets/status.multi_repo_disallowed_repo.json b/mock_secrets/status.multi_repo_disallowed_repo.json new file mode 100644 index 00000000..81211dc6 --- /dev/null +++ b/mock_secrets/status.multi_repo_disallowed_repo.json @@ -0,0 +1,4 @@ +{ + "allowed_repos": ["https://github.com/Codertocat/Hello-World"], + "slack_access_token": "" +} diff --git a/mock_secrets/status.multi_repo_independent_status_state.json b/mock_secrets/status.multi_repo_independent_status_state.json new file mode 100644 index 00000000..24c307cc --- /dev/null +++ b/mock_secrets/status.multi_repo_independent_status_state.json @@ -0,0 +1,19 @@ +{ + "repos": { + "https://github.com/ahrefs/runner": { + "pipeline_statuses": { + "buildkite/pipeline2": { + "master": "success" + } + } + }, + "https://example.org/ahrefs/monorepo": { + "pipeline_statuses": { + "buildkite/pipeline2": { + "master": "failure" + } + } + } + }, + "slack_access_token": "" +} diff --git a/test/slack_payloads.expected b/test/slack_payloads.expected index 4e02646b..e7f4798c 100644 --- a/test/slack_payloads.expected +++ b/test/slack_payloads.expected @@ -514,6 +514,29 @@ will notify #default ] } ===== file ../mock_payloads/status.merge_develop.json ===== +===== file ../mock_payloads/status.multi_repo_disallowed_repo.json ===== +===== file ../mock_payloads/status.multi_repo_independent_status_state.json ===== +will notify #default +{ + "channel": "default", + "attachments": [ + { + "fallback": + " CI Build Status notification for : success", + "mrkdwn_in": [ "fields", "text" ], + "color": "good", + "pretext": + " CI Build Status notification for : success", + "text": "*Description*: Build #2 passed (5 minutes, 19 seconds).", + "fields": [ + { + "value": + "*Commit*: `` Update README.md - wileecoyote\n*Branch*: master" + } + ] + } + ] +} ===== file ../mock_payloads/status.pending_test.json ===== ===== file ../mock_payloads/status.state_hide_success_test.json ===== ===== file ../mock_payloads/status.state_hide_success_test_disallowed_pipeline.json =====