diff --git a/bin/lib/amazon.py b/bin/lib/amazon.py index a83b402dc..18d621caa 100644 --- a/bin/lib/amazon.py +++ b/bin/lib/amazon.py @@ -206,6 +206,11 @@ def release_for(releases, s3_key): return None +def get_current_release(cfg: Config) -> Optional[Release]: + current = get_current_key(cfg) + return release_for(get_releases(), current) if current else None + + def get_events_file(cfg: Config) -> str: try: o = s3_client.get_object(Bucket="compiler-explorer", Key=events_file_for(cfg)) @@ -221,6 +226,7 @@ def save_event_file(cfg: Config, contents: str): Body=contents, ACL="public-read", CacheControl="public, max-age=60", + ContentType="application/json", ) @@ -355,6 +361,7 @@ def put_bouncelock_file(cfg: Config): Body="", ACL="public-read", CacheControl="public, max-age=60", + ContentType="text/plain", ) @@ -368,3 +375,38 @@ def has_bouncelock_file(cfg: Config): return True except s3_client.exceptions.NoSuchKey: return False + + +def notify_file_name(): + return "ce-notify-file" + + +def has_notify_file(): + try: + s3_client.head_object(Bucket="compiler-explorer", Key=notify_file_name()) + return True + except (s3_client.exceptions.NoSuchKey, botocore.exceptions.ClientError): + return False + + +def put_notify_file(body: str): + s3_client.put_object( + Bucket="compiler-explorer", Key=notify_file_name(), Body=body, ACL="public-read", ContentType="text/plain" + ) + + +def delete_notify_file(): + s3_client.delete_object(Bucket="compiler-explorer", Key=notify_file_name()) + + +def set_current_notify(sha: str): + if not has_notify_file(): + put_notify_file(sha) + + +def get_current_notify(): + try: + o = s3_client.get_object(Bucket="compiler-explorer", Key=notify_file_name()) + return o["Body"].read().decode("utf-8") + except s3_client.exceptions.NoSuchKey: + return None diff --git a/bin/lib/cli/builds.py b/bin/lib/cli/builds.py index 47cf78272..027eeeb38 100644 --- a/bin/lib/cli/builds.py +++ b/bin/lib/cli/builds.py @@ -28,6 +28,7 @@ put_bouncelock_file, delete_bouncelock_file, has_bouncelock_file, + set_current_notify, ) from lib.cdn import DeploymentJob from lib.ce_utils import describe_current_release, are_you_sure, display_releases, confirm_branch, confirm_action @@ -120,6 +121,9 @@ def builds_set_current(cfg: Config, branch: Optional[str], version: str, raw: bo old_deploy_staticfiles(branch, to_set) set_current_key(cfg, to_set) if release: + if cfg.env.value == cfg.env.PROD: + print("Logging for notifications") + set_current_notify(release.hash.hash) print("Marking as a release in sentry...") token = get_ssm_param("/compiler-explorer/sentryAuthToken") result = requests.post( diff --git a/bin/lib/cli/environment.py b/bin/lib/cli/environment.py index 8225b4aa8..945e736e6 100644 --- a/bin/lib/cli/environment.py +++ b/bin/lib/cli/environment.py @@ -2,11 +2,20 @@ import click -from lib.amazon import get_autoscaling_groups_for, as_client +from lib.amazon import ( + get_autoscaling_groups_for, + as_client, + get_current_release, + get_current_notify, + get_ssm_param, + delete_notify_file, +) from lib.ce_utils import are_you_sure, describe_current_release, set_update_message from lib.cli import cli from lib.env import Config, Environment +from lib.notify import handle_notify + @cli.group() def environment(): @@ -54,14 +63,17 @@ def environment_start(cfg: Config): help="Set the message of the day used during refresh", show_default=True, ) +@click.option("--notify/--no-notify", help="Send GitHub notifications for newly released PRs", default=True) @click.pass_obj -def environment_refresh(cfg: Config, min_healthy_percent: int, motd: str): +def environment_refresh(cfg: Config, min_healthy_percent: int, motd: str, notify: bool): """Refreshes an environment. This replaces all the instances in the ASGs associated with an environment with new instances (with the latest code), while ensuring there are some left to handle the traffic while we update.""" set_update_message(cfg, motd) + current_release = get_current_release(cfg) + for asg in get_autoscaling_groups_for(cfg): group_name = asg["AutoScalingGroupName"] if asg["DesiredCapacity"] == 0: @@ -105,6 +117,12 @@ def environment_refresh(cfg: Config, min_healthy_percent: int, motd: str): last_log = log if status in ("Successful", "Failed", "Cancelled"): break + if cfg.env.value == cfg.env.PROD and notify: + current_notify = get_current_notify() + if current_notify is not None and current_release is not None: + gh_token = get_ssm_param("/compiler-explorer/githubAuthToken") + handle_notify(current_notify, current_release.hash.hash, gh_token) + delete_notify_file() set_update_message(cfg, "") diff --git a/bin/lib/cli/notify.py b/bin/lib/cli/notify.py new file mode 100644 index 000000000..03673265a --- /dev/null +++ b/bin/lib/cli/notify.py @@ -0,0 +1,19 @@ +import click + +from lib.amazon import has_notify_file, delete_notify_file, set_current_notify +from lib.cli import cli + + +@cli.group() +def notify(): + """Now-live notification manipulation commands.""" + + +@notify.command(name="set_base") +@click.argument("sha", type=str) +def notify_current(sha: str): + """Sets the first commit from which to start notifying to a specific one. + The commit hash is not validated, so make sure to add a valid one""" + if has_notify_file(): + delete_notify_file() + set_current_notify(sha) diff --git a/bin/lib/notify.py b/bin/lib/notify.py new file mode 100644 index 000000000..770da1198 --- /dev/null +++ b/bin/lib/notify.py @@ -0,0 +1,189 @@ +import urllib.request +import urllib.parse +import json +from typing import List + +OWNER_REPO = "compiler-explorer/compiler-explorer" +USER_AGENT = "CE Live Now Notification Bot" + +NOW_LIVE_LABEL = "live" +NOW_LIVE_MESSAGE = "This is now live" + + +def post(entity: str, token: str, query: dict = None, dry=False) -> dict: + try: + if query is None: + query = {} + path = entity + querystring = json.dumps(query).encode() + print(f"Posting {path}") + req = urllib.request.Request( + f"https://api.github.com/{path}", + data=querystring, + headers={ + "User-Agent": USER_AGENT, + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + }, + ) + if dry: + return {} + result = urllib.request.urlopen(req) + # It's ok not to check for error codes here. We'll throw either way + return json.loads(result.read()) + except Exception as e: + raise RuntimeError(f"Error while posting {entity}") from e + + +def get(entity: str, token: str, query: dict = None) -> dict: + try: + if query is None: + query = {} + path = entity + if query: + querystring = urllib.parse.urlencode(query) + path += f"?{querystring}" + print(f"Getting {path}") + req = urllib.request.Request( + f"https://api.github.com/{path}", + None, + { + "User-Agent": USER_AGENT, + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + }, + ) + result = urllib.request.urlopen(req) + # It's ok not to check for error codes here. We'll throw either way + return json.loads(result.read()) + except Exception as e: + raise RuntimeError(f"Error while getting {entity}") from e + + +def paginated_get(entity: str, token: str, query: dict = None) -> List[dict]: + if query is None: + query = {} + result: List[dict] = [] + results_per_page = 50 + query["page"] = 1 + query["per_page"] = results_per_page + while True: + current_page_results = get(entity, token, query) + result.extend(current_page_results) + if len(current_page_results) == results_per_page: + query["page"] += 1 + else: + break + return result + + +def list_inbetween_commits(end_commit: str, new_commit: str, token: str) -> List[dict]: + commits = get(f"repos/{OWNER_REPO}/compare/{end_commit}...{new_commit}", token=token) + return commits["commits"] + + +def get_linked_pr(commit: str, token: str) -> dict: + """Returns a list whose items are the PR associated to each commit""" + pr = get(f"repos/{OWNER_REPO}/commits/{commit}/pulls", token=token) + return pr[0] if len(pr) == 1 else {} + + +def get_linked_issues(pr: str, token: str): + query = ( + """ +query { + repository(owner: "compiler-explorer", name: "compiler-explorer") { + pullRequest(number: %s) { + closingIssuesReferences(first: 10) { + edges { + node { + labels(first: 10) { + edges { + node { + name + } + } + } + number + } + } + } + } + } +} + """ + % pr + ) + return post("graphql", token, {"query": query}) + + +def get_issue_comments(issue: str, token: str) -> List[dict]: + return paginated_get(f"repos/{OWNER_REPO}/issues/{issue}/comments", token) + + +def comment_on_issue(issue: str, msg: str, token: str): + result = post(f"repos/{OWNER_REPO}/issues/{issue}/comments", token, {"body": msg}) + return result + + +def set_issue_labels(issue: str, labels: List[str], token: str): + post(f"repos/{OWNER_REPO}/issues/{issue}/labels", token, {"labels": labels}) + + +def should_send_comment_to_issue(issue: str, token: str): + """Only send a comment to the issue if nothing like the live message is there already""" + comments = get_issue_comments(issue, token) + return all([NOW_LIVE_MESSAGE not in comment["body"] for comment in comments]) + + +def send_live_message(issue: str, token: str): + set_issue_labels(issue, [NOW_LIVE_LABEL], token) + if should_send_comment_to_issue(issue, token): + comment_on_issue(issue, NOW_LIVE_MESSAGE, token) + + +def get_edges(issue: dict) -> List[dict]: + return issue["data"]["repository"]["pullRequest"]["closingIssuesReferences"]["edges"] + + +def should_process_pr(pr_labels): + """Only process PRs that do not have the live label already set""" + return all([label["name"] != NOW_LIVE_LABEL for label in pr_labels]) + + +def should_notify_issue(edge) -> bool: + """We want to notify the issue if: + - there's one linked ("number" in edge) AND + - either: + - the linked issue has no labels ("labels" not in edge["node"]) OR + - the NOW_LIVE_LABEL label is not among its labels""" + return "number" in edge and ( + ("labels" not in edge) or all([label["node"]["name"] != NOW_LIVE_LABEL for label in edge["labels"]["edges"]]) + ) + + +def handle_notify(base, new, token): + print(f"Checking for live notifications from {base} to {new}") + + commits = list_inbetween_commits(base, new, token) + prs = [get_linked_pr(commit["sha"], token) for commit in commits] + + for pr_data in prs: + pr_id = pr_data["number"] + if should_process_pr(pr_data["labels"]): + print(f"Notifying PR {pr_id}") + send_live_message(pr_id, token) + + linked_issues = get_linked_issues(pr_id, token) + issues_edges = get_edges(linked_issues) + if len(issues_edges) == 1 and "node" in issues_edges[0]: + edge = issues_edges[0]["node"] + if should_notify_issue(edge): + print(f"Notifying issue {edge['number']}") + send_live_message(edge["number"], token) + else: + print(f"Skipping notifying issue {edge['number']}") + else: + print(f"No issues in which to notify for PR {pr_id}") + else: + print(f"Skipping notifying PR {pr_id}")