From d7b39eba37e81ab4304425a801dd7405613d3ba8 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Wed, 18 Oct 2023 15:20:54 -0700 Subject: [PATCH] Log warnings and skipped models in addition to errored out models --- parsons/utilities/dbt.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/parsons/utilities/dbt.py b/parsons/utilities/dbt.py index c607099356..13eb041311 100644 --- a/parsons/utilities/dbt.py +++ b/parsons/utilities/dbt.py @@ -59,6 +59,7 @@ def __init__( self.start = time.time() self.log_messages = [] self.error_messages = [] + self.warn_messages = [] self.done_messages = [] self.slack_channel = slack_channel self.slack_webhook = slack_webhook @@ -86,7 +87,14 @@ def seconds_to_time_string(self, seconds: int): return time_str - def record_result(self, command: str, error_messages: list[str], done_message: str): + def record_result( + self, + command: str, + error_messages: list[str], + warn_messages: list[str], + skip_messages: list[str], + done_message: str, + ): command_seconds = int( self._command_times[command]["end"] - self._command_times[command]["start"] ) @@ -95,6 +103,9 @@ def record_result(self, command: str, error_messages: list[str], done_message: s if error_messages: log_message += ":red_circle:" status = "Error" + elif warn_messages: + log_message += ":large_orange_circle:" + status = "Warning" else: log_message += ":large_green_circle:" status = "Success" @@ -106,9 +117,22 @@ def record_result(self, command: str, error_messages: list[str], done_message: s if error_messages: log_message += "\nError messages:\n```{}```".format( - "\n".join(error_messages) + "\n\n".join(error_messages) + ) + + if warn_messages: + log_message += "\nWarning messages:\n```{}```".format( + "\n\n".join(warn_messages) ) + if skip_messages: + skips = [ + msg.split(" ")[5].split(".")[1] + for msg in skip_messages + if msg.split(" ")[4] == "relation" + ] + log_message += "\nSkipped:\n```{}```".format(", ".join(skips)) + self.log_messages.append(log_message) def send_to_slack(self) -> None: @@ -153,6 +177,8 @@ def log_results(self, command_str: str, stdout: str, stderr: str) -> None: log_messages = [] error_messages = [] + warn_messages = [] + skip_messages = [] for row in parsed_rows: if not row["info"]["msg"]: @@ -164,6 +190,12 @@ def log_results(self, command_str: str, stdout: str, stderr: str) -> None: if row["info"]["level"] == "error": logger.error(log_message) error_messages.append(log_message) + elif row["info"]["level"] == "warn": + logger.warning(log_message) + warn_messages.append(log_message) + elif "SKIP " in row["info"]["msg"]: + logger.warning(log_message) + skip_messages.append(log_message) else: logger.info(log_message) @@ -173,7 +205,9 @@ def log_results(self, command_str: str, stdout: str, stderr: str) -> None: else: done_message = "" - self.record_result(command_str, error_messages, done_message) + self.record_result( + command_str, error_messages, warn_messages, skip_messages, done_message + ) class dbtRunner: