feat: add logging module

eth_validator_watcher/config.py

@@ -24,9 +24,13 @@ class Config(BaseSettings):
     beacon_url: Optional[str] = None
     beacon_timeout_sec: Optional[int] = None
     metrics_port: Optional[int] = None
-    start_at: Optional[int] = None
     watched_keys: Optional[List[WatchedKeyConfig]] = None
 
+    slack_token: Optional[str] = None
+    slack_channel: Optional[str] = None
+
+    start_at: Optional[int] = None
+
 
 def _default_config() -> Config:
     """Returns the default configuration.

eth_validator_watcher/entrypoint.py

@@ -16,7 +16,7 @@
 from .clock import BeaconClock
 from .beacon import Beacon, NoBlockError
 from .config import load_config, WatchedKeyConfig
-from .log import log_details
+from .log import log_details, slack_send
 from .metrics import get_prometheus_metrics, compute_validator_metrics
 from .blocks import process_block, process_finalized_block, process_future_blocks
 from .models import BlockIdentierType, Validators
@@ -103,7 +103,7 @@ def _update_metrics(self, watched_validators: WatchedValidators, epoch: int, slo
 
         metrics = compute_validator_metrics(watched_validators.get_validators(), slot)
 
-        log_details(self._cfg, watched_validators, metrics)
+        log_details(self._cfg, watched_validators, metrics, slot)
 
         for label, m in metrics.items():
             for status in Validators.DataItem.StatusEnum:
@@ -151,6 +151,8 @@ def run(self) -> None:
         rewards = None
         last_processed_finalized_slot = None
 
+        slack_send(self._cfg, f'🚀 Ethereum Validator Watcher started on {self._cfg.network}, watching {len(self._cfg.watched_keys)} validators 🚀')
+
         while True:
             logging.info(f'🔨 Processing slot {slot}')
 

eth_validator_watcher/log.py

@@ -0,0 +1,110 @@
+import collections
+import logging
+
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
+
+from eth_validator_watcher_ext import MetricsByLabel
+from .config import Config
+from .utils import LABEL_SCOPE_WATCHED
+from .watched_validators import WatchedValidators
+
+# We colorize anything related to validators so that it's easy to spot
+# in the log noise from the watcher from actual issues.
+COLOR_GREEN = "\x1b[32;20m"
+COLOR_BOLD_GREEN = "\x1b[32;1m"
+COLOR_YELLOW = "\x1b[33;20m"
+COLOR_RED     = "\x1b[31;20m"
+COLOR_BOLD_RED = "\x1b[31;1m"
+COLOR_RESET = "\x1b[0m"
+
+
+def shorten_validator(validator_pubkey: str) -> str:
+    """Shorten a validator name
+    """
+    return f"{validator_pubkey[:10]}"
+
+
+def slack_send(cfg: Config, msg: str) -> None:
+    """Attempts to send a message to the configured slack channel."""
+    if not (cfg.slack_channel and cfg.slack_token):
+        return
+
+    try:
+        w = WebClient(token=cfg.slack_token)
+        w.chat_postMessage(channel=cfg.slack_channel, text=msg)
+    except SlackApiError as e:
+        logging.warning(f'😿 Unable to send slack notification: {e.response["error"]}')
+
+
+
+def log_single_entry(cfg: Config, validator: str, registry: WatchedValidators, msg: str, emoji: str, color: str) -> None:
+    """Logs a single validator entry.
+    """
+    v = registry.get_validator_by_pubkey(validator)
+
+    label_msg = ''
+    if v:
+        labels = [label for label in v.labels if not label.startswith('scope:')]
+        if labels:
+            label_msg = f' ({", ".join(labels)})'
+
+    msg_slack = f'{emoji} Validator {shorten_validator(validator)}{label_msg} {msg}'
+    msg_shell = f'{color}{msg_slack}{COLOR_RESET}'
+
+    logging.info(msg_shell)
+    slack_send(cfg, msg_slack)
+
+
+def log_multiple_entries(cfg: Config, validators: list[str], registry: WatchedValidators, msg: str, emoji: str, color: str) -> None:
+    """Logs a multiple validator entries.
+    """
+
+    impacted_labels = collections.defaultdict(int)
+    for validator in validators:
+        v = registry.get_validator_by_pubkey(validator)
+        if v:
+            for label in v.labels:
+                if not label.startswith('scope'):
+                    impacted_labels[label] += 1
+    top_labels = sorted(impacted_labels, key=impacted_labels.get, reverse=True)[:5]
+
+    label_msg = ''
+    if top_labels:
+        label_msg = f' ({", ".join(top_labels)}...)'
+
+    msg_validators = f'{", ".join([shorten_validator(v) for v in validators])} and more'
+
+    msg_slack = f'{emoji} Validator(s) {msg_validators}{label_msg} {msg}'
+    msg_shell = f'{color}{msg_slack}{COLOR_RESET}'
+
+    logging.info(msg_shell)
+    slack_send(cfg, msg_slack)
+
+
+def log_details(cfg: Config, registry: WatchedValidators, metrics: MetricsByLabel, current_slot: int):
+    """Log details about watched validators
+    """
+    m = metrics.get(LABEL_SCOPE_WATCHED)
+    if not m:
+        return None
+
+    for slot, validator in m.details_future_blocks:
+        # Only log once per epoch future block proposals.
+        if current_slot % 32 == 0:
+            log_single_entry(cfg, validator, registry, f'will propose a block on slot {slot}', '🙏', COLOR_GREEN)
+
+    for slot, validator in m.details_proposed_blocks:
+        log_single_entry(cfg, validator, registry, f'proposed a block on slot {slot}', '🏅', COLOR_BOLD_GREEN)
+
+    for slot, validator in m.details_missed_blocks:
+        log_single_entry(cfg, validator, registry, f'likely missed a block on slot {slot}', '😩', COLOR_RED)
+
+    for slot, validator in m.details_missed_blocks_finalized:
+        log_single_entry(cfg, validator, registry, f'missed a block for real on slot {slot}', '😭', COLOR_BOLD_RED)
+
+    for validator in m.details_missed_attestations:
+        log_single_entry(cfg, validator, registry, f'missed a block for real on slot {slot}', '😭', COLOR_BOLD_RED)        
+
+    if m.details_missed_attestations:
+        log_multiple_entries(cfg, m.details_missed_attestations, registry, f'missed an attestation', '😞', COLOR_YELLOW)