diff --git a/eth_validator_watcher/config.py b/eth_validator_watcher/config.py index 899147e..39425c4 100644 --- a/eth_validator_watcher/config.py +++ b/eth_validator_watcher/config.py @@ -24,9 +24,13 @@ class Config(BaseSettings): beacon_url: Optional[str] = None beacon_timeout_sec: Optional[int] = None metrics_port: Optional[int] = None - start_at: Optional[int] = None watched_keys: Optional[List[WatchedKeyConfig]] = None + slack_token: Optional[str] = None + slack_channel: Optional[str] = None + + start_at: Optional[int] = None + def _default_config() -> Config: """Returns the default configuration. diff --git a/eth_validator_watcher/entrypoint.py b/eth_validator_watcher/entrypoint.py index 0428dc1..ad0f8a5 100644 --- a/eth_validator_watcher/entrypoint.py +++ b/eth_validator_watcher/entrypoint.py @@ -16,7 +16,7 @@ from .clock import BeaconClock from .beacon import Beacon, NoBlockError from .config import load_config, WatchedKeyConfig -from .log import log_details +from .log import log_details, slack_send from .metrics import get_prometheus_metrics, compute_validator_metrics from .blocks import process_block, process_finalized_block, process_future_blocks from .models import BlockIdentierType, Validators @@ -103,7 +103,7 @@ def _update_metrics(self, watched_validators: WatchedValidators, epoch: int, slo metrics = compute_validator_metrics(watched_validators.get_validators(), slot) - log_details(self._cfg, watched_validators, metrics) + log_details(self._cfg, watched_validators, metrics, slot) for label, m in metrics.items(): for status in Validators.DataItem.StatusEnum: @@ -151,6 +151,8 @@ def run(self) -> None: rewards = None last_processed_finalized_slot = None + slack_send(self._cfg, f'🚀 Ethereum Validator Watcher started on {self._cfg.network}, watching {len(self._cfg.watched_keys)} validators 🚀') + while True: logging.info(f'🔨 Processing slot {slot}') diff --git a/eth_validator_watcher/log.py b/eth_validator_watcher/log.py new file mode 100644 index 0000000..9a6701d --- /dev/null +++ b/eth_validator_watcher/log.py @@ -0,0 +1,110 @@ +import collections +import logging + +from slack_sdk import WebClient +from slack_sdk.errors import SlackApiError + +from eth_validator_watcher_ext import MetricsByLabel +from .config import Config +from .utils import LABEL_SCOPE_WATCHED +from .watched_validators import WatchedValidators + +# We colorize anything related to validators so that it's easy to spot +# in the log noise from the watcher from actual issues. +COLOR_GREEN = "\x1b[32;20m" +COLOR_BOLD_GREEN = "\x1b[32;1m" +COLOR_YELLOW = "\x1b[33;20m" +COLOR_RED = "\x1b[31;20m" +COLOR_BOLD_RED = "\x1b[31;1m" +COLOR_RESET = "\x1b[0m" + + +def shorten_validator(validator_pubkey: str) -> str: + """Shorten a validator name + """ + return f"{validator_pubkey[:10]}" + + +def slack_send(cfg: Config, msg: str) -> None: + """Attempts to send a message to the configured slack channel.""" + if not (cfg.slack_channel and cfg.slack_token): + return + + try: + w = WebClient(token=cfg.slack_token) + w.chat_postMessage(channel=cfg.slack_channel, text=msg) + except SlackApiError as e: + logging.warning(f'😿 Unable to send slack notification: {e.response["error"]}') + + + +def log_single_entry(cfg: Config, validator: str, registry: WatchedValidators, msg: str, emoji: str, color: str) -> None: + """Logs a single validator entry. + """ + v = registry.get_validator_by_pubkey(validator) + + label_msg = '' + if v: + labels = [label for label in v.labels if not label.startswith('scope:')] + if labels: + label_msg = f' ({", ".join(labels)})' + + msg_slack = f'{emoji} Validator {shorten_validator(validator)}{label_msg} {msg}' + msg_shell = f'{color}{msg_slack}{COLOR_RESET}' + + logging.info(msg_shell) + slack_send(cfg, msg_slack) + + +def log_multiple_entries(cfg: Config, validators: list[str], registry: WatchedValidators, msg: str, emoji: str, color: str) -> None: + """Logs a multiple validator entries. + """ + + impacted_labels = collections.defaultdict(int) + for validator in validators: + v = registry.get_validator_by_pubkey(validator) + if v: + for label in v.labels: + if not label.startswith('scope'): + impacted_labels[label] += 1 + top_labels = sorted(impacted_labels, key=impacted_labels.get, reverse=True)[:5] + + label_msg = '' + if top_labels: + label_msg = f' ({", ".join(top_labels)}...)' + + msg_validators = f'{", ".join([shorten_validator(v) for v in validators])} and more' + + msg_slack = f'{emoji} Validator(s) {msg_validators}{label_msg} {msg}' + msg_shell = f'{color}{msg_slack}{COLOR_RESET}' + + logging.info(msg_shell) + slack_send(cfg, msg_slack) + + +def log_details(cfg: Config, registry: WatchedValidators, metrics: MetricsByLabel, current_slot: int): + """Log details about watched validators + """ + m = metrics.get(LABEL_SCOPE_WATCHED) + if not m: + return None + + for slot, validator in m.details_future_blocks: + # Only log once per epoch future block proposals. + if current_slot % 32 == 0: + log_single_entry(cfg, validator, registry, f'will propose a block on slot {slot}', '🙏', COLOR_GREEN) + + for slot, validator in m.details_proposed_blocks: + log_single_entry(cfg, validator, registry, f'proposed a block on slot {slot}', '🏅', COLOR_BOLD_GREEN) + + for slot, validator in m.details_missed_blocks: + log_single_entry(cfg, validator, registry, f'likely missed a block on slot {slot}', '😩', COLOR_RED) + + for slot, validator in m.details_missed_blocks_finalized: + log_single_entry(cfg, validator, registry, f'missed a block for real on slot {slot}', '😭', COLOR_BOLD_RED) + + for validator in m.details_missed_attestations: + log_single_entry(cfg, validator, registry, f'missed a block for real on slot {slot}', '😭', COLOR_BOLD_RED) + + if m.details_missed_attestations: + log_multiple_entries(cfg, m.details_missed_attestations, registry, f'missed an attestation', '😞', COLOR_YELLOW)