Skip to content

Commit

Permalink
cloud_functions: add alarming guardian heartbeats
Browse files Browse the repository at this point in the history
  • Loading branch information
panoel committed Sep 7, 2024
1 parent 358dab0 commit bce4fa0
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 1 deletion.
7 changes: 6 additions & 1 deletion cloud_functions/scripts/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@ if [ -z "$CLOUD_FUNCTIONS_BLOCK_INCREMENT" ]; then
exit 1
fi

if [ -z "$FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION" ]; then
echo "FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION must be specified"
exit 1
fi

if [ -z "$FIRESTORE_LATEST_COLLECTION" ]; then
echo "FIRESTORE_LATEST_COLLECTION must be specified"
exit 1
Expand Down Expand Up @@ -121,7 +126,7 @@ fi
# Context: Of all the cloud functions, there are some that only go into MAINNET and some that go into both MAINNET and TESTNET.
# There are no cloud functions that only go into TESTNET.
# First, deploy the functions that are common to both MAINNET and TESTNET
gcloud functions --project "$GCP_PROJECT" deploy alarm-missing-vaas --entry-point alarmMissingVaas --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars MISSING_VAA_SLACK_CHANNEL_ID=$MISSING_VAA_SLACK_CHANNEL_ID,MISSING_VAA_SLACK_POST_URL=$MISSING_VAA_SLACK_POST_URL,MISSING_VAA_SLACK_BOT_TOKEN=$MISSING_VAA_SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,FIRESTORE_GOVERNOR_STATUS_COLLECTION=$FIRESTORE_GOVERNOR_STATUS_COLLECTION,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION,NETWORK=$NETWORK,FUNCTION=alarmMissingVaas
gcloud functions --project "$GCP_PROJECT" deploy alarm-missing-vaas --entry-point alarmMissingVaas --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars MISSING_VAA_SLACK_CHANNEL_ID=$MISSING_VAA_SLACK_CHANNEL_ID,MISSING_VAA_SLACK_POST_URL=$MISSING_VAA_SLACK_POST_URL,MISSING_VAA_SLACK_BOT_TOKEN=$MISSING_VAA_SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,FIRESTORE_GOVERNOR_STATUS_COLLECTION=$FIRESTORE_GOVERNOR_STATUS_COLLECTION,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION,FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION=$FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION,NETWORK=$NETWORK,FUNCTION=alarmMissingVaas
gcloud functions --project "$GCP_PROJECT" deploy compute-message-count-history --entry-point computeMessageCountHistory --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,BIGTABLE_SIGNED_VAAS_TABLE_ID=$BIGTABLE_SIGNED_VAAS_TABLE_ID,FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION=$FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION,NETWORK=$NETWORK,FUNCTION=computeMessageCountHistory
gcloud functions --project "$GCP_PROJECT" deploy compute-message-counts --entry-point computeMessageCounts --gen2 --runtime nodejs18 --trigger-http --allow-unauthenticated --timeout 300 --memory 4GB --region europe-west3 --set-env-vars BIGTABLE_TABLE_ID=$BIGTABLE_TABLE_ID,BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL,NETWORK=$NETWORK,FUNCTION=computeMessageCounts
gcloud functions --project "$GCP_PROJECT" deploy compute-missing-vaas --entry-point computeMissingVaas --gen2 --runtime nodejs18 --trigger-http --allow-unauthenticated --timeout 300 --memory 4GB --region europe-west3 --set-env-vars BIGTABLE_TABLE_ID=$BIGTABLE_TABLE_ID,BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL,NETWORK=$NETWORK,FUNCTION=computeMissingVaas
Expand Down
37 changes: 37 additions & 0 deletions cloud_functions/src/alarmMissingVaas.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ export async function alarmMissingVaas(req: any, res: any) {
// Alarm any watchers that are behind by more than 24 hours
await alarmOldBlockTimes(refTimes);

// Alarm any guardians that have not sent a heartbeat in more than 1 hour
await alarmOldHeartbeats();

// attempting to retrieve missing VAAs...
const messages: MissingVaasByChain = await commonGetMissingVaas();
if (messages) {
Expand Down Expand Up @@ -399,6 +402,40 @@ async function storeAlarmedChains(alarms: AlarmedChainTime[]): Promise<void> {
await alarmedChains.set({ times: alarms });
}

async function alarmOldHeartbeats(): Promise<void> {
// Get Guardian Heartbeats from firestore
const now: Date = new Date();
const firestore = new Firestore();
const collectionRef = firestore.collection(
assertEnvironmentVariable('FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION')
);
const snapshot = await collectionRef.get();
// Walk all the documents in the collection
const documents = snapshot.docs;
for (const doc of documents) {
const data = doc.data();
if (data) {
// Only need to look at the timestamp field, which is in nanoseconds
const timestamp: string = data.timestamp;
// Convert the timestamp to a milliseconds
const timestampMs: number = Math.floor(Number(timestamp) / 1_000_000);
const heartbeatTime: Date = new Date(timestampMs);
const deltaTime: number = (now.getTime() - heartbeatTime.getTime()) / (1000 * 60 * 60); // hours
if (deltaTime >= 1) {
// Send a message to slack
const alarmSlackInfo: SlackInfo = {
channelId: assertEnvironmentVariable('MISSING_VAA_SLACK_CHANNEL_ID'),
postUrl: assertEnvironmentVariable('MISSING_VAA_SLACK_POST_URL'),
botToken: assertEnvironmentVariable('MISSING_VAA_SLACK_BOT_TOKEN'),
bannerTxt: 'Wormhole Missing VAA Alarm',
msg: `The guardian ${doc.id} has not sent a heartbeat in ${deltaTime} hours.`,
};
await formatAndSendToSlack(alarmSlackInfo);
}
}
}
}

type FirestoreVAA = {
chain: string;
txHash: string;
Expand Down

0 comments on commit bce4fa0

Please sign in to comment.