diff --git a/cloud_functions/scripts/deploy.sh b/cloud_functions/scripts/deploy.sh index cb74d443..54932a14 100755 --- a/cloud_functions/scripts/deploy.sh +++ b/cloud_functions/scripts/deploy.sh @@ -68,6 +68,11 @@ if [ -z "$CLOUD_FUNCTIONS_BLOCK_INCREMENT" ]; then exit 1 fi +if [ -z "$FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION" ]; then + echo "FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION must be specified" + exit 1 +fi + if [ -z "$FIRESTORE_LATEST_COLLECTION" ]; then echo "FIRESTORE_LATEST_COLLECTION must be specified" exit 1 @@ -121,7 +126,7 @@ fi # Context: Of all the cloud functions, there are some that only go into MAINNET and some that go into both MAINNET and TESTNET. # There are no cloud functions that only go into TESTNET. # First, deploy the functions that are common to both MAINNET and TESTNET -gcloud functions --project "$GCP_PROJECT" deploy alarm-missing-vaas --entry-point alarmMissingVaas --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars MISSING_VAA_SLACK_CHANNEL_ID=$MISSING_VAA_SLACK_CHANNEL_ID,MISSING_VAA_SLACK_POST_URL=$MISSING_VAA_SLACK_POST_URL,MISSING_VAA_SLACK_BOT_TOKEN=$MISSING_VAA_SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,FIRESTORE_GOVERNOR_STATUS_COLLECTION=$FIRESTORE_GOVERNOR_STATUS_COLLECTION,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION,NETWORK=$NETWORK,FUNCTION=alarmMissingVaas +gcloud functions --project "$GCP_PROJECT" deploy alarm-missing-vaas --entry-point alarmMissingVaas --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars MISSING_VAA_SLACK_CHANNEL_ID=$MISSING_VAA_SLACK_CHANNEL_ID,MISSING_VAA_SLACK_POST_URL=$MISSING_VAA_SLACK_POST_URL,MISSING_VAA_SLACK_BOT_TOKEN=$MISSING_VAA_SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION,FIRESTORE_GOVERNOR_STATUS_COLLECTION=$FIRESTORE_GOVERNOR_STATUS_COLLECTION,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION,FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION=$FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION,NETWORK=$NETWORK,FUNCTION=alarmMissingVaas gcloud functions --project "$GCP_PROJECT" deploy compute-message-count-history --entry-point computeMessageCountHistory --gen2 --runtime nodejs18 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 1GB --region europe-west3 --set-env-vars BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,BIGTABLE_SIGNED_VAAS_TABLE_ID=$BIGTABLE_SIGNED_VAAS_TABLE_ID,FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION=$FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION,NETWORK=$NETWORK,FUNCTION=computeMessageCountHistory gcloud functions --project "$GCP_PROJECT" deploy compute-message-counts --entry-point computeMessageCounts --gen2 --runtime nodejs18 --trigger-http --allow-unauthenticated --timeout 300 --memory 4GB --region europe-west3 --set-env-vars BIGTABLE_TABLE_ID=$BIGTABLE_TABLE_ID,BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL,NETWORK=$NETWORK,FUNCTION=computeMessageCounts gcloud functions --project "$GCP_PROJECT" deploy compute-missing-vaas --entry-point computeMissingVaas --gen2 --runtime nodejs18 --trigger-http --allow-unauthenticated --timeout 300 --memory 4GB --region europe-west3 --set-env-vars BIGTABLE_TABLE_ID=$BIGTABLE_TABLE_ID,BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL,NETWORK=$NETWORK,FUNCTION=computeMissingVaas diff --git a/cloud_functions/src/alarmMissingVaas.ts b/cloud_functions/src/alarmMissingVaas.ts index ea6c6afb..12aa966f 100644 --- a/cloud_functions/src/alarmMissingVaas.ts +++ b/cloud_functions/src/alarmMissingVaas.ts @@ -88,6 +88,9 @@ export async function alarmMissingVaas(req: any, res: any) { // Alarm any watchers that are behind by more than 24 hours await alarmOldBlockTimes(refTimes); + // Alarm any guardians that have not sent a heartbeat in more than 1 hour + await alarmOldHeartbeats(); + // attempting to retrieve missing VAAs... const messages: MissingVaasByChain = await commonGetMissingVaas(); if (messages) { @@ -399,6 +402,40 @@ async function storeAlarmedChains(alarms: AlarmedChainTime[]): Promise { await alarmedChains.set({ times: alarms }); } +async function alarmOldHeartbeats(): Promise { + // Get Guardian Heartbeats from firestore + const now: Date = new Date(); + const firestore = new Firestore(); + const collectionRef = firestore.collection( + assertEnvironmentVariable('FIRESTORE_GUARDIAN_HEARTBEAT_COLLECTION') + ); + const snapshot = await collectionRef.get(); + // Walk all the documents in the collection + const documents = snapshot.docs; + for (const doc of documents) { + const data = doc.data(); + if (data) { + // Only need to look at the timestamp field, which is in nanoseconds + const timestamp: string = data.timestamp; + // Convert the timestamp to a milliseconds + const timestampMs: number = Math.floor(Number(timestamp) / 1_000_000); + const heartbeatTime: Date = new Date(timestampMs); + const deltaTime: number = (now.getTime() - heartbeatTime.getTime()) / (1000 * 60 * 60); // hours + if (deltaTime >= 1) { + // Send a message to slack + const alarmSlackInfo: SlackInfo = { + channelId: assertEnvironmentVariable('MISSING_VAA_SLACK_CHANNEL_ID'), + postUrl: assertEnvironmentVariable('MISSING_VAA_SLACK_POST_URL'), + botToken: assertEnvironmentVariable('MISSING_VAA_SLACK_BOT_TOKEN'), + bannerTxt: 'Wormhole Missing VAA Alarm', + msg: `The guardian ${doc.id} has not sent a heartbeat in ${deltaTime} hours.`, + }; + await formatAndSendToSlack(alarmSlackInfo); + } + } + } +} + type FirestoreVAA = { chain: string; txHash: string;