cloud_functions/src: add snoozing

cloud_functions/scripts/deploy.sh

@@ -109,6 +109,26 @@ if [ -z "$FIRESTORE_MESSAGE_COUNT_HISTORY_COLLECTION" ]; then
     exit 1
 fi
 
+if [ -z "$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION" ]; then
+    echo "FIRESTORE_ALARM_MISSING_VAAS_COLLECTION must be specified"
+    exit 1
+fi
+
+if [ -z "$SLACK_CHANNEL_ID" ]; then
+    echo "SLACK_CHANNEL_ID must be specified"
+    exit 1
+fi
+
+if [ -z "$SLACK_POST_URL" ]; then
+    echo "SLACK_POST_URL must be specified"
+    exit 1
+fi
+
+if [ -z "$SLACK_BOT_TOKEN" ]; then
+    echo "SLACK_BOT_TOKEN must be specified"
+    exit 1
+fi
+
 # Hack to make these packages available in the GCP build until they're published
 npm pack --silent --workspace @wormhole-foundation/wormhole-monitor-common --pack-destination ./dist/src
 npm pack --silent --workspace @wormhole-foundation/wormhole-monitor-database --pack-destination ./dist/src
@@ -119,7 +139,7 @@ gcloud functions deploy compute-message-counts --entry-point computeMessageCount
 gcloud functions deploy latest-blocks --entry-point getLatestBlocks --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL,FIRESTORE_LATEST_COLLECTION=$FIRESTORE_LATEST_COLLECTION
 gcloud functions deploy compute-missing-vaas --entry-point computeMissingVaas --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 2GB --region europe-west3 --set-env-vars BIGTABLE_TABLE_ID=$BIGTABLE_TABLE_ID,BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL=$CLOUD_FUNCTIONS_REFRESH_TIME_INTERVAL
 gcloud functions deploy missing-vaas --entry-point getMissingVaas --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3
-gcloud functions deploy alarm-missing-vaas --entry-point alarmMissingVaas --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars SLACK_CHANNEL_ID=$SLACK_CHANNEL_ID,SLACK_POST_URL=$SLACK_POST_URL,SLACK_BOT_TOKEN=$SLACK_BOT_TOKEN
+gcloud functions deploy alarm-missing-vaas --entry-point alarmMissingVaas --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars SLACK_CHANNEL_ID=$SLACK_CHANNEL_ID,SLACK_POST_URL=$SLACK_POST_URL,SLACK_BOT_TOKEN=$SLACK_BOT_TOKEN,FIRESTORE_ALARM_MISSING_VAAS_COLLECTION=$FIRESTORE_ALARM_MISSING_VAAS_COLLECTION
 gcloud functions deploy vaas-by-tx-hash --entry-point getVaasByTxHash --runtime nodejs16 --trigger-http --allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,BIGTABLE_SIGNED_VAAS_TABLE_ID=$BIGTABLE_SIGNED_VAAS_TABLE_ID,BIGTABLE_VAAS_BY_TX_HASH_TABLE_ID=$BIGTABLE_VAAS_BY_TX_HASH_TABLE_ID
 gcloud functions deploy process-vaa --entry-point processVaa --runtime nodejs16 --timeout 300 --memory 256MB --region europe-west3 --trigger-topic $PUBSUB_SIGNED_VAA_TOPIC --set-env-vars BIGTABLE_INSTANCE_ID=$BIGTABLE_INSTANCE_ID,BIGTABLE_SIGNED_VAAS_TABLE_ID=$BIGTABLE_SIGNED_VAAS_TABLE_ID,BIGTABLE_VAAS_BY_TX_HASH_TABLE_ID=$BIGTABLE_VAAS_BY_TX_HASH_TABLE_ID,PG_USER=$PG_USER,PG_PASSWORD=$PG_PASSWORD,PG_DATABASE=$PG_DATABASE,PG_HOST=$PG_HOST,PG_TOKEN_TRANSFER_TABLE=$PG_TOKEN_TRANSFER_TABLE,PG_ATTEST_MESSAGE_TABLE=$PG_ATTEST_MESSAGE_TABLE,PG_TOKEN_METADATA_TABLE=$PG_TOKEN_METADATA_TABLE
 gcloud functions deploy refresh-todays-token-prices --entry-point refreshTodaysTokenPrices --runtime nodejs16 --trigger-http --no-allow-unauthenticated --timeout 300 --memory 256MB --region europe-west3 --set-env-vars PG_USER=$PG_USER,PG_PASSWORD=$PG_PASSWORD,PG_DATABASE=$PG_DATABASE,PG_HOST=$PG_HOST,PG_TOKEN_METADATA_TABLE=$PG_TOKEN_METADATA_TABLE,PG_TOKEN_PRICE_HISTORY_TABLE=$PG_TOKEN_PRICE_HISTORY_TABLE

cloud_functions/src/alarmMissingVaas.ts

@@ -1,8 +1,9 @@
 import { CHAIN_ID_TO_NAME, ChainId, ChainName } from '@certusone/wormhole-sdk';
 import { MissingVaasByChain, commonGetMissingVaas } from './getMissingVaas';
-import { formatAndSendToSlack } from './utils';
+import { assertEnvironmentVariable, formatAndSendToSlack } from './utils';
 import { ObservedMessage } from './types';
 import { explorerBlock, explorerTx } from '@wormhole-foundation/wormhole-monitor-common';
+import { Firestore } from 'firebase-admin/firestore';
 
 export async function alarmMissingVaas(req: any, res: any) {
   res.set('Access-Control-Allow-Origin', '*');
@@ -14,21 +15,39 @@ export async function alarmMissingVaas(req: any, res: any) {
     res.status(204).send('');
     return;
   }
+  let firestoreVAAs: FirestoreVAA[] = [];
   try {
+    // Get the current VAAs in the firestore holding area that we want to keep there.
+    // The key is the vaaKey
+    let firestoreMap: Map<string, FirestoreVAA> = await getAndProcessFirestore();
+
+    // Pre fill out firestoreVAAS with the VAAs we know we want to keep.
+    firestoreMap.forEach((vaa) => {
+      firestoreVAAs.push(vaa);
+    });
+
     // attempting to retrieve missing VAAs...
     const messages: MissingVaasByChain = await commonGetMissingVaas();
     if (messages) {
       const now = new Date();
-      now.setHours(now.getHours() - 2);
-      const twoHoursAgo = now.toISOString();
+      const thePast = now;
+      thePast.setHours(now.getHours() - 2);
+      const twoHoursAgo = thePast.toISOString();
       for (const chain of Object.keys(messages)) {
         const chainId = chain as unknown as ChainId;
         const msgs = messages[chainId];
         if (msgs && msgs.messages) {
           for (let i = 0; i < msgs.messages.length; i++) {
             // Check the timestamp and only send messages that are older than 2 hours
-            if (msgs.messages[i].timestamp < twoHoursAgo) {
-              await formatAndSendToSlack(formatMessage(msgs.messages[i]));
+            const msg: ObservedMessage = msgs.messages[i];
+            if (msg.timestamp < twoHoursAgo) {
+              let vaaKey: string = `${msg.chain}/${msg.emitter}/${msg.seq}`;
+              if (!firestoreMap.has(vaaKey)) {
+                let firestoreMsg: FirestoreVAA = convert(msg);
+                firestoreMap.set(vaaKey, firestoreMsg);
+                firestoreVAAs.push(firestoreMsg);
+                await formatAndSendToSlack(formatMessage(msg));
+              }
             }
           }
         } else {
@@ -40,10 +59,69 @@ export async function alarmMissingVaas(req: any, res: any) {
     console.log('could not get missing VAAs', e);
     res.sendStatus(500);
   }
+  await updateFirestore(firestoreVAAs);
   res.status(200).send('successfully alarmed missing VAAS');
   return;
 }
 
+// This function gets all the VAAs in the firestore table,
+// checks the timestamp (keeping any that are less than 2 hours old),
+// and returns a map of those VAAs.
+async function getAndProcessFirestore(): Promise<Map<string, FirestoreVAA>> {
+  // Get VAAs in the firestore holding area.
+  const firestore = new Firestore();
+  const collection = firestore.collection(
+    assertEnvironmentVariable('FIRESTORE_ALARM_MISSING_VAAS_COLLECTION')
+  );
+  let current = new Map<string, FirestoreVAA>();
+  const now = new Date();
+  const thePast = now;
+  thePast.setHours(now.getHours() - 2);
+  const twoHoursAgo = thePast.toISOString();
+  await collection
+    .doc('VAAs')
+    .get()
+    .then((doc) => {
+      if (doc.exists) {
+        const data = doc.data();
+        if (data) {
+          // if VAA < 2 hours old, leave in firestore
+          const vaas: FirestoreVAA[] = data.VAAs;
+          vaas.forEach((vaa) => {
+            if (vaa.noticedTS > twoHoursAgo) {
+              // console.log('keeping VAA in firestore', vaa.vaaKey);
+              current.set(vaa.vaaKey, vaa);
+            }
+          });
+        }
+      }
+    })
+    .catch((error) => {
+      console.log('Error getting document:', error);
+    });
+  return current;
+}
+
+async function updateFirestore(vaas: FirestoreVAA[]): Promise<void> {
+  const firestore = new Firestore();
+  const collection = firestore.collection(
+    assertEnvironmentVariable('FIRESTORE_ALARM_MISSING_VAAS_COLLECTION')
+  );
+  const doc = collection.doc('VAAs');
+  await doc.set({ VAAs: vaas });
+}
+
+function convert(msg: ObservedMessage): FirestoreVAA {
+  return {
+    chain: msg.chain.toString(),
+    txHash: msg.txHash,
+    vaaKey: `${msg.chain}/${msg.emitter}/${msg.seq}`,
+    block: msg.block.toString(),
+    blockTS: msg.timestamp,
+    noticedTS: new Date().toISOString(),
+  };
+}
+
 function formatMessage(msg: ObservedMessage): string {
   const cName: string = CHAIN_ID_TO_NAME[msg.chain as ChainId] as ChainName;
   // const vaaKeyUrl: string = `https://wormhole.com/explorer/?emitterChain=${msg.chain}&emitterAddress=${msg.emitter}&sequence=${msg.seq}`;
@@ -53,3 +131,12 @@ function formatMessage(msg: ObservedMessage): string {
   const formattedMsg = `*Chain:* ${cName}(${msg.chain})\n*TxHash:* <${txHashUrl}|${msg.txHash}>\n*VAA Key:* <${vaaKeyUrl}|${msg.chain}/${msg.emitter}/${msg.seq}> \n*Block:* <${blockUrl}|${msg.block}> \n*Timestamp:* ${msg.timestamp}`;
   return formattedMsg;
 }
+
+type FirestoreVAA = {
+  chain: string;
+  txHash: string;
+  vaaKey: string;
+  block: string;
+  blockTS: string;
+  noticedTS: string;
+};