cloud_functions: alarm if chains get behind

cloud_functions/src/alarmMissingVaas.ts

@@ -61,6 +61,9 @@ export async function alarmMissingVaas(req: any, res: any) {
     // Get reference times
     const refTimes: LatestTimeByChain = await getLastBlockTimeFromFirestore();
 
+    // Alarm any watchers that are behind by more than 24 hours
+    await alarmOldBlockTimes(refTimes);
+
     // attempting to retrieve missing VAAs...
     const messages: MissingVaasByChain = await commonGetMissingVaas();
     if (messages) {
@@ -235,6 +238,28 @@ async function getLastBlockTimeFromFirestore(): Promise<LatestTimeByChain> {
   return values;
 }
 
+async function alarmOldBlockTimes(latestTimes: LatestTimeByChain): Promise<void> {
+  // Walk all chains and check the latest block time.
+  for (const chain of Object.keys(latestTimes)) {
+    const chainId = chain as any as ChainId;
+    const latestTime = latestTimes[chainId]?.latestTime;
+    if (latestTime) {
+      const now = new Date();
+      const thePast = now;
+      thePast.setHours(now.getHours() - 24);
+      const oneDayAgo = thePast.toISOString();
+      if (latestTime < oneDayAgo) {
+        // Send a message to slack
+        const lTime: Date = new Date(latestTime);
+        const cName: string = CHAIN_ID_TO_NAME[chainId] as ChainName;
+        const deltaTime: number = (now.getTime() - lTime.getTime()) / (1000 * 60 * 60);
+        const formattedMsg = `*Chain:* ${cName}(${chainId})\nThe watcher is behind by ${deltaTime} days.`;
+        await formatAndSendToSlack(formattedMsg);
+      }
+    }
+  }
+}
+
 type FirestoreVAA = {
   chain: string;
   txHash: string;