From ce1bc8e8fadb1b05a9f2d2aad7f33a33defd2714 Mon Sep 17 00:00:00 2001 From: Joey Organisak Date: Fri, 3 Jul 2020 16:46:03 -0400 Subject: [PATCH 1/4] send error notification to slack --- src/lib/slack/errorNotification.js | 39 ++++++++++++++++++++++++ src/lib/strings/locales/en/slackapp.json | 2 +- src/workers/airtable-sync/worker.js | 10 +++++- 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/lib/slack/errorNotification.js diff --git a/src/lib/slack/errorNotification.js b/src/lib/slack/errorNotification.js new file mode 100644 index 00000000..0e60b102 --- /dev/null +++ b/src/lib/slack/errorNotification.js @@ -0,0 +1,39 @@ +const slackapi = require("~slack/webApi"); +const { TECH_CHANNEL } = require("~slack/constants"); +const { findChannelByName } = require("~slack/channels"); + +const sentErrors = []; + +module.exports = async function sendErrorNotification(error) { + /** + * + */ + const now = new Date(); + + const twoHoursAgo = new Date(now.getTime() - 120 * 60000); + + let shouldSend = true; + + for (const sentError of sentErrors) { + // dont send if error occured within last 2 hours + if ( + sentError[0] === error.message && + sentError[1].getTime() > twoHoursAgo.getTime() + ) { + shouldSend = false; + } + } + + if (shouldSend) { + let errText = `${error.stack}`; + errText = ` - We've got an error!\n\`\`\`${errText}\`\`\``; + + const wgTechChannel = await findChannelByName(TECH_CHANNEL); + await slackapi.chat.postMessage({ + channel: wgTechChannel.id, + text: errText + }); + + sentErrors.push([error.message, now]); + } +}; diff --git a/src/lib/strings/locales/en/slackapp.json b/src/lib/strings/locales/en/slackapp.json index 102b5164..ae99b8cc 100644 --- a/src/lib/strings/locales/en/slackapp.json +++ b/src/lib/strings/locales/en/slackapp.json @@ -7,7 +7,7 @@ "intakeVolunteers": "intake_volunteers", "deliveryVolunteers": "delivery_volunteers", "flyering": "flyer_squad", - "tech": "tech", + "tech": "wg_tech", "communityReimbursements": "community_reimbursement", "hatHolders": "hat_holders" }, diff --git a/src/workers/airtable-sync/worker.js b/src/workers/airtable-sync/worker.js index 80c70231..4eecf7fb 100644 --- a/src/workers/airtable-sync/worker.js +++ b/src/workers/airtable-sync/worker.js @@ -1,14 +1,21 @@ +const { EventEmitter } = require("events"); const ChangeDetector = require("airtable-change-detector"); const { table: requestsTable, fields: requestFields, SENSITIVE_FIELDS: sensitiveRequestFields } = require("~airtable/tables/requests"); +const sendErrorNotification = require("~slack/errorNotification"); const updateMessageContent = require("./actions/updateMessageContent"); const notifyManyc = require("./actions/notifyManyc"); const defaultInterval = 10000; +const errorEmitter = new EventEmitter(); +errorEmitter.on("error", error => { + sendErrorNotification(error); +}); + function startWorker(interval) { let pollInterval = interval; if (pollInterval < defaultInterval) { @@ -19,7 +26,8 @@ function startWorker(interval) { } const sharedDetectorOptions = { writeDelayMs: 100, - lastProcessedFieldName: "Last Processed" + lastProcessedFieldName: "Last Processed", + errorEmitter }; const requestChanges = new ChangeDetector(requestsTable, { From 7a95c33cc3b7aecc07e96ae516191940b7610829 Mon Sep 17 00:00:00 2001 From: Joey Organisak Date: Fri, 3 Jul 2020 17:46:17 -0400 Subject: [PATCH 2/4] use err func, not emitter --- src/workers/airtable-sync/paymentWorker.js | 29 ++++++++++++++-------- src/workers/airtable-sync/worker.js | 18 ++++++-------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/workers/airtable-sync/paymentWorker.js b/src/workers/airtable-sync/paymentWorker.js index 6abdb9df..1e7c46a0 100644 --- a/src/workers/airtable-sync/paymentWorker.js +++ b/src/workers/airtable-sync/paymentWorker.js @@ -2,19 +2,24 @@ const ChangeDetector = require("airtable-change-detector"); const { donorPaymentsTable, donorPaymentsFields, - donorPaymentsSensitiveFields + donorPaymentsSensitiveFields, } = require("~airtable/tables/donorPayments"); const { paymentRequestsTable, paymentRequestsFields, - paymentRequestsSensitiveFields + paymentRequestsSensitiveFields, } = require("~airtable/tables/paymentRequests"); +const sendErrorNotification = require("~slack/errorNotification"); const newExternalDonorPayment = require("./actions/payments/newExternalDonorPayment"); const newPaymentRequest = require("./actions/payments/newPaymentRequest"); const updateReimbursementMessage = require("./actions/payments/updateReimbursementStatus"); const defaultInterval = 10000; +const errFunc = (error) => { + sendErrorNotification(error); +}; + function startWorker(interval) { let pollInterval = interval; if (pollInterval < defaultInterval) { @@ -25,20 +30,20 @@ function startWorker(interval) { } const sharedDetectorOptions = { writeDelayMs: 100, - lastProcessedFieldName: "Last Processed" + lastProcessedFieldName: "Last Processed", }; const paymentRequestChanges = new ChangeDetector(paymentRequestsTable, { senstiveFields: paymentRequestsSensitiveFields, - ...sharedDetectorOptions + ...sharedDetectorOptions, }); paymentRequestChanges.pollWithInterval( "airtable-sync.payment-requests", interval + 3000, // Stagger polling to avoid rate limit - async recordsChanged => { + async (recordsChanged) => { console.info(`Found ${recordsChanged.length} changes in PaymentRequests`); const promises = []; - recordsChanged.forEach(record => { + recordsChanged.forEach((record) => { if ( record.didChange(paymentRequestsFields.id) && !record.getPrior(paymentRequestsFields.id) @@ -50,20 +55,21 @@ function startWorker(interval) { } }); return Promise.all(promises); - } + }, + errFunc ); const donorSignupChanges = new ChangeDetector(donorPaymentsTable, { senstiveFields: donorPaymentsSensitiveFields, - ...sharedDetectorOptions + ...sharedDetectorOptions, }); donorSignupChanges.pollWithInterval( "airtable-sync.donor-payments", interval, - async recordsChanged => { + async (recordsChanged) => { console.info(`Found ${recordsChanged.length} changes in Donor Payments`); const promises = []; - recordsChanged.forEach(record => { + recordsChanged.forEach((record) => { const isExternal = !record.get(donorPaymentsFields.donorSlackId); if (isExternal) { // this logic is needed because the donorPayment isn't created at once @@ -82,7 +88,8 @@ function startWorker(interval) { } }); return Promise.all(promises); - } + }, + errFunc ); } diff --git a/src/workers/airtable-sync/worker.js b/src/workers/airtable-sync/worker.js index 4eecf7fb..495c509e 100644 --- a/src/workers/airtable-sync/worker.js +++ b/src/workers/airtable-sync/worker.js @@ -1,9 +1,8 @@ -const { EventEmitter } = require("events"); const ChangeDetector = require("airtable-change-detector"); const { table: requestsTable, fields: requestFields, - SENSITIVE_FIELDS: sensitiveRequestFields + SENSITIVE_FIELDS: sensitiveRequestFields, } = require("~airtable/tables/requests"); const sendErrorNotification = require("~slack/errorNotification"); const updateMessageContent = require("./actions/updateMessageContent"); @@ -11,10 +10,9 @@ const notifyManyc = require("./actions/notifyManyc"); const defaultInterval = 10000; -const errorEmitter = new EventEmitter(); -errorEmitter.on("error", error => { +const errFunc = (error) => { sendErrorNotification(error); -}); +}; function startWorker(interval) { let pollInterval = interval; @@ -27,20 +25,19 @@ function startWorker(interval) { const sharedDetectorOptions = { writeDelayMs: 100, lastProcessedFieldName: "Last Processed", - errorEmitter }; const requestChanges = new ChangeDetector(requestsTable, { senstiveFields: sensitiveRequestFields, - ...sharedDetectorOptions + ...sharedDetectorOptions, }); requestChanges.pollWithInterval( "airtable-sync.requests", interval, - async recordsChanged => { + async (recordsChanged) => { console.info(`Found ${recordsChanged.length} changes in Requests`); const promises = []; - recordsChanged.forEach(record => { + recordsChanged.forEach((record) => { if (record.didChange(requestFields.status)) { const status = record.get(requestFields.status); const newStatus = record.getPrior(requestFields.status); @@ -65,7 +62,8 @@ function startWorker(interval) { } }); return Promise.all(promises); - } + }, + errFunc ); } From ec5bcc2d5dcb61a9503b24145844caa6dc7374c1 Mon Sep 17 00:00:00 2001 From: Joey Organisak Date: Sun, 5 Jul 2020 14:50:09 -0400 Subject: [PATCH 3/4] lint fix --- src/lib/slack/errorNotification.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/slack/errorNotification.js b/src/lib/slack/errorNotification.js index 0e60b102..d4a851da 100644 --- a/src/lib/slack/errorNotification.js +++ b/src/lib/slack/errorNotification.js @@ -31,7 +31,7 @@ module.exports = async function sendErrorNotification(error) { const wgTechChannel = await findChannelByName(TECH_CHANNEL); await slackapi.chat.postMessage({ channel: wgTechChannel.id, - text: errText + text: errText, }); sentErrors.push([error.message, now]); From 5673b911286455688da82a1541d2f0e1be464105 Mon Sep 17 00:00:00 2001 From: Joey Organisak Date: Sat, 18 Jul 2020 16:05:07 -0400 Subject: [PATCH 4/4] async errFunc --- src/workers/airtable-sync/paymentWorker.js | 2 +- src/workers/airtable-sync/worker.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/workers/airtable-sync/paymentWorker.js b/src/workers/airtable-sync/paymentWorker.js index 1e7c46a0..45b66445 100644 --- a/src/workers/airtable-sync/paymentWorker.js +++ b/src/workers/airtable-sync/paymentWorker.js @@ -16,7 +16,7 @@ const updateReimbursementMessage = require("./actions/payments/updateReimburseme const defaultInterval = 10000; -const errFunc = (error) => { +const errFunc = async (error) => { sendErrorNotification(error); }; diff --git a/src/workers/airtable-sync/worker.js b/src/workers/airtable-sync/worker.js index 495c509e..f144ce07 100644 --- a/src/workers/airtable-sync/worker.js +++ b/src/workers/airtable-sync/worker.js @@ -10,8 +10,8 @@ const notifyManyc = require("./actions/notifyManyc"); const defaultInterval = 10000; -const errFunc = (error) => { - sendErrorNotification(error); +const errFunc = async (error) => { + await sendErrorNotification(error); }; function startWorker(interval) {