From 5848e028759e081b325c787854beb09c8d72930d Mon Sep 17 00:00:00 2001 From: cherman27 Date: Tue, 15 Oct 2024 16:27:53 -0400 Subject: [PATCH 1/8] vendoring pmap and updating dependencies --- scrapers/classes/parsersxe/bannerv9Parser.ts | 2 +- scrapers/classes/parsersxe/termParser.ts | 2 +- scripts/migrate_major_data.ts | 2 +- services/updater.ts | 2 +- utils/elastic.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapers/classes/parsersxe/bannerv9Parser.ts b/scrapers/classes/parsersxe/bannerv9Parser.ts index ff89f37d..518b49b2 100644 --- a/scrapers/classes/parsersxe/bannerv9Parser.ts +++ b/scrapers/classes/parsersxe/bannerv9Parser.ts @@ -4,7 +4,7 @@ */ import _ from "lodash"; -import pMap from "p-map"; +import pMap from "../../../vendor/p-map/index"; import moment from "moment"; import Request from "../../request"; import macros from "../../../utils/macros"; diff --git a/scrapers/classes/parsersxe/termParser.ts b/scrapers/classes/parsersxe/termParser.ts index 58850f1c..54722635 100644 --- a/scrapers/classes/parsersxe/termParser.ts +++ b/scrapers/classes/parsersxe/termParser.ts @@ -4,7 +4,7 @@ */ import _ from "lodash"; -import pMap from "p-map"; +import pMap from "../../../vendor/p-map/index"; import keys from "../../../utils/keys"; import macros from "../../../utils/macros"; import Request from "../../request"; diff --git a/scripts/migrate_major_data.ts b/scripts/migrate_major_data.ts index 61234469..e391c626 100644 --- a/scripts/migrate_major_data.ts +++ b/scripts/migrate_major_data.ts @@ -4,7 +4,7 @@ */ import fs from "fs-extra"; import path from "path"; -import pMap from "p-map"; +import pMap from "../vendor/p-map/index"; import { Major, Prisma } from "@prisma/client"; import prisma from "../services/prisma"; diff --git a/services/updater.ts b/services/updater.ts index 4352c98f..68226c52 100644 --- a/services/updater.ts +++ b/services/updater.ts @@ -3,7 +3,7 @@ * See the license file in the root folder for details. */ -import pMap from "p-map"; +import pMap from "../vendor/p-map/index"; import { Course, Section, User } from "@prisma/client"; import macros from "../utils/macros"; diff --git a/utils/elastic.ts b/utils/elastic.ts index f853400b..f2fccec5 100755 --- a/utils/elastic.ts +++ b/utils/elastic.ts @@ -5,7 +5,7 @@ */ import { Client } from "@elastic/elasticsearch"; -import pMap from "p-map"; +import pMap from "../vendor/p-map/index"; import _ from "lodash"; import macros from "./macros"; import { From 507e642f62338cadae783076e91e0e2e192275dd Mon Sep 17 00:00:00 2001 From: cherman27 Date: Wed, 23 Oct 2024 19:56:34 -0400 Subject: [PATCH 2/8] Added healthchecks service. --- services/healthchecks.ts | 14 ++++++++++++++ services/updater.ts | 3 +++ 2 files changed, 17 insertions(+) create mode 100644 services/healthchecks.ts diff --git a/services/healthchecks.ts b/services/healthchecks.ts new file mode 100644 index 00000000..ae76819d --- /dev/null +++ b/services/healthchecks.ts @@ -0,0 +1,14 @@ +import axios from "axios"; + +export async function sendHealthCheck(): Promise { + const pingURL = " https://hc-ping.com/7d9197be-db67-4cc5-be00-1a3df56d0918"; + /** + * Sends out a ping to the pingURL, if there is an error with the updater, + * a message will be sent to the Search-Support channel in the sandbox slack. + */ + try { + await axios.get(pingURL); + } catch (error) { + console.log("Health checks service is not responding properly " + error); + } +} diff --git a/services/updater.ts b/services/updater.ts index 68226c52..1ba189d3 100644 --- a/services/updater.ts +++ b/services/updater.ts @@ -21,6 +21,7 @@ import { } from "../types/scraperTypes"; import processor from "../scrapers/classes/main"; import filters from "../scrapers/filters"; +import { sendHealthCheck } from "./healthchecks"; /* At most, there are 12 terms that we want to update - if we're in the spring & summer semesters have been posted @@ -362,6 +363,8 @@ class Updater { await this.sendUserNotifications(sections); // Save the data in our database await this.saveDataToDatabase(sections); + // Send out healthcheck ping + await sendHealthCheck(); const totalTime = Date.now() - startTime; macros.log( From 50023bd18edf807e6b4ac8b02addb660e3f62701 Mon Sep 17 00:00:00 2001 From: cherman27 Date: Thu, 31 Oct 2024 11:28:52 -0400 Subject: [PATCH 3/8] Modify to use macros when healthchecks does not respond. --- services/healthchecks.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/healthchecks.ts b/services/healthchecks.ts index ae76819d..9d9e3ce2 100644 --- a/services/healthchecks.ts +++ b/services/healthchecks.ts @@ -1,4 +1,5 @@ import axios from "axios"; +import macros from "../utils/macros"; export async function sendHealthCheck(): Promise { const pingURL = " https://hc-ping.com/7d9197be-db67-4cc5-be00-1a3df56d0918"; @@ -9,6 +10,6 @@ export async function sendHealthCheck(): Promise { try { await axios.get(pingURL); } catch (error) { - console.log("Health checks service is not responding properly " + error); + macros.warn("Health checks service is not responding properly: " + error); } } From d78a36cd8abff55b7dba8f77e9c89e3077c1919f Mon Sep 17 00:00:00 2001 From: cherman27 Date: Thu, 31 Oct 2024 11:32:47 -0400 Subject: [PATCH 4/8] pmap is a part of another commit, was on this branch accidentally. --- scrapers/classes/parsersxe/bannerv9Parser.ts | 2 +- scrapers/classes/parsersxe/termParser.ts | 2 +- scripts/migrate_major_data.ts | 2 +- utils/elastic.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapers/classes/parsersxe/bannerv9Parser.ts b/scrapers/classes/parsersxe/bannerv9Parser.ts index 518b49b2..ff89f37d 100644 --- a/scrapers/classes/parsersxe/bannerv9Parser.ts +++ b/scrapers/classes/parsersxe/bannerv9Parser.ts @@ -4,7 +4,7 @@ */ import _ from "lodash"; -import pMap from "../../../vendor/p-map/index"; +import pMap from "p-map"; import moment from "moment"; import Request from "../../request"; import macros from "../../../utils/macros"; diff --git a/scrapers/classes/parsersxe/termParser.ts b/scrapers/classes/parsersxe/termParser.ts index 54722635..58850f1c 100644 --- a/scrapers/classes/parsersxe/termParser.ts +++ b/scrapers/classes/parsersxe/termParser.ts @@ -4,7 +4,7 @@ */ import _ from "lodash"; -import pMap from "../../../vendor/p-map/index"; +import pMap from "p-map"; import keys from "../../../utils/keys"; import macros from "../../../utils/macros"; import Request from "../../request"; diff --git a/scripts/migrate_major_data.ts b/scripts/migrate_major_data.ts index e391c626..61234469 100644 --- a/scripts/migrate_major_data.ts +++ b/scripts/migrate_major_data.ts @@ -4,7 +4,7 @@ */ import fs from "fs-extra"; import path from "path"; -import pMap from "../vendor/p-map/index"; +import pMap from "p-map"; import { Major, Prisma } from "@prisma/client"; import prisma from "../services/prisma"; diff --git a/utils/elastic.ts b/utils/elastic.ts index f2fccec5..f853400b 100755 --- a/utils/elastic.ts +++ b/utils/elastic.ts @@ -5,7 +5,7 @@ */ import { Client } from "@elastic/elasticsearch"; -import pMap from "../vendor/p-map/index"; +import pMap from "p-map"; import _ from "lodash"; import macros from "./macros"; import { From 5c471bdc00bf695173195692090b66f30bebe060 Mon Sep 17 00:00:00 2001 From: cherman27 Date: Thu, 31 Oct 2024 11:34:20 -0400 Subject: [PATCH 5/8] pmap is a part of another commit, was on this branch accidentally. --- services/updater.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/updater.ts b/services/updater.ts index 1ba189d3..43e102d5 100644 --- a/services/updater.ts +++ b/services/updater.ts @@ -3,7 +3,7 @@ * See the license file in the root folder for details. */ -import pMap from "../vendor/p-map/index"; +import pMap from "p-map"; import { Course, Section, User } from "@prisma/client"; import macros from "../utils/macros"; From 4fcb6e56d073d3e45c95f9c859e5c6bffc172c65 Mon Sep 17 00:00:00 2001 From: cherman27 Date: Thu, 31 Oct 2024 12:18:20 -0400 Subject: [PATCH 6/8] healthchecks ping only sends out in prod --- services/updater.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/updater.ts b/services/updater.ts index 43e102d5..b1f5c9b7 100644 --- a/services/updater.ts +++ b/services/updater.ts @@ -364,7 +364,7 @@ class Updater { // Save the data in our database await this.saveDataToDatabase(sections); // Send out healthcheck ping - await sendHealthCheck(); + macros.PROD ? await sendHealthCheck() : null; const totalTime = Date.now() - startTime; macros.log( From 0de727d9aa8d77e67c5fe56cac0f9af41b7eaf2e Mon Sep 17 00:00:00 2001 From: cherman27 Date: Sun, 3 Nov 2024 09:30:11 -0500 Subject: [PATCH 7/8] Change ping to env variable. Modify ternary to if statement. --- services/healthchecks.ts | 3 +-- services/updater.ts | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/services/healthchecks.ts b/services/healthchecks.ts index 9d9e3ce2..c47f5b43 100644 --- a/services/healthchecks.ts +++ b/services/healthchecks.ts @@ -2,13 +2,12 @@ import axios from "axios"; import macros from "../utils/macros"; export async function sendHealthCheck(): Promise { - const pingURL = " https://hc-ping.com/7d9197be-db67-4cc5-be00-1a3df56d0918"; /** * Sends out a ping to the pingURL, if there is an error with the updater, * a message will be sent to the Search-Support channel in the sandbox slack. */ try { - await axios.get(pingURL); + await axios.get(process.env.HEALTHCHECK_PING); } catch (error) { macros.warn("Health checks service is not responding properly: " + error); } diff --git a/services/updater.ts b/services/updater.ts index b1f5c9b7..ffb3fd46 100644 --- a/services/updater.ts +++ b/services/updater.ts @@ -363,8 +363,10 @@ class Updater { await this.sendUserNotifications(sections); // Save the data in our database await this.saveDataToDatabase(sections); - // Send out healthcheck ping - macros.PROD ? await sendHealthCheck() : null; + // Send out healthcheck ping in prod + if (macros.PROD) { + await sendHealthCheck(); + } const totalTime = Date.now() - startTime; macros.log( From 9e844a83d8eef8fec5e4a7aafb8b58e9cc3adfb2 Mon Sep 17 00:00:00 2001 From: cherman27 Date: Sun, 3 Nov 2024 09:36:48 -0500 Subject: [PATCH 8/8] Include ping to env template --- template.env | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/template.env b/template.env index 368b4869..c5b09966 100644 --- a/template.env +++ b/template.env @@ -9,4 +9,7 @@ CLIENT_ORIGIN=http://localhost:5000 JWT_SECRET= # This is only used for local development -POSTGRES_PASSWORD=default_password \ No newline at end of file +POSTGRES_PASSWORD=default_password + +# Healtchecks ping server +HEALTHCHECK_PING = https://hc-ping.com/7d9197be-db67-4cc5-be00-1a3df56d0918 \ No newline at end of file