From 35ba776b8a06e256e08aeda6f540c691241e99c9 Mon Sep 17 00:00:00 2001 From: Paul Furbacher Date: Fri, 23 Apr 2021 22:15:40 -0400 Subject: [PATCH 1/4] Initial commit (#248) --- site-scrapers/Walmart/config.js | 9 ++ site-scrapers/Walmart/index.js | 157 +++++++++++++++++++++++++ site-scrapers/Walmart/stores.js | 150 ++++++++++++++++++++++++ test/Walmart/noAvailability.js | 13 +++ test/Walmart/someAvailability.js | 189 +++++++++++++++++++++++++++++++ test/WalmartTest.js | 58 ++++++++++ 6 files changed, 576 insertions(+) create mode 100644 site-scrapers/Walmart/config.js create mode 100644 site-scrapers/Walmart/index.js create mode 100644 site-scrapers/Walmart/stores.js create mode 100644 test/Walmart/noAvailability.js create mode 100644 test/Walmart/someAvailability.js create mode 100644 test/WalmartTest.js diff --git a/site-scrapers/Walmart/config.js b/site-scrapers/Walmart/config.js new file mode 100644 index 00000000..703cc447 --- /dev/null +++ b/site-scrapers/Walmart/config.js @@ -0,0 +1,9 @@ +const entityName = "Walmart"; + +const loginUrl = + "https://www.walmart.com/account/login?returnUrl=%2Fpharmacy%2Fclinical-services%2Fimmunization%2Fscheduled%3FimzType%3Dcovid"; + +module.exports = { + entityName, + loginUrl, +}; diff --git a/site-scrapers/Walmart/index.js b/site-scrapers/Walmart/index.js new file mode 100644 index 00000000..0fab819d --- /dev/null +++ b/site-scrapers/Walmart/index.js @@ -0,0 +1,157 @@ +const { entityName, loginUrl } = require("./config"); +// const { parseAvailability, getStores } = require("./walmartBase"); + +const moment = require("moment"); + +module.exports = async function GetAvailableAppointments( + browser, + fetchService = liveFetchService() +) { + console.log(`${entityName} starting.`); + + const websiteData = await ScrapeWebsiteData(browser, fetchService); + + const results = { + parentLocationName: `${entityName}`, + isChain: true, + timestamp: moment().format(), + individualLocationData: websiteData, + }; + + console.log(`${entityName} done.`); + + return results; +}; + +/** + * Dependency injection: in live scraping, the fetchAvailability() in this module is used. + * In testing, mocks of these functions are injected. + */ +function liveFetchService() { + return { + getStores() { + return getStores(); + }, + async login(page) { + return await login(page); + }, + async fetchStoreAvailability(page, storeNumber) { + return await fetchStoreAvailability(page, storeNumber); + }, + }; +} + +async function ScrapeWebsiteData(browser, fetchService) { + const page = await browser.newPage(); + + const loginSucceeded = await fetchService.login(page); + + const results = []; + + const stores = fetchService.getStores(); + if (loginSucceeded) { + Promise.all([ + Object.keys(stores).forEach(async (storeNumber) => { + let response = await fetchService.fetchStoreAvailability( + page, + storeNumber + ); + + // accumulate the store results into the encompassing results object + const availability = parseAvailability(response); + + results.push({ + ...stores[storeNumber], + ...availability, + hasAvailability: + Object.keys(availability.availability).length > 0, + }); + }), + ]); + } + + return results; +} + +/** + * Uses browser fetch() to get JSON for store availability. + * + * @param {Page} page + * @param {String} storeNumber + * @return response JSON + */ +async function fetchStoreAvailability(page, storeNumber) { + const accountId = process.env.WALMART_CUSTOMER_ACCOUNT_ID; + const inventoryUrl = `https://www.walmart.com/pharmacy/v2/clinical-services/inventory/store/${storeNumber}/${accountId}?type=imz`; + return await page.evaluate(async (storeNumber, inventoryUrl) => { + fetch(inventoryUrl) + .then((res) => res.json()) + .then((json) => { + return json; + }) + .catch((error) => console.log(`Argh! ${error}`)); + }); +} + +/** + * Login using puppeteer. No direct access to data URLs. + * After logging in, access to availability can be done via fetch within the browser. + * Fetch within the browser applies login credentials. + * + * @param {Page} page + * @returns + */ +async function login(page) { + await page.goto(loginUrl); + + await page.waitForSelector("input#email"); + + await page.type("input#email", process.env.WALMART_EMAIL); + await page.waitForTimeout(200); + await page.type("input#password", process.env.WALMART_PASSWORD); + + await page.click("#sign-in-form > button[type='submit']"); + + const success = await page.waitForNavigation().then( + () => { + return true; + }, + (err) => { + page.screenshot({ path: "walmart.png" }); // login failed + return false; + } + ); + return success; +} + +function getStores() { + const { stores } = require("./stores"); + return stores; +} + +function parseAvailability(response) { + const result = { + availability: {}, + hasAvailability: false, + signUpLink: loginUrl, + timestamp: moment(), + }; + + // slotDays is an array of objects: + // {slotDate: "04212021", slots: [...], message} + + response?.data?.slotDays.forEach((day) => { + if (day.slots.length > 0) { + const dateParts = day.slotDate.match(/(\d{2})(\d{2})(\d{4})/); + result.availability[ + `${dateParts[1]}/${dateParts[2]}/${dateParts[3]}` + ] = { + numberAvailableAppointments: day.slots.length, + hasAvailability: day.slots.length > 0, + }; + result.hasAvailability = true; + } + }); + + return result; +} diff --git a/site-scrapers/Walmart/stores.js b/site-scrapers/Walmart/stores.js new file mode 100644 index 00000000..a92e109a --- /dev/null +++ b/site-scrapers/Walmart/stores.js @@ -0,0 +1,150 @@ +const stores = { + 1906: { + street: "42 Fairhaven Commons Way", + city: "Fairhaven", + zip: "02719", + }, + 1918: { + street: "250 Hartford Ave", + city: "Bellingham", + zip: "02019", + }, + 1967: { + street: "1105 Boston Rd", + city: "Springfield", + zip: "01119", + }, + 1970: { street: "280 Washington St", city: "Hudson", zip: "01749" }, + 1984: { + street: "1415 Curran Memorial Hwy", + city: "North Adams", + zip: "01247", + }, + 2012: { street: "742 Main St", city: "North Oxford", zip: "01537" }, + 2021: { street: "36 Paramount Dr", city: "Raynham", zip: "02767" }, + 2095: { street: "15 Tobey Rd", city: "Wareham", zip: "02571" }, + 2103: { + street: "550 Providence Hwy", + city: "Walpole", + zip: "02081", + }, + 2118: { + street: "301 Massachusetts Ave", + city: "Lunenburg", + zip: "01462", + }, + 2122: { street: "30 Memorial Dr", city: "Avon", zip: "02322" }, + 2128: { street: "295 Plymouth St", city: "Halifax", zip: "02338" }, + 2139: { street: "780 Lynnway", city: "Lynn", zip: "01905" }, + 2155: { street: "677 Timpany Blvd", city: "Gardner", zip: "01440" }, + 2157: { + street: "506 State Rd", + city: "North Dartmouth", + zip: "02747", + }, + 2158: { street: "200 Otis St", city: "Northborough", zip: "01532" }, + 2174: { + street: "141 Springfield Rd", + city: "Westfield", + zip: "01085", + }, + 2180: { + street: "55 Brooksby Village Way", + city: "Danvers", + zip: "01923", + }, + 2184: { + street: "1180 Fall River Ave", + city: "Seekonk", + zip: "02771", + }, + 2222: { street: "333 Main St", city: "Tewksbury", zip: "01876" }, + 2227: { + street: "777 Brockton Ave", + city: "Abington", + zip: "02351", + }, + 2228: { + street: "555 Hubbard Ave Ste 12", + city: "Pittsfield", + zip: "01201", + }, + 2267: { + street: "137 W Boylston St", + city: "West Boylston", + zip: "01583", + }, + 2329: { street: "555 E Main St", city: "Orange", zip: "01364" }, + 2336: { + street: "300 Colony Place Rd", + city: "Plymouth", + zip: "02360", + }, + 2341: { street: "301 Falls Blvd", city: "Quincy", zip: "02169" }, + 2366: { + street: "1470 S Washington St", + city: "North Attleboro", + zip: "02760", + }, + 2386: { street: "352 Palmer Rd", city: "Ware", zip: "01082" }, + 2629: { + street: "100 Valley Pkwy", + city: "Whitinsville", + zip: "01588", + }, + 2640: { street: "450 Highland Ave", city: "Salem", zip: "01970" }, + 2660: { street: "72 Main St", city: "North Reading", zip: "01864" }, + 2683: { street: "337 Russell St", city: "Hadley", zip: "01035" }, + 2797: { + street: "100 Charlton Rd", + city: "Sturbridge", + zip: "01566", + }, + 2901: { + street: "180 N King St", + city: "Northampton", + zip: "01060", + }, + 2902: { + street: "121 Worcester Rd", + city: "Framingham", + zip: "01701", + }, + 2903: { + street: "66 Parkhurst Rd", + city: "Chelmsford", + zip: "01824", + }, + 2904: { street: "700 Oak St", city: "Brockton", zip: "02301" }, + 2953: { street: "54 Cousineau Dr", city: "Swansea", zip: "02777" }, + 2964: { street: "11 Jungle Rd", city: "Leominster", zip: "01453" }, + 3114: { street: "770 Broadway", city: "Saugus", zip: "01906" }, + 3200: { street: "740 Middle St", city: "Weymouth", zip: "02188" }, + 3409: { street: "20 Soojian Dr", city: "Leicester", zip: "01524" }, + 3491: { + street: "70 Pleasant Valley St", + city: "Methuen", + zip: "01844", + }, + 3560: { + street: "638 Quequechan Street", + city: "Fall River", + zip: "02721", + }, + 3561: { + street: "137 Teaticket Hwy", + city: "Teaticket", + zip: "02536", + }, + 4387: { + street: "25 Tobias Boland Way", + city: "Worcester", + zip: "01607", + }, + 5278: { street: "591 Memorial Dr", city: "Chicopee", zip: "01020" }, + 5448: { street: "160 Broadway", city: "Raynham", zip: "02767" }, +}; + +module.exports = { + stores, +}; diff --git a/test/Walmart/noAvailability.js b/test/Walmart/noAvailability.js new file mode 100644 index 00000000..4f6a6d80 --- /dev/null +++ b/test/Walmart/noAvailability.js @@ -0,0 +1,13 @@ +const noAvailability = { + status: "1", + message: "OK", + data: { + startDate: "04212021", + endDate: "04272021", + accessPointId: "607052ab-8410-47ec-ba1c-388a1597cfc2", + }, +}; + +module.exports = { + noData: noAvailability, +}; diff --git a/test/Walmart/someAvailability.js b/test/Walmart/someAvailability.js new file mode 100644 index 00000000..86e1e8b8 --- /dev/null +++ b/test/Walmart/someAvailability.js @@ -0,0 +1,189 @@ +const someAvailability = { + status: "1", + message: "OK", + data: { + startDate: "04212021", + endDate: "04272021", + accessPointId: "607052ab-8410-47ec-ba1c-388a1597cfc2", + slotDays: [ + { + slotDate: "04212021", + slots: [], + message: + "There are no appointments available today. Please select a different day.", + }, + { + slotDate: "04222021", + slots: [], + message: + "There are no appointments available for Thu, 22 Apr. Please select a different day.", + }, + { + slotDate: "04232021", + slots: [], + message: + "There are no appointments available for Fri, 23 Apr. Please select a different day.", + }, + { + slotDate: "04242021", + slots: [], + message: + "There are no appointments available for Sat, 24 Apr. Please select a different day.", + }, + { + slotDate: "04252021", + slots: [], + message: + "There are no appointments available for Sun, 25 Apr. Please select a different day.", + }, + { + slotDate: "04262021", + slots: [], + message: + "There are no appointments available for Mon, 26 Apr. Please select a different day.", + }, + { + slotDate: "04272021", + slots: [ + { + slotId: + "bc21bc61-e02a-4b9a-aba4-d578bc29dc28-2021-04-27", + startTime: "6:20", + endTime: "6:40", + }, + { + slotId: + "a0867a2c-4732-414d-a862-7717379e3fe2-2021-04-27", + startTime: "6:40", + endTime: "7:00", + }, + { + slotId: + "f8f3b999-dfea-4a55-8d06-0acbfb7c3b7b-2021-04-27", + startTime: "7:00", + endTime: "7:20", + }, + { + slotId: + "4e66c0a4-02c6-4ef9-88c1-e109c895e124-2021-04-27", + startTime: "7:20", + endTime: "7:40", + }, + { + slotId: + "f6fe381b-0efa-4e75-b81c-615860796944-2021-04-27", + startTime: "7:40", + endTime: "8:00", + }, + { + slotId: + "429f9392-77d6-4db8-a03a-2b8d168dc02e-2021-04-27", + startTime: "8:00", + endTime: "8:20", + }, + { + slotId: + "f985aaef-10d5-4a33-8c32-74f8565e59c4-2021-04-27", + startTime: "8:20", + endTime: "8:40", + }, + { + slotId: + "f7408a63-4c1f-411c-b970-325cca3507a1-2021-04-27", + startTime: "8:40", + endTime: "9:00", + }, + { + slotId: + "d358ee26-35a7-4660-a21b-7e1fa0c81777-2021-04-27", + startTime: "9:00", + endTime: "9:20", + }, + { + slotId: + "cacab50e-fd61-4230-b519-1b16a6cc50a6-2021-04-27", + startTime: "9:20", + endTime: "9:40", + }, + { + slotId: + "62c8884a-c7e7-4c88-937a-eafa03e5327c-2021-04-27", + startTime: "9:40", + endTime: "10:00", + }, + { + slotId: + "19bf48b5-95c4-49ce-b172-03d895fc74c8-2021-04-27", + startTime: "10:00", + endTime: "10:20", + }, + { + slotId: + "36c6ad32-e86e-4b31-b6e7-a8fd8a6606d0-2021-04-27", + startTime: "10:20", + endTime: "10:40", + }, + { + slotId: + "ba2b597d-0381-49b0-84cb-ec3bdab24bd9-2021-04-27", + startTime: "11:00", + endTime: "11:20", + }, + { + slotId: + "c5b225c7-a9ff-4b6f-bfac-667a4a28fcd7-2021-04-27", + startTime: "11:20", + endTime: "11:40", + }, + { + slotId: + "d191dc3f-e437-4236-bfa9-b5396de9c14c-2021-04-27", + startTime: "11:40", + endTime: "12:00", + }, + { + slotId: + "744cd421-c4be-4439-b755-3d6eab0e2357-2021-04-27", + startTime: "12:00", + endTime: "12:20", + }, + { + slotId: + "a49fdd12-7106-4f64-ba18-cc20bab0009b-2021-04-27", + startTime: "12:20", + endTime: "12:40", + }, + { + slotId: + "37387661-b6bf-480a-810a-3819fbeb3ea5-2021-04-27", + startTime: "15:00", + endTime: "15:20", + }, + { + slotId: + "05f0ed4b-546b-4b44-a54f-61682b677668-2021-04-27", + startTime: "15:40", + endTime: "16:00", + }, + { + slotId: + "81520dc4-c62c-4a4e-885c-f06599d2f510-2021-04-27", + startTime: "16:00", + endTime: "16:20", + }, + { + slotId: + "98a45321-5ac8-43f9-a244-a4d13cf4e8c4-2021-04-27", + startTime: "16:20", + endTime: "16:40", + }, + ], + message: null, + }, + ], + }, +}; + +module.exports = { + someAvailability, +}; diff --git a/test/WalmartTest.js b/test/WalmartTest.js new file mode 100644 index 00000000..ed9cf18c --- /dev/null +++ b/test/WalmartTest.js @@ -0,0 +1,58 @@ +const scraper = require("../site-scrapers/Walmart/index"); +const { someAvailability } = require("./Walmart/someAvailability"); +const { noAvailability } = require("./Walmart/noAvailability"); +const { expect } = require("chai"); +const file = require("../lib/file"); + +describe("WalmartTest :: testing JSON response", function () { + function* jsonGenerator() { + yield someAvailability; + yield noAvailability; + } + const jsonSource = jsonGenerator(); + + const testFetchService = { + /** Limit the number of stores, one with availability, the other without. + * Allows inspection of either type of availability record in output. + */ + getStores() { + return { + 5278: { + street: "591 Memorial Dr", + city: "Chicopee", + zip: "01020", + }, + 5448: { street: "160 Broadway", city: "Raynham", zip: "02767" }, + }; + }, + /** Testing uses mock data, so no need to log-in. */ + async login() { + return true; + }, + /** + * Mock data, with and without availability, is provided by + * the generator which serves up the appropriate data from files. + */ + async fetchStoreAvailability() { + const json = jsonSource.next().value; + return json; + }, + }; + + it("should show availability in one store, and none in the other", async () => { + const results = await scraper(browser, testFetchService); + + expect(Object.keys(results.individualLocationData).length).equals(2); + expect( + results.individualLocationData[0].availability["04/27/2021"] + .numberAvailableAppointments + ).equals(22); + + if (process.env.DEVELOPMENT) { + file.write( + `${process.cwd()}/out.json`, + `${JSON.stringify(results, null, " ")}` + ); + } + }); +}); From 84936de1ce438922c7018b26231c039ce0a6b5ee Mon Sep 17 00:00:00 2001 From: Paul Furbacher Date: Sat, 8 May 2021 13:55:54 -0400 Subject: [PATCH 2/4] Revert "Merge branch 'walmart' of https://github.com/livgust/covid-vaccine-scrapers" This reverts commit a7ea3b69d966fbd70e8070bf236a484c96619376, reversing changes made to 1b86d44d237f2cd5a5db3d293f10b9db829f0a5f. --- site-scrapers/Walmart/config.js | 9 -- site-scrapers/Walmart/index.js | 157 ------------------------- site-scrapers/Walmart/stores.js | 150 ------------------------ test/Walmart/noAvailability.js | 13 --- test/Walmart/someAvailability.js | 189 ------------------------------- test/WalmartTest.js | 58 ---------- 6 files changed, 576 deletions(-) delete mode 100644 site-scrapers/Walmart/config.js delete mode 100644 site-scrapers/Walmart/index.js delete mode 100644 site-scrapers/Walmart/stores.js delete mode 100644 test/Walmart/noAvailability.js delete mode 100644 test/Walmart/someAvailability.js delete mode 100644 test/WalmartTest.js diff --git a/site-scrapers/Walmart/config.js b/site-scrapers/Walmart/config.js deleted file mode 100644 index 703cc447..00000000 --- a/site-scrapers/Walmart/config.js +++ /dev/null @@ -1,9 +0,0 @@ -const entityName = "Walmart"; - -const loginUrl = - "https://www.walmart.com/account/login?returnUrl=%2Fpharmacy%2Fclinical-services%2Fimmunization%2Fscheduled%3FimzType%3Dcovid"; - -module.exports = { - entityName, - loginUrl, -}; diff --git a/site-scrapers/Walmart/index.js b/site-scrapers/Walmart/index.js deleted file mode 100644 index 0fab819d..00000000 --- a/site-scrapers/Walmart/index.js +++ /dev/null @@ -1,157 +0,0 @@ -const { entityName, loginUrl } = require("./config"); -// const { parseAvailability, getStores } = require("./walmartBase"); - -const moment = require("moment"); - -module.exports = async function GetAvailableAppointments( - browser, - fetchService = liveFetchService() -) { - console.log(`${entityName} starting.`); - - const websiteData = await ScrapeWebsiteData(browser, fetchService); - - const results = { - parentLocationName: `${entityName}`, - isChain: true, - timestamp: moment().format(), - individualLocationData: websiteData, - }; - - console.log(`${entityName} done.`); - - return results; -}; - -/** - * Dependency injection: in live scraping, the fetchAvailability() in this module is used. - * In testing, mocks of these functions are injected. - */ -function liveFetchService() { - return { - getStores() { - return getStores(); - }, - async login(page) { - return await login(page); - }, - async fetchStoreAvailability(page, storeNumber) { - return await fetchStoreAvailability(page, storeNumber); - }, - }; -} - -async function ScrapeWebsiteData(browser, fetchService) { - const page = await browser.newPage(); - - const loginSucceeded = await fetchService.login(page); - - const results = []; - - const stores = fetchService.getStores(); - if (loginSucceeded) { - Promise.all([ - Object.keys(stores).forEach(async (storeNumber) => { - let response = await fetchService.fetchStoreAvailability( - page, - storeNumber - ); - - // accumulate the store results into the encompassing results object - const availability = parseAvailability(response); - - results.push({ - ...stores[storeNumber], - ...availability, - hasAvailability: - Object.keys(availability.availability).length > 0, - }); - }), - ]); - } - - return results; -} - -/** - * Uses browser fetch() to get JSON for store availability. - * - * @param {Page} page - * @param {String} storeNumber - * @return response JSON - */ -async function fetchStoreAvailability(page, storeNumber) { - const accountId = process.env.WALMART_CUSTOMER_ACCOUNT_ID; - const inventoryUrl = `https://www.walmart.com/pharmacy/v2/clinical-services/inventory/store/${storeNumber}/${accountId}?type=imz`; - return await page.evaluate(async (storeNumber, inventoryUrl) => { - fetch(inventoryUrl) - .then((res) => res.json()) - .then((json) => { - return json; - }) - .catch((error) => console.log(`Argh! ${error}`)); - }); -} - -/** - * Login using puppeteer. No direct access to data URLs. - * After logging in, access to availability can be done via fetch within the browser. - * Fetch within the browser applies login credentials. - * - * @param {Page} page - * @returns - */ -async function login(page) { - await page.goto(loginUrl); - - await page.waitForSelector("input#email"); - - await page.type("input#email", process.env.WALMART_EMAIL); - await page.waitForTimeout(200); - await page.type("input#password", process.env.WALMART_PASSWORD); - - await page.click("#sign-in-form > button[type='submit']"); - - const success = await page.waitForNavigation().then( - () => { - return true; - }, - (err) => { - page.screenshot({ path: "walmart.png" }); // login failed - return false; - } - ); - return success; -} - -function getStores() { - const { stores } = require("./stores"); - return stores; -} - -function parseAvailability(response) { - const result = { - availability: {}, - hasAvailability: false, - signUpLink: loginUrl, - timestamp: moment(), - }; - - // slotDays is an array of objects: - // {slotDate: "04212021", slots: [...], message} - - response?.data?.slotDays.forEach((day) => { - if (day.slots.length > 0) { - const dateParts = day.slotDate.match(/(\d{2})(\d{2})(\d{4})/); - result.availability[ - `${dateParts[1]}/${dateParts[2]}/${dateParts[3]}` - ] = { - numberAvailableAppointments: day.slots.length, - hasAvailability: day.slots.length > 0, - }; - result.hasAvailability = true; - } - }); - - return result; -} diff --git a/site-scrapers/Walmart/stores.js b/site-scrapers/Walmart/stores.js deleted file mode 100644 index a92e109a..00000000 --- a/site-scrapers/Walmart/stores.js +++ /dev/null @@ -1,150 +0,0 @@ -const stores = { - 1906: { - street: "42 Fairhaven Commons Way", - city: "Fairhaven", - zip: "02719", - }, - 1918: { - street: "250 Hartford Ave", - city: "Bellingham", - zip: "02019", - }, - 1967: { - street: "1105 Boston Rd", - city: "Springfield", - zip: "01119", - }, - 1970: { street: "280 Washington St", city: "Hudson", zip: "01749" }, - 1984: { - street: "1415 Curran Memorial Hwy", - city: "North Adams", - zip: "01247", - }, - 2012: { street: "742 Main St", city: "North Oxford", zip: "01537" }, - 2021: { street: "36 Paramount Dr", city: "Raynham", zip: "02767" }, - 2095: { street: "15 Tobey Rd", city: "Wareham", zip: "02571" }, - 2103: { - street: "550 Providence Hwy", - city: "Walpole", - zip: "02081", - }, - 2118: { - street: "301 Massachusetts Ave", - city: "Lunenburg", - zip: "01462", - }, - 2122: { street: "30 Memorial Dr", city: "Avon", zip: "02322" }, - 2128: { street: "295 Plymouth St", city: "Halifax", zip: "02338" }, - 2139: { street: "780 Lynnway", city: "Lynn", zip: "01905" }, - 2155: { street: "677 Timpany Blvd", city: "Gardner", zip: "01440" }, - 2157: { - street: "506 State Rd", - city: "North Dartmouth", - zip: "02747", - }, - 2158: { street: "200 Otis St", city: "Northborough", zip: "01532" }, - 2174: { - street: "141 Springfield Rd", - city: "Westfield", - zip: "01085", - }, - 2180: { - street: "55 Brooksby Village Way", - city: "Danvers", - zip: "01923", - }, - 2184: { - street: "1180 Fall River Ave", - city: "Seekonk", - zip: "02771", - }, - 2222: { street: "333 Main St", city: "Tewksbury", zip: "01876" }, - 2227: { - street: "777 Brockton Ave", - city: "Abington", - zip: "02351", - }, - 2228: { - street: "555 Hubbard Ave Ste 12", - city: "Pittsfield", - zip: "01201", - }, - 2267: { - street: "137 W Boylston St", - city: "West Boylston", - zip: "01583", - }, - 2329: { street: "555 E Main St", city: "Orange", zip: "01364" }, - 2336: { - street: "300 Colony Place Rd", - city: "Plymouth", - zip: "02360", - }, - 2341: { street: "301 Falls Blvd", city: "Quincy", zip: "02169" }, - 2366: { - street: "1470 S Washington St", - city: "North Attleboro", - zip: "02760", - }, - 2386: { street: "352 Palmer Rd", city: "Ware", zip: "01082" }, - 2629: { - street: "100 Valley Pkwy", - city: "Whitinsville", - zip: "01588", - }, - 2640: { street: "450 Highland Ave", city: "Salem", zip: "01970" }, - 2660: { street: "72 Main St", city: "North Reading", zip: "01864" }, - 2683: { street: "337 Russell St", city: "Hadley", zip: "01035" }, - 2797: { - street: "100 Charlton Rd", - city: "Sturbridge", - zip: "01566", - }, - 2901: { - street: "180 N King St", - city: "Northampton", - zip: "01060", - }, - 2902: { - street: "121 Worcester Rd", - city: "Framingham", - zip: "01701", - }, - 2903: { - street: "66 Parkhurst Rd", - city: "Chelmsford", - zip: "01824", - }, - 2904: { street: "700 Oak St", city: "Brockton", zip: "02301" }, - 2953: { street: "54 Cousineau Dr", city: "Swansea", zip: "02777" }, - 2964: { street: "11 Jungle Rd", city: "Leominster", zip: "01453" }, - 3114: { street: "770 Broadway", city: "Saugus", zip: "01906" }, - 3200: { street: "740 Middle St", city: "Weymouth", zip: "02188" }, - 3409: { street: "20 Soojian Dr", city: "Leicester", zip: "01524" }, - 3491: { - street: "70 Pleasant Valley St", - city: "Methuen", - zip: "01844", - }, - 3560: { - street: "638 Quequechan Street", - city: "Fall River", - zip: "02721", - }, - 3561: { - street: "137 Teaticket Hwy", - city: "Teaticket", - zip: "02536", - }, - 4387: { - street: "25 Tobias Boland Way", - city: "Worcester", - zip: "01607", - }, - 5278: { street: "591 Memorial Dr", city: "Chicopee", zip: "01020" }, - 5448: { street: "160 Broadway", city: "Raynham", zip: "02767" }, -}; - -module.exports = { - stores, -}; diff --git a/test/Walmart/noAvailability.js b/test/Walmart/noAvailability.js deleted file mode 100644 index 4f6a6d80..00000000 --- a/test/Walmart/noAvailability.js +++ /dev/null @@ -1,13 +0,0 @@ -const noAvailability = { - status: "1", - message: "OK", - data: { - startDate: "04212021", - endDate: "04272021", - accessPointId: "607052ab-8410-47ec-ba1c-388a1597cfc2", - }, -}; - -module.exports = { - noData: noAvailability, -}; diff --git a/test/Walmart/someAvailability.js b/test/Walmart/someAvailability.js deleted file mode 100644 index 86e1e8b8..00000000 --- a/test/Walmart/someAvailability.js +++ /dev/null @@ -1,189 +0,0 @@ -const someAvailability = { - status: "1", - message: "OK", - data: { - startDate: "04212021", - endDate: "04272021", - accessPointId: "607052ab-8410-47ec-ba1c-388a1597cfc2", - slotDays: [ - { - slotDate: "04212021", - slots: [], - message: - "There are no appointments available today. Please select a different day.", - }, - { - slotDate: "04222021", - slots: [], - message: - "There are no appointments available for Thu, 22 Apr. Please select a different day.", - }, - { - slotDate: "04232021", - slots: [], - message: - "There are no appointments available for Fri, 23 Apr. Please select a different day.", - }, - { - slotDate: "04242021", - slots: [], - message: - "There are no appointments available for Sat, 24 Apr. Please select a different day.", - }, - { - slotDate: "04252021", - slots: [], - message: - "There are no appointments available for Sun, 25 Apr. Please select a different day.", - }, - { - slotDate: "04262021", - slots: [], - message: - "There are no appointments available for Mon, 26 Apr. Please select a different day.", - }, - { - slotDate: "04272021", - slots: [ - { - slotId: - "bc21bc61-e02a-4b9a-aba4-d578bc29dc28-2021-04-27", - startTime: "6:20", - endTime: "6:40", - }, - { - slotId: - "a0867a2c-4732-414d-a862-7717379e3fe2-2021-04-27", - startTime: "6:40", - endTime: "7:00", - }, - { - slotId: - "f8f3b999-dfea-4a55-8d06-0acbfb7c3b7b-2021-04-27", - startTime: "7:00", - endTime: "7:20", - }, - { - slotId: - "4e66c0a4-02c6-4ef9-88c1-e109c895e124-2021-04-27", - startTime: "7:20", - endTime: "7:40", - }, - { - slotId: - "f6fe381b-0efa-4e75-b81c-615860796944-2021-04-27", - startTime: "7:40", - endTime: "8:00", - }, - { - slotId: - "429f9392-77d6-4db8-a03a-2b8d168dc02e-2021-04-27", - startTime: "8:00", - endTime: "8:20", - }, - { - slotId: - "f985aaef-10d5-4a33-8c32-74f8565e59c4-2021-04-27", - startTime: "8:20", - endTime: "8:40", - }, - { - slotId: - "f7408a63-4c1f-411c-b970-325cca3507a1-2021-04-27", - startTime: "8:40", - endTime: "9:00", - }, - { - slotId: - "d358ee26-35a7-4660-a21b-7e1fa0c81777-2021-04-27", - startTime: "9:00", - endTime: "9:20", - }, - { - slotId: - "cacab50e-fd61-4230-b519-1b16a6cc50a6-2021-04-27", - startTime: "9:20", - endTime: "9:40", - }, - { - slotId: - "62c8884a-c7e7-4c88-937a-eafa03e5327c-2021-04-27", - startTime: "9:40", - endTime: "10:00", - }, - { - slotId: - "19bf48b5-95c4-49ce-b172-03d895fc74c8-2021-04-27", - startTime: "10:00", - endTime: "10:20", - }, - { - slotId: - "36c6ad32-e86e-4b31-b6e7-a8fd8a6606d0-2021-04-27", - startTime: "10:20", - endTime: "10:40", - }, - { - slotId: - "ba2b597d-0381-49b0-84cb-ec3bdab24bd9-2021-04-27", - startTime: "11:00", - endTime: "11:20", - }, - { - slotId: - "c5b225c7-a9ff-4b6f-bfac-667a4a28fcd7-2021-04-27", - startTime: "11:20", - endTime: "11:40", - }, - { - slotId: - "d191dc3f-e437-4236-bfa9-b5396de9c14c-2021-04-27", - startTime: "11:40", - endTime: "12:00", - }, - { - slotId: - "744cd421-c4be-4439-b755-3d6eab0e2357-2021-04-27", - startTime: "12:00", - endTime: "12:20", - }, - { - slotId: - "a49fdd12-7106-4f64-ba18-cc20bab0009b-2021-04-27", - startTime: "12:20", - endTime: "12:40", - }, - { - slotId: - "37387661-b6bf-480a-810a-3819fbeb3ea5-2021-04-27", - startTime: "15:00", - endTime: "15:20", - }, - { - slotId: - "05f0ed4b-546b-4b44-a54f-61682b677668-2021-04-27", - startTime: "15:40", - endTime: "16:00", - }, - { - slotId: - "81520dc4-c62c-4a4e-885c-f06599d2f510-2021-04-27", - startTime: "16:00", - endTime: "16:20", - }, - { - slotId: - "98a45321-5ac8-43f9-a244-a4d13cf4e8c4-2021-04-27", - startTime: "16:20", - endTime: "16:40", - }, - ], - message: null, - }, - ], - }, -}; - -module.exports = { - someAvailability, -}; diff --git a/test/WalmartTest.js b/test/WalmartTest.js deleted file mode 100644 index ed9cf18c..00000000 --- a/test/WalmartTest.js +++ /dev/null @@ -1,58 +0,0 @@ -const scraper = require("../site-scrapers/Walmart/index"); -const { someAvailability } = require("./Walmart/someAvailability"); -const { noAvailability } = require("./Walmart/noAvailability"); -const { expect } = require("chai"); -const file = require("../lib/file"); - -describe("WalmartTest :: testing JSON response", function () { - function* jsonGenerator() { - yield someAvailability; - yield noAvailability; - } - const jsonSource = jsonGenerator(); - - const testFetchService = { - /** Limit the number of stores, one with availability, the other without. - * Allows inspection of either type of availability record in output. - */ - getStores() { - return { - 5278: { - street: "591 Memorial Dr", - city: "Chicopee", - zip: "01020", - }, - 5448: { street: "160 Broadway", city: "Raynham", zip: "02767" }, - }; - }, - /** Testing uses mock data, so no need to log-in. */ - async login() { - return true; - }, - /** - * Mock data, with and without availability, is provided by - * the generator which serves up the appropriate data from files. - */ - async fetchStoreAvailability() { - const json = jsonSource.next().value; - return json; - }, - }; - - it("should show availability in one store, and none in the other", async () => { - const results = await scraper(browser, testFetchService); - - expect(Object.keys(results.individualLocationData).length).equals(2); - expect( - results.individualLocationData[0].availability["04/27/2021"] - .numberAvailableAppointments - ).equals(22); - - if (process.env.DEVELOPMENT) { - file.write( - `${process.cwd()}/out.json`, - `${JSON.stringify(results, null, " ")}` - ); - } - }); -}); From 579ba68e113af2c7425ee9ea6fbb8cb51dacb282 Mon Sep 17 00:00:00 2001 From: Paul Furbacher Date: Sat, 8 May 2021 14:03:14 -0400 Subject: [PATCH 3/4] Initial commit --- site-scrapers/SouthcoastHealth/config.js | 36 + site-scrapers/SouthcoastHealth/index.js | 111 +++ .../SouthcoastHealth/responseParser.js | 43 ++ test/SouthcoastHealth/sampleFallRiver.json | 729 ++++++++++++++++++ .../sampleNoAvailability.json | 6 + 5 files changed, 925 insertions(+) create mode 100644 site-scrapers/SouthcoastHealth/config.js create mode 100644 site-scrapers/SouthcoastHealth/index.js create mode 100644 site-scrapers/SouthcoastHealth/responseParser.js create mode 100644 test/SouthcoastHealth/sampleFallRiver.json create mode 100644 test/SouthcoastHealth/sampleNoAvailability.json diff --git a/site-scrapers/SouthcoastHealth/config.js b/site-scrapers/SouthcoastHealth/config.js new file mode 100644 index 00000000..e3bfdfb6 --- /dev/null +++ b/site-scrapers/SouthcoastHealth/config.js @@ -0,0 +1,36 @@ +const entityName = "Southcoast Health"; + +const sites = [ + { + name: "Southcoast Health (Fall River)", + street: "20 Star Street", + city: "Fall River", + zip: "02724", + signUpLink: + "https://www.southcoast.org/covid-19-vaccine-scheduling/#/slot-search/covid/resource-type/98C5A8BE-25D1-4125-9AD5-1EE64AD164D2", + }, + { + name: "Southcoast Health (North Dartmouth)", + street: "375 Faunce Corner Road", + city: "North Dartmouth", + zip: "02747", + signUpLink: + "https://www.southcoast.org/covid-19-vaccine-scheduling/#/slot-search/covid/resource-type/98C5A8BE-25D1-4125-9AD5-1EE64AD164D2", + }, + { + name: "Southcoast Health (Wareham)", + street: "48 Marion Road", + city: "Wareham", + zip: "02571", + signUpLink: + "https://www.southcoast.org/covid-19-vaccine-scheduling/#/slot-search/covid/resource-type/98C5A8BE-25D1-4125-9AD5-1EE64AD164D2", + }, +]; + +const siteUrl = + "https://www.southcoast.org/covid-19-vaccine-scheduling/#/slot-search/covid/resource-type/98C5A8BE-25D1-4125-9AD5-1EE64AD164D2"; +module.exports = { + entityName, + sites, + siteUrl, +}; diff --git a/site-scrapers/SouthcoastHealth/index.js b/site-scrapers/SouthcoastHealth/index.js new file mode 100644 index 00000000..3a1f1771 --- /dev/null +++ b/site-scrapers/SouthcoastHealth/index.js @@ -0,0 +1,111 @@ +const { siteUrl, sites, entityName } = require("./config"); +const { parseJson } = require("./responseParser.js"); +const moment = require("moment"); + +module.exports = async function GetAvailableAppointments(browser) { + console.log(`${entityName} starting.`); + const siteData = await ScrapeWebsiteData(browser, sites); + console.log(`${entityName} done.`); + return { + parentLocationName: entityName, + timestamp: moment().format(), + individualLocationData: siteData, + }; +}; + +/** + * + * @param {*} browser + * @param {*} pageService + * @param {*} sites + * @returns array of site details and availability + */ +async function ScrapeWebsiteData(browser, sites) { + const page = await browser.newPage(); + + const submitButtonSelector = "button.btn.btn-default.btn-lg.center-block"; + await Promise.all([ + page.goto(siteUrl), + await page.waitForSelector(submitButtonSelector, { + visible: true, + timeout: 15000, + }), + ]); + + await page.evaluate((submitButtonSelector) => { + document.querySelector(submitButtonSelector).click(); + }, submitButtonSelector); + + await page.waitForSelector("#oas-scheduler"); + + const siteData = []; + + for (const site of sites) { + const availabilityContainer = await getAvailabilityForSite(page, site); + + siteData.push({ + ...site, + availability: availabilityContainer, + hasAvailability: Object.keys(availabilityContainer).length > 0, + }); + } + + page.close(); + return siteData; +} + +async function getAvailabilityForSite(page, site) { + const responseJson = await fetchDataForSite(page, site); + const availabilityContainer = parseJson(responseJson); + return availabilityContainer; +} + +async function fetchDataForSite(page, site) { + const location = site.city; + + // 2021-05-08T02:34:12.058Z + const startDateStr = moment().toISOString(); + // 2021-05-21T02:34:12.058Z 13 days + const endDateStr = moment().add(13, "days").toISOString(); + + const response = await page.evaluate( + async (location, startDateStr, endDateStr) => { + const json = await fetch( + "https://southcoastapps.southcoast.org/OnlineAppointmentSchedulingApi/api/resourceTypes/slots/search", + { + headers: { + accept: "application/json, text/plain, */*", + "accept-language": "en-US,en;q=0.9", + "content-type": "application/json", + "sec-ch-ua": + '" Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"', + "sec-ch-ua-mobile": "?0", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-site", + sessiontoken: "5d886340-71e9-4bd2-9449-80d5eb3263e9", + }, + referrer: "https://www.southcoast.org/", + referrerPolicy: "strict-origin-when-cross-origin", + body: `{"ProviderCriteria":{"SpecialtyID":null,"ConcentrationID":null},"ResourceTypeId":"98C5A8BE-25D1-4125-9AD5-1EE64AD164D2","StartDate":"${startDateStr}","EndDate":"${endDateStr}","Location":"${location}"}`, + method: "POST", + mode: "cors", + credentials: "omit", + } + ) + .then((res) => res.json()) + .then((json) => { + return json; + }) + .catch((error) => + console.log(`error fetching site data: ${error}`) + ); + return json; + }, + location, + startDateStr, + endDateStr + ); + + return response; +} diff --git a/site-scrapers/SouthcoastHealth/responseParser.js b/site-scrapers/SouthcoastHealth/responseParser.js new file mode 100644 index 00000000..7f55e309 --- /dev/null +++ b/site-scrapers/SouthcoastHealth/responseParser.js @@ -0,0 +1,43 @@ +/* + Functionality here is unit testable. +*/ +/** + * Sample data format: + {... + "dateToSlots": { + "2021-05-08": {}, + "2021-05-09": {}, + "2021-05-10": { + "e01e1f56-0029-4256-be54-a8f2a33d6011": { + "slots": [...] + } + }, ... + * + * @param {JSON} json + * @returns if no slots -> {}; otherwise { MM-DD-YYYY: { numberAvailableAppointments: x, hasAvailability: y}, ... } + */ +function parseJson(json) { + const dateToSlots = json.dateToSlots; + + if (!dateToSlots) { + return {}; + } + const availabilityContainer = {}; + // Otherwise, get dates with slots: keys are dates + for (const [date, slotsObj] of Object.entries(dateToSlots)) { + const slots = Object.values(slotsObj); + + if (slots.length > 0) { + availabilityContainer[date] = { + numberAvailableAppointments: slots[0].slots.length, + hasAvailability: true, + }; + } + } + + return availabilityContainer; +} + +module.exports = { + parseJson, +}; diff --git a/test/SouthcoastHealth/sampleFallRiver.json b/test/SouthcoastHealth/sampleFallRiver.json new file mode 100644 index 00000000..c3902085 --- /dev/null +++ b/test/SouthcoastHealth/sampleFallRiver.json @@ -0,0 +1,729 @@ +{ + "providersInfoById": { + "a801e0fa-5887-4583-82c8-20e9be284d40": { + "name": "FS2 COVID VACCINE", + "externalId": "605386", + "oasId": "a801e0fa-5887-4583-82c8-20e9be284d40", + "scUrl": "", + "imgUrl": null, + "visitTypeExternalId": "1170571", + "visitTypeOasId": "55e5bf82-ee58-4725-a52f-cf1b42ad6c07", + "patientTypeOasId": "f1d4f5db-1f72-4255-b8f2-9ab3cdee678a", + "starRating": 0, + "ageRestrictionMessage": null + } + }, + "dateToSlots": { + "2021-05-08": {}, + "2021-05-09": {}, + "2021-05-10": { + "e01e1f56-0029-4256-be54-a8f2a33d6011": { + "slots": [ + { + "time": "07:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "07:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "07:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "07:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "07:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "08:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "08:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "08:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "08:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "08:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "08:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "09:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "09:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "09:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "09:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "09:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "09:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "10:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "10:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "10:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "10:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "10:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "10:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "11:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "11:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "11:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "11:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "11:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "11:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "12:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "12:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "12:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "12:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "12:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "12:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "13:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "13:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "13:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "13:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "13:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "13:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "14:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "14:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "14:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "14:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "14:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "14:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + } + ] + } + }, + "2021-05-11": { + "e01e1f56-0029-4256-be54-a8f2a33d6011": { + "slots": [ + { + "time": "16:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + } + ] + } + }, + "2021-05-12": { + "e01e1f56-0029-4256-be54-a8f2a33d6011": { + "slots": [ + { + "time": "16:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "16:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "17:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "18:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:05:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:15:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:25:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:35:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:45:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + }, + { + "time": "19:55:00", + "departmentId": "10090001", + "resourceIds": [ + "a801e0fa-5887-4583-82c8-20e9be284d40" + ] + } + ] + } + }, + "2021-05-13": {}, + "2021-05-14": {}, + "2021-05-15": {}, + "2021-05-16": {}, + "2021-05-17": {}, + "2021-05-18": {}, + "2021-05-19": {}, + "2021-05-20": {}, + "2021-05-21": {} + }, + "addressByDepartmentId": { + "10090001": { + "displayName": "Southcoast Hospitals Group", + "state": "MA", + "city": "Fall River", + "address": "20 Star Street" + } + }, + "departmentsInfoById": { + "e01e1f56-0029-4256-be54-a8f2a33d6011": { + "departmentId": "e01e1f56-0029-4256-be54-a8f2a33d6011", + "label": "20 Star Street", + "externalId": "10090001", + "externalIdType": "EXTERNAL", + "url": "https://goo.gl/maps/UUwnM26jU9ehbcnJ6" + } + } +} \ No newline at end of file diff --git a/test/SouthcoastHealth/sampleNoAvailability.json b/test/SouthcoastHealth/sampleNoAvailability.json new file mode 100644 index 00000000..8c0649bd --- /dev/null +++ b/test/SouthcoastHealth/sampleNoAvailability.json @@ -0,0 +1,6 @@ +{ + "providersInfoById": {}, + "dateToSlots": {}, + "addressByDepartmentId": {}, + "departmentsInfoById": {} +} \ No newline at end of file From 0d476acaf5e3584b3b5b8214fd821900cd6c3f76 Mon Sep 17 00:00:00 2001 From: Paul Furbacher Date: Sat, 8 May 2021 18:39:55 -0400 Subject: [PATCH 4/4] Initial commit of unit tests --- test/SouthcoastHealthTest.js | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 test/SouthcoastHealthTest.js diff --git a/test/SouthcoastHealthTest.js b/test/SouthcoastHealthTest.js new file mode 100644 index 00000000..b05fceb0 --- /dev/null +++ b/test/SouthcoastHealthTest.js @@ -0,0 +1,31 @@ +const { + parseJson, +} = require("../site-scrapers/SouthcoastHealth/responseParser"); +const { expect } = require("chai"); + +describe("Southcoast Health :: test availability JSON", () => { + it("should show availability", () => { + const json = require("./SouthcoastHealth/sampleFallRiver.json"); + + const results = parseJson(json); + + expect(Object.keys(results).length).equals(3); + + // Count the number times the key "time" occurs in the JSON file. + // That should equal the number of appointments (slots). + const timesStrCount = JSON.stringify(json).match(/time/g).length; + const totalSlots = Object.values(results).reduce((acc, value) => { + acc += value.numberAvailableAppointments; + return acc; + }, 0); + + expect(totalSlots).equals(timesStrCount); + }); + + it("should show no availability", () => { + const json = require("./SouthcoastHealth/sampleNoAvailability.json"); + + const results = parseJson(json); + expect(results).deep.equals({}); + }); +});