diff --git a/CHANGELOG.md b/CHANGELOG.md index 852053a8..226bc8f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog Changes to adapters will be documented in this file. + +## 11/01/2023 +### DELETED +- au_sa.js (replaced by southaustralia.js) + ## 10/31/2023 ### Set active: true - Mexico Sinaica (Added to deployments) @@ -10,6 +15,7 @@ Changes to adapters will be documented in this file. ### Set active: false - Mexico Sinaica - Turkiye (Down) +- Rwanda (Needs work, down?) ## 08/23/2023 ### Set active: false diff --git a/src/adapters/acumar.js b/src/adapters/acumar.js index 10c31370..53fd4432 100644 --- a/src/adapters/acumar.js +++ b/src/adapters/acumar.js @@ -1,9 +1,9 @@ 'use strict'; -import got from 'got'; +import client from '../lib/requests.js'; +import log from '../lib/logger.js'; import { load } from 'cheerio'; import { DateTime } from 'luxon'; -import log from '../lib/logger.js'; const stations = [ { @@ -25,11 +25,11 @@ let offset; export const name = 'acumar'; export async function fetchData (source, cb) { - try { - if (source.datetime) { - log.debug(`Fetching data with ${source.datetime}`); - const dateLuxon = source.datetime.toFormat('dd/MM/yy'); - const hourLuxon = source.datetime.toFormat('HH'); + try { + if (source.datetime) { + log.debug(`Fetching data with ${source.datetime}`); + const dateLuxon = source.datetime.toFormat('dd/MM/yy'); + const hourLuxon = source.datetime.toFormat('HH'); const results = await Promise.all( stations.map((station) => @@ -64,7 +64,7 @@ export async function fetchData (source, cb) { } } -async function getPollutionData ( +async function getPollutionData( station, dateLuxon, hourLuxon, @@ -83,16 +83,7 @@ async function getPollutionData ( let results = []; try { - const response = await got(station.url, { - timeout: { - request: 5000, - connect: 1000, - secureConnect: 1000, - socket: 5000, - response: 5000, - send: 1000, - }, - }); + const response = await client(station.url); const $ = load(response.body); if (dateLuxon && hourLuxon) { diff --git a/src/adapters/adairquality.js b/src/adapters/adairquality.js index a68b835a..3c50f526 100644 --- a/src/adapters/adairquality.js +++ b/src/adapters/adairquality.js @@ -4,8 +4,7 @@ */ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; -import got from 'got'; +import client from '../lib/requests.js'; import { DateTime } from 'luxon'; import { parallel } from 'async'; import flatMap from 'lodash/flatMap.js'; @@ -204,11 +203,7 @@ const stations = [ export function fetchData(source, cb) { const requests = stations.map((station) => { return (done) => { - got(`${source.url}${station.slug}`, { - timeout: { - request: REQUEST_TIMEOUT - }, - }) + client(`${source.url}${station.slug}`) .then((response) => { if (response.statusCode !== 200) { return done({ diff --git a/src/adapters/au_act.js b/src/adapters/au_act.js index 5797c640..dcebf26d 100644 --- a/src/adapters/au_act.js +++ b/src/adapters/au_act.js @@ -1,68 +1,73 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; +import client from '../lib/requests.js'; +import log from '../lib/logger.js'; import cloneDeep from 'lodash/cloneDeep.js'; import flatten from 'lodash/flatten.js'; -import { DateTime } from 'luxon' - -import got from 'got'; +import { DateTime } from 'luxon'; export const name = 'au_act'; -export function fetchData (source, cb) { - const timeAgo = DateTime.now().setZone('Australia/Sydney').minus({ days: 1 }).toFormat('yyyy-LL-dd\'T\'HH:mm:ss'); +export function fetchData(source, cb) { + const timeAgo = DateTime.now() + .setZone('Australia/Sydney') + .minus({ days: 1 }) + .toFormat("yyyy-LL-dd'T'HH:mm:ss"); - got(source.url, { + client(source.url, { searchParams: { - '$query': `select *, :id where (\`datetime\` > '${timeAgo}') order by \`datetime\` desc limit 1000` + $query: `select *, :id where (\`datetime\` > '${timeAgo}') order by \`datetime\` desc limit 1000`, }, - timeout: { - request: REQUEST_TIMEOUT - } - }).then(res => { - const body = res.body; - - try { - const data = formatData(JSON.parse(body), source); - if (data === undefined) { - return cb({message: 'Failure to parse data.'}); + }) + .then((res) => { + const body = res.body; + + try { + const data = formatData(JSON.parse(body), source); + if (data === undefined) { + return cb({ message: 'Failure to parse data.' }); + } + cb(null, data); + } catch (e) { + return cb({ message: 'Unknown adapter error.' }); } - cb(null, data); - } catch (e) { - return cb({message: 'Unknown adapter error.'}); - } - }).catch(err => { - console.error('Error:', err.message, err.response && err.response.body); - return cb({message: 'Failure to load data url.'}); - }); - + }) + .catch((err) => { + log.error( + 'Error:', + err.message, + err.response && err.response.body + ); + return cb({ message: 'Failure to load data url.' }); + }); } - const formatData = function (data, source) { const parseDate = function (string) { - const date = DateTime.fromISO(string, { zone: 'Australia/Sydney' }); + const date = DateTime.fromISO(string, { + zone: 'Australia/Sydney', + }); return { utc: date.toUTC().toFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), - local: date.toFormat("yyyy-MM-dd'T'HH:mm:ssZZ") + local: date.toFormat("yyyy-MM-dd'T'HH:mm:ssZZ"), }; }; // mapping of types from source data into OpenAQ format const types = { - 'no2': 'no2', - 'o3_1hr': 'o3', - 'co': 'co', - 'pm10': 'pm10', - 'pm2_5': 'pm25' + no2: 'no2', + o3_1hr: 'o3', + co: 'co', + pm10: 'pm10', + pm2_5: 'pm25', }; const units = { - 'no2': 'ppm', - 'o3': 'ppm', - 'co': 'ppm', - 'pm10': 'µg/m³', - 'pm25': 'µg/m³' + no2: 'ppm', + o3: 'ppm', + co: 'ppm', + pm10: 'µg/m³', + pm25: 'µg/m³', }; const measurements = []; @@ -79,13 +84,15 @@ const formatData = function (data, source) { mobile: false, coordinates: { latitude: parseFloat(row.gps.latitude), - longitude: parseFloat(row.gps.longitude) + longitude: parseFloat(row.gps.longitude), }, - attribution: [{ - name: 'Health Protection Service, ACT Government', - url: 'https://www.data.act.gov.au/Environment/Air-Quality-Monitoring-Data/94a5-zqnn' - }], - averagingPeriod: {'value': 1, 'unit': 'hours'} + attribution: [ + { + name: 'Health Protection Service, ACT Government', + url: 'https://www.data.act.gov.au/Environment/Air-Quality-Monitoring-Data/94a5-zqnn', + }, + ], + averagingPeriod: { value: 1, unit: 'hours' }, }; Object.keys(types).forEach(function (type) { @@ -103,6 +110,6 @@ const formatData = function (data, source) { return { name: 'unused', - measurements: flatten(measurements) + measurements: flatten(measurements), }; }; diff --git a/src/adapters/au_sa.js b/src/adapters/au_sa.js deleted file mode 100644 index 99a0db70..00000000 --- a/src/adapters/au_sa.js +++ /dev/null @@ -1,185 +0,0 @@ -'use strict'; - -import { REQUEST_TIMEOUT } from '../lib/constants.js'; -import { default as baseRequest } from 'request'; -import cloneDeep from 'lodash/cloneDeep.js'; -import { default as moment } from 'moment-timezone'; - -import { - FetchError, - AdapterError, - MeasurementValidationError, - DATA_URL_ERROR, - DATA_PARSE_ERROR -} from '../lib/errors.js'; - -// note: this is the 'synchronous' version (lost hours to this!) -import { parse } from 'csv-parse/sync'; -const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT}); -import log from '../lib/logger.js'; - -export const name = 'au_sa'; - -export function fetchData (source, cb) { - // Fetch the data - request(source.url, function (err, res, body) { - if (err || res.statusCode !== 200) { - //return cb({message: 'Failure to load data url.'}); - throw new FetchError(DATA_URL_ERROR, source, null); - } - // Wrap everything in a try/catch in case something goes wrong - try { - // Format the data - var data = formatData(body, source); - if (data === undefined) { - //return cb({message: 'Failure to parse data.'}); - throw new FetchError(DATA_URL_ERROR, source, null); - } - cb(null, data); - } catch (e) { - return cb({message: 'Unknown adapter error.'}); - //throw new AdapterError(source, e.message, e); - } - }); -}; - -// site locations retrieved from https://data.sa.gov.au/data/dataset/a768c1f5-9714-4576-90bd-9dddaaa66ce4 -var siteLocations = { - 'chr': [ 138.4951966, -35.1349443 ], - 'eli': [ 138.6957631, -34.6984929 ], - 'ken': [ 138.6650977, -34.9214075 ], - 'lef1': [ 138.4963475, -34.8386688 ], - 'net': [ 138.549098, -34.9438035 ], - 'nor': [ 138.6229313, -34.8620143 ], - 'ptp_o': [ 138.0198974, -33.1947947 ], - 'why_s': [ 137.5332255, -33.023596 ], - 'lef2': [ 138.4978642, -34.79128 ], - 'cbd': [ 138.6010841, -34.92889 ], - 'ptp_t': [ 138.0037294, -33.1711633 ], - 'why_w': [ 137.5860979, -33.0361164 ], - 'pta': [ 137.7868467, -32.5100065 ] -}; - -var siteCities = { - 'chr': 'Adelaide', - 'eli': 'Adelaide', - 'ken': 'Adelaide', - 'lef1': 'Adelaide', - 'net': 'Adelaide', - 'nor': 'Adelaide', - 'ptp_o': 'Port Pirie', - 'why_s': 'Whyalla', - 'lef2': 'Adelaide', - 'cbd': 'Adelaide', - 'ptp_t': 'Port Pirie', - 'why_w': 'Whyalla', - 'pta': 'Port Augusta' -}; - -// remove non numeric values -function parseValue (value) { - if (value === null || value === 'NM' || value === 'NA') { - return null; - } - - var number = Number(value); - if (Number.isNaN(number)) { - return null; - } - - return number; -} - -var formatData = function (data, source) { - var units = { - 'no2': 'ppm', - 'o3': 'ppm', - 'co': 'ppm', - 'so2': 'ppm', - 'pm10': 'µg/m³', - 'pm25': 'µg/m³' - }; - - var measurements = []; - - // parse the csv feed, exclude # lines - var rows = parse(data, { - trim: true, - comment: '#', - relax_column_count: true - }); - - // header row contains the date/time - var day = rows[0][1]; - var month = rows[0][2]; - var year = rows[0][3]; - var time = rows[0][4]; - - // according to https://data.sa.gov.au/data/dataset/recent-air-quality/resource/d8abf079-9c51-4a0c-b827-dca926c4e95b - // "Times shown ... are Australian Central Standard Time (ACST). During - // daylight savings an hour will need to be added to the times shown." - // Hence we specifiy the date time is in +09:30 ie. ACST, this then gets - // correctly formatted to local time in Australia/Adelaide - var date = moment.tz(`${day} ${month} ${year} ${time} +09:30`, 'DD MMMM YYYY HH:mm ZZ', 'Australia/Adelaide'); - var dateObject = {utc: date.toDate(), local: date.format()}; - - // loop through the remaining csv rows, which each contain a location - // the second and third rows now have the column names - for (var i = 3; i < rows.length; i++) { - - var row = rows[i]; - var siteName = row[2]; - var siteRef = row[3].replace(/_dm.jpg$/, ''); - - var parameterValues = { - 'o3': parseValue(row[4]), - 'co': parseValue(row[5]), - 'no2': parseValue(row[6]), - 'so2': parseValue(row[7]), - 'pm10': parseValue(row[8]), - 'pm25': parseValue(row[9]) - }; - - const location = siteLocations[siteRef]; - - if (!location) { - throw new MeasurementValidationError(source, `Cannot find location`, row); - } - - // base measurement properties - var baseMeasurement = { - location: siteName, - city: siteCities[siteRef], - country: 'AU', - date: dateObject, - sourceName: source.name, - sourceType: 'government', - mobile: false, - coordinates: { - latitude: location[1], - longitude: location[0] - }, - attribution: [{ - name: 'Environment Protection Authority (EPA), South Australia', - url: source.sourceURL - }], - averagingPeriod: {'value': 1, 'unit': 'hours'} - }; - - Object.keys(parameterValues).forEach(function (parameter) { - if (parameterValues[parameter] !== null) { - var measurement = cloneDeep(baseMeasurement); - measurement.parameter = parameter; - measurement.value = parameterValues[parameter]; - measurement.unit = units[parameter]; - - measurements.push(measurement); - } - }); - } - - return { - name: 'unused', - measurements: measurements - }; -}; diff --git a/src/adapters/buenos-aires.js b/src/adapters/buenos-aires.js index 0ccd0c56..d5a3af51 100644 --- a/src/adapters/buenos-aires.js +++ b/src/adapters/buenos-aires.js @@ -1,24 +1,22 @@ 'use strict'; -import got from 'got'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; +import client from '../lib/requests.js'; import { acceptableParameters, convertUnits } from '../lib/utils.js'; -import cheerio from 'cheerio'; +import { load } from 'cheerio'; import { parallel } from 'async'; import flattenDeep from 'lodash/flattenDeep.js'; import { DateTime } from 'luxon'; -const getter = got.extend({ timeout: { request: REQUEST_TIMEOUT } }); const timezone = 'America/Argentina/Buenos_Aires'; export const name = 'buenos-aires'; export function fetchData(source, callback) { - getter(source.url) + client(source.url) .then((response) => { const body = response.body; let tasks = []; - let $ = cheerio.load(body); + let $ = load(body); const stations = $('#estacion option') .filter(function (i, el) { @@ -103,7 +101,7 @@ const makeStationQuery = (sourceUrl, station, parameter, date) => { const handleStation = (url, station, parameter, today) => { return (done) => { - getter(url) + client(url) .then((response) => { const body = response.body; const results = formatData(body, station, parameter, today); @@ -116,7 +114,7 @@ const handleStation = (url, station, parameter, today) => { }; const formatData = (body, station, parameter, today) => { - const $ = cheerio.load(body); + const $ = load(body); let measurements = []; const averagingPeriod = getAveragingPeriod(parameter); diff --git a/src/adapters/canterbury.js b/src/adapters/canterbury.js index d6bb2203..3b1e9d88 100644 --- a/src/adapters/canterbury.js +++ b/src/adapters/canterbury.js @@ -8,9 +8,8 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; +import client from '../lib/requests.js'; import { DateTime } from 'luxon'; -import got from 'got'; import async from 'async'; export const name = 'canterbury'; @@ -33,7 +32,7 @@ export function fetchData(source, cb) { .replace('$date', date); return function (cb) { - got(url, { timeout: { request: REQUEST_TIMEOUT } }) + client(url) .then((response) => { let body = JSON.parse(response.body); body = body.data.item[body.data.item.length - 1]; // get the last item in the array diff --git a/src/adapters/catalonia.js b/src/adapters/catalonia.js index 2b73865d..0385246c 100644 --- a/src/adapters/catalonia.js +++ b/src/adapters/catalonia.js @@ -1,35 +1,38 @@ /** * This code is responsible for implementing all methods related to fetching * and returning data for the Catalonian data sources. credit to @magsyg - * based off of openaq-fetch PR #711 + * based off of openaq-fetch PR #711 */ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; -import { default as baseRequest } from 'request'; +import client from '../lib/requests.js'; import { DateTime } from 'luxon'; -const request = baseRequest.defaults({timeout: REQUEST_TIMEOUT}); export const name = 'catalonia'; -export function fetchData (source, cb) { - const fetchURL = (source.url) - request(fetchURL, function (err, res, body) { - if (err || res.statusCode !== 200) { - return cb({message: 'Failure to load data url.'}); +export async function fetchData(source, cb) { + const fetchURL = source.url; + + try { + const response = await client(fetchURL); + const body = response.body; + + if (response.statusCode !== 200) { + return cb({ message: 'Failure to load data url.' }); } - try { - const data = formatData(body); - if (data === undefined) { - return cb({message: 'Failure to parse data.'}); - } - cb(null, data); - } catch (e) { - return cb({message: 'Unknown adapter error.'}); + + const data = formatData(body); + if (data === undefined) { + return cb({ message: 'Failure to parse data.' }); } - }); -}; + + cb(null, data); + } catch (error) { + console.error('Error fetching data:', error); + return cb({ message: 'Unknown adapter error.' }); + } +} function formatData(data) { try { @@ -40,38 +43,51 @@ function formatData(data) { const aqRepack = (item) => { let aq = []; - let dateLuxon = DateTime.fromISO(item.data, { zone: 'Europe/Madrid' }); + let dateLuxon = DateTime.fromISO(item.data, { + zone: 'Europe/Madrid', + }); const param = item.contaminant.toLowerCase().replace('.', ''); - if (String(param).localeCompare('h2s') !== 0 && + if ( + String(param).localeCompare('h2s') !== 0 && String(param).localeCompare('c6h6') !== 0 && String(param).localeCompare('cl2') !== 0 && - String(param).localeCompare('hg') !== 0) { - + String(param).localeCompare('hg') !== 0 + ) { const template = { - location: ('nom_estaci' in item) ? item.nom_estaci : item.municipi, + location: + 'nom_estaci' in item ? item.nom_estaci : item.municipi, city: item.municipi, parameter: param, coordinates: { latitude: parseFloat(item.latitud), - longitude: parseFloat(item.longitud) + longitude: parseFloat(item.longitud), }, unit: item.unitats, - attribution: [{ name: 'GENCAT', url: 'https://mediambient.gencat.cat/ca/05_ambits_dactuacio/atmosfera/qualitat_de_laire/vols-saber-que-respires/visor-de-dades/' }], - averagingPeriod: { unit: 'hours', value: 1 } + attribution: [ + { + name: 'GENCAT', + url: 'https://mediambient.gencat.cat/ca/05_ambits_dactuacio/atmosfera/qualitat_de_laire/vols-saber-que-respires/visor-de-dades/', + }, + ], + averagingPeriod: { unit: 'hours', value: 1 }, }; for (let i = 1; i < 25; i++) { dateLuxon = dateLuxon.plus({ hours: 1 }); - let valueKey = (i < 10) ? ('h0' + i.toString()) : ('h' + i.toString()); + let valueKey = + i < 10 ? 'h0' + i.toString() : 'h' + i.toString(); if (valueKey in item) { - let temp = Object.assign({ - value: parseFloat(item[valueKey]), - date: { - utc: dateLuxon.toUTC().toJSDate(), - local: dateLuxon.toFormat("yyyy-MM-dd'T'HH:mm:ssZZ") - } - }, template); + let temp = Object.assign( + { + value: parseFloat(item[valueKey]), + date: { + utc: dateLuxon.toUTC().toJSDate(), + local: dateLuxon.toFormat("yyyy-MM-dd'T'HH:mm:ssZZ"), + }, + }, + template + ); aq.push(temp); } @@ -89,33 +105,41 @@ function formatData(data) { }); return results; } - + const allData = concatAll(Object.values(data.map(aqRepack))); - let measurements = getLatestMeasurements(allData); - measurements = filterDuplicates(measurements, eeaCataloniaDuplicateStations) // remove duplicates of EEA stations! + let measurements = getLatestMeasurements(allData); + measurements = filterDuplicates( + measurements, + eeaCataloniaDuplicateStations + ); // remove duplicates of EEA stations! return { name: 'unused', measurements: measurements }; - } const getLatestMeasurements = function (measurements) { const latestMeasurements = {}; - + measurements.forEach((measurement) => { - const key = measurement.location + ' ' + measurement.parameter; - if (!latestMeasurements[key] || measurement.date.local > latestMeasurements[key].date.local) { + const key = measurement.location + ' ' + measurement.parameter; + if ( + !latestMeasurements[key] || + measurement.date.local > latestMeasurements[key].date.local + ) { latestMeasurements[key] = measurement; } }); return Object.values(latestMeasurements); -} +}; // remove stations deemed to be duplicates of EEA stations function filterDuplicates(measurements, criteria) { - return measurements.filter(measurement => { - return !criteria.some(criterion => { - const matchLocation = measurement.location === criterion.location; + return measurements.filter((measurement) => { + return !criteria.some((criterion) => { + const matchLocation = + measurement.location === criterion.location; const matchCoordinates = - measurement.coordinates.latitude === criterion.coordinates.latitude && - measurement.coordinates.longitude === criterion.coordinates.longitude; + measurement.coordinates.latitude === + criterion.coordinates.latitude && + measurement.coordinates.longitude === + criterion.coordinates.longitude; return matchLocation && matchCoordinates; }); }); @@ -124,66 +148,66 @@ function filterDuplicates(measurements, criteria) { // this deny list is used to exclude stations that are within 0.1 km of an EEA station const eeaCataloniaDuplicateStations = [ { - "location": "Alcover", - "coordinates": { - "latitude": 41.278687, - "longitude": 1.1798977 - } + location: 'Alcover', + coordinates: { + latitude: 41.278687, + longitude: 1.1798977, }, - { - "location": "Viladecans", - "coordinates": { - "latitude": 41.31335, - "longitude": 2.0136087 - } + }, + { + location: 'Viladecans', + coordinates: { + latitude: 41.31335, + longitude: 2.0136087, }, - { - "location": "Tona", - "coordinates": { - "latitude": 41.84666, - "longitude": 2.2175014 - } + }, + { + location: 'Tona', + coordinates: { + latitude: 41.84666, + longitude: 2.2175014, }, - { - "location": "Sort", - "coordinates": { - "latitude": 42.405407, - "longitude": 1.1299014 - } + }, + { + location: 'Sort', + coordinates: { + latitude: 42.405407, + longitude: 1.1299014, }, - { - "location": "Barcelona", - "coordinates": { - "latitude": 41.37878, - "longitude": 2.133099 - } + }, + { + location: 'Barcelona', + coordinates: { + latitude: 41.37878, + longitude: 2.133099, }, - { - "location": "Tarragona", - "coordinates": { - "latitude": 41.15951, - "longitude": 1.2396973 - } + }, + { + location: 'Tarragona', + coordinates: { + latitude: 41.15951, + longitude: 1.2396973, }, - { - "location": "Barcelona", - "coordinates": { - "latitude": 41.386406, - "longitude": 2.1873982 - } + }, + { + location: 'Barcelona', + coordinates: { + latitude: 41.386406, + longitude: 2.1873982, }, - { - "location": "Barcelona", - "coordinates": { - "latitude": 41.42611, - "longitude": 2.1480017 - } + }, + { + location: 'Barcelona', + coordinates: { + latitude: 41.42611, + longitude: 2.1480017, }, - { - "location": "Sabadell", - "coordinates": { - "latitude": 41.561214, - "longitude": 2.1011107 - } - } - ] \ No newline at end of file + }, + { + location: 'Sabadell', + coordinates: { + latitude: 41.561214, + longitude: 2.1011107, + }, + }, +]; diff --git a/src/adapters/chile.js b/src/adapters/chile.js index 030a65ab..668ab3a2 100644 --- a/src/adapters/chile.js +++ b/src/adapters/chile.js @@ -7,17 +7,12 @@ */ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import _ from 'lodash'; import async from 'async'; -import cheerio from 'cheerio'; +import { load } from 'cheerio'; import { DateTime } from 'luxon'; -import got from 'got'; - -const gotExtended = got.extend({ - retry: { limit: 3 }, - timeout: { request: REQUEST_TIMEOUT }, -}); +import client from '../lib/requests.js'; +import log from '../lib/logger.js'; // Adding in certs to get around unverified connection issue // 2022-04-29 - this should no longer be needed @@ -39,7 +34,7 @@ export function fetchData (source, cb) { _.forEach(sources, function (e) { var task = function (cb) { - gotExtended(e) + client(e) .then((response) => { if (response.statusCode !== 200) { return cb(response); @@ -141,7 +136,7 @@ const formatData = function (results) { local: date.toISO({ suppressMilliseconds: true }), }; } catch (error) { - console.error('Error parsing date:', error); + log.error('Error parsing date:', error); throw new Error('Error parsing date'); } }; @@ -151,7 +146,7 @@ const formatData = function (results) { * @return {string} It's pretty! */ const parseUnit = function (u) { - var $ = cheerio.load(u, { decodeEntities: false }); + var $ = load(u, { decodeEntities: false }); var str = $.text(); return str.indexOf('µg⁄m3') > -1 ? 'µg/m³' : null; }; diff --git a/src/adapters/cyprus.js b/src/adapters/cyprus.js index c778cf07..a9b67c04 100644 --- a/src/adapters/cyprus.js +++ b/src/adapters/cyprus.js @@ -5,7 +5,7 @@ import { removeUnwantedParameters } from '../lib/utils.js'; import { DateTime } from 'luxon'; -import fetch from 'node-fetch'; +import client from '../lib/requests.js'; export const name = 'cyprus'; @@ -15,99 +15,98 @@ export const name = 'cyprus'; * @param {function} cb A callback of the form cb(err, data) */ -export async function fetchData (source, cb) { - try { - const response = await fetch(source.url); - let data = await response.json(); - let out = formatData(data); - return cb(null, out); - } catch (error) { - return cb(error); - } +export function fetchData(source, cb) { + client(source.url) + .then((response) => { + const data = JSON.parse(response.body); + const formattedData = formatData(data); + + if (formattedData === undefined) { + throw new Error('Failure to parse data.'); + } + cb(null, formattedData); + }) + .catch((error) => { + cb(error); + }); } const formatData = (input) => { - try { - let time; - let measurements = []; - let data = input.data; - - Object.values(data).map(o => { - let station = { - location: o.name_en, - city: o.name_el, - latitude: parseFloat(o.latitude), - longitude: parseFloat(o.longitude) - } + let time; + let measurements = []; + let data = input.data; - // the first key in o.pollutants is the datetime for all the measurements of the station - for (const [key, value] of Object.entries(o.pollutants)) { - if (key === 'date_time') { + Object.values(data).map((o) => { + let station = { + location: o.name_en, + city: o.name_el, + latitude: parseFloat(o.latitude), + longitude: parseFloat(o.longitude), + }; - time = DateTime.fromFormat(value, 'yyyy-MM-dd HH:mm:ss', { zone: 'Europe/Nicosia' }); // UTC+2 - - } else { - - let parameter = correctParam(value.notation) - - const measurement = { - location: station.location, - city: station.city, - parameter: parameter, - value: parseFloat(value.value), - unit: "µg/m³", // the unit is always µg/m³ on the website, unavailable in the api - date: { - utc: time.toUTC().toISO({suppressMilliseconds: true}), - local: time.toISO({suppressMilliseconds: true}) - }, - coordinates: { - latitude: station.latitude, - longitude: station.longitude - }, - attribution: [ - { - name: "Republic of Cyprus Department of Labor Inspection", - url: "https://www.data.gov.cy/dataset/%CF%84%CF%81%CE%AD%CF%87%CE%BF%CF%85%CF%83%CE%B5%CF%82-%CE%BC%CE%B5%CF%84%CF%81%CE%AE%CF%83%CE%B5%CE%B9%CF%82-%CE%B1%CF%84%CE%BC%CE%BF%CF%83%CF%86%CE%B1%CE%B9%CF%81%CE%B9%CE%BA%CF%8E%CE%BD-%CF%81%CF%8D%CF%80%CF%89%CE%BD-api" - } - ], - averagingPeriod: { - unit: "hours", - value: 1 - } - } + // the first key in o.pollutants is the datetime for all the measurements of the station + for (const [key, value] of Object.entries(o.pollutants)) { + if (key === 'date_time') { + time = DateTime.fromFormat(value, 'yyyy-MM-dd HH:mm:ss', { + zone: 'Europe/Nicosia', + }); // UTC+2 + } else { + let parameter = correctParam(value.notation); - measurements.push(measurement); - } - } - }) + const measurement = { + location: station.location, + city: station.city, + parameter: parameter, + value: parseFloat(value.value), + unit: 'µg/m³', // the unit is always µg/m³ on the website, unavailable in the api + date: { + utc: time.toUTC().toISO({ suppressMilliseconds: true }), + local: time.toISO({ suppressMilliseconds: true }), + }, + coordinates: { + latitude: station.latitude, + longitude: station.longitude, + }, + attribution: [ + { + name: 'Republic of Cyprus Department of Labor Inspection', + url: 'https://www.data.gov.cy/dataset/%CF%84%CF%81%CE%AD%CF%87%CE%BF%CF%85%CF%83%CE%B5%CF%82-%CE%BC%CE%B5%CF%84%CF%81%CE%AE%CF%83%CE%B5%CE%B9%CF%82-%CE%B1%CF%84%CE%BC%CE%BF%CF%83%CF%86%CE%B1%CE%B9%CF%81%CE%B9%CE%BA%CF%8E%CE%BD-%CF%81%CF%8D%CF%80%CF%89%CE%BD-api', + }, + ], + averagingPeriod: { + unit: 'hours', + value: 1, + }, + }; - measurements = removeUnwantedParameters(measurements); - return {name: 'unused', measurements: measurements} - - } catch (error) { - throw error; + measurements.push(measurement); + } } -} + }); + + measurements = removeUnwantedParameters(measurements); + return { name: 'unused', measurements: measurements }; +}; function correctParam(name) { - switch (name) { - case 'SO₂': - return 'so2'; - case 'PM₁₀': - return 'pm10'; - case 'O₃': - return 'o3'; - case 'NO₂': - return 'no2'; - case 'NOx': - return 'nox'; - case 'CO': - return 'co'; - case 'PM₂.₅': - return 'pm25'; - case 'NO': - return 'no'; - default: - return name; - } - } \ No newline at end of file + switch (name) { + case 'SO₂': + return 'so2'; + case 'PM₁₀': + return 'pm10'; + case 'O₃': + return 'o3'; + case 'NO₂': + return 'no2'; + case 'NOx': + return 'nox'; + case 'CO': + return 'co'; + case 'PM₂.₅': + return 'pm25'; + case 'NO': + return 'no'; + default: + return name; + } +} diff --git a/src/adapters/defra.js b/src/adapters/defra.js index e6a3b231..8809158b 100644 --- a/src/adapters/defra.js +++ b/src/adapters/defra.js @@ -1,35 +1,28 @@ 'use strict'; -//import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { DateTime } from 'luxon'; -import cheerio from 'cheerio'; -//import got from 'got'; +import { load } from 'cheerio'; import log from '../lib/logger.js'; import client from '../lib/requests.js'; export const name = 'defra'; export function fetchData(source, cb) { - client(source, cb) - .then((response) => { - // Wrap everything in a try/catch in case something goes wrong - try { - // Format the data - const data = formatData(source, response.body); - // Make sure the data is valid - if (data === undefined) { - return cb({ message: 'Failure to parse data.' }); - } - cb(null, data); - } catch (e) { - return cb({ message: 'Unknown adapter error.' }); + client(source, cb).then((response) => { + try { + const data = formatData(source, response.body); + if (data === undefined) { + return cb({ message: 'Failure to parse data.' }); } - }) + cb(null, data); + } catch (e) { + return cb({ message: 'Unknown adapter error.' }); + } + }); } let formatData = function (source, data) { let measurements = []; - // Load the html into Cheerio - let $ = cheerio.load(data); + let $ = load(data); $('.current_levels_table').each((i, e) => { $('tr', $(e)).each((i, e) => { handleLocation(e); @@ -42,29 +35,25 @@ let formatData = function (source, data) { function sanitizeDate (dateString) { try { - // Create DateTime object with timezone using the fromFormat() method const date = DateTime.fromFormat( dateString, - 'dd/MM/yyyyHH:mm:ss', + 'dd/MM/yyyyHH:mm', { zone: 'Europe/London', } ); - // Return UTC and local ISO strings return { - utc: date.toUTC().toISO({ - suppressMilliseconds: true, - }), - local: date.toISO({ suppressMilliseconds: true }), + utc: date.toUTC().toFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"), + local: date.toFormat("yyyy-MM-dd'T'HH:mm:ssZZ"), }; } catch (error) { - console.error('Error parsing date:', error); + log.error('Error parsing date:', error); throw new Error('Error parsing date'); } - }; + } - function getValue (measuredValue) { + function getValue(measuredValue) { if (measuredValue === 'n/a' || measuredValue === 'n/m') { return NaN; } @@ -73,7 +62,7 @@ let formatData = function (source, data) { return parseFloat(measuredValue.substring(0, idx)); } - function handleMeasurement (parameter, el, period, base) { + function handleMeasurement(parameter, el, period, base) { let m = Object.assign({}, base); m.value = getValue($(el).text()); m.parameter = parameter; @@ -86,15 +75,18 @@ let formatData = function (source, data) { return m; } - function handleLocation (row) { - // Create base + function handleLocation(row) { let base = { - location: sanitizeName($($('a', $('td', $(row)).get(0)).get(0)).text()), + location: sanitizeName( + $($('a', $('td', $(row)).get(0)).get(0)).text() + ), date: sanitizeDate($($('td', $(row)).get(6)).text()), - attribution: [{ - name: 'Department for Environmental Food & Rural Affairs', - url: source.url - }] + attribution: [ + { + name: 'Department for Environmental Food & Rural Affairs', + url: source.url, + }, + ], }; // Do nothing if we have a nav item @@ -109,7 +101,7 @@ let formatData = function (source, data) { let o3 = handleMeasurement( 'o3', $($('td', $(row)).get(1)), - {'value': 8, 'unit': 'hours'}, + { value: 8, unit: 'hours' }, base ); if (o3) { @@ -120,7 +112,7 @@ let formatData = function (source, data) { let no2 = handleMeasurement( 'no2', $($('td', $(row)).get(2)), - {'value': 1, 'unit': 'hours'}, + { value: 1, unit: 'hours' }, base ); if (no2) { @@ -131,7 +123,7 @@ let formatData = function (source, data) { let so2 = handleMeasurement( 'so2', $($('td', $(row)).get(3)), - {'value': 0.25, 'unit': 'hours'}, + { value: 0.25, unit: 'hours' }, base ); if (so2) { @@ -142,7 +134,7 @@ let formatData = function (source, data) { let pm25 = handleMeasurement( 'pm25', $($('td', $(row)).get(4)), - {'value': 24, 'unit': 'hours'}, + { value: 24, unit: 'hours' }, base ); if (pm25) { @@ -152,7 +144,7 @@ let formatData = function (source, data) { let pm10 = handleMeasurement( 'pm10', $($('td', $(row)).get(5)), - {'value': 24, 'unit': 'hours'}, + { value: 24, unit: 'hours' }, base ); if (pm10) { @@ -161,496 +153,658 @@ let formatData = function (source, data) { } return { name: 'unused', - measurements: (measurements || []).filter(i => i.city) + measurements: (measurements || []).filter((i) => i.city), }; }; // Values from location pages at https://uk-air.defra.gov.uk/latest/currentlevels let metadata = { - 'Sheffield Barnsley Road': - { city: 'Yorkshire & Humberside', - coordinates: { latitude: 53.404950, longitude: -1.455815 } }, - 'Cannock A5190 Roadside': - { city: 'West Midlands', - coordinates: { latitude: 52.687298, longitude: -1.980821 } }, - 'Christchurch Barrack Road': - { city: 'South West', - coordinates: { latitude: 50.735454, longitude: -1.780888 } }, - 'St Helens Linkway': - { city: 'North West & Merseyside', - coordinates: { latitude: 53.451826, longitude: -2.742134 } }, - 'Greenock A8 Roadside': - { city: 'Central Scotland', - coordinates: { latitude: 55.944079, longitude: -4.734421 } }, - 'Birkenhead Borough Road': - { city: 'North West & Merseyside', - coordinates: { latitude: 53.388511, longitude: -3.025014 } }, - 'Worthing A27 Roadside': - { city: 'South East', - coordinates: { latitude: 50.832947, longitude: -0.379916 } }, - 'Birmingham A4540 Roadside': - { city: 'West Midlands', - coordinates: { latitude: 52.476090, longitude: -1.875024 } }, - 'Bush Estate': - { city: 'Bush Estate', - coordinates: { latitude: 55.862281, longitude: -3.205782 } }, - 'Dumbarton Roadside': - { city: 'Dumbarton', - coordinates: { latitude: 55.943197, longitude: -4.55973 } }, - 'Glasgow Great Western Road': - { city: 'Glasgow', - coordinates: { latitude: 55.872038, longitude: -4.270936 } }, - 'Edinburgh St Leonards': - { city: 'Edinburgh', - coordinates: { latitude: 55.945589, longitude: -3.182186 } }, - 'Auchencorth Moss': - { city: 'Auchencorth', - coordinates: { latitude: 55.79216, longitude: -3.2429 } }, - 'Glasgow High Street': - { city: 'Glasgow', - coordinates: { latitude: 55.860936, longitude: -4.238214 } }, - 'Glasgow Kerbside': - { city: 'Glasgow', - coordinates: { latitude: 55.85917, longitude: -4.258889 } }, - 'Glasgow Townhead': - { city: 'Glasgow', - coordinates: { latitude: 55.865782, longitude: -4.243631 } }, - Grangemouth: - { city: 'Grangemouth', - coordinates: { latitude: 56.010319, longitude: -3.704399 } }, - 'Grangemouth Moray': - { city: 'Grangemouth', - coordinates: { latitude: 56.013142, longitude: -3.710833 } }, - Bottesford: - { city: 'Bottesford', - coordinates: { latitude: 52.93028, longitude: -0.814722 } }, - 'Chesterfield Roadside': - { city: 'Chesterfield', - coordinates: { latitude: 53.231722, longitude: -1.456944 } }, - 'Chesterfield Loundsley Green': - { city: 'Chesterfield', - coordinates: { latitude: 53.244131, longitude: -1.454946 } }, - Ladybower: - { city: 'Ladybower', - coordinates: { latitude: 53.40337, longitude: -1.752006 } }, - 'Leicester University': - { city: 'Leicester', - coordinates: { latitude: 52.619823, longitude: -1.127311 } }, - 'Northampton Kingsthorpe': - { city: 'Northampton', - coordinates: { latitude: 52.271886, longitude: -0.879898 } }, - 'Market Harborough': - { city: 'Market Harborough', - coordinates: { latitude: 52.554444, longitude: -0.772222 } }, - 'Lincoln Canwick Rd.': - { city: 'Lincoln', - coordinates: { latitude: 53.221373, longitude: -0.534189 } }, - 'Leicester A594 Roadside': - { city: 'Leicester', - coordinates: { latitude: 52.638677, longitude: -1.124228 } }, - 'Nottingham Western Boulevard': - { city: 'Nottingham', - coordinates: { latitude: 52.969377, longitude: -1.188851 } }, - 'Nottingham Centre': - { city: 'Nottingham', - coordinates: { latitude: 52.95473, longitude: -1.146447 } }, - 'Luton A505 Roadside': - { city: 'Luton', - coordinates: { latitude: 51.892293, longitude: -0.46211 } }, - 'Cambridge Roadside': - { city: 'Cambridge', - coordinates: { latitude: 52.20237, longitude: 0.124456 } }, - 'Norwich Lakenfields': - { city: 'Norwich', - coordinates: { latitude: 52.614193, longitude: 1.301976 } }, - 'Sandy Roadside': - { city: 'Sandy', - coordinates: { latitude: 52.132417, longitude: -0.300306 } }, - Sibton: - { city: 'Sibton', - coordinates: { latitude: 52.2944, longitude: 1.463497 } }, - 'Southend-on-Sea': - { city: 'London', - coordinates: { latitude: 51.544206, longitude: 0.678408 } }, - 'St Osyth': - { city: 'St Osyth', - coordinates: { latitude: 51.77798, longitude: 1.049031 } }, - 'Stanford-le-Hope Roadside': - { city: 'Stanford-le-Hope', - coordinates: { latitude: 51.518167, longitude: 0.439548 } }, - Thurrock: - { city: 'London', - coordinates: { latitude: 51.47707, longitude: 0.317969 } }, - Weybourne: - { city: 'Weybourne', - coordinates: { latitude: 52.95049, longitude: 1.122017 } }, - 'Wicken Fen': - { city: 'Wicken Fen', - coordinates: { latitude: 52.2985, longitude: 0.290917 } }, - 'Ealing Horn Lane': - { city: 'London', - coordinates: { latitude: 51.51895, longitude: -0.265617 } }, - 'Camden Kerbside': - { city: 'London', - coordinates: { latitude: 51.54421, longitude: -0.175269 } }, - 'Haringey Roadside': - { city: 'London', - coordinates: { latitude: 51.5993, longitude: -0.068218 } }, - 'London Bexley': - { city: 'London', - coordinates: { latitude: 51.46603, longitude: 0.184806 } }, - 'London Bloomsbury': - { city: 'London', - coordinates: { latitude: 51.52229, longitude: -0.125889 } }, - 'London Eltham': - { city: 'London', - coordinates: { latitude: 51.45258, longitude: 0.070766 } }, - 'London Haringey Priory Park South': - { city: 'London', - coordinates: { latitude: 51.584128, longitude: -0.125254 } }, - 'London Harlington': - { city: 'London', - coordinates: { latitude: 51.48879, longitude: -0.441614 } }, - 'London Harrow Stanmore': - { city: 'London', - coordinates: { latitude: 51.617333, longitude: -0.298777 } }, - 'London Hillingdon': - { city: 'London', - coordinates: { latitude: 51.49633, longitude: -0.460861 } }, - 'London N. Kensington': - { city: 'London', - coordinates: { latitude: 51.52105, longitude: -0.213492 } }, - 'London Teddington': - { city: 'London', - coordinates: { latitude: 51.42099, longitude: -0.339647 } }, - 'London Marylebone Road': - { city: 'London', - coordinates: { latitude: 51.52253, longitude: -0.154611 } }, - 'London Teddington Bushy Park': - { city: 'London', - coordinates: { latitude: 51.425286, longitude: -0.345606 } }, - 'London Westminster': - { city: 'London', - coordinates: { latitude: 51.49467, longitude: -0.131931 } }, - 'Southwark A2 Old Kent Road': - { city: 'London', - coordinates: { latitude: 51.480499, longitude: -0.05955 } }, - 'Tower Hamlets Roadside': - { city: 'London', - coordinates: { latitude: 51.52253, longitude: -0.042155 } }, - 'Fort William': - { city: 'Fort William', - coordinates: { latitude: 56.82266, longitude: -5.101102 } }, - Inverness: - { city: 'Inverness', - coordinates: { latitude: 57.481308, longitude: -4.241451 } }, - Lerwick: - { city: 'Lerwick', - coordinates: { latitude: 60.13922, longitude: -1.185319 } }, - Strathvaich: - { city: 'Strath Vaich', - coordinates: { latitude: 57.734456, longitude: -4.776583 } }, - Billingham: - { city: 'Billingham', - coordinates: { latitude: 54.60537, longitude: -1.275039 } }, - Middlesbrough: - { city: 'Middlesbrough', - coordinates: { latitude: 54.569297, longitude: -1.220874 } }, - 'Newcastle Centre': - { city: 'Newcastle', - coordinates: { latitude: 54.97825, longitude: -1.610528 } }, - 'Newcastle Cradlewell Roadside': - { city: 'Newcastle', - coordinates: { latitude: 54.986405, longitude: -1.595362 } }, - 'Stockton-on-Tees A1305 Roadside': - { city: 'Stockton-on-Tees', - coordinates: { latitude: 54.565819, longitude: -1.3159 } }, - 'Stockton-on-Tees Eaglescliffe': - { city: 'Stockton-on-Tees', - coordinates: { latitude: 54.516667, longitude: -1.358547 } }, - 'Sunderland Silksworth': - { city: 'Sunderland', - coordinates: { latitude: 54.88361, longitude: -1.406878 } }, - 'Sunderland Wessington Way': - { city: 'Sunderland', - coordinates: { latitude: 54.91839, longitude: -1.408391 } }, - Aberdeen: - { city: 'Aberdeen', - coordinates: { latitude: 57.157360, longitude: -2.094278 } }, - 'Aberdeen Union Street Roadside': - { city: 'Aberdeen', - coordinates: { latitude: 57.144555, longitude: -2.106472 } }, - 'Aberdeen Wellington Road': - { city: 'Aberdeen', - coordinates: { latitude: 57.133888, longitude: -2.094198 } }, - 'Aston Hill': - { city: 'Aston Hill', - coordinates: { latitude: 52.50385, longitude: -3.034178 } }, - Wrexham: - { city: 'Wrexham', - coordinates: { latitude: 53.04222, longitude: -3.002778 } }, - 'Blackburn Accrington Road': - { city: 'Blackburn', - coordinates: { latitude: 53.747751, longitude: -2.452724 } }, - 'Blackpool Marton': - { city: 'Blackpool', - coordinates: { latitude: 53.80489, longitude: -3.007175 } }, - 'Bury Whitefield Roadside': - { city: 'Bury', - coordinates: { latitude: 53.559029, longitude: -2.293772 } }, - 'Carlisle Roadside': - { city: 'Carlisle', - coordinates: { latitude: 54.894834, longitude: -2.945307 } }, - Glazebury: - { city: 'Glazebury', - coordinates: { latitude: 53.46008, longitude: -2.472056 } }, - 'Liverpool Queen\'s Drive Roadside': - { city: 'Liverpool', - coordinates: { latitude: 53.446944, longitude: -2.9625 } }, - 'Great Dun Fell': - { city: 'Great Dun Fell', - coordinates: { latitude: 54.684233, longitude: -2.450799 } }, - 'Liverpool Speke': - { city: 'Liverpool', - coordinates: { latitude: 53.34633, longitude: -2.844333 } }, - 'Manchester Piccadilly': - { city: 'Manchester', - coordinates: { latitude: 53.48152, longitude: -2.237881 } }, - 'Manchester Sharston': - { city: 'Manchester', - coordinates: { latitude: 53.371306, longitude: -2.239218 } }, - Preston: - { city: 'Preston', - coordinates: { latitude: 53.76559, longitude: -2.680353 } }, - 'Salford Eccles': - { city: 'Manchester', - coordinates: { latitude: 53.48481, longitude: -2.334139 } }, - 'Shaw Crompton Way': - { city: 'Crompton Way OL2 8AQ', - coordinates: { latitude: 53.579283, longitude: -2.093786 } }, - Warrington: - { city: 'Warrington', - coordinates: { latitude: 53.38928, longitude: -2.615358 } }, - 'Widnes Milton Road': - { city: 'Widnes', - coordinates: { latitude: 53.365391, longitude: -2.73168 } }, - 'Wigan Centre': - { city: 'Wigan', - coordinates: { latitude: 53.54914, longitude: -2.638139 } }, - 'Armagh Roadside': - { city: 'Armagh', - coordinates: { latitude: 54.353728, longitude: -6.654558 } }, - 'Wirral Tranmere': - { city: 'Liverpool', - coordinates: { latitude: 53.37287, longitude: -3.022722 } }, - 'Ballymena Ballykeel': - { city: 'Ballymena', - coordinates: { latitude: 54.861595, longitude: -6.250873 } }, - 'Belfast Centre': - { city: 'Belfast', - coordinates: { latitude: 54.59965, longitude: -5.928833 } }, - 'Belfast Stockman\'s Lane': - { city: 'Belfast', - coordinates: { latitude: 54.572586, longitude: -5.974944 } }, - 'Derby St Alkmund\'s Way': - { city: 'Derby', - coordinates: { latitude: 52.922983, longitude: -1.469507 } }, - Derry: - { city: 'Derry', - coordinates: { latitude: 55.001225, longitude: -7.329115 } }, - 'Derry Rosemount': - { city: 'Derry', - coordinates: { latitude: 55.002818, longitude: -7.331179 } }, - Dumfries: - { city: 'Dumfries', - coordinates: { latitude: 55.070033, longitude: -3.614233 } }, - 'Lough Navar': - { city: 'Lough Navar', - coordinates: { latitude: 54.43951, longitude: -7.900328 } }, - Eskdalemuir: - { city: 'Eskdalemuir', - coordinates: { latitude: 55.31531, longitude: -3.206111 } }, - Peebles: - { city: 'Peebles', - coordinates: { latitude: 55.657472, longitude: -3.196527 } }, - 'Brighton Preston Park': - { city: 'Brighton', - coordinates: { latitude: 50.840836, longitude: -0.147572 } }, - Canterbury: - { city: 'Canterbury', - coordinates: { latitude: 51.27399, longitude: 1.098061 } }, - 'Chatham Roadside': - { city: 'Chatham', - coordinates: { latitude: 51.374264, longitude: 0.54797 } }, - Eastbourne: - { city: 'Eastbourne', - coordinates: { latitude: 50.805778, longitude: 0.271611 } }, - Horley: - { city: 'Horley', - coordinates: { latitude: 51.165865, longitude: -0.167734 } }, - 'Chilbolton Observatory': - { city: 'Stockbridge', - coordinates: { latitude: 51.149617, longitude: -1.438228 } }, - 'Lullington Heath': - { city: 'Lullington Heath', - coordinates: { latitude: 50.7937, longitude: 0.18125 } }, - 'Oxford Centre Roadside': - { city: 'Oxford', - coordinates: { latitude: 51.751745, longitude: -1.257463 } }, - 'Oxford St Ebbes': - { city: 'Oxford', - coordinates: { latitude: 51.744806, longitude: -1.260278 } }, - Portsmouth: - { city: 'Portsmouth', - coordinates: { latitude: 50.82881, longitude: -1.068583 } }, - 'Reading New Town': - { city: 'Reading', - coordinates: { latitude: 51.45309, longitude: -0.944067 } }, - 'Reading London Rd.': - { city: 'Reading', - coordinates: { latitude: 51.454896, longitude: -0.940382 } }, - 'Rochester Stoke': - { city: 'Rochester', - coordinates: { latitude: 51.45617, longitude: 0.634889 } }, - 'Southampton A33': - { city: 'Southampton', - coordinates: { latitude: 50.920265, longitude: -1.463484 } }, - 'Southampton Centre': - { city: 'Southampton', - coordinates: { latitude: 50.90814, longitude: -1.395778 } }, - 'Storrington Roadside': - { city: 'Storrington', - coordinates: { latitude: 50.916932, longitude: -0.449548 } }, - 'Cardiff Centre': - { city: 'Cardiff', - coordinates: { latitude: 51.48178, longitude: -3.17625 } }, - 'Chepstow A48': - { city: 'Chepstow', - coordinates: { latitude: 51.638094, longitude: -2.678731 } }, - Cwmbran: - { city: 'Cardiff', - coordinates: { latitude: 51.6538, longitude: -3.006953 } }, - 'Hafod-yr-ynys Roadside': - { city: 'Swfrryd', - coordinates: { latitude: 51.680579, longitude: -3.133508 } }, - Narberth: - { city: 'Narberth', - coordinates: { latitude: 51.781784, longitude: -4.691462 } }, - Newport: - { city: 'Newport', - coordinates: { latitude: 51.601203, longitude: -2.977281 } }, - 'Port Talbot Margam': - { city: 'Port Talbot', - coordinates: { latitude: 51.58395, longitude: -3.770822 } }, - 'Swansea Roadside': - { city: 'Swansea', - coordinates: { latitude: 51.632696, longitude: -3.947374 } }, - 'Barnstaple A39': - { city: 'Barnstaple', - coordinates: { latitude: 51.074793, longitude: -4.041924 } }, - 'Bath Roadside': - { city: 'Bath', - coordinates: { latitude: 51.391127, longitude: -2.354155 } }, - Bournemouth: - { city: 'Bournemouth', - coordinates: { latitude: 50.73957, longitude: -1.826744 } }, - 'Charlton Mackrell': - { city: 'Charlton', - coordinates: { latitude: 51.05625, longitude: -2.68345 } }, - 'Bristol St Paul\'s': - { city: 'Bristol', - coordinates: { latitude: 51.462839, longitude: -2.584482 } }, - 'Exeter Roadside': - { city: 'Exeter', - coordinates: { latitude: 50.725083, longitude: -3.532465 } }, - Honiton: - { city: 'Honiton', - coordinates: { latitude: 50.792287, longitude: -3.196702 } }, - 'Saltash Callington Road': - { city: 'Saltash', - coordinates: { latitude: 50.411463, longitude: -4.227678 } }, - 'Plymouth Centre': - { city: 'Plymouth', - coordinates: { latitude: 50.37167, longitude: -4.142361 } }, - 'Plymouth Tavistock Road.': - { city: 'Plymouth', - coordinates: { latitude: 50.411058, longitude: -4.130288 } }, - 'Mace Head': - { city: 'Mace Head', - coordinates: { latitude: 53.326444, longitude: -9.903917 } }, - 'Yarner Wood': - { city: 'Yarner Wood', - coordinates: { latitude: 50.5976, longitude: -3.71651 } }, - 'Birmingham Acocks Green': - { city: 'Birmingham', - coordinates: { latitude: 52.437165, longitude: -1.829999 } }, - 'Birmingham Tyburn': - { city: 'Birmingham', - coordinates: { latitude: 52.511722, longitude: -1.830583 } }, - 'Leamington Spa': - { city: 'Leamington Spa', - coordinates: { latitude: 52.28881, longitude: -1.533119 } }, - 'Coventry Allesley': - { city: 'Coventry', - coordinates: { latitude: 52.411563, longitude: -1.560228 } }, - 'Birmingham Tyburn Roadside': - { city: 'Birmingham', - coordinates: { latitude: 52.512194, longitude: -1.830861 } }, - 'Leamington Spa Rugby Road': - { city: 'Leamington Spa', - coordinates: { latitude: 52.294884, longitude: -1.542911 } }, - Leominster: - { city: 'Leominster', - coordinates: { latitude: 52.22174, longitude: -2.736665 } }, - 'Oldbury Birmingham Road': - { city: 'Oldbury', - coordinates: { latitude: 52.502436, longitude: -2.003497 } }, - 'Stoke-on-Trent A50 Roadside': - { city: 'Stoke-on-Trent', - coordinates: { latitude: 52.980436, longitude: -2.111898 } }, - 'Stoke-on-Trent Centre': - { city: 'Stoke-on-Trent', - coordinates: { latitude: 53.02821, longitude: -2.175133 } }, - 'Walsall Woodlands': - { city: 'Willenhall', - coordinates: { latitude: 52.605621, longitude: -2.030523 } }, - 'Barnsley Gawber': - { city: 'Barnsley', - coordinates: { latitude: 53.56292, longitude: -1.510436 } }, - 'Bradford Mayo Avenue': - { city: 'Bradford', - coordinates: { latitude: 53.771245, longitude: -1.759774 } }, - 'Doncaster A630 Cleveland Street': - { city: 'Doncaster', - coordinates: { latitude: 53.518868, longitude: -1.138073 } }, - 'High Muffles': - { city: 'High Muffles', - coordinates: { latitude: 54.334944, longitude: -0.80855 } }, - 'Hull Freetown': - { city: 'Hull', - coordinates: { latitude: 53.74878, longitude: -0.341222 } }, - 'Hull Holderness Road': - { city: 'Yorkshire', - coordinates: { latitude: 53.758971, longitude: -0.305749 } }, - 'Leeds Centre': - { city: 'Leeds', - coordinates: { latitude: 53.80378, longitude: -1.546472 } }, - 'Leeds Headingley Kerbside': - { city: 'Leeds', - coordinates: { latitude: 53.819972, longitude: -1.576361 } }, - 'Scunthorpe Town': - { city: 'Scunthorpe', - coordinates: { latitude: 53.58634, longitude: -0.636811 } }, - 'Sheffield Devonshire Green': - { city: 'Sheffield', - coordinates: { latitude: 53.378622, longitude: -1.478096 } }, - 'Sheffield Tinsley': - { city: 'Sheffield', - coordinates: { latitude: 53.41058, longitude: -1.396139 } }, - 'York Bootham': - { city: 'York', - coordinates: { latitude: 53.967513, longitude: -1.086514 } }, - 'York Fishergate': - { city: 'York', - coordinates: { latitude: 53.951889, longitude: -1.075861 } } + 'Sheffield Barnsley Road': { + city: 'Yorkshire & Humberside', + coordinates: { latitude: 53.40495, longitude: -1.455815 }, + }, + 'Cannock A5190 Roadside': { + city: 'West Midlands', + coordinates: { latitude: 52.687298, longitude: -1.980821 }, + }, + 'Christchurch Barrack Road': { + city: 'South West', + coordinates: { latitude: 50.735454, longitude: -1.780888 }, + }, + 'St Helens Linkway': { + city: 'North West & Merseyside', + coordinates: { latitude: 53.451826, longitude: -2.742134 }, + }, + 'Greenock A8 Roadside': { + city: 'Central Scotland', + coordinates: { latitude: 55.944079, longitude: -4.734421 }, + }, + 'Birkenhead Borough Road': { + city: 'North West & Merseyside', + coordinates: { latitude: 53.388511, longitude: -3.025014 }, + }, + 'Worthing A27 Roadside': { + city: 'South East', + coordinates: { latitude: 50.832947, longitude: -0.379916 }, + }, + 'Birmingham A4540 Roadside': { + city: 'West Midlands', + coordinates: { latitude: 52.47609, longitude: -1.875024 }, + }, + 'Bush Estate': { + city: 'Bush Estate', + coordinates: { latitude: 55.862281, longitude: -3.205782 }, + }, + 'Dumbarton Roadside': { + city: 'Dumbarton', + coordinates: { latitude: 55.943197, longitude: -4.55973 }, + }, + 'Glasgow Great Western Road': { + city: 'Glasgow', + coordinates: { latitude: 55.872038, longitude: -4.270936 }, + }, + 'Edinburgh St Leonards': { + city: 'Edinburgh', + coordinates: { latitude: 55.945589, longitude: -3.182186 }, + }, + 'Auchencorth Moss': { + city: 'Auchencorth', + coordinates: { latitude: 55.79216, longitude: -3.2429 }, + }, + 'Glasgow High Street': { + city: 'Glasgow', + coordinates: { latitude: 55.860936, longitude: -4.238214 }, + }, + 'Glasgow Kerbside': { + city: 'Glasgow', + coordinates: { latitude: 55.85917, longitude: -4.258889 }, + }, + 'Glasgow Townhead': { + city: 'Glasgow', + coordinates: { latitude: 55.865782, longitude: -4.243631 }, + }, + Grangemouth: { + city: 'Grangemouth', + coordinates: { latitude: 56.010319, longitude: -3.704399 }, + }, + 'Grangemouth Moray': { + city: 'Grangemouth', + coordinates: { latitude: 56.013142, longitude: -3.710833 }, + }, + Bottesford: { + city: 'Bottesford', + coordinates: { latitude: 52.93028, longitude: -0.814722 }, + }, + 'Chesterfield Roadside': { + city: 'Chesterfield', + coordinates: { latitude: 53.231722, longitude: -1.456944 }, + }, + 'Chesterfield Loundsley Green': { + city: 'Chesterfield', + coordinates: { latitude: 53.244131, longitude: -1.454946 }, + }, + Ladybower: { + city: 'Ladybower', + coordinates: { latitude: 53.40337, longitude: -1.752006 }, + }, + 'Leicester University': { + city: 'Leicester', + coordinates: { latitude: 52.619823, longitude: -1.127311 }, + }, + 'Northampton Kingsthorpe': { + city: 'Northampton', + coordinates: { latitude: 52.271886, longitude: -0.879898 }, + }, + 'Market Harborough': { + city: 'Market Harborough', + coordinates: { latitude: 52.554444, longitude: -0.772222 }, + }, + 'Lincoln Canwick Rd.': { + city: 'Lincoln', + coordinates: { latitude: 53.221373, longitude: -0.534189 }, + }, + 'Leicester A594 Roadside': { + city: 'Leicester', + coordinates: { latitude: 52.638677, longitude: -1.124228 }, + }, + 'Nottingham Western Boulevard': { + city: 'Nottingham', + coordinates: { latitude: 52.969377, longitude: -1.188851 }, + }, + 'Nottingham Centre': { + city: 'Nottingham', + coordinates: { latitude: 52.95473, longitude: -1.146447 }, + }, + 'Luton A505 Roadside': { + city: 'Luton', + coordinates: { latitude: 51.892293, longitude: -0.46211 }, + }, + 'Cambridge Roadside': { + city: 'Cambridge', + coordinates: { latitude: 52.20237, longitude: 0.124456 }, + }, + 'Norwich Lakenfields': { + city: 'Norwich', + coordinates: { latitude: 52.614193, longitude: 1.301976 }, + }, + 'Sandy Roadside': { + city: 'Sandy', + coordinates: { latitude: 52.132417, longitude: -0.300306 }, + }, + Sibton: { + city: 'Sibton', + coordinates: { latitude: 52.2944, longitude: 1.463497 }, + }, + 'Southend-on-Sea': { + city: 'London', + coordinates: { latitude: 51.544206, longitude: 0.678408 }, + }, + 'St Osyth': { + city: 'St Osyth', + coordinates: { latitude: 51.77798, longitude: 1.049031 }, + }, + 'Stanford-le-Hope Roadside': { + city: 'Stanford-le-Hope', + coordinates: { latitude: 51.518167, longitude: 0.439548 }, + }, + Thurrock: { + city: 'London', + coordinates: { latitude: 51.47707, longitude: 0.317969 }, + }, + Weybourne: { + city: 'Weybourne', + coordinates: { latitude: 52.95049, longitude: 1.122017 }, + }, + 'Wicken Fen': { + city: 'Wicken Fen', + coordinates: { latitude: 52.2985, longitude: 0.290917 }, + }, + 'Ealing Horn Lane': { + city: 'London', + coordinates: { latitude: 51.51895, longitude: -0.265617 }, + }, + 'Camden Kerbside': { + city: 'London', + coordinates: { latitude: 51.54421, longitude: -0.175269 }, + }, + 'Haringey Roadside': { + city: 'London', + coordinates: { latitude: 51.5993, longitude: -0.068218 }, + }, + 'London Bexley': { + city: 'London', + coordinates: { latitude: 51.46603, longitude: 0.184806 }, + }, + 'London Bloomsbury': { + city: 'London', + coordinates: { latitude: 51.52229, longitude: -0.125889 }, + }, + 'London Eltham': { + city: 'London', + coordinates: { latitude: 51.45258, longitude: 0.070766 }, + }, + 'London Haringey Priory Park South': { + city: 'London', + coordinates: { latitude: 51.584128, longitude: -0.125254 }, + }, + 'London Harlington': { + city: 'London', + coordinates: { latitude: 51.48879, longitude: -0.441614 }, + }, + 'London Harrow Stanmore': { + city: 'London', + coordinates: { latitude: 51.617333, longitude: -0.298777 }, + }, + 'London Hillingdon': { + city: 'London', + coordinates: { latitude: 51.49633, longitude: -0.460861 }, + }, + 'London N. Kensington': { + city: 'London', + coordinates: { latitude: 51.52105, longitude: -0.213492 }, + }, + 'London Teddington': { + city: 'London', + coordinates: { latitude: 51.42099, longitude: -0.339647 }, + }, + 'London Marylebone Road': { + city: 'London', + coordinates: { latitude: 51.52253, longitude: -0.154611 }, + }, + 'London Teddington Bushy Park': { + city: 'London', + coordinates: { latitude: 51.425286, longitude: -0.345606 }, + }, + 'London Westminster': { + city: 'London', + coordinates: { latitude: 51.49467, longitude: -0.131931 }, + }, + 'Southwark A2 Old Kent Road': { + city: 'London', + coordinates: { latitude: 51.480499, longitude: -0.05955 }, + }, + 'Tower Hamlets Roadside': { + city: 'London', + coordinates: { latitude: 51.52253, longitude: -0.042155 }, + }, + 'Fort William': { + city: 'Fort William', + coordinates: { latitude: 56.82266, longitude: -5.101102 }, + }, + Inverness: { + city: 'Inverness', + coordinates: { latitude: 57.481308, longitude: -4.241451 }, + }, + Lerwick: { + city: 'Lerwick', + coordinates: { latitude: 60.13922, longitude: -1.185319 }, + }, + Strathvaich: { + city: 'Strath Vaich', + coordinates: { latitude: 57.734456, longitude: -4.776583 }, + }, + Billingham: { + city: 'Billingham', + coordinates: { latitude: 54.60537, longitude: -1.275039 }, + }, + Middlesbrough: { + city: 'Middlesbrough', + coordinates: { latitude: 54.569297, longitude: -1.220874 }, + }, + 'Newcastle Centre': { + city: 'Newcastle', + coordinates: { latitude: 54.97825, longitude: -1.610528 }, + }, + 'Newcastle Cradlewell Roadside': { + city: 'Newcastle', + coordinates: { latitude: 54.986405, longitude: -1.595362 }, + }, + 'Stockton-on-Tees A1305 Roadside': { + city: 'Stockton-on-Tees', + coordinates: { latitude: 54.565819, longitude: -1.3159 }, + }, + 'Stockton-on-Tees Eaglescliffe': { + city: 'Stockton-on-Tees', + coordinates: { latitude: 54.516667, longitude: -1.358547 }, + }, + 'Sunderland Silksworth': { + city: 'Sunderland', + coordinates: { latitude: 54.88361, longitude: -1.406878 }, + }, + 'Sunderland Wessington Way': { + city: 'Sunderland', + coordinates: { latitude: 54.91839, longitude: -1.408391 }, + }, + Aberdeen: { + city: 'Aberdeen', + coordinates: { latitude: 57.15736, longitude: -2.094278 }, + }, + 'Aberdeen Union Street Roadside': { + city: 'Aberdeen', + coordinates: { latitude: 57.144555, longitude: -2.106472 }, + }, + 'Aberdeen Wellington Road': { + city: 'Aberdeen', + coordinates: { latitude: 57.133888, longitude: -2.094198 }, + }, + 'Aston Hill': { + city: 'Aston Hill', + coordinates: { latitude: 52.50385, longitude: -3.034178 }, + }, + Wrexham: { + city: 'Wrexham', + coordinates: { latitude: 53.04222, longitude: -3.002778 }, + }, + 'Blackburn Accrington Road': { + city: 'Blackburn', + coordinates: { latitude: 53.747751, longitude: -2.452724 }, + }, + 'Blackpool Marton': { + city: 'Blackpool', + coordinates: { latitude: 53.80489, longitude: -3.007175 }, + }, + 'Bury Whitefield Roadside': { + city: 'Bury', + coordinates: { latitude: 53.559029, longitude: -2.293772 }, + }, + 'Carlisle Roadside': { + city: 'Carlisle', + coordinates: { latitude: 54.894834, longitude: -2.945307 }, + }, + Glazebury: { + city: 'Glazebury', + coordinates: { latitude: 53.46008, longitude: -2.472056 }, + }, + "Liverpool Queen's Drive Roadside": { + city: 'Liverpool', + coordinates: { latitude: 53.446944, longitude: -2.9625 }, + }, + 'Great Dun Fell': { + city: 'Great Dun Fell', + coordinates: { latitude: 54.684233, longitude: -2.450799 }, + }, + 'Liverpool Speke': { + city: 'Liverpool', + coordinates: { latitude: 53.34633, longitude: -2.844333 }, + }, + 'Manchester Piccadilly': { + city: 'Manchester', + coordinates: { latitude: 53.48152, longitude: -2.237881 }, + }, + 'Manchester Sharston': { + city: 'Manchester', + coordinates: { latitude: 53.371306, longitude: -2.239218 }, + }, + Preston: { + city: 'Preston', + coordinates: { latitude: 53.76559, longitude: -2.680353 }, + }, + 'Salford Eccles': { + city: 'Manchester', + coordinates: { latitude: 53.48481, longitude: -2.334139 }, + }, + 'Shaw Crompton Way': { + city: 'Crompton Way OL2 8AQ', + coordinates: { latitude: 53.579283, longitude: -2.093786 }, + }, + Warrington: { + city: 'Warrington', + coordinates: { latitude: 53.38928, longitude: -2.615358 }, + }, + 'Widnes Milton Road': { + city: 'Widnes', + coordinates: { latitude: 53.365391, longitude: -2.73168 }, + }, + 'Wigan Centre': { + city: 'Wigan', + coordinates: { latitude: 53.54914, longitude: -2.638139 }, + }, + 'Armagh Roadside': { + city: 'Armagh', + coordinates: { latitude: 54.353728, longitude: -6.654558 }, + }, + 'Wirral Tranmere': { + city: 'Liverpool', + coordinates: { latitude: 53.37287, longitude: -3.022722 }, + }, + 'Ballymena Ballykeel': { + city: 'Ballymena', + coordinates: { latitude: 54.861595, longitude: -6.250873 }, + }, + 'Belfast Centre': { + city: 'Belfast', + coordinates: { latitude: 54.59965, longitude: -5.928833 }, + }, + "Belfast Stockman's Lane": { + city: 'Belfast', + coordinates: { latitude: 54.572586, longitude: -5.974944 }, + }, + "Derby St Alkmund's Way": { + city: 'Derby', + coordinates: { latitude: 52.922983, longitude: -1.469507 }, + }, + Derry: { + city: 'Derry', + coordinates: { latitude: 55.001225, longitude: -7.329115 }, + }, + 'Derry Rosemount': { + city: 'Derry', + coordinates: { latitude: 55.002818, longitude: -7.331179 }, + }, + Dumfries: { + city: 'Dumfries', + coordinates: { latitude: 55.070033, longitude: -3.614233 }, + }, + 'Lough Navar': { + city: 'Lough Navar', + coordinates: { latitude: 54.43951, longitude: -7.900328 }, + }, + Eskdalemuir: { + city: 'Eskdalemuir', + coordinates: { latitude: 55.31531, longitude: -3.206111 }, + }, + Peebles: { + city: 'Peebles', + coordinates: { latitude: 55.657472, longitude: -3.196527 }, + }, + 'Brighton Preston Park': { + city: 'Brighton', + coordinates: { latitude: 50.840836, longitude: -0.147572 }, + }, + Canterbury: { + city: 'Canterbury', + coordinates: { latitude: 51.27399, longitude: 1.098061 }, + }, + 'Chatham Roadside': { + city: 'Chatham', + coordinates: { latitude: 51.374264, longitude: 0.54797 }, + }, + Eastbourne: { + city: 'Eastbourne', + coordinates: { latitude: 50.805778, longitude: 0.271611 }, + }, + Horley: { + city: 'Horley', + coordinates: { latitude: 51.165865, longitude: -0.167734 }, + }, + 'Chilbolton Observatory': { + city: 'Stockbridge', + coordinates: { latitude: 51.149617, longitude: -1.438228 }, + }, + 'Lullington Heath': { + city: 'Lullington Heath', + coordinates: { latitude: 50.7937, longitude: 0.18125 }, + }, + 'Oxford Centre Roadside': { + city: 'Oxford', + coordinates: { latitude: 51.751745, longitude: -1.257463 }, + }, + 'Oxford St Ebbes': { + city: 'Oxford', + coordinates: { latitude: 51.744806, longitude: -1.260278 }, + }, + Portsmouth: { + city: 'Portsmouth', + coordinates: { latitude: 50.82881, longitude: -1.068583 }, + }, + 'Reading New Town': { + city: 'Reading', + coordinates: { latitude: 51.45309, longitude: -0.944067 }, + }, + 'Reading London Rd.': { + city: 'Reading', + coordinates: { latitude: 51.454896, longitude: -0.940382 }, + }, + 'Rochester Stoke': { + city: 'Rochester', + coordinates: { latitude: 51.45617, longitude: 0.634889 }, + }, + 'Southampton A33': { + city: 'Southampton', + coordinates: { latitude: 50.920265, longitude: -1.463484 }, + }, + 'Southampton Centre': { + city: 'Southampton', + coordinates: { latitude: 50.90814, longitude: -1.395778 }, + }, + 'Storrington Roadside': { + city: 'Storrington', + coordinates: { latitude: 50.916932, longitude: -0.449548 }, + }, + 'Cardiff Centre': { + city: 'Cardiff', + coordinates: { latitude: 51.48178, longitude: -3.17625 }, + }, + 'Chepstow A48': { + city: 'Chepstow', + coordinates: { latitude: 51.638094, longitude: -2.678731 }, + }, + Cwmbran: { + city: 'Cardiff', + coordinates: { latitude: 51.6538, longitude: -3.006953 }, + }, + 'Hafod-yr-ynys Roadside': { + city: 'Swfrryd', + coordinates: { latitude: 51.680579, longitude: -3.133508 }, + }, + Narberth: { + city: 'Narberth', + coordinates: { latitude: 51.781784, longitude: -4.691462 }, + }, + Newport: { + city: 'Newport', + coordinates: { latitude: 51.601203, longitude: -2.977281 }, + }, + 'Port Talbot Margam': { + city: 'Port Talbot', + coordinates: { latitude: 51.58395, longitude: -3.770822 }, + }, + 'Swansea Roadside': { + city: 'Swansea', + coordinates: { latitude: 51.632696, longitude: -3.947374 }, + }, + 'Barnstaple A39': { + city: 'Barnstaple', + coordinates: { latitude: 51.074793, longitude: -4.041924 }, + }, + 'Bath Roadside': { + city: 'Bath', + coordinates: { latitude: 51.391127, longitude: -2.354155 }, + }, + Bournemouth: { + city: 'Bournemouth', + coordinates: { latitude: 50.73957, longitude: -1.826744 }, + }, + 'Charlton Mackrell': { + city: 'Charlton', + coordinates: { latitude: 51.05625, longitude: -2.68345 }, + }, + "Bristol St Paul's": { + city: 'Bristol', + coordinates: { latitude: 51.462839, longitude: -2.584482 }, + }, + 'Exeter Roadside': { + city: 'Exeter', + coordinates: { latitude: 50.725083, longitude: -3.532465 }, + }, + Honiton: { + city: 'Honiton', + coordinates: { latitude: 50.792287, longitude: -3.196702 }, + }, + 'Saltash Callington Road': { + city: 'Saltash', + coordinates: { latitude: 50.411463, longitude: -4.227678 }, + }, + 'Plymouth Centre': { + city: 'Plymouth', + coordinates: { latitude: 50.37167, longitude: -4.142361 }, + }, + 'Plymouth Tavistock Road.': { + city: 'Plymouth', + coordinates: { latitude: 50.411058, longitude: -4.130288 }, + }, + 'Mace Head': { + city: 'Mace Head', + coordinates: { latitude: 53.326444, longitude: -9.903917 }, + }, + 'Yarner Wood': { + city: 'Yarner Wood', + coordinates: { latitude: 50.5976, longitude: -3.71651 }, + }, + 'Birmingham Acocks Green': { + city: 'Birmingham', + coordinates: { latitude: 52.437165, longitude: -1.829999 }, + }, + 'Birmingham Tyburn': { + city: 'Birmingham', + coordinates: { latitude: 52.511722, longitude: -1.830583 }, + }, + 'Leamington Spa': { + city: 'Leamington Spa', + coordinates: { latitude: 52.28881, longitude: -1.533119 }, + }, + 'Coventry Allesley': { + city: 'Coventry', + coordinates: { latitude: 52.411563, longitude: -1.560228 }, + }, + 'Birmingham Tyburn Roadside': { + city: 'Birmingham', + coordinates: { latitude: 52.512194, longitude: -1.830861 }, + }, + 'Leamington Spa Rugby Road': { + city: 'Leamington Spa', + coordinates: { latitude: 52.294884, longitude: -1.542911 }, + }, + Leominster: { + city: 'Leominster', + coordinates: { latitude: 52.22174, longitude: -2.736665 }, + }, + 'Oldbury Birmingham Road': { + city: 'Oldbury', + coordinates: { latitude: 52.502436, longitude: -2.003497 }, + }, + 'Stoke-on-Trent A50 Roadside': { + city: 'Stoke-on-Trent', + coordinates: { latitude: 52.980436, longitude: -2.111898 }, + }, + 'Stoke-on-Trent Centre': { + city: 'Stoke-on-Trent', + coordinates: { latitude: 53.02821, longitude: -2.175133 }, + }, + 'Walsall Woodlands': { + city: 'Willenhall', + coordinates: { latitude: 52.605621, longitude: -2.030523 }, + }, + 'Barnsley Gawber': { + city: 'Barnsley', + coordinates: { latitude: 53.56292, longitude: -1.510436 }, + }, + 'Bradford Mayo Avenue': { + city: 'Bradford', + coordinates: { latitude: 53.771245, longitude: -1.759774 }, + }, + 'Doncaster A630 Cleveland Street': { + city: 'Doncaster', + coordinates: { latitude: 53.518868, longitude: -1.138073 }, + }, + 'High Muffles': { + city: 'High Muffles', + coordinates: { latitude: 54.334944, longitude: -0.80855 }, + }, + 'Hull Freetown': { + city: 'Hull', + coordinates: { latitude: 53.74878, longitude: -0.341222 }, + }, + 'Hull Holderness Road': { + city: 'Yorkshire', + coordinates: { latitude: 53.758971, longitude: -0.305749 }, + }, + 'Leeds Centre': { + city: 'Leeds', + coordinates: { latitude: 53.80378, longitude: -1.546472 }, + }, + 'Leeds Headingley Kerbside': { + city: 'Leeds', + coordinates: { latitude: 53.819972, longitude: -1.576361 }, + }, + 'Scunthorpe Town': { + city: 'Scunthorpe', + coordinates: { latitude: 53.58634, longitude: -0.636811 }, + }, + 'Sheffield Devonshire Green': { + city: 'Sheffield', + coordinates: { latitude: 53.378622, longitude: -1.478096 }, + }, + 'Sheffield Tinsley': { + city: 'Sheffield', + coordinates: { latitude: 53.41058, longitude: -1.396139 }, + }, + 'York Bootham': { + city: 'York', + coordinates: { latitude: 53.967513, longitude: -1.086514 }, + }, + 'York Fishergate': { + city: 'York', + coordinates: { latitude: 53.951889, longitude: -1.075861 }, + }, }; diff --git a/src/adapters/hong-kong.js b/src/adapters/hong-kong.js index 75d5bad8..d6b9cc29 100644 --- a/src/adapters/hong-kong.js +++ b/src/adapters/hong-kong.js @@ -5,16 +5,11 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { convertUnits } from '../lib/utils.js'; import { load } from 'cheerio'; import { DateTime } from 'luxon'; -import got from 'got'; - -const gotExtended = got.extend({ - retry: { limit: 3 }, - timeout: { request: REQUEST_TIMEOUT }, -}); +import client from '../lib/requests.js'; +import log from '../lib/logger.js'; export const name = 'hong-kong'; @@ -22,9 +17,9 @@ const timeZone = 'Asia/Hong_Kong'; export const fetchData = async (source, cb) => { try { - const response = await gotExtended(`${source.url}/24pc_Eng.xml`); + const response = await client(`${source.url}/24pc_Eng.xml`); if (response.statusCode !== 200) { - console.log('bad request'); + log.debug('bad request'); return; } @@ -35,12 +30,11 @@ export const fetchData = async (source, cb) => { } return cb(null, data); } catch (err) { - console.log('Request error:', err); + log.debug('Request error:', err); return cb({ message: 'Unknown adapter error.' }); } }; - const formatData = function (data) { let measurements = []; const $ = load(data, { xmlMode: true }); diff --git a/src/adapters/hungary.js b/src/adapters/hungary.js index 9304b854..701760c0 100644 --- a/src/adapters/hungary.js +++ b/src/adapters/hungary.js @@ -3,46 +3,55 @@ * and returning data for the Hungary data sources. */ +import client from '../lib/requests.js'; +import log from '../lib/logger.js'; import { removeUnwantedParameters } from '../lib/utils.js'; import { DateTime } from 'luxon'; -import fetch from 'node-fetch' // Get the current time in Hungary let dt = DateTime.utc(); -const { year, month, day } = dt.toObject({ year: 'numeric', month: 'numeric', day: 'numeric' }); +const { year, month, day } = dt.toObject({ + year: 'numeric', + month: 'numeric', + day: 'numeric', +}); -export const name = 'hungary' +export const name = 'hungary'; -export async function fetchData (source, cb) { +export async function fetchData(source, cb) { /** - * Fetches the data for a given source and returns an appropriate object - * @param {object} source A valid source object - * @param {function} cb A callback of the form cb(err, data) - */ + * Fetches the data for a given source and returns an appropriate object + * @param {object} source A valid source object + * @param {function} cb A callback of the form cb(err, data) + */ try { let stations = await fetchStations(source.url); // Map through station objects and create data request for each one - let requests = stations.data.map(station => { + let requests = stations.data.map((station) => { if (station.hasOwnProperty('stationId')) { const stationId = station.stationId; - const url = `${source.url}${stationId}` - return fetch(url) - .then(response => response.json()) - .then(data => { + const url = `${source.url}${stationId}`; + return client (url) + .then((response) => JSON.parse(response.body)) + .then((data) => { // create dateTime object with each station's hour const date = DateTime.fromISO( - `${year}-${month.toString().padStart(2, '0')}-${day.toString().padStart(2, '0')}T${data.data.lastHour}`, + `${year}-${month.toString().padStart(2, '0')}-${day + .toString() + .padStart(2, '0')}T${data.data.lastHour}`, { zone: 'utc' } ); // Add fetched data to station object station.station = data.data.stationName; station.month = data.data.month; station.utc = date.toFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - station.local = date.setZone("Europe/Budapest").toFormat("yyyy-MM-dd'T'HH:mm:ssZZ"); + station.local = date + .setZone('Europe/Budapest') + .toFormat("yyyy-MM-dd'T'HH:mm:ssZZ"); station.measurements = data.data.lastHourValues; return station; }) - .catch(error => { + .catch((error) => { throw error; }); } @@ -50,30 +59,33 @@ export async function fetchData (source, cb) { // Wait for all fetch requests to complete in parallel let allStationData = await Promise.all(requests); - let out = await formatData(allStationData) + let out = await formatData(allStationData); return cb(null, out); } catch (error) { return cb(error); } } - -async function fetchStations (stationUrl) { + +async function fetchStations(stationUrl) { try { - let response = await fetch (stationUrl); - let stations = await response.json(); - return stations + let response = await client(stationUrl); + let stations = await JSON.parse(response.body); + return stations; } catch (error) { - console.log('Failed to resolve stations URL.') + log.debug('Failed to resolve stations URL.'); throw error; } } async function formatData(input) { let measurements = []; - input.forEach(o => { - Object.values(o.measurements).forEach(param => { + input.forEach((o) => { + Object.values(o.measurements).forEach((param) => { let parameter = correctParam(param.name); - const [value, unit] = [parseFloat(param.value), param.value.split(' ')[1]]; + const [value, unit] = [ + parseFloat(param.value), + param.value.split(' ')[1], + ]; let measurement = { location: o.station, city: o.station, @@ -82,47 +94,55 @@ async function formatData(input) { unit, date: { utc: o.utc, - local: o.local + local: o.local, }, coordinates: { latitude: parseFloat(o.latitude), - longitude: parseFloat(o.longitude) + longitude: parseFloat(o.longitude), }, attribution: [ { - name: "Hungary National Meteorological Service", - url: "https://legszennyezettseg.met.hu/" - } + name: 'Hungary National Meteorological Service', + url: 'https://legszennyezettseg.met.hu/', + }, ], averagingPeriod: { - unit: "hours", - value: 1 - } - } - measurements.push(measurement) + unit: 'hours', + value: 1, + }, + }; + measurements.push(measurement); }); }); measurements = removeUnwantedParameters(measurements); measurements = filterMeasurements(measurements); measurements = getLatestMeasurements(measurements); - measurements = filterDuplicates(measurements, eeaHungaryDuplicateStations) - return { name: 'unused', measurements: measurements } -}; + measurements = filterDuplicates( + measurements, + eeaHungaryDuplicateStations + ); + return { name: 'unused', measurements: measurements }; +} function filterMeasurements(measurements) { return measurements.filter((measurement) => { - return (measurement.value !== undefined && - measurement.value === measurement.value && - measurement.unit !== undefined && - measurement.unit === measurement.unit); + return ( + measurement.value !== undefined && + measurement.value === measurement.value && + measurement.unit !== undefined && + measurement.unit === measurement.unit + ); }); } - + function getLatestMeasurements(measurements) { const latestMeasurements = {}; measurements.forEach((measurement) => { const key = measurement.parameter + measurement.location; - if (!latestMeasurements[key] || measurement.date.utc > latestMeasurements[key].date.utc) { + if ( + !latestMeasurements[key] || + measurement.date.utc > latestMeasurements[key].date.utc + ) { latestMeasurements[key] = measurement; } }); @@ -131,35 +151,38 @@ function getLatestMeasurements(measurements) { function correctParam(name) { switch (name) { - case 'SO₂': - return 'so2'; - case 'PM₁₀': - return 'pm10'; - case 'O₃': - return 'o3'; - case 'NO₂': - return 'no2'; - case 'NOx': - return 'nox'; - case 'CO': - return 'co'; - case 'PM₂,₅': - return 'pm25'; - case 'NO': - return 'no'; - default: - return name; - } + case 'SO₂': + return 'so2'; + case 'PM₁₀': + return 'pm10'; + case 'O₃': + return 'o3'; + case 'NO₂': + return 'no2'; + case 'NOx': + return 'nox'; + case 'CO': + return 'co'; + case 'PM₂,₅': + return 'pm25'; + case 'NO': + return 'no'; + default: + return name; + } } // remove stations deemed to be duplicates of EEA stations function filterDuplicates(measurements, criteria) { - return measurements.filter(measurement => { - return !criteria.some(criterion => { - const matchLocation = measurement.location === criterion.location; + return measurements.filter((measurement) => { + return !criteria.some((criterion) => { + const matchLocation = + measurement.location === criterion.location; const matchCoordinates = - measurement.coordinates.latitude === criterion.coordinates.latitude && - measurement.coordinates.longitude === criterion.coordinates.longitude; + measurement.coordinates.latitude === + criterion.coordinates.latitude && + measurement.coordinates.longitude === + criterion.coordinates.longitude; return matchLocation && matchCoordinates; }); @@ -169,73 +192,73 @@ function filterDuplicates(measurements, criteria) { // this deny list is used to exclude stations that are within 0.1 km of EEA stations const eeaHungaryDuplicateStations = [ { - "location": "Budapest Teleki tér", - "coordinates": { - "latitude": 47.492104, - "longitude": 19.087778 - } -}, -{ - "location": "Esztergom", - "coordinates": { - "latitude": 47.79044, - "longitude": 18.74582 - } -}, -{ - "location": "Győr 1 Szent István", - "coordinates": { - "latitude": 47.68537, - "longitude": 17.63955 - } -}, -{ - "location": "Sarród", - "coordinates": { - "latitude": 47.67148, - "longitude": 16.83955 - } -}, -{ - "location": "Pécs Szabadság u.", - "coordinates": { - "latitude": 46.07098, - "longitude": 18.22527 - } -}, -{ - "location": "Budapest Pesthidegkút", - "coordinates": { - "latitude": 47.561738, - "longitude": 18.960876 - } -}, -{ - "location": "Sajószentpéter", - "coordinates": { - "latitude": 48.21819, - "longitude": 20.70334 - } -}, -{ - "location": "Sopron", - "coordinates": { - "latitude": 47.6913, - "longitude": 16.57548 - } -}, -{ - "location": "Debrecen Kalotaszeg tér", - "coordinates": { - "latitude": 47.513384, - "longitude": 21.624621 - } -}, -{ - "location": "Budapest Széna tér", - "coordinates": { - "latitude": 47.508605, - "longitude": 19.02764 - } -} -] \ No newline at end of file + location: 'Budapest Teleki tér', + coordinates: { + latitude: 47.492104, + longitude: 19.087778, + }, + }, + { + location: 'Esztergom', + coordinates: { + latitude: 47.79044, + longitude: 18.74582, + }, + }, + { + location: 'Győr 1 Szent István', + coordinates: { + latitude: 47.68537, + longitude: 17.63955, + }, + }, + { + location: 'Sarród', + coordinates: { + latitude: 47.67148, + longitude: 16.83955, + }, + }, + { + location: 'Pécs Szabadság u.', + coordinates: { + latitude: 46.07098, + longitude: 18.22527, + }, + }, + { + location: 'Budapest Pesthidegkút', + coordinates: { + latitude: 47.561738, + longitude: 18.960876, + }, + }, + { + location: 'Sajószentpéter', + coordinates: { + latitude: 48.21819, + longitude: 20.70334, + }, + }, + { + location: 'Sopron', + coordinates: { + latitude: 47.6913, + longitude: 16.57548, + }, + }, + { + location: 'Debrecen Kalotaszeg tér', + coordinates: { + latitude: 47.513384, + longitude: 21.624621, + }, + }, + { + location: 'Budapest Széna tér', + coordinates: { + latitude: 47.508605, + longitude: 19.02764, + }, + }, +]; diff --git a/src/adapters/japan.js b/src/adapters/japan.js index 75ddd403..024df59c 100644 --- a/src/adapters/japan.js +++ b/src/adapters/japan.js @@ -6,11 +6,11 @@ 'use strict'; import { acceptableParameters } from '../lib/utils.js'; -import log from '../lib/logger.js'; import { DateTime } from 'luxon'; import { parse } from 'csv-parse'; import Bottleneck from 'bottleneck'; -import got from 'got'; +import client from '../lib/requests.js'; +import log from '../lib/logger.js'; const limiter = new Bottleneck({ minTime: 50, // Minimum time between requests (ms) @@ -50,7 +50,7 @@ export async function fetchData (source, cb) { async function fetchStations (stationsCsvUrl) { try { - const response = await got(stationsCsvUrl); + const response = await client(stationsCsvUrl); return new Promise((resolve, reject) => { parse( response.body, @@ -85,7 +85,7 @@ async function fetchStationData (latestDataUrl, stationId, unixTimeStamp) { url.pathname += `${stationId}/today.csv`; url.searchParams.append('_', unixTimeStamp); - const response = await got(url.href); + const response = await client(url.href); return new Promise((resolve, reject) => { parse(response.body, { columns: true }, (err, records) => { err ? reject(err) : resolve(records); diff --git a/src/adapters/laqn.js b/src/adapters/laqn.js index 5c1b38c9..08ca3bf9 100644 --- a/src/adapters/laqn.js +++ b/src/adapters/laqn.js @@ -1,7 +1,7 @@ 'use strict'; import _ from 'lodash'; -import got from 'got'; +import client from '../lib/requests.js'; import log from '../lib/logger.js'; import { DateTime } from 'luxon'; import { @@ -30,14 +30,14 @@ export async function fetchData(source, cb) { const startDate = dateNow.toFormat('dd LLL yyyy'); const endDate = dateNow.plus({ days: 1 }).toFormat('dd LLL yyyy'); - const siteCodesResponse = await got( + const siteCodesResponse = await client( `${source.url}/AirQuality/Information/MonitoringSites/GroupName=All/Json` ); let allSites = JSON.parse(siteCodesResponse.body).Sites.Site; allSites = allSites.filter((s) => !s['@DateClosed']); const siteLookup = _.keyBy(allSites, '@SiteCode'); const dataPromises = allSites.map((site) => - got( + client( `${source.url}/AirQuality/Data/Site/SiteCode=${site['@SiteCode']}/StartDate=${startDate}/EndDate=${endDate}/Json` ) .catch((error) => { diff --git a/src/adapters/medellin.js b/src/adapters/medellin.js index aadbb83c..9e9c7310 100644 --- a/src/adapters/medellin.js +++ b/src/adapters/medellin.js @@ -7,12 +7,12 @@ 'use strict'; +import client from '../lib/requests.js'; +import log from '../lib/logger.js'; import { DateTime } from 'luxon'; -import got from 'got'; -import _ from 'lodash'; import { convertUnits } from '../lib/utils.js'; -import log from '../lib/logger.js'; import { REQUEST_TIMEOUT } from '../lib/constants.js'; +import _ from 'lodash'; export const name = 'medellin'; @@ -36,7 +36,7 @@ export async function fetchData (source, cb) { const url = `${source.url}EntregaData1/Datos_SIATA_Aire_AQ_${p}_Last.json`; try { - const response = await got.get(url, options); + const response = await client(url, options); return JSON.parse(response.body); } catch (error) { log.warn(error || `Unable to load data for parameter: ${p} for adapter ${source.name}`); diff --git a/src/adapters/montenegro.js b/src/adapters/montenegro.js index 4573fb55..3c972b6b 100644 --- a/src/adapters/montenegro.js +++ b/src/adapters/montenegro.js @@ -5,54 +5,48 @@ * This is a two-stage adapter requiring loading multiple urls before parsing * data. */ + 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { DateTime } from 'luxon'; import { load } from 'cheerio'; -import got from 'got'; +import client from '../lib/requests.js'; +import log from '../lib/logger.js'; import { removeUnwantedParameters, unifyMeasurementUnits, unifyParameters, } from '../lib/utils.js'; -const gotInstance = got.extend({ timeout: { request: REQUEST_TIMEOUT } }); - export const name = 'montenegro'; export async function fetchData(source, cb) { let tasks = []; for (let i = 1; i < 20; i++) { - try { - await gotInstance(source.url + i); - let task = async function () { - try { - const response = await gotInstance(source.url + i); - return response.body; - } catch (error) { - console.error('Error in task:', error.message); - throw error; - } - }; - tasks.push(task); - } catch (error) { - console.error('Error while creating tasks:', error.message); - continue; - } + let task = async function () { + try { + const response = await client(source.url + i); + return response.body; + } catch (error) { + log.debug(`Error fetching data from URL: ${source.url + i}. Giving up after retries.`, error.message); + return null; + } + }; + tasks.push(task); } try { const results = await Promise.all(tasks.map((task) => task())); - const data = formatData(results); + const filteredResults = results.filter(result => result); + const data = formatData(filteredResults); if (data === undefined) { - console.error('Failed to parse data'); + log.debug('Failed to parse data'); return cb({ message: 'Failure to parse data.' }); } cb(null, data); } catch (error) { - console.error('Error in async.parallel:', error.message); + log.debug('Error in async.parallel:', error.message); return cb({ message: 'Failure to load data urls.' }); } } @@ -61,7 +55,7 @@ const formatData = function (results) { const parseLocation = (location, template) => { try { if (location === undefined || location === null) { - console.error('Location is undefined or null'); + log.debug('Location is undefined or null'); return; } location = location.split('|'); @@ -69,8 +63,7 @@ const formatData = function (results) { location.length < 2 || location[0].includes('Otvori veću kartu') ) { - // Add this check - console.error('Invalid location format'); + log.debug('Invalid location format'); return; } const city = location[0].split(','); @@ -86,19 +79,17 @@ const formatData = function (results) { }; template.coordinates = coordinates; } catch (e) { - console.error('Error in parseLocation:', e); + log.debug('Error in parseLocation:', e); } }; const parseDate = function (date, template) { - // Validate input if (typeof date !== 'string' || date.trim() === '') { throw new Error('Invalid date string'); } try { - // Create DateTime object with timezone using the fromFormat() method date = date.replace('Pregled mjerenja za', '').replace('h', ''); - date = date.trim(); // remove whitespace + date = date.trim(); const dateLuxon = DateTime.fromFormat( date, 'dd.MM.yyyy HH:mm', @@ -106,7 +97,6 @@ const formatData = function (results) { zone: 'Europe/Podgorica', } ); - // Return UTC and local ISO strings const utc = dateLuxon .toUTC() .toISO({ suppressMilliseconds: true }); @@ -138,7 +128,7 @@ const formatData = function (results) { value.substring(0, splitPos).replace(',', '.').trim() ); } catch (e) { - console.error('Error in parseValueAndUnit:', e); + log.debug('Error in parseValueAndUnit:', e); } }; @@ -156,7 +146,7 @@ const formatData = function (results) { $('.col-6.col-12-medium').each((i, e) => { $('h6 a', e).each((i, e) => { const text = $(e).text(); - // console.log('Text:', text); // Add this line for additional logging + // log.debug('Text:', text); if (text.search('|') !== -1 && text.charAt(0) !== '*') { parseLocation(text, template); } diff --git a/src/adapters/netherlands.js b/src/adapters/netherlands.js index 716c3ab1..7eb4716d 100644 --- a/src/adapters/netherlands.js +++ b/src/adapters/netherlands.js @@ -5,20 +5,18 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { removeUnwantedParameters } from '../lib/utils.js'; - -import got from 'got'; -import _ from 'lodash'; import { DateTime } from 'luxon'; import { load } from 'cheerio'; +import client from '../lib/requests.js'; +import _ from 'lodash'; import async from 'async'; export const name = 'netherlands'; export function fetchData(source, cb) { const finalURL = source.url; - got(finalURL, { timeout: { request: REQUEST_TIMEOUT } }) + client(finalURL) .then((response) => { const body = response.body; @@ -36,7 +34,7 @@ export function fetchData(source, cb) { _.forEach(recentFiles, function (f) { let task = function (cb) { // download the xml - got(f.url, { timeout: { request: REQUEST_TIMEOUT } }) + client(f.url) .then((response) => { const body = response.body; @@ -224,7 +222,10 @@ const formatData = function (name, data) { zone: 'Europe/Amsterdam', }); - return { utc: date.toUTC().toISO({ suppressMilliseconds: true }), local: date.toISO({ suppressMilliseconds: true }) }; + return { + utc: date.toUTC().toISO({ suppressMilliseconds: true }), + local: date.toISO({ suppressMilliseconds: true }), + }; }; const getStationId = function (string) { diff --git a/src/adapters/norway.js b/src/adapters/norway.js index 8eaaea5a..7dcc6ee9 100644 --- a/src/adapters/norway.js +++ b/src/adapters/norway.js @@ -1,13 +1,12 @@ /** * This code is responsible for implementing all methods related to fetching - * and returning data for the Norwegian data sources. + * and returning data for the Norway data sources. */ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { DateTime } from 'luxon'; -import got from 'got'; +import client from '../lib/requests.js'; export const name = 'norway'; @@ -17,9 +16,9 @@ export const name = 'norway'; * @param {function} cb A callback of the form cb(err, data) */ -export async function fetchData (source, cb) { +export async function fetchData(source, cb) { try { - const response = await got(source.url, { timeout: { request: REQUEST_TIMEOUT } }); + const response = await client(source.url); if (response.statusCode !== 200) { return cb({ message: 'Failure to load data url.' }); @@ -63,23 +62,30 @@ const formatData = function (data) { * @return {object} a repacked object */ const aqRepack = (item) => { - const dateLuxon = DateTime.fromISO(item.toTime, { zone: 'Europe/Oslo' }); + const dateLuxon = DateTime.fromISO(item.toTime, { + zone: 'Europe/Oslo', + }); const template = { location: item.station, city: item.area, parameter: item.component.toLowerCase().replace('.', ''), date: { utc: dateLuxon.toUTC().toISO({ suppressMilliseconds: true }), - local: dateLuxon.toISO({ suppressMilliseconds: true }) + local: dateLuxon.toISO({ suppressMilliseconds: true }), }, coordinates: { latitude: item.latitude, - longitude: item.longitude + longitude: item.longitude, }, value: parseFloat(item.value), unit: item.unit, - attribution: [{ name: 'Luftkvalitet.info', url: 'http://www.luftkvalitet.info/home.aspx' }], - averagingPeriod: { unit: 'hours', value: 1 } + attribution: [ + { + name: 'Luftkvalitet.info', + url: 'http://www.luftkvalitet.info/home.aspx', + }, + ], + averagingPeriod: { unit: 'hours', value: 1 }, }; return template; diff --git a/src/adapters/queensland.js b/src/adapters/queensland.js index 63ac3796..8329276a 100644 --- a/src/adapters/queensland.js +++ b/src/adapters/queensland.js @@ -1,16 +1,15 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { removeUnwantedParameters } from '../lib/utils.js'; import _ from 'lodash'; import { DateTime } from 'luxon'; import { load } from 'cheerio'; -import got from 'got'; +import client from '../lib/requests.js'; export const name = 'queensland'; export function fetchData (source, cb) { - got(source.url, { timeout: { request: REQUEST_TIMEOUT } }) + client(source.url) .then((response) => { try { const data = formatData(response.body, source); diff --git a/src/adapters/senamhi.js b/src/adapters/senamhi.js index f86d63b3..fc5eec9b 100644 --- a/src/adapters/senamhi.js +++ b/src/adapters/senamhi.js @@ -1,9 +1,8 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { convertUnits } from '../lib/utils.js'; -import got from 'got'; +import client from '../lib/requests.js'; import { load } from 'cheerio'; import { DateTime } from 'luxon'; import { join, dirname } from 'path'; @@ -26,8 +25,7 @@ export const name = 'senamhi'; export async function fetchData (source, cb) { try { - const response = await got(source.url, { - timeout: { request: REQUEST_TIMEOUT }, + const response = await client(source.url, { https: { certificateAuthority: rootCas, }, diff --git a/src/adapters/serbia.js b/src/adapters/serbia.js index 33c06fc6..fae26d60 100644 --- a/src/adapters/serbia.js +++ b/src/adapters/serbia.js @@ -6,8 +6,7 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; -import got from 'got'; +import client from '../lib/requests.js'; import { DateTime } from 'luxon'; import { unifyMeasurementUnits } from '../lib/utils.js'; @@ -21,7 +20,7 @@ export const name = 'serbia'; export async function fetchData (source, cb) { try { - const response = await got(source.url, { timeout: { request: REQUEST_TIMEOUT } }); + const response = await client(source.url); if (response.statusCode !== 200) { return cb({ message: 'Failure to load data url.' }); diff --git a/src/adapters/slovenia.js b/src/adapters/slovenia.js index ac12062e..34b07c0f 100644 --- a/src/adapters/slovenia.js +++ b/src/adapters/slovenia.js @@ -1,14 +1,13 @@ -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { convertUnits } from '../lib/utils.js'; import cloneDeep from 'lodash/cloneDeep.js'; import { DateTime } from 'luxon'; import { load } from 'cheerio'; -import got from 'got'; +import client from '../lib/requests.js'; export const name = 'slovenia'; export function fetchData (source, cb) { - got(source.url, { timeout: { request: REQUEST_TIMEOUT } }) + client(source.url) .then((response) => { if (response.statusCode !== 200) { throw new Error('Failure to load data url.'); diff --git a/src/adapters/southafrica.js b/src/adapters/southafrica.js index 94ff5507..d566ed1f 100644 --- a/src/adapters/southafrica.js +++ b/src/adapters/southafrica.js @@ -5,7 +5,6 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { unifyMeasurementUnits, removeUnwantedParameters, @@ -14,8 +13,8 @@ import { import log from '../lib/logger.js'; import { DateTime } from 'luxon'; -import got from 'got'; import _ from 'lodash'; +import client from '../lib/requests.js'; export const name = 'southafrica'; @@ -28,9 +27,7 @@ export const name = 'southafrica'; export async function fetchData (source, cb) { try { - const response = await got(source.url, { - timeout: { request: REQUEST_TIMEOUT }, - }); + const response = await client(source.url); if (response.statusCode !== 200) { log.error('Request error:', response.statusCode); // Log the error status code diff --git a/src/adapters/southaustralia.js b/src/adapters/southaustralia.js index 8b1eb8ca..6446d188 100644 --- a/src/adapters/southaustralia.js +++ b/src/adapters/southaustralia.js @@ -7,45 +7,92 @@ import { acceptableParameters } from '../lib/utils.js'; import { DateTime } from 'luxon'; -import Parser from 'rss-parser' +import Parser from 'rss-parser'; import { parse } from 'node-html-parser'; +import log from '../lib/logger.js'; -let parser = new Parser(); +const parser = new Parser(); -export const name = 'southaustralia' +process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; -export async function fetchData (source, cb) { - /** - * Fetches the data for a given source and returns an appropriate object - * @param {object} source A valid source object - * @param {function} cb A callback of the form cb(err, data) - */ +export const name = 'southaustralia'; + +export async function fetchData(source, cb) { try { - let parsedRss = await getMeasurements(source.url); - let data = await formatData(parsedRss); + log.debug('Starting fetchData...'); + const parsedRss = await getMeasurements(source.url); + if (!parsedRss) { + log.debug('No parsed RSS data received.'); + return cb({ message: 'Failed to parse RSS data.' }); + } + const data = await formatData(parsedRss); if (data === undefined) { - return cb({ message: 'Failure to parse data.' }); - } - return cb(null, data); - } - catch (error) { - return cb(error); + log.debug('Failed to format data.'); + return cb({ message: 'Failure to parse data.' }); } + return cb(null, data); + } catch (error) { + log.debug('Error in fetchData:', error); + return cb(error); + } } async function getMeasurements(path) { + log.debug('Fetching RSS data from:', path); + try { const feed = await parser.parseURL(path); - if (feed.status !== 'ok') { - console.log('Something went wrong, failed to resolve feed.'); - } - const sites = new Set(); - feed.items.forEach((item) => { + log.debug('Processing RSS item:', item.title); + + const dateMatch = item.title.match( + /\d{2}\/\d{2}\/\d{4} \d{1,2}:\d{2} [ap]m/ + ); + if (!dateMatch) { + log.debug('Failed to extract date from title:', item.title); + return; + } + const dateStr = dateMatch[0]; + + // Extract date components + const matches = dateStr.match( + /(\d{2})\/(\d{2})\/(\d{4}) (\d{1,2}):(\d{2}) ([ap]m)/ + ); + if (!matches) { + log.debug( + "Date string doesn't match the expected format:", + dateStr + ); + return; + } + + const [_, day, month, year, hour, minute, ampm] = matches; + let parsedHour = parseInt(hour, 10); + if (ampm === 'pm' && parsedHour !== 12) parsedHour += 12; + if (ampm === 'am' && parsedHour === 12) parsedHour = 0; + + const date = DateTime.fromObject({ + day: parseInt(day, 10), + month: parseInt(month, 10), + year: parseInt(year, 10), + hour: parsedHour, + minute: parseInt(minute, 10), + }).setZone('Australia/Adelaide'); + + if (!date.isValid) { + log.debug( + 'Error constructing date:', + date.invalidExplanation + ); + return; + } + const root = parse(item.content); - const head = root.querySelector('thead'); - const keys = head.childNodes[0].childNodes.map((o) => o.text); + const head = root.querySelector('thead'); + const keys = head.childNodes[0].childNodes.map((o) => o.text); + log.debug('Parsed keys:', keys); + const tbody = root.querySelector('tbody'); const rows = tbody.childNodes; for (const row of rows) { @@ -53,181 +100,183 @@ async function getMeasurements(path) { const site = Object.assign( ...keys.map((k, i) => ({ [k]: cells[i].text })) ); + site.time = date; // Assign the parsed date to the site object + log.debug('Parsed site data:', site); sites.add(site); } }); - + let uniqueSites = [...sites]; - uniqueSites.forEach((site) => { - site.time = DateTime.fromFormat(site['Date/time'], 'yyyy-MM-dd HH:mm:ss', { zone: 'Australia/Adelaide' }); - }); - uniqueSites = getLatestEntries(uniqueSites); - + log.debug('Fetched and parsed measurements:', uniqueSites); return uniqueSites; + } catch (error) { + log.debug('Error fetching RSS data:', error); + return null; + } } - -async function getLatestEntries(sites) { - const latestEntries = sites.reduce((result, site) => { - const key = site.Site; - const currentLatest = result[key]; - - if (!currentLatest || site.time > currentLatest.time) { - result[key] = site; - } - return result; - }, {}); - - return Object.values(latestEntries); +async function getLatestEntries(sites) { + log.debug('Getting latest entries...'); + const latestEntries = sites.reduce((result, site) => { + const key = site.Site; + const currentLatest = result[key]; + if (!currentLatest || site.time > currentLatest.time) { + result[key] = site; + } + return result; + }, {}); + return Object.values(latestEntries); } async function formatData(data) { - try { - data.forEach(site => { - const location = locations.find(location => location.label === site.Site); - if (location) { - site.lat = location.lat; - site.lng = location.lng; - } - Object.keys(site).forEach(key => { - const newKey = correctParam(key); - if (newKey !== key) { - site[newKey] = site[key]; - delete site[key]; - } - }); - }); - - let filteredData = []; - - data.forEach(obj => { - acceptableParameters.forEach(param => { - filteredData.push({ - location: obj.Site, - city: obj.Region, - - parameter: param, - value: obj.hasOwnProperty(param) ? parseFloat(obj[param]) : null, - unit: param === ('pm10' || 'pm25') ? 'µg/m³': 'ppm', - date: { - utc: obj.time.toUTC().toISO({suppressMilliseconds: true}), - local: obj.time.toISO({suppressMilliseconds: true}) - }, - coordinates: { - latitude: parseFloat(obj.lat), - longitude: parseFloat(obj.lng) - }, - attribution: [ - { - name: "South Australia Environmental Protection Authority (EPA)", - url: "https://data.sa.gov.au/data/dataset/recent-air-quality" - } - ], - averagingPeriod: { - unit: "hours", - value: param === ('co') ? 8 : 1, - } - }); - }); - }); - let measurements = filteredData.filter(obj => obj.value !== null && !isNaN(obj.value)); - return { name: 'unused', measurements: measurements } - } catch (error) { - throw error; + log.debug('Formatting data...'); + data.forEach((site) => { + const location = locations.find( + (location) => location.label === site.Site + ); + if (location) { + site.lat = location.lat; + site.lng = location.lng; + } else { + log.debug(`Location not found for site: ${site.Site}`); } + Object.keys(site).forEach((key) => { + const newKey = correctParam(key); + if (newKey !== key) { + site[newKey] = site[key]; + delete site[key]; + } + }); + }); + + let filteredData = []; + data.forEach((obj) => { + acceptableParameters.forEach((param) => { + filteredData.push({ + location: obj.Site, + city: obj.Region, + parameter: param, + value: obj.hasOwnProperty(param) + ? parseFloat(obj[param]) + : null, + unit: param === 'pm10' || param === 'pm25' ? 'µg/m³' : 'ppm', + date: { + utc: obj.time.toUTC().toISO({ suppressMilliseconds: true }), + local: obj.time.toISO({ suppressMilliseconds: true }), + }, + coordinates: { + latitude: parseFloat(obj.lat), + longitude: parseFloat(obj.lng), + }, + attribution: [ + { + name: 'South Australia Environmental Protection Authority (EPA)', + url: 'https://data.sa.gov.au/data/dataset/recent-air-quality', + }, + ], + averagingPeriod: { + unit: 'hours', + value: param === 'co' ? 8 : 1, + }, + }); + }); + }); + const measurements = filteredData.filter( + (obj) => obj.value !== null && !isNaN(obj.value) + ); + log.debug('Formatted data:', measurements); + return { name: 'unused', measurements: measurements }; } function correctParam(name) { - switch (name) { - case 'Sulfur dioxide (SO2) 1Hr': - return 'so2'; - case 'Particles (PM10) 1Hr': - return 'pm10'; - case 'Ozone (O3) 1Hr': - return 'o3'; - case 'Nitrogen dioxide (NO2) 1Hr': - return 'no2'; - case 'NOx': - return 'nox'; - case 'Carbon monoxide (CO) 8Hr': - return 'co'; - case 'Particles (PM2.5) 1Hr': - return 'pm25'; - case 'NO': - return 'no'; - default: - return name; - } + switch (name) { + case 'SO2 1 Hour': + return 'so2'; + case 'PM10 1 Hour': + return 'pm10'; + case 'O3 1 Hour': + return 'o3'; + case 'NO2 1 Hour': + return 'no2'; + case 'NOx': + return 'nox'; + case 'CO 8 Hours': + return 'co'; + case 'PM2.5 1 Hour': + return 'pm25'; + case 'NO': + return 'no'; + default: + return name; } +} - let locations = [ - { - "name": "Mt Barker", - "label": "Wood smoke program", - "lat": "-35.073352", - "lng": "138.864943" - }, - { - "name": "Adelaide CBD", - "label": "CBD", - "lat": "-34.928853", - "lng": "138.600943" - }, - { - "name": "Le Fevre 1 Birkenhead", - "label": "Birkenhead", - "lat": "-34.838654", - "lng": "138.496351" - }, - { - "name": "Le Fevre 2 North Haven", - "label": "North Haven", - "lat": "-34.791288", - "lng": "138.497860", - }, - { - "name": "Netley", - "label": "Netley", - "lat": "-34.9438", - "lng": "138.5491" - }, - { - "name": "Northfield", - "label": "Northfield", - "lat": "-34.862004", - "lng": "138.622932" - }, - { - "name": "Elizabeth", - "label": "Elizabeth", - "lat": "-34.698472", - "lng": "138.695751" - }, - { - "name": "Christies", - "label": "Christies", - "lat": "-35.134927", - "lng": "138.495159" - }, - { - "name": "Port Pirie Oliver St", - "label": "Oliver St", - "lat": "-33.194818", - "lng": "138.020014" - }, - { - "name": "Whyalla Schulz Res", - "label": "Schulz Reserve", - "lat": "-33.023595", - "lng": "137.533239" - }, - { - "name": "Whyalla Walls St", - "label": "Walls St", - "lat": "-33.036096", - "lng": "137.586088" - } +let locations = [ + { + name: 'Mt Barker', + label: 'Wood smoke program', + lat: '-35.073352', + lng: '138.864943', + }, + { + name: 'Adelaide CBD', + label: 'CBD', + lat: '-34.928853', + lng: '138.600943', + }, + { + name: 'Le Fevre 1 Birkenhead', + label: 'Birkenhead', + lat: '-34.838654', + lng: '138.496351', + }, + { + name: 'Le Fevre 2 North Haven', + label: 'North Haven', + lat: '-34.791288', + lng: '138.497860', + }, + { + name: 'Netley', + label: 'Netley', + lat: '-34.9438', + lng: '138.5491', + }, + { + name: 'Northfield', + label: 'Northfield', + lat: '-34.862004', + lng: '138.622932', + }, + { + name: 'Elizabeth', + label: 'Elizabeth', + lat: '-34.698472', + lng: '138.695751', + }, + { + name: 'Christies', + label: 'Christies', + lat: '-35.134927', + lng: '138.495159', + }, + { + name: 'Port Pirie Oliver St', + label: 'Oliver St', + lat: '-33.194818', + lng: '138.020014', + }, + { + name: 'Whyalla Schulz Res', + label: 'Schulz Reserve', + lat: '-33.023595', + lng: '137.533239', + }, + { + name: 'Whyalla Walls St', + label: 'Walls St', + lat: '-33.036096', + lng: '137.586088', + }, ]; - - \ No newline at end of file diff --git a/src/adapters/stateair.js b/src/adapters/stateair.js index 4fa3e8dd..73476915 100644 --- a/src/adapters/stateair.js +++ b/src/adapters/stateair.js @@ -1,23 +1,20 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { convertUnits } from '../lib/utils.js'; -import got from 'got'; +import client from '../lib/requests.js'; import _ from 'lodash'; import { DateTime } from 'luxon'; import { load } from 'cheerio'; import { parallel } from 'async'; -const getter = got.extend({ timeout: { request: REQUEST_TIMEOUT } }); - export const name = 'stateair'; export async function fetchData (source, cb) { // Generic fetch function const getData = async (url, done) => { try { - const response = await getter(url); + const response = await client(url); return done(null, response.body); } catch (error) { if (error.response && error.response.statusCode === 404) { diff --git a/src/adapters/stockholm.js b/src/adapters/stockholm.js index 9d7a286a..fcce1752 100644 --- a/src/adapters/stockholm.js +++ b/src/adapters/stockholm.js @@ -1,17 +1,16 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { acceptableParameters } from '../lib/utils.js'; import log from '../lib/logger.js'; import { DateTime } from 'luxon'; import { load } from 'cheerio'; -import got from 'got'; +import client from '../lib/requests.js'; export const name = 'stockholm'; -export function fetchData(source, cb) { - got(source.url, { timeout: { request: REQUEST_TIMEOUT } }) +export function fetchData (source, cb) { + client(source.url) .then(response => { try { if (response.statusCode !== 200) { @@ -27,7 +26,7 @@ export function fetchData(source, cb) { cb(null, data); } catch (error) { - log.error(error); + log.debug(error); cb(error, { message: 'Unknown adapter error.' }, null); } }); @@ -217,7 +216,7 @@ const formatData = function (result) { if (!isNaN(base.value) && base.coordinates) { measurements.push(base); } else { - log.warn(`Unable to load data for ${base.location}`); + log.debug(`Unable to load data for ${base.location}`); } } } diff --git a/src/adapters/tuzlanski.js b/src/adapters/tuzlanski.js index 3b3fc956..5d22997a 100644 --- a/src/adapters/tuzlanski.js +++ b/src/adapters/tuzlanski.js @@ -1,6 +1,5 @@ 'use strict'; -import { REQUEST_TIMEOUT } from '../lib/constants.js'; import { convertUnits, unifyMeasurementUnits, @@ -11,13 +10,13 @@ import { DateTime } from 'luxon'; import flattenDeep from 'lodash/flattenDeep.js'; import { parallel } from 'async'; import log from '../lib/logger.js'; -import got from 'got'; +import client from '../lib/requests.js'; export const name = 'tuzlanski'; export function fetchData(source, cb) { // Load initial page to get active stations - got(source.url, { timeout: { request: REQUEST_TIMEOUT } }) + client(source.url) .then(response => { const body = response.body; const $ = load(body); @@ -48,7 +47,7 @@ export function fetchData(source, cb) { const handleStation = function (stationUrl) { return function (done) { log.debug(`Fetching data for ${stationUrl}`); - got(stationUrl, { timeout: { request: REQUEST_TIMEOUT } }) + client(stationUrl) .then(response => { const body = response.body; formatData(body, (measurements) => { diff --git a/src/adapters/ust-ist.js b/src/adapters/ust-ist.js index b9bd9adb..83121f48 100644 --- a/src/adapters/ust-ist.js +++ b/src/adapters/ust-ist.js @@ -6,16 +6,15 @@ import { acceptableParameters } from '../lib/utils.js'; import { DateTime } from 'luxon'; -import got from 'got'; - +import client from '../lib/requests.js'; export const name = 'ust-ist'; export async function fetchData (source, cb) { try { - const allDataResponse = await got(source.url); + const allDataResponse = await client(source.url); const allData = JSON.parse(allDataResponse.body); - const allMetaResponse = await got('https://api.ust.is/aq/a/getStations'); + const allMetaResponse = await client('https://api.ust.is/aq/a/getStations'); const allMeta = JSON.parse(allMetaResponse.body); // Generate an array of station IDs there is data for. diff --git a/src/sources/rw.json b/src/sources/rw.json index eb835c77..4ae3059b 100644 --- a/src/sources/rw.json +++ b/src/sources/rw.json @@ -9,6 +9,6 @@ "contacts": [ "info@openaq.org" ], - "active": true + "active": false } ] \ No newline at end of file