Updated adapters to use client, bug and logging fixes (#1067)

* updated adapter to use client * updated adapters to use client --------- Co-authored-by: Gabriel Fosse <[email protected]>
openaq · Nov 2, 2023 · 5eb6c31 · 5eb6c31
1 parent 90c2901
commit 5eb6c31
Show file tree

Hide file tree

Showing 30 changed files with 1,463 additions and 1,431 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Changelog
 
 Changes to adapters will be documented in this file.
+
+## 11/01/2023
+### DELETED
+- au_sa.js (replaced by southaustralia.js)
+
 ## 10/31/2023
 ### Set active: true
 - Mexico Sinaica (Added to deployments)
@@ -10,6 +15,7 @@ Changes to adapters will be documented in this file.
 ### Set active: false
 - Mexico Sinaica
 - Turkiye (Down)
+- Rwanda (Needs work, down?)
 
 ## 08/23/2023
 ### Set active: false

diff --git a/src/adapters/acumar.js b/src/adapters/acumar.js
@@ -1,9 +1,9 @@
 'use strict';
 
-import got from 'got';
+import client from '../lib/requests.js';
+import log from '../lib/logger.js';
 import { load } from 'cheerio';
 import { DateTime } from 'luxon';
-import log from '../lib/logger.js';
 
 const stations = [
   {
@@ -25,11 +25,11 @@ let offset;
 export const name = 'acumar';
 
 export async function fetchData (source, cb) {
-	try {
-		if (source.datetime) {
-			log.debug(`Fetching data with ${source.datetime}`);
-			const dateLuxon = source.datetime.toFormat('dd/MM/yy');
-			const hourLuxon = source.datetime.toFormat('HH');
+  try {
+    if (source.datetime) {
+      log.debug(`Fetching data with ${source.datetime}`);
+      const dateLuxon = source.datetime.toFormat('dd/MM/yy');
+      const hourLuxon = source.datetime.toFormat('HH');
 
       const results = await Promise.all(
         stations.map((station) =>
@@ -64,7 +64,7 @@ export async function fetchData (source, cb) {
   }
 }
 
-async function getPollutionData (
+async function getPollutionData(
   station,
   dateLuxon,
   hourLuxon,
@@ -83,16 +83,7 @@ async function getPollutionData (
   let results = [];
 
   try {
-    const response = await got(station.url, {
-      timeout: {
-        request: 5000,
-        connect: 1000,
-        secureConnect: 1000,
-        socket: 5000,
-        response: 5000,
-        send: 1000,
-      },
-    });
+    const response = await client(station.url);
     const $ = load(response.body);
 
     if (dateLuxon && hourLuxon) {

diff --git a/src/adapters/adairquality.js b/src/adapters/adairquality.js
@@ -4,8 +4,7 @@
  */
 'use strict';
 
-import { REQUEST_TIMEOUT } from '../lib/constants.js';
-import got from 'got';
+import client from '../lib/requests.js';
 import { DateTime } from 'luxon';
 import { parallel } from 'async';
 import flatMap from 'lodash/flatMap.js';
@@ -204,11 +203,7 @@ const stations = [
 export function fetchData(source, cb) {
   const requests = stations.map((station) => {
     return (done) => {
-      got(`${source.url}${station.slug}`, {
-        timeout: {
-          request: REQUEST_TIMEOUT
-        },
-      })
+      client(`${source.url}${station.slug}`)
         .then((response) => {
           if (response.statusCode !== 200) {
             return done({

diff --git a/src/adapters/au_act.js b/src/adapters/au_act.js
@@ -1,68 +1,73 @@
 'use strict';
 
-import { REQUEST_TIMEOUT } from '../lib/constants.js';
+import client from '../lib/requests.js';
+import log from '../lib/logger.js';
 import cloneDeep from 'lodash/cloneDeep.js';
 import flatten from 'lodash/flatten.js';
-import { DateTime } from 'luxon'
-
-import got from 'got';
+import { DateTime } from 'luxon';
 
 export const name = 'au_act';
 
-export function fetchData (source, cb) {
-  const timeAgo = DateTime.now().setZone('Australia/Sydney').minus({ days: 1 }).toFormat('yyyy-LL-dd\'T\'HH:mm:ss');
+export function fetchData(source, cb) {
+  const timeAgo = DateTime.now()
+    .setZone('Australia/Sydney')
+    .minus({ days: 1 })
+    .toFormat("yyyy-LL-dd'T'HH:mm:ss");
 
-  got(source.url, {
+  client(source.url, {
     searchParams: {
-      '$query': `select *, :id where (\`datetime\` > '${timeAgo}') order by \`datetime\` desc limit 1000`
+      $query: `select *, :id where (\`datetime\` > '${timeAgo}') order by \`datetime\` desc limit 1000`,
     },
-    timeout: {
-      request: REQUEST_TIMEOUT
-    }
-  }).then(res => {
-    const body = res.body;
-
-    try {
-      const data = formatData(JSON.parse(body), source);
-      if (data === undefined) {
-        return cb({message: 'Failure to parse data.'});
+  })
+    .then((res) => {
+      const body = res.body;
+
+      try {
+        const data = formatData(JSON.parse(body), source);
+        if (data === undefined) {
+          return cb({ message: 'Failure to parse data.' });
+        }
+        cb(null, data);
+      } catch (e) {
+        return cb({ message: 'Unknown adapter error.' });
       }
-      cb(null, data);
-    } catch (e) {
-      return cb({message: 'Unknown adapter error.'});
-    }
-  }).catch(err => {
-    console.error('Error:', err.message, err.response && err.response.body);
-    return cb({message: 'Failure to load data url.'});
-  });
-
+    })
+    .catch((err) => {
+      log.error(
+        'Error:',
+        err.message,
+        err.response && err.response.body
+      );
+      return cb({ message: 'Failure to load data url.' });
+    });
 }
 
-
 const formatData = function (data, source) {
   const parseDate = function (string) {
-    const date = DateTime.fromISO(string, { zone: 'Australia/Sydney' });
+    const date = DateTime.fromISO(string, {
+      zone: 'Australia/Sydney',
+    });
     return {
       utc: date.toUTC().toFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"),
-      local: date.toFormat("yyyy-MM-dd'T'HH:mm:ssZZ")
+      local: date.toFormat("yyyy-MM-dd'T'HH:mm:ssZZ"),
     };
   };
 
   // mapping of types from source data into OpenAQ format
   const types = {
-    'no2': 'no2',
-    'o3_1hr': 'o3',
-    'co': 'co',
-    'pm10': 'pm10',
-    'pm2_5': 'pm25'
+    no2: 'no2',
+    o3_1hr: 'o3',
+    co: 'co',
+    pm10: 'pm10',
+    pm2_5: 'pm25',
   };
 
   const units = {
-    'no2': 'ppm',
-    'o3': 'ppm',
-    'co': 'ppm',
-    'pm10': 'µg/m³',
-    'pm25': 'µg/m³'
+    no2: 'ppm',
+    o3: 'ppm',
+    co: 'ppm',
+    pm10: 'µg/m³',
+    pm25: 'µg/m³',
   };
 
   const measurements = [];
@@ -79,13 +84,15 @@ const formatData = function (data, source) {
       mobile: false,
       coordinates: {
         latitude: parseFloat(row.gps.latitude),
-        longitude: parseFloat(row.gps.longitude)
+        longitude: parseFloat(row.gps.longitude),
       },
-      attribution: [{
-        name: 'Health Protection Service, ACT Government',
-        url: 'https://www.data.act.gov.au/Environment/Air-Quality-Monitoring-Data/94a5-zqnn'
-      }],
-      averagingPeriod: {'value': 1, 'unit': 'hours'}
+      attribution: [
+        {
+          name: 'Health Protection Service, ACT Government',
+          url: 'https://www.data.act.gov.au/Environment/Air-Quality-Monitoring-Data/94a5-zqnn',
+        },
+      ],
+      averagingPeriod: { value: 1, unit: 'hours' },
     };
 
     Object.keys(types).forEach(function (type) {
@@ -103,6 +110,6 @@ const formatData = function (data, source) {
 
   return {
     name: 'unused',
-    measurements: flatten(measurements)
+    measurements: flatten(measurements),
   };
 };