diff --git a/CHANGELOG.md b/CHANGELOG.md index b659cc15..5d88ec7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,26 @@ Changes to adapters will be documented in this file. -## 11/26/2024 +## 02/14/2024 +### Removed deprecated adapters +- agaar_mn.js +- airnow-ftp.js +- eea.js +## 02/13/2024 +### Removed unused files +- docker-compose.yml +- index.adapter.sh +- index.sh +- accra.js +- arpae-locations.js +- arpae-parameters.js +- arpalazio.js +- defra-locations.js +- israel.js +- moscow-locations.js +- ards-bay.js + +## 11/26/2023 ### Removed eslint - Removed eslint from package.json - Removed .eslintrc diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 716d6f69..00000000 --- a/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -FROM ubuntu:14.04 - -# Replace shell with bash so we can source files -RUN rm /bin/sh && ln -s /bin/bash /bin/sh - -# Set debconf to run non-interactively -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - -# Install base dependencies -RUN apt-get update && apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - build-essential \ - ca-certificates \ - curl \ - git \ - libssl-dev \ - python \ - rsync \ - && rm -rf /var/lib/apt/lists/* - -# Install nvm with node and npm -# http://stackoverflow.com/questions/25899912/install-nvm-in-docker -ENV NVM_DIR /usr/local/nvm -ENV NODE_VERSION 8 -RUN mkdir -p ${NVM_DIR} \ - && curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash \ - && source $NVM_DIR/nvm.sh \ - && nvm install $NODE_VERSION \ - && nvm alias default $NODE_VERSION \ - && nvm use default -ENV PATH $NVM_BIN:$PATH - -# Go ahead and install nodemon for convenience while developing -RUN source $NVM_DIR/nvm.sh - -########################### -# App-specific stuff - -# mongo uses kerberos -RUN apt-get update && apt-get install -y libkrb5-dev - -# install jq for json processing -RUN apt-get -y install jq - -# Install NPM dependencies. Do this first so that if package.json hasn't -# changed we don't have to re-run npm install during `docker build` -COPY package.json /app/package.json -COPY package-lock.json /app/package-lock.json -WORKDIR /app -RUN source $NVM_DIR/nvm.sh; npm install -# Copy the app -COPY ["index.js", ".eslintrc", ".eslintignore", ".babelrc", "knexfile.js", "/app/"] -COPY ["fetch.js", "/app/"] -COPY lib /app/lib/ -COPY certs /app/certs/ -COPY test /app/test/ -COPY sources /app/sources/ -COPY adapters /app/adapters/ -COPY migrations /app/migrations/ -COPY index.sh /app/ -COPY index.adapter.sh /app/ - -############################# -# entrypoint -# -RUN source $NVM_DIR/nvm.sh -ADD .build_scripts/entrypoint.sh / -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md index 7580335c..7f713feb 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This is the main data ingest pipeline for the [OpenAQ](https://openaq.org) proje Starting with `index.js`, there is an ingest mechanism to gather global air quality measurements from a variety of sources. This is currently run every 10 minutes and saves all unique measurements to a database. -[openaq-api](https://github.com/openaq/openaq-api) powers the API and more information on the data format can be found in [openaq-data-format](https://github.com/openaq/openaq-data-format). +[openaq-api-v2](https://github.com/openaq/openaq-api-v2) powers the API and more information on the data format can be found in [openaq-data-format](https://github.com/openaq/openaq-data-format). For more info see the [OpenAQ-Fetch documentation index](docs/index.md). @@ -24,18 +24,11 @@ Now you can get started with: `node index.js --help` -For a full development quick start (with database setup etc.), please see the [dev-quick-start doc](docs/dev-quick-start.md). - For production deployment, you will need to have certain environment variables set as in the table below - | Name | Description | Default | |---|---|---| -| SENDGRID_PASSWORD | Email service password | not set | -| SENDGRID_USERNAME | Email service username | not set | | API_URL | URL of openaq-api | http://localhost:3004/v1/webhooks | | WEBHOOK_KEY | Secret key to interact with openaq-api | '123' | -| AIRNOW_FTP_USER | User for AirNow FTP | not set | -| AIRNOW_FTP_PASSWORD | Password for AirNow FTP | not set | | EEA_TOKEN | API token for EEA API | not set | | DATA_GOV_IN_TOKEN | API token for data.gov.in | not set | | EPA_VICTORIA_TOKEN | API token for portal.api.epa.vic.gov.au | not set | diff --git a/data_scripts/accra.js b/data_scripts/accra.js deleted file mode 100644 index 131deb72..00000000 --- a/data_scripts/accra.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Script to convert data dump from Accra to proper format for inclusion - * in the platform. Output of this script needs to be uploaded manually - * to data bucket. - */ - -'use strict'; - -const parse = require('csv-parse'); -const fs = require('fs'); -const moment = require('moment-timezone'); - -const input = fs.readFileSync('accra.csv'); -let records = ''; -parse(input, (err, output) => { - if (err) { - return console.error(err); - } - - output.forEach((m) => { - // Create expected format for ndjson files - let base = { - date: { - utc: moment.utc(moment.tz(m[9], 'MM/DD/YY H:mm', 'Africa/Accra')).toDate(), - local: moment.tz(m[9], 'MM/DD/YY H:mm', 'Africa/Accra').format('YYYY-MM-DDTHH:mm:ssZ') - }, - parameter: m[0].toLowerCase().replace('.', ''), - location: `${m[7]} - ${m[6]}`, - value: Number(m[2]), - unit: 'µg/m³', - city: 'Accra', - attribution: [{ - name: 'Dr. Raphael E. Arku and Colleagues', - url: m[13] - }], - averagingPeriod: { - value: Number(m[3]), - unit: 'hours' - }, - coordinates: { - latitude: Number(m[11]), - longitude: Number(m[12]) - }, - country: 'GH', - sourceName: 'Dr. Raphael E. Arku and Colleagues', - sourceType: 'research', - mobile: false - }; - - records += `${JSON.stringify(base)}\n`; - }); - - fs.writeFileSync('accra.ndjson', records); -}); diff --git a/data_scripts/arpae-locations.js b/data_scripts/arpae-locations.js deleted file mode 100644 index eeaaa075..00000000 --- a/data_scripts/arpae-locations.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict'; - -/* - A script for generating a locations mapping - object from the ARPAE parameters source. - - */ - -import { default as parse } from 'csv-parse/lib/sync'; -import { uniqBy } from 'lodash'; -var request = require('request'); - -const csvUrl = 'https://docs.google.com/spreadsheets/d/1GlY3Pu9GDpLDk8Spl9yV1wjCRPvOI1m7BFxumfcuGcE/export?format=csv'; - -request(csvUrl, (err, res, data) => { - if (err) { - console.log(err); - } - - const parsed = parse(data, { columns: true }); - let locations = {}; - uniqBy(parsed, 'Cod_staz').forEach((o) => { - locations[o.Cod_staz] = { - station: o.Stazione, - comune: o.COMUNE, - coordinates: { - latitude: Number(o.Lat), - longitude: Number(o.Lon) - } - }; - }); - console.log(locations); -}); diff --git a/data_scripts/arpae-parameters.js b/data_scripts/arpae-parameters.js deleted file mode 100644 index b12307c7..00000000 --- a/data_scripts/arpae-parameters.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -/* - A script for generating a parameter mapping - object from the ARPAE parameters source. - - */ - -import { default as parse } from 'csv-parse/lib/sync'; -var request = require('request'); - -const csvUrl = 'https://docs.google.com/spreadsheets/d/13QcqldwA3EQ_4E17Hqggd2ZcMgUA5UwACttAWEkaU28/export?format=csv'; - -request(csvUrl, (err, res, data) => { - if (err) { - console.err(err); - } - const parsed = parse(data, { columns: true }); - let parameters = {}; - parsed.forEach((el) => { - parameters[el.IdParametro] = { - parameter: el.PARAMETRO.split('(')[0].toLowerCase().trim().replace('.', ''), - unit: el.UM - }; - }); - console.log(parameters); -}); diff --git a/data_scripts/arpalazio.js b/data_scripts/arpalazio.js deleted file mode 100644 index 386bbd05..00000000 --- a/data_scripts/arpalazio.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict'; - -/* - Get ARPALAZIO station coordinates - from project Calicantus source - - Run from project root with: - node run-scripts.js ./data_scripts/arpalazio.js -*/ - -import {default as parse} from 'csv-parse/lib/sync'; -const request = require('request'); - -const metadataURL = 'https://raw.githubusercontent.com/jobonaf/calicantus/master/data/sites-info/metadata.ARPA-Lazio.csv'; - -request(metadataURL, (err, response, body) => { - if (err) { - return console.error(err); - } - - let results = {}; - parse(body, {columns: true}).forEach((rowObject) => { - results[rowObject.ID] = { - name: rowObject.NOME, - latitude: Number(rowObject.LAT), - longitude: Number(rowObject.LON) - }; - }); - console.log(results); -}); diff --git a/data_scripts/defra-locations.js b/data_scripts/defra-locations.js deleted file mode 100644 index a9116036..00000000 --- a/data_scripts/defra-locations.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict'; - -import cheerio from 'cheerio'; -import { parallelLimit } from 'async'; -var request = require('request'); - -function handleLink (url) { - return function (done) { - request(url, (err, res, body) => { - if (err) { - done(err); - } - let $ = cheerio.load(body); - let city = $($('#tab_info').find('p').get(5)).html(); - city = city.split('')[1].trim(); - let coords = $($('#tab_info').find('p').get(8)).html(); - coords = coords.split('')[1].trim(); - coords = { - latitude: Number(coords.split(',')[0]), - longitude: Number(coords.split(',')[1]) - }; - - done(null, {city: city, coordinates: coords}); - }); - }; -} - -request('http://uk-air.defra.gov.uk/latest/currentlevels', (err, res, body) => { - if (err) { - return console.error(err); - } - - // Get links from main page - console.info('Grabbing main page to get locations.'); - let $ = cheerio.load(body); - let links = {}; - $('.current_levels_table').each((i, e) => { - $('tr', $(e)).each((i, e) => { - let link = $($(e).find('td a')).attr('href'); - let name = $($(e).find('td a')).html(); - if (link) { - links[name] = handleLink(link.replace('../', 'http://uk-air.defra.gov.uk/')); - } - }); - }); - - // Get info from each link - console.info(`Grabbing data for ${Object.keys(links).length} locations.`); - // links = {'Auchencorth Moss': links['Auchencorth Moss']}; - parallelLimit(links, 5, (err, results) => { - console.info('Grabbing all data completed!'); - if (err) { - return console.error(err); - } - - console.info(results); - }); -}); diff --git a/data_scripts/israel.js b/data_scripts/israel.js deleted file mode 100644 index 442267cf..00000000 --- a/data_scripts/israel.js +++ /dev/null @@ -1,108 +0,0 @@ -'use strict'; - -/* - script to get coordinates for israel stations. -*/ - -const headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Content-Type': 'text/html; charset=utf-8' -}; - -var async = require('async'); -var request = require('request'); -var cheerio = require('cheerio'); -var _ = require('lodash'); - -let baseUrl = 'http://www.svivaaqm.net/'; - -request.get({ - headers: headers, - url: 'http://www.svivaaqm.net/MenuSite.aspx' -}, (err, res, body) => { - if (err || res.statusCode !== 200) { - return; - } - let tasks = []; - let regionLinks = body.match(/DynamicTable.aspx\?G_ID=(\d*)/g); - while (regionLinks.length > 0) { - let link = baseUrl + regionLinks.pop(); - tasks.push(getStations(baseUrl, link)); - } - async.parallel(tasks, (err, results) => { - if (err) { - console.log(err); - } - results = _.flattenDeep(results); - results = Object.assign({}, ...results); - console.log(results); - }); -}); - -let getStations = function (baseUrl, link) { - return function (done) { - request.get({ - headers: headers, - url: link - }, (err, res, body) => { - if (err || res.statusCode !== 200) { - return done(null, []); - } - let tasks = []; - const stationURLs = body.match(/StationInfo5?.aspx\?ST_ID=(\d*)/g); - while (stationURLs.length > 0) { - const stationURL = stationURLs.pop(); - const stationID = stationURL.match(/StationInfo5?.aspx\?ST_ID=(\d*)/)[1]; - const link = `${baseUrl}${stationURL}`; - tasks.push(getCoordinates(link, stationID)); - } - async.parallel(tasks, (err, results) => { - if (err) { done(null, []); } - results = _.flattenDeep(results); - return done(err, results); - }); - }); - }; -}; - -let getCoordinates = function (link, stationID) { - return function (done) { - request.get({ - url: link, - headers: headers - }, (err, res, body) => { - if (err) { - return done(null, []); - } - - let $ = cheerio.load(body); - const stationData = $('#stationInfoDiv > table').children().toArray().map((tr) => { - let toReturn = tr.children.find((td) => { - return td.attribs && td.attribs.class === 'key' && _.includes(['שם תחנה', 'קו אורך', 'קו רוחב'], td.children[0].data); - }); - if (toReturn !== undefined) { - return $(toReturn.next).text(); - } - }).filter((data) => { return data !== undefined; }); - if (stationData.length === 3) { - if (!(_.includes([stationData[1], stationData[2]], undefined)) || !(_.includes([stationData[1], stationData[2]], 0))) { - if (Number(stationData[2]) === 0 && Number(stationData[1]) === 0) { - return done(null, []); - } - const stationObj = {}; - stationObj[stationID] = { - coordinates: { - latitude: Number(stationData[2]), - longitude: Number(stationData[1]) - - } - }; - return done(null, [stationObj]); - } - } else { - return done(null, []); - } - }); - }; -}; diff --git a/data_scripts/moscow-locations.js b/data_scripts/moscow-locations.js deleted file mode 100644 index e8248479..00000000 --- a/data_scripts/moscow-locations.js +++ /dev/null @@ -1,43 +0,0 @@ -'use strict'; - -/* - A script for getting names and coordinates - from a KML file at Google Maps. - - It uses the Moscow map, - but could be adapted for other uses. -*/ - -var request = require('request'); -var cheerio = require('cheerio'); - -let midValue = '1ve9zrTQE0ONlSkk6tC5qyglpoME'; -let kmlURL = 'http://www.google.com/maps/d/kml?forcekml=1&mid=' + midValue; - -request(kmlURL, (err, res, body) => { - if (err) { - return console.error(err); - } - - let $ = cheerio.load(body, { xmlMode: true }); - let results = {}; - - $('Placemark').each(function (index, element) { - let points = $(this).find('Point coordinates').text(); - points = points.trim(); - - let description = $(this).find('description').text(); - let stationRegexp = /air-today\/station\/(\w*)\//; - let stationId = stationRegexp.exec(description)[1]; - - results[stationId] = { - coordinates: { - longitude: Number(points.split(',')[0]), - latitude: Number(points.split(',')[1]) - } - }; - }); - - console.log(results); - // console.log(JSON.stringify(results, null, '\t')); -}); diff --git a/data_scripts/richards-bay.js b/data_scripts/richards-bay.js deleted file mode 100644 index 6fa1e0d3..00000000 --- a/data_scripts/richards-bay.js +++ /dev/null @@ -1,67 +0,0 @@ -'use strict'; - -/* - A script to collect coordinates - for Richards Bay stations -*/ - -var async = require('async'); -var request = require('request'); -var cheerio = require('cheerio'); - -let stationIds = [1, 2, 3, 4, 5, 6, 9, 11, 13, 15, 16]; -let baseUrl = 'http://live.rbcaa.org.za/StationDetails.aspx?ST_ID='; - -let findTextGetNext = function ($, text) { - let textNode = $('.TitleLabel').filter(function (i, el) { - return $(this).text() === text; - }); - return $(textNode).parent().next().text(); -}; - -let getCoordinates = function (stationId) { - return function (done) { - let url = baseUrl + stationId; - request(url, (err, res, body) => { - if (err) { - return console.error(err); - } - - let $ = cheerio.load(body); - let stationName = findTextGetNext($, 'Station Name'); - let longitude = findTextGetNext($, 'Longitude'); - let latitude = findTextGetNext($, 'Latitude'); - if (longitude && latitude) { - return done(null, [stationName, longitude, latitude]); - } else { - return done(null, []); - } - }); - }; -}; - -let tasks = []; -while (stationIds.length > 0) { - tasks.push(getCoordinates(stationIds.pop())); -} - -async.parallel(tasks, function (err, results) { - if (err) { - console.error(err); - } - - let locations = {}; - while (results.length > 0) { - let result = results.pop(); - if (!result[0]) { - continue; - } - locations[result[0]] = { - coordinates: { - longitude: Number(result[1]), - latitude: Number(result[2]) - } - }; - } - console.log(locations); -}); diff --git a/data_scripts/update-db-sources.js b/data_scripts/update-db-sources.js deleted file mode 100644 index 760262f3..00000000 --- a/data_scripts/update-db-sources.js +++ /dev/null @@ -1,97 +0,0 @@ -import knex from 'knex'; -import config from '../knexfile.js'; -import sources from '../src/sources/index.cjs'; - -const db = knex(config); - -console.log(`There are currently ${sources.length} sources locally`); - -// check to see what keys are in the sources -// console.log( -// sources.map(data => Object.keys(data)) -// .flat() -// .reduce((acc, curr) => { -// acc[curr] = -~acc[curr]; -// return acc; -// }, {}) -// ); - -// reshape to match what is in the database -const sources_db = sources - .filter(d => d.name!='test_adapter') - .map(data => { - let interval; - switch (data.resolution) { - case '10 min': - interval = 10; - break; - case '15 min': - interval = 15; - break; - default: - interval = 60; - } - data.interval = interval; - return { - label: data.name, - description: data.description, - source_name: data.name, - export_prefix: data.adapter, - adapter: data.adapter, - active: data.active, - metadata: data, - }; -}); - - -(async () => { - - if(!(await db.schema.hasTable('providers_temp'))) { - await db.schema.createTable('providers_temp', table => { - table.string('label'); - table.string('description'); - table.string('source_name'); - table.string('export_prefix'); - table.string('adapter'); - table.boolean('active'); - table.json('metadata'); - }); - }; - - let upserts = await db('providers_temp') - .insert(sources_db) - .returning("*"); - - await db.raw( - `INSERT INTO adapters (name, description, handler) - SELECT adapter - , 'Automatically added from fetcher data' - , 'openaq-fetch' - FROM providers_temp - ON CONFLICT DO NOTHING` - ); - - await db.raw( - `UPDATE providers - SET metadata = p.metadata - , is_active = p.active - FROM providers_temp p - WHERE providers.source_name = p.source_name` - ); - - await db.raw( - `UPDATE providers - SET adapters_id = a.adapters_id - FROM adapters a - WHERE providers.metadata->>'adapter' = a.name` - ); - - await db.schema.dropTable('providers_temp'); - - console.log(`Upserted ${upserts.length} providers to the db`); - - // example showing how to just query the table - // let rows = await db('sources_tmp'); - - process.exit(0); -})(); diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c832e920..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,5 +0,0 @@ -fetch: - build: . - volumes: - - .:/local - env_file: ./local.env diff --git a/docs/adapter.md b/docs/adapter.md index 449cd1fb..815ae977 100644 --- a/docs/adapter.md +++ b/docs/adapter.md @@ -124,7 +124,7 @@ These are available internally via `utils.acceptableParameters` All other values will be ignored. Date should be provided as UTC and local, and should be the end of the averaging period. -[moment.js](http://momentjs.com/) can be used to handle time zone conversions. List of valid zone names at +[Luxon](https://www.npmjs.com/package/luxon) can be used to handle time zone conversions. List of valid zone names at https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. ### Benchmark diff --git a/docs/dev-quick-start.md b/docs/dev-quick-start.md deleted file mode 100644 index 8fdcb84a..00000000 --- a/docs/dev-quick-start.md +++ /dev/null @@ -1,35 +0,0 @@ -How to quick start development? -================================= - -What you'll need ------------------- - -* Windows, Mac, Linux system -* Approx `200MB` of free disk space -* At least `1.8GB` free memory -* `2` core CPU is recommended -* `node.js >= 8.6` -* Either of: - * `docker` or - * `postgresql >= 10` with `postgis >= 2` extensions. - -Environment -------------- - -Follow these steps to create a full local development environment (this is probably the quickest way to start from scratch): - -1. Install fairly recent [node.js](https://nodejs.org/en/download/) [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [Docker](https://docs.docker.com/install/). -2. Start PostgreSQL with PostGIS `docker run --env POSTGRES_PASSWD=pgpassword --name postgis -dit --restart unless-stopped -p 5432:5432 geographica/postgis:quick_quail` -3. Pull the data from repo `git clone git@github.com:openaq/openaq-fetch.git` -4. Install dependencies `cd openaq-fetch && npm install` -5. Create a user and db in postgres in `docker exec -ti postgis psql -U postgres` -```sql -CREATE ROLE openaq WITH LOGIN PASSWORD 'openaq-pass'; -CREATE DATABASE "openaq-local" OWNER openaq; -\connect "openaq-local" -CREATE EXTENSION postgis; -\quit -``` -6. Create a local knexfile `cp knexfile.js knexfile-local.js && editor knexfile-local.js` - -And you're set. diff --git a/index.adapter.sh b/index.adapter.sh deleted file mode 100755 index f5f9a84e..00000000 --- a/index.adapter.sh +++ /dev/null @@ -1,11 +0,0 @@ -# !/usr/bin/env bash -# Script to fetch data adapter by adapter -adapters=adapters.list -for file in ./sources/*.json; do - jq '.[] | select(.active == true ).name' $file | sed -r 's/^"|"$//g' >>$adapters -done - -while read adapter; do - echo "================================> $adapter <================================" - node index.js --source "$adapter" -b -done <$adapters diff --git a/index.sh b/index.sh deleted file mode 100755 index ede93f48..00000000 --- a/index.sh +++ /dev/null @@ -1,31 +0,0 @@ -# !/usr/bin/env bash -set -x - -############################################################################### -#### Main fetch data process -############################################################################### - -time timeout 10m npm start - -############################################################################### -#### Adapters that consume lot memory and need to be run one by one -#### The follow adapters are currently disabled, in "sources/", but actually we are requesting data. We disable because -#### those adapter request huge amount of data, that sometimes made failing the fetch process and we are running them one by one. -############################################################################### - -# # ARPALAZIO adapter requests ~7.5k items -# time timeout 3m node index.js --source="ARPALAZIO" - -# # 'London Air Quality Network' adapter requests ~7.8k items -# time timeout 3m node index.js --source="London Air Quality Network" - -# # GIOS adapter requests ~2k items -# time timeout 3m node index.js --source="GIOS" - -############################################################################### -#### Adapters that are running in fargate -#### Reasons: -#### - Because the site response very slow, took more that 5 minutes to complete the fetch process -############################################################################### -# caaqm adapter requests ~1.5k items -# time timeout 3m node index.js --source="caaqm" diff --git a/knexfile.js b/knexfile.js deleted file mode 100644 index fe0d1bc9..00000000 --- a/knexfile.js +++ /dev/null @@ -1,28 +0,0 @@ -import _env from './src/lib/env.js'; - -const { - psqlHost, - psqlPort, - psqlUser, - psqlPassword, - psqlDatabase, - psqlPoolMin, - psqlPoolMax -} = _env(); - -export default { - client: 'pg', - connection: { - host: psqlHost, - port: psqlPort, - user: psqlUser, - password: psqlPassword, - database: psqlDatabase, - //ssl: { rejectUnauthorized: false }, - }, - pool: { - min: psqlPoolMin, - max: psqlPoolMax - }, - acquireConnectionTimeout: 600000 -}; diff --git a/package-lock.json b/package-lock.json index 0ac9b1e8..bf78dfb9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,6 @@ "ftp": "^0.3.10", "got": "^12.6.0", "jsonschema": "^1.2.0", - "knex": "^3.1.0", "lodash": "^4.17.10", "luxon": "^3.0.4", "moment-timezone": "^0.5.21", @@ -4422,6 +4421,7 @@ }, "node_modules/debug": { "version": "4.3.4", + "dev": true, "license": "MIT", "dependencies": { "ms": "2.1.2" @@ -4756,13 +4756,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/esm": { - "version": "3.2.25", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/esprima": { "version": "4.0.1", "dev": true, @@ -5018,13 +5011,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-package-type": { - "version": "0.1.0", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/get-stream": { "version": "6.0.1", "license": "MIT", @@ -5035,10 +5021,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/getopts": { - "version": "2.3.0", - "license": "MIT" - }, "node_modules/getpass": { "version": "0.1.7", "license": "MIT", @@ -5359,13 +5341,6 @@ "dev": true, "license": "ISC" }, - "node_modules/interpret": { - "version": "2.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/is-arguments": { "version": "1.1.1", "license": "MIT", @@ -5408,6 +5383,7 @@ }, "node_modules/is-core-module": { "version": "2.13.1", + "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.0" @@ -6012,6 +5988,7 @@ }, "node_modules/ms": { "version": "2.1.2", + "dev": true, "license": "MIT" }, "node_modules/multimatch": { @@ -6220,6 +6197,7 @@ }, "node_modules/path-parse": { "version": "1.0.7", + "dev": true, "license": "MIT" }, "node_modules/path-type": { @@ -6419,16 +6397,6 @@ "node": ">=8.10.0" } }, - "node_modules/rechoir": { - "version": "0.8.0", - "license": "MIT", - "dependencies": { - "resolve": "^1.20.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, "node_modules/regenerate": { "version": "1.4.2", "dev": true, @@ -6564,6 +6532,7 @@ }, "node_modules/resolve": { "version": "1.22.8", + "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.13.0", @@ -6595,6 +6564,7 @@ }, "node_modules/resolve-from": { "version": "5.0.0", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -6864,6 +6834,7 @@ }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6872,24 +6843,10 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/tarn": { - "version": "3.0.2", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/text-hex": { "version": "1.0.0", "license": "MIT" }, - "node_modules/tildify": { - "version": "2.0.0", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/to-fast-properties": { "version": "2.0.0", "dev": true, diff --git a/package.json b/package.json index a30a1a3a..1544492b 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "ftp": "^0.3.10", "got": "^12.6.0", "jsonschema": "^1.2.0", - "knex": "^3.1.0", "lodash": "^4.17.10", "luxon": "^3.0.4", "moment-timezone": "^0.5.21", diff --git a/src/fetch.js b/src/fetch.js index 1a48a5a5..7eb2a8db 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -37,8 +37,6 @@ const { DataStream } = sj; const env = _env(); const { - apiURL, - webhookKey, processTimeout, maxParallelAdapters, strict, @@ -182,8 +180,6 @@ export function handler (event, context) { fetchReport, sourcesArray, env, - apiURL, - webhookKey ) ) ); diff --git a/src/lib/env.js b/src/lib/env.js index b38320c2..ac347d7b 100644 --- a/src/lib/env.js +++ b/src/lib/env.js @@ -9,15 +9,6 @@ const yargs = _yargs(hideBin(process.argv)); * @extends Object * @param {number} logLevel * @param {boolean} logColor - * @param {string} psqlHost - * @param {number} psqlPort - * @param {string} psqlUser - * @param {string} psqlPassword - * @param {string} psqlDatabase - * @param {number} psqlPoolMin - * @param {number} psqlPoolMax - * @param {string} apiURL - * @param {string} webhookKey * @param {number} processTimeout * @param {string} bucketName * @param {number} s3ChunkSize @@ -172,8 +163,6 @@ export default () => { adapter = _env.ADAPTER; } - const apiURL = _env.API_URL || 'http://localhost:3004/v1/webhooks'; // The url to ping on completion - const webhookKey = _env.WEBHOOK_KEY || '123'; // Secret key to auth with API const processTimeout = _env.PROCESS_TIMEOUT || 14 * 60 * 1000; // Kill the process after a certain time in case it hangs const bucketName = _env.AWS_BUCKET_NAME || ''; const doSaveToS3 = _env.SAVE_TO_S3 === 'true' || +_env.SAVE_TO_S3; @@ -181,13 +170,6 @@ export default () => { const maxParallelAdapters = +_env.MAX_PARALLEL_ADAPTERS || 4; const s3ChunkSize = +_env.S3_CHUNK_SIZE || 1048576; - const psqlHost = _env.PSQL_HOST || '127.0.0.1'; - const psqlPort = _env.PSQL_PORT || 5432; - const psqlUser = _env.PSQL_USER || 'openaq'; - const psqlPassword = _env.PSQL_PASSWORD || 'openaq-pass'; - const psqlDatabase = _env.PSQL_DATABASE || 'openaq-local'; - const psqlPoolMin = +_env.PSQL_POOL_MIN || 2; - const psqlPoolMax = +_env.PSQL_POOL_MAX || 20; const suffix = _env.SUFFIX || ''; offset = +(offset || _env.OFFSET); @@ -206,15 +188,6 @@ export default () => { return { logLevel, logColor, - psqlHost, - psqlPort, - psqlUser, - psqlPassword, - psqlDatabase, - psqlPoolMin, - psqlPoolMax, - apiURL, - webhookKey, processTimeout, bucketName, s3ChunkSize, diff --git a/src/lib/notification.js b/src/lib/notification.js index ee0b3e00..badb0238 100644 --- a/src/lib/notification.js +++ b/src/lib/notification.js @@ -49,7 +49,6 @@ export function reportAndRecordFetch (fetchReport, sources, env, apiURL, webhook return acc; }, {}); - const failures = fetchReport.results .filter(r => !r.count); @@ -59,12 +58,15 @@ export function reportAndRecordFetch (fetchReport, sources, env, apiURL, webhook failures.map(r => { log.debug(r); }); - log.info(`Dry run finished with ${successes.length} successes and ${failures.length} failures in ${(fetchReport.timeEnded - fetchReport.timeStarted)/1000} seconds`); + log.info(`Finished with ${successes.length} successes and ${failures.length} failures in ${(fetchReport.timeEnded - fetchReport.timeStarted)/1000} seconds`); if (!env.dryrun) { await publish(fetchReport.results, 'fetcher/success'); + } else { + // for dev purposes + failures.map(r => console.warn(`No results`, r)); + fetchReport.results.map( r => log.debug(`${r.locations} locations from ${r.from} - ${r.to} | Parameters for ${r.sourceName}`, r.parameters)); } - return 0; }; -} +} \ No newline at end of file diff --git a/src/package-lock.json b/src/package-lock.json index 79421b84..b1b46456 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -21,7 +21,6 @@ "ftp": "^0.3.10", "got": "^12.6.0", "jsonschema": "^1.2.0", - "knex": "^1.0.6", "lodash": "^4.17.10", "luxon": "^3.0.4", "moment-timezone": "^0.5.21", diff --git a/src/sources/it.json b/src/sources/it.json index 51c47593..4a139bdb 100644 --- a/src/sources/it.json +++ b/src/sources/it.json @@ -11,7 +11,7 @@ "info@openaq.org" ], "active": true, - "--comment1": "// metadata generated with ../data_scripts/arpalazio.js", + "--comment1": "// metadata generated from https://raw.githubusercontent.com/jobonaf/calicantus/master/data/sites-info/metadata.ARPA-Lazio.csv", "metadata": { "2": { "name": "Preneste", "latitude": 41.886018, "longitude": 12.541614 }, "3": { "name": "Francia", "latitude": 41.947447, "longitude": 12.469588 },