From 5a6c5b3e6b1f9284bbdfd73f822cb217faf060d0 Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Thu, 8 Feb 2024 15:48:10 -0800 Subject: [PATCH 1/9] cleaned up env.js and removed knexfile Dockerfile and associated docs --- Dockerfile | 69 ---------------------- data_scripts/update-db-sources.js | 97 ------------------------------- docs/dev-quick-start.md | 35 ----------- knexfile.js | 28 --------- src/lib/env.js | 21 ------- 5 files changed, 250 deletions(-) delete mode 100644 Dockerfile delete mode 100644 data_scripts/update-db-sources.js delete mode 100644 docs/dev-quick-start.md delete mode 100644 knexfile.js diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 716d6f69..00000000 --- a/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -FROM ubuntu:14.04 - -# Replace shell with bash so we can source files -RUN rm /bin/sh && ln -s /bin/bash /bin/sh - -# Set debconf to run non-interactively -RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections - -# Install base dependencies -RUN apt-get update && apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - build-essential \ - ca-certificates \ - curl \ - git \ - libssl-dev \ - python \ - rsync \ - && rm -rf /var/lib/apt/lists/* - -# Install nvm with node and npm -# http://stackoverflow.com/questions/25899912/install-nvm-in-docker -ENV NVM_DIR /usr/local/nvm -ENV NODE_VERSION 8 -RUN mkdir -p ${NVM_DIR} \ - && curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.11/install.sh | bash \ - && source $NVM_DIR/nvm.sh \ - && nvm install $NODE_VERSION \ - && nvm alias default $NODE_VERSION \ - && nvm use default -ENV PATH $NVM_BIN:$PATH - -# Go ahead and install nodemon for convenience while developing -RUN source $NVM_DIR/nvm.sh - -########################### -# App-specific stuff - -# mongo uses kerberos -RUN apt-get update && apt-get install -y libkrb5-dev - -# install jq for json processing -RUN apt-get -y install jq - -# Install NPM dependencies. Do this first so that if package.json hasn't -# changed we don't have to re-run npm install during `docker build` -COPY package.json /app/package.json -COPY package-lock.json /app/package-lock.json -WORKDIR /app -RUN source $NVM_DIR/nvm.sh; npm install -# Copy the app -COPY ["index.js", ".eslintrc", ".eslintignore", ".babelrc", "knexfile.js", "/app/"] -COPY ["fetch.js", "/app/"] -COPY lib /app/lib/ -COPY certs /app/certs/ -COPY test /app/test/ -COPY sources /app/sources/ -COPY adapters /app/adapters/ -COPY migrations /app/migrations/ -COPY index.sh /app/ -COPY index.adapter.sh /app/ - -############################# -# entrypoint -# -RUN source $NVM_DIR/nvm.sh -ADD .build_scripts/entrypoint.sh / -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/data_scripts/update-db-sources.js b/data_scripts/update-db-sources.js deleted file mode 100644 index 760262f3..00000000 --- a/data_scripts/update-db-sources.js +++ /dev/null @@ -1,97 +0,0 @@ -import knex from 'knex'; -import config from '../knexfile.js'; -import sources from '../src/sources/index.cjs'; - -const db = knex(config); - -console.log(`There are currently ${sources.length} sources locally`); - -// check to see what keys are in the sources -// console.log( -// sources.map(data => Object.keys(data)) -// .flat() -// .reduce((acc, curr) => { -// acc[curr] = -~acc[curr]; -// return acc; -// }, {}) -// ); - -// reshape to match what is in the database -const sources_db = sources - .filter(d => d.name!='test_adapter') - .map(data => { - let interval; - switch (data.resolution) { - case '10 min': - interval = 10; - break; - case '15 min': - interval = 15; - break; - default: - interval = 60; - } - data.interval = interval; - return { - label: data.name, - description: data.description, - source_name: data.name, - export_prefix: data.adapter, - adapter: data.adapter, - active: data.active, - metadata: data, - }; -}); - - -(async () => { - - if(!(await db.schema.hasTable('providers_temp'))) { - await db.schema.createTable('providers_temp', table => { - table.string('label'); - table.string('description'); - table.string('source_name'); - table.string('export_prefix'); - table.string('adapter'); - table.boolean('active'); - table.json('metadata'); - }); - }; - - let upserts = await db('providers_temp') - .insert(sources_db) - .returning("*"); - - await db.raw( - `INSERT INTO adapters (name, description, handler) - SELECT adapter - , 'Automatically added from fetcher data' - , 'openaq-fetch' - FROM providers_temp - ON CONFLICT DO NOTHING` - ); - - await db.raw( - `UPDATE providers - SET metadata = p.metadata - , is_active = p.active - FROM providers_temp p - WHERE providers.source_name = p.source_name` - ); - - await db.raw( - `UPDATE providers - SET adapters_id = a.adapters_id - FROM adapters a - WHERE providers.metadata->>'adapter' = a.name` - ); - - await db.schema.dropTable('providers_temp'); - - console.log(`Upserted ${upserts.length} providers to the db`); - - // example showing how to just query the table - // let rows = await db('sources_tmp'); - - process.exit(0); -})(); diff --git a/docs/dev-quick-start.md b/docs/dev-quick-start.md deleted file mode 100644 index 8fdcb84a..00000000 --- a/docs/dev-quick-start.md +++ /dev/null @@ -1,35 +0,0 @@ -How to quick start development? -================================= - -What you'll need ------------------- - -* Windows, Mac, Linux system -* Approx `200MB` of free disk space -* At least `1.8GB` free memory -* `2` core CPU is recommended -* `node.js >= 8.6` -* Either of: - * `docker` or - * `postgresql >= 10` with `postgis >= 2` extensions. - -Environment -------------- - -Follow these steps to create a full local development environment (this is probably the quickest way to start from scratch): - -1. Install fairly recent [node.js](https://nodejs.org/en/download/) [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [Docker](https://docs.docker.com/install/). -2. Start PostgreSQL with PostGIS `docker run --env POSTGRES_PASSWD=pgpassword --name postgis -dit --restart unless-stopped -p 5432:5432 geographica/postgis:quick_quail` -3. Pull the data from repo `git clone git@github.com:openaq/openaq-fetch.git` -4. Install dependencies `cd openaq-fetch && npm install` -5. Create a user and db in postgres in `docker exec -ti postgis psql -U postgres` -```sql -CREATE ROLE openaq WITH LOGIN PASSWORD 'openaq-pass'; -CREATE DATABASE "openaq-local" OWNER openaq; -\connect "openaq-local" -CREATE EXTENSION postgis; -\quit -``` -6. Create a local knexfile `cp knexfile.js knexfile-local.js && editor knexfile-local.js` - -And you're set. diff --git a/knexfile.js b/knexfile.js deleted file mode 100644 index fe0d1bc9..00000000 --- a/knexfile.js +++ /dev/null @@ -1,28 +0,0 @@ -import _env from './src/lib/env.js'; - -const { - psqlHost, - psqlPort, - psqlUser, - psqlPassword, - psqlDatabase, - psqlPoolMin, - psqlPoolMax -} = _env(); - -export default { - client: 'pg', - connection: { - host: psqlHost, - port: psqlPort, - user: psqlUser, - password: psqlPassword, - database: psqlDatabase, - //ssl: { rejectUnauthorized: false }, - }, - pool: { - min: psqlPoolMin, - max: psqlPoolMax - }, - acquireConnectionTimeout: 600000 -}; diff --git a/src/lib/env.js b/src/lib/env.js index 158e404f..ba191c67 100644 --- a/src/lib/env.js +++ b/src/lib/env.js @@ -9,13 +9,6 @@ const yargs = _yargs(hideBin(process.argv)); * @extends Object * @param {number} logLevel * @param {boolean} logColor - * @param {string} psqlHost - * @param {number} psqlPort - * @param {string} psqlUser - * @param {string} psqlPassword - * @param {string} psqlDatabase - * @param {number} psqlPoolMin - * @param {number} psqlPoolMax * @param {string} apiURL * @param {string} webhookKey * @param {number} processTimeout @@ -181,13 +174,6 @@ export default () => { const maxParallelAdapters = +_env.MAX_PARALLEL_ADAPTERS || 1024; const s3ChunkSize = +_env.S3_CHUNK_SIZE || 1048576; - const psqlHost = _env.PSQL_HOST || '127.0.0.1'; - const psqlPort = _env.PSQL_PORT || 5432; - const psqlUser = _env.PSQL_USER || 'openaq'; - const psqlPassword = _env.PSQL_PASSWORD || 'openaq-pass'; - const psqlDatabase = _env.PSQL_DATABASE || 'openaq-local'; - const psqlPoolMin = +_env.PSQL_POOL_MIN || 2; - const psqlPoolMax = +_env.PSQL_POOL_MAX || 20; const suffix = _env.SUFFIX || ''; offset = +(offset || _env.OFFSET); @@ -206,13 +192,6 @@ export default () => { return { logLevel, logColor, - psqlHost, - psqlPort, - psqlUser, - psqlPassword, - psqlDatabase, - psqlPoolMin, - psqlPoolMax, apiURL, webhookKey, processTimeout, From 815703de07007ba04b9bb1d0f43f24172c32d316 Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Thu, 8 Feb 2024 15:51:54 -0800 Subject: [PATCH 2/9] removed unused dependency 'knex' --- package-lock.json | 121 +++--------------------------------------- package.json | 1 - src/package-lock.json | 1 - 3 files changed, 7 insertions(+), 116 deletions(-) diff --git a/package-lock.json b/package-lock.json index be31f50e..8998ca15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,6 @@ "ftp": "^0.3.10", "got": "^12.6.0", "jsonschema": "^1.2.0", - "knex": "^1.0.6", "lodash": "^4.17.10", "luxon": "^3.0.4", "moment-timezone": "^0.5.21", @@ -3749,10 +3748,6 @@ "simple-swizzle": "^0.2.2" } }, - "node_modules/colorette": { - "version": "2.0.16", - "license": "MIT" - }, "node_modules/colorspace": { "version": "1.1.4", "license": "MIT", @@ -3771,13 +3766,6 @@ "node": ">= 0.8" } }, - "node_modules/commander": { - "version": "9.5.0", - "license": "MIT", - "engines": { - "node": "^12.20.0 || >=14" - } - }, "node_modules/concat-map": { "version": "0.0.1", "dev": true, @@ -3887,6 +3875,7 @@ }, "node_modules/debug": { "version": "4.3.4", + "dev": true, "license": "MIT", "dependencies": { "ms": "2.1.2" @@ -4221,13 +4210,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/esm": { - "version": "3.2.25", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/esprima": { "version": "4.0.1", "dev": true, @@ -4483,13 +4465,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/get-package-type": { - "version": "0.1.0", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/get-stream": { "version": "6.0.1", "license": "MIT", @@ -4500,10 +4475,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/getopts": { - "version": "2.3.0", - "license": "MIT" - }, "node_modules/getpass": { "version": "0.1.7", "license": "MIT", @@ -4824,13 +4795,6 @@ "dev": true, "license": "ISC" }, - "node_modules/interpret": { - "version": "2.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/is-arguments": { "version": "1.1.1", "license": "MIT", @@ -4873,6 +4837,7 @@ }, "node_modules/is-core-module": { "version": "2.13.1", + "dev": true, "license": "MIT", "dependencies": { "hasown": "^2.0.0" @@ -5097,55 +5062,6 @@ "json-buffer": "3.0.1" } }, - "node_modules/knex": { - "version": "1.0.7", - "license": "MIT", - "dependencies": { - "colorette": "2.0.16", - "commander": "^9.1.0", - "debug": "4.3.4", - "escalade": "^3.1.1", - "esm": "^3.2.25", - "get-package-type": "^0.1.0", - "getopts": "2.3.0", - "interpret": "^2.2.0", - "lodash": "^4.17.21", - "pg-connection-string": "2.5.0", - "rechoir": "^0.8.0", - "resolve-from": "^5.0.0", - "tarn": "^3.0.2", - "tildify": "2.0.0" - }, - "bin": { - "knex": "bin/cli.js" - }, - "engines": { - "node": ">=12" - }, - "peerDependenciesMeta": { - "@vscode/sqlite3": { - "optional": true - }, - "better-sqlite3": { - "optional": true - }, - "mysql": { - "optional": true - }, - "mysql2": { - "optional": true - }, - "pg": { - "optional": true - }, - "pg-native": { - "optional": true - }, - "tedious": { - "optional": true - } - } - }, "node_modules/kuler": { "version": "2.0.0", "license": "MIT" @@ -5476,6 +5392,7 @@ }, "node_modules/ms": { "version": "2.1.2", + "dev": true, "license": "MIT" }, "node_modules/multimatch": { @@ -5684,6 +5601,7 @@ }, "node_modules/path-parse": { "version": "1.0.7", + "dev": true, "license": "MIT" }, "node_modules/path-type": { @@ -5706,10 +5624,6 @@ "version": "2.1.0", "license": "MIT" }, - "node_modules/pg-connection-string": { - "version": "2.5.0", - "license": "MIT" - }, "node_modules/picocolors": { "version": "1.0.0", "dev": true, @@ -5882,16 +5796,6 @@ "node": ">=8.10.0" } }, - "node_modules/rechoir": { - "version": "0.8.0", - "license": "MIT", - "dependencies": { - "resolve": "^1.20.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, "node_modules/regenerate": { "version": "1.4.2", "dev": true, @@ -6027,6 +5931,7 @@ }, "node_modules/resolve": { "version": "1.22.8", + "dev": true, "license": "MIT", "dependencies": { "is-core-module": "^2.13.0", @@ -6058,6 +5963,7 @@ }, "node_modules/resolve-from": { "version": "5.0.0", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -6327,6 +6233,7 @@ }, "node_modules/supports-preserve-symlinks-flag": { "version": "1.0.0", + "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -6335,24 +6242,10 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/tarn": { - "version": "3.0.2", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/text-hex": { "version": "1.0.0", "license": "MIT" }, - "node_modules/tildify": { - "version": "2.0.0", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/to-fast-properties": { "version": "2.0.0", "dev": true, diff --git a/package.json b/package.json index f0304d75..2aed6a83 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,6 @@ "ftp": "^0.3.10", "got": "^12.6.0", "jsonschema": "^1.2.0", - "knex": "^1.0.6", "lodash": "^4.17.10", "luxon": "^3.0.4", "moment-timezone": "^0.5.21", diff --git a/src/package-lock.json b/src/package-lock.json index 79421b84..b1b46456 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -21,7 +21,6 @@ "ftp": "^0.3.10", "got": "^12.6.0", "jsonschema": "^1.2.0", - "knex": "^1.0.6", "lodash": "^4.17.10", "luxon": "^3.0.4", "moment-timezone": "^0.5.21", From 348d3ad416ea396b84efe0863f1fa34913a40f86 Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Tue, 13 Feb 2024 11:33:01 -0800 Subject: [PATCH 3/9] Remove unused files and update README.md --- CHANGELOG.md | 16 ++++- README.md | 9 +-- data_scripts/accra.js | 54 ---------------- data_scripts/arpae-locations.js | 33 ---------- data_scripts/arpae-parameters.js | 27 -------- data_scripts/arpalazio.js | 30 --------- data_scripts/defra-locations.js | 58 ----------------- data_scripts/israel.js | 108 ------------------------------- data_scripts/moscow-locations.js | 43 ------------ data_scripts/richards-bay.js | 67 ------------------- docker-compose.yml | 5 -- index.adapter.sh | 11 ---- index.sh | 31 --------- src/sources/it.json | 2 +- 14 files changed, 17 insertions(+), 477 deletions(-) delete mode 100644 data_scripts/accra.js delete mode 100644 data_scripts/arpae-locations.js delete mode 100644 data_scripts/arpae-parameters.js delete mode 100644 data_scripts/arpalazio.js delete mode 100644 data_scripts/defra-locations.js delete mode 100644 data_scripts/israel.js delete mode 100644 data_scripts/moscow-locations.js delete mode 100644 data_scripts/richards-bay.js delete mode 100644 docker-compose.yml delete mode 100755 index.adapter.sh delete mode 100755 index.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index b659cc15..879258ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,21 @@ Changes to adapters will be documented in this file. -## 11/26/2024 +## 02/13/2024 +### Removed unused files +- docker-compose.yml +- index.adapter.sh +- index.sh +- accra.js +- arpae-locations.js +- arpae-parameters.js +- arpalazio.js +- defra-locations.js +- israel.js +- moscow-locations.js +- ards-bay.js + +## 11/26/2023 ### Removed eslint - Removed eslint from package.json - Removed .eslintrc diff --git a/README.md b/README.md index 7580335c..7f713feb 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This is the main data ingest pipeline for the [OpenAQ](https://openaq.org) proje Starting with `index.js`, there is an ingest mechanism to gather global air quality measurements from a variety of sources. This is currently run every 10 minutes and saves all unique measurements to a database. -[openaq-api](https://github.com/openaq/openaq-api) powers the API and more information on the data format can be found in [openaq-data-format](https://github.com/openaq/openaq-data-format). +[openaq-api-v2](https://github.com/openaq/openaq-api-v2) powers the API and more information on the data format can be found in [openaq-data-format](https://github.com/openaq/openaq-data-format). For more info see the [OpenAQ-Fetch documentation index](docs/index.md). @@ -24,18 +24,11 @@ Now you can get started with: `node index.js --help` -For a full development quick start (with database setup etc.), please see the [dev-quick-start doc](docs/dev-quick-start.md). - For production deployment, you will need to have certain environment variables set as in the table below - | Name | Description | Default | |---|---|---| -| SENDGRID_PASSWORD | Email service password | not set | -| SENDGRID_USERNAME | Email service username | not set | | API_URL | URL of openaq-api | http://localhost:3004/v1/webhooks | | WEBHOOK_KEY | Secret key to interact with openaq-api | '123' | -| AIRNOW_FTP_USER | User for AirNow FTP | not set | -| AIRNOW_FTP_PASSWORD | Password for AirNow FTP | not set | | EEA_TOKEN | API token for EEA API | not set | | DATA_GOV_IN_TOKEN | API token for data.gov.in | not set | | EPA_VICTORIA_TOKEN | API token for portal.api.epa.vic.gov.au | not set | diff --git a/data_scripts/accra.js b/data_scripts/accra.js deleted file mode 100644 index 131deb72..00000000 --- a/data_scripts/accra.js +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Script to convert data dump from Accra to proper format for inclusion - * in the platform. Output of this script needs to be uploaded manually - * to data bucket. - */ - -'use strict'; - -const parse = require('csv-parse'); -const fs = require('fs'); -const moment = require('moment-timezone'); - -const input = fs.readFileSync('accra.csv'); -let records = ''; -parse(input, (err, output) => { - if (err) { - return console.error(err); - } - - output.forEach((m) => { - // Create expected format for ndjson files - let base = { - date: { - utc: moment.utc(moment.tz(m[9], 'MM/DD/YY H:mm', 'Africa/Accra')).toDate(), - local: moment.tz(m[9], 'MM/DD/YY H:mm', 'Africa/Accra').format('YYYY-MM-DDTHH:mm:ssZ') - }, - parameter: m[0].toLowerCase().replace('.', ''), - location: `${m[7]} - ${m[6]}`, - value: Number(m[2]), - unit: 'µg/m³', - city: 'Accra', - attribution: [{ - name: 'Dr. Raphael E. Arku and Colleagues', - url: m[13] - }], - averagingPeriod: { - value: Number(m[3]), - unit: 'hours' - }, - coordinates: { - latitude: Number(m[11]), - longitude: Number(m[12]) - }, - country: 'GH', - sourceName: 'Dr. Raphael E. Arku and Colleagues', - sourceType: 'research', - mobile: false - }; - - records += `${JSON.stringify(base)}\n`; - }); - - fs.writeFileSync('accra.ndjson', records); -}); diff --git a/data_scripts/arpae-locations.js b/data_scripts/arpae-locations.js deleted file mode 100644 index eeaaa075..00000000 --- a/data_scripts/arpae-locations.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict'; - -/* - A script for generating a locations mapping - object from the ARPAE parameters source. - - */ - -import { default as parse } from 'csv-parse/lib/sync'; -import { uniqBy } from 'lodash'; -var request = require('request'); - -const csvUrl = 'https://docs.google.com/spreadsheets/d/1GlY3Pu9GDpLDk8Spl9yV1wjCRPvOI1m7BFxumfcuGcE/export?format=csv'; - -request(csvUrl, (err, res, data) => { - if (err) { - console.log(err); - } - - const parsed = parse(data, { columns: true }); - let locations = {}; - uniqBy(parsed, 'Cod_staz').forEach((o) => { - locations[o.Cod_staz] = { - station: o.Stazione, - comune: o.COMUNE, - coordinates: { - latitude: Number(o.Lat), - longitude: Number(o.Lon) - } - }; - }); - console.log(locations); -}); diff --git a/data_scripts/arpae-parameters.js b/data_scripts/arpae-parameters.js deleted file mode 100644 index b12307c7..00000000 --- a/data_scripts/arpae-parameters.js +++ /dev/null @@ -1,27 +0,0 @@ -'use strict'; - -/* - A script for generating a parameter mapping - object from the ARPAE parameters source. - - */ - -import { default as parse } from 'csv-parse/lib/sync'; -var request = require('request'); - -const csvUrl = 'https://docs.google.com/spreadsheets/d/13QcqldwA3EQ_4E17Hqggd2ZcMgUA5UwACttAWEkaU28/export?format=csv'; - -request(csvUrl, (err, res, data) => { - if (err) { - console.err(err); - } - const parsed = parse(data, { columns: true }); - let parameters = {}; - parsed.forEach((el) => { - parameters[el.IdParametro] = { - parameter: el.PARAMETRO.split('(')[0].toLowerCase().trim().replace('.', ''), - unit: el.UM - }; - }); - console.log(parameters); -}); diff --git a/data_scripts/arpalazio.js b/data_scripts/arpalazio.js deleted file mode 100644 index 386bbd05..00000000 --- a/data_scripts/arpalazio.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict'; - -/* - Get ARPALAZIO station coordinates - from project Calicantus source - - Run from project root with: - node run-scripts.js ./data_scripts/arpalazio.js -*/ - -import {default as parse} from 'csv-parse/lib/sync'; -const request = require('request'); - -const metadataURL = 'https://raw.githubusercontent.com/jobonaf/calicantus/master/data/sites-info/metadata.ARPA-Lazio.csv'; - -request(metadataURL, (err, response, body) => { - if (err) { - return console.error(err); - } - - let results = {}; - parse(body, {columns: true}).forEach((rowObject) => { - results[rowObject.ID] = { - name: rowObject.NOME, - latitude: Number(rowObject.LAT), - longitude: Number(rowObject.LON) - }; - }); - console.log(results); -}); diff --git a/data_scripts/defra-locations.js b/data_scripts/defra-locations.js deleted file mode 100644 index a9116036..00000000 --- a/data_scripts/defra-locations.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict'; - -import cheerio from 'cheerio'; -import { parallelLimit } from 'async'; -var request = require('request'); - -function handleLink (url) { - return function (done) { - request(url, (err, res, body) => { - if (err) { - done(err); - } - let $ = cheerio.load(body); - let city = $($('#tab_info').find('p').get(5)).html(); - city = city.split('')[1].trim(); - let coords = $($('#tab_info').find('p').get(8)).html(); - coords = coords.split('')[1].trim(); - coords = { - latitude: Number(coords.split(',')[0]), - longitude: Number(coords.split(',')[1]) - }; - - done(null, {city: city, coordinates: coords}); - }); - }; -} - -request('http://uk-air.defra.gov.uk/latest/currentlevels', (err, res, body) => { - if (err) { - return console.error(err); - } - - // Get links from main page - console.info('Grabbing main page to get locations.'); - let $ = cheerio.load(body); - let links = {}; - $('.current_levels_table').each((i, e) => { - $('tr', $(e)).each((i, e) => { - let link = $($(e).find('td a')).attr('href'); - let name = $($(e).find('td a')).html(); - if (link) { - links[name] = handleLink(link.replace('../', 'http://uk-air.defra.gov.uk/')); - } - }); - }); - - // Get info from each link - console.info(`Grabbing data for ${Object.keys(links).length} locations.`); - // links = {'Auchencorth Moss': links['Auchencorth Moss']}; - parallelLimit(links, 5, (err, results) => { - console.info('Grabbing all data completed!'); - if (err) { - return console.error(err); - } - - console.info(results); - }); -}); diff --git a/data_scripts/israel.js b/data_scripts/israel.js deleted file mode 100644 index 442267cf..00000000 --- a/data_scripts/israel.js +++ /dev/null @@ -1,108 +0,0 @@ -'use strict'; - -/* - script to get coordinates for israel stations. -*/ - -const headers = { - 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Content-Type': 'text/html; charset=utf-8' -}; - -var async = require('async'); -var request = require('request'); -var cheerio = require('cheerio'); -var _ = require('lodash'); - -let baseUrl = 'http://www.svivaaqm.net/'; - -request.get({ - headers: headers, - url: 'http://www.svivaaqm.net/MenuSite.aspx' -}, (err, res, body) => { - if (err || res.statusCode !== 200) { - return; - } - let tasks = []; - let regionLinks = body.match(/DynamicTable.aspx\?G_ID=(\d*)/g); - while (regionLinks.length > 0) { - let link = baseUrl + regionLinks.pop(); - tasks.push(getStations(baseUrl, link)); - } - async.parallel(tasks, (err, results) => { - if (err) { - console.log(err); - } - results = _.flattenDeep(results); - results = Object.assign({}, ...results); - console.log(results); - }); -}); - -let getStations = function (baseUrl, link) { - return function (done) { - request.get({ - headers: headers, - url: link - }, (err, res, body) => { - if (err || res.statusCode !== 200) { - return done(null, []); - } - let tasks = []; - const stationURLs = body.match(/StationInfo5?.aspx\?ST_ID=(\d*)/g); - while (stationURLs.length > 0) { - const stationURL = stationURLs.pop(); - const stationID = stationURL.match(/StationInfo5?.aspx\?ST_ID=(\d*)/)[1]; - const link = `${baseUrl}${stationURL}`; - tasks.push(getCoordinates(link, stationID)); - } - async.parallel(tasks, (err, results) => { - if (err) { done(null, []); } - results = _.flattenDeep(results); - return done(err, results); - }); - }); - }; -}; - -let getCoordinates = function (link, stationID) { - return function (done) { - request.get({ - url: link, - headers: headers - }, (err, res, body) => { - if (err) { - return done(null, []); - } - - let $ = cheerio.load(body); - const stationData = $('#stationInfoDiv > table').children().toArray().map((tr) => { - let toReturn = tr.children.find((td) => { - return td.attribs && td.attribs.class === 'key' && _.includes(['שם תחנה', 'קו אורך', 'קו רוחב'], td.children[0].data); - }); - if (toReturn !== undefined) { - return $(toReturn.next).text(); - } - }).filter((data) => { return data !== undefined; }); - if (stationData.length === 3) { - if (!(_.includes([stationData[1], stationData[2]], undefined)) || !(_.includes([stationData[1], stationData[2]], 0))) { - if (Number(stationData[2]) === 0 && Number(stationData[1]) === 0) { - return done(null, []); - } - const stationObj = {}; - stationObj[stationID] = { - coordinates: { - latitude: Number(stationData[2]), - longitude: Number(stationData[1]) - - } - }; - return done(null, [stationObj]); - } - } else { - return done(null, []); - } - }); - }; -}; diff --git a/data_scripts/moscow-locations.js b/data_scripts/moscow-locations.js deleted file mode 100644 index e8248479..00000000 --- a/data_scripts/moscow-locations.js +++ /dev/null @@ -1,43 +0,0 @@ -'use strict'; - -/* - A script for getting names and coordinates - from a KML file at Google Maps. - - It uses the Moscow map, - but could be adapted for other uses. -*/ - -var request = require('request'); -var cheerio = require('cheerio'); - -let midValue = '1ve9zrTQE0ONlSkk6tC5qyglpoME'; -let kmlURL = 'http://www.google.com/maps/d/kml?forcekml=1&mid=' + midValue; - -request(kmlURL, (err, res, body) => { - if (err) { - return console.error(err); - } - - let $ = cheerio.load(body, { xmlMode: true }); - let results = {}; - - $('Placemark').each(function (index, element) { - let points = $(this).find('Point coordinates').text(); - points = points.trim(); - - let description = $(this).find('description').text(); - let stationRegexp = /air-today\/station\/(\w*)\//; - let stationId = stationRegexp.exec(description)[1]; - - results[stationId] = { - coordinates: { - longitude: Number(points.split(',')[0]), - latitude: Number(points.split(',')[1]) - } - }; - }); - - console.log(results); - // console.log(JSON.stringify(results, null, '\t')); -}); diff --git a/data_scripts/richards-bay.js b/data_scripts/richards-bay.js deleted file mode 100644 index 6fa1e0d3..00000000 --- a/data_scripts/richards-bay.js +++ /dev/null @@ -1,67 +0,0 @@ -'use strict'; - -/* - A script to collect coordinates - for Richards Bay stations -*/ - -var async = require('async'); -var request = require('request'); -var cheerio = require('cheerio'); - -let stationIds = [1, 2, 3, 4, 5, 6, 9, 11, 13, 15, 16]; -let baseUrl = 'http://live.rbcaa.org.za/StationDetails.aspx?ST_ID='; - -let findTextGetNext = function ($, text) { - let textNode = $('.TitleLabel').filter(function (i, el) { - return $(this).text() === text; - }); - return $(textNode).parent().next().text(); -}; - -let getCoordinates = function (stationId) { - return function (done) { - let url = baseUrl + stationId; - request(url, (err, res, body) => { - if (err) { - return console.error(err); - } - - let $ = cheerio.load(body); - let stationName = findTextGetNext($, 'Station Name'); - let longitude = findTextGetNext($, 'Longitude'); - let latitude = findTextGetNext($, 'Latitude'); - if (longitude && latitude) { - return done(null, [stationName, longitude, latitude]); - } else { - return done(null, []); - } - }); - }; -}; - -let tasks = []; -while (stationIds.length > 0) { - tasks.push(getCoordinates(stationIds.pop())); -} - -async.parallel(tasks, function (err, results) { - if (err) { - console.error(err); - } - - let locations = {}; - while (results.length > 0) { - let result = results.pop(); - if (!result[0]) { - continue; - } - locations[result[0]] = { - coordinates: { - longitude: Number(result[1]), - latitude: Number(result[2]) - } - }; - } - console.log(locations); -}); diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c832e920..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,5 +0,0 @@ -fetch: - build: . - volumes: - - .:/local - env_file: ./local.env diff --git a/index.adapter.sh b/index.adapter.sh deleted file mode 100755 index f5f9a84e..00000000 --- a/index.adapter.sh +++ /dev/null @@ -1,11 +0,0 @@ -# !/usr/bin/env bash -# Script to fetch data adapter by adapter -adapters=adapters.list -for file in ./sources/*.json; do - jq '.[] | select(.active == true ).name' $file | sed -r 's/^"|"$//g' >>$adapters -done - -while read adapter; do - echo "================================> $adapter <================================" - node index.js --source "$adapter" -b -done <$adapters diff --git a/index.sh b/index.sh deleted file mode 100755 index ede93f48..00000000 --- a/index.sh +++ /dev/null @@ -1,31 +0,0 @@ -# !/usr/bin/env bash -set -x - -############################################################################### -#### Main fetch data process -############################################################################### - -time timeout 10m npm start - -############################################################################### -#### Adapters that consume lot memory and need to be run one by one -#### The follow adapters are currently disabled, in "sources/", but actually we are requesting data. We disable because -#### those adapter request huge amount of data, that sometimes made failing the fetch process and we are running them one by one. -############################################################################### - -# # ARPALAZIO adapter requests ~7.5k items -# time timeout 3m node index.js --source="ARPALAZIO" - -# # 'London Air Quality Network' adapter requests ~7.8k items -# time timeout 3m node index.js --source="London Air Quality Network" - -# # GIOS adapter requests ~2k items -# time timeout 3m node index.js --source="GIOS" - -############################################################################### -#### Adapters that are running in fargate -#### Reasons: -#### - Because the site response very slow, took more that 5 minutes to complete the fetch process -############################################################################### -# caaqm adapter requests ~1.5k items -# time timeout 3m node index.js --source="caaqm" diff --git a/src/sources/it.json b/src/sources/it.json index 05fd5a9a..3899eae6 100644 --- a/src/sources/it.json +++ b/src/sources/it.json @@ -11,7 +11,7 @@ "info@openaq.org" ], "active": true, - "--comment1": "// metadata generated with ../data_scripts/arpalazio.js", + "--comment1": "// metadata generated from https://raw.githubusercontent.com/jobonaf/calicantus/master/data/sites-info/metadata.ARPA-Lazio.csv", "metadata": { "2": { "name": "Preneste", "latitude": 41.886018, "longitude": 12.541614 }, "3": { "name": "Francia", "latitude": 41.947447, "longitude": 12.469588 }, From 1d4ef7cd9e491937745c913de5be21c9d44972e2 Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Tue, 20 Feb 2024 08:33:29 -0800 Subject: [PATCH 4/9] Remove deprecated adapters and update changelog --- CHANGELOG.md | 5 +++++ docs/adapter.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 879258ac..5d88ec7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ Changes to adapters will be documented in this file. +## 02/14/2024 +### Removed deprecated adapters +- agaar_mn.js +- airnow-ftp.js +- eea.js ## 02/13/2024 ### Removed unused files - docker-compose.yml diff --git a/docs/adapter.md b/docs/adapter.md index 449cd1fb..815ae977 100644 --- a/docs/adapter.md +++ b/docs/adapter.md @@ -124,7 +124,7 @@ These are available internally via `utils.acceptableParameters` All other values will be ignored. Date should be provided as UTC and local, and should be the end of the averaging period. -[moment.js](http://momentjs.com/) can be used to handle time zone conversions. List of valid zone names at +[Luxon](https://www.npmjs.com/package/luxon) can be used to handle time zone conversions. List of valid zone names at https://en.wikipedia.org/wiki/List_of_tz_database_time_zones. ### Benchmark From 625a906906d143fae96598ceac55f966982e058c Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Wed, 20 Mar 2024 08:14:52 -0700 Subject: [PATCH 5/9] cleanujp aipurl and webhookjey --- src/lib/notification.js | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/lib/notification.js b/src/lib/notification.js index f48cdf5c..8933f80e 100644 --- a/src/lib/notification.js +++ b/src/lib/notification.js @@ -2,17 +2,6 @@ import request from 'request'; import log from './logger.js'; import { promisify } from 'util'; -/** -* Ping openaq-api to let it know cause fetching is complete -* @param {function} cb A function of form func(cause) called on completion -*/ -async function sendUpdatedWebhook (apiURL, webhookKey) { - var form = { - key: webhookKey, - action: 'DATABASE_UPDATED' - }; - return promisify(request.post)(apiURL, { form: form }); -} /** * Reports and saves fetch information. @@ -20,10 +9,8 @@ async function sendUpdatedWebhook (apiURL, webhookKey) { * @param {FetchReport} fetchReport * @param {Source[]} sources * @param {Object} argv - * @param {URL} apiURL - * @param {String} webhookKey */ -export function reportAndRecordFetch (fetchReport, sources, argv, apiURL, webhookKey) { +export function reportAndRecordFetch (fetchReport, sources, argv) { return async (results) => { fetchReport.results = results; fetchReport.timeEnded = Date.now(); @@ -39,9 +26,5 @@ export function reportAndRecordFetch (fetchReport, sources, argv, apiURL, webhoo log.info('Dry run ended.'); return 0; } - - await sendUpdatedWebhook(apiURL, webhookKey); - log.info('Webhook posted, have a good day!'); - return 0; }; } From b69a2e08cc4a1e6df824cdf30fb6fb659480bbdc Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Wed, 20 Mar 2024 08:32:14 -0700 Subject: [PATCH 6/9] Remove unused variables and parameters --- src/fetch.js | 4 ---- src/lib/env.js | 6 ------ 2 files changed, 10 deletions(-) diff --git a/src/fetch.js b/src/fetch.js index 1a48a5a5..7eb2a8db 100644 --- a/src/fetch.js +++ b/src/fetch.js @@ -37,8 +37,6 @@ const { DataStream } = sj; const env = _env(); const { - apiURL, - webhookKey, processTimeout, maxParallelAdapters, strict, @@ -182,8 +180,6 @@ export function handler (event, context) { fetchReport, sourcesArray, env, - apiURL, - webhookKey ) ) ); diff --git a/src/lib/env.js b/src/lib/env.js index bb766288..ac347d7b 100644 --- a/src/lib/env.js +++ b/src/lib/env.js @@ -9,8 +9,6 @@ const yargs = _yargs(hideBin(process.argv)); * @extends Object * @param {number} logLevel * @param {boolean} logColor - * @param {string} apiURL - * @param {string} webhookKey * @param {number} processTimeout * @param {string} bucketName * @param {number} s3ChunkSize @@ -165,8 +163,6 @@ export default () => { adapter = _env.ADAPTER; } - const apiURL = _env.API_URL || 'http://localhost:3004/v1/webhooks'; // The url to ping on completion - const webhookKey = _env.WEBHOOK_KEY || '123'; // Secret key to auth with API const processTimeout = _env.PROCESS_TIMEOUT || 14 * 60 * 1000; // Kill the process after a certain time in case it hangs const bucketName = _env.AWS_BUCKET_NAME || ''; const doSaveToS3 = _env.SAVE_TO_S3 === 'true' || +_env.SAVE_TO_S3; @@ -192,8 +188,6 @@ export default () => { return { logLevel, logColor, - apiURL, - webhookKey, processTimeout, bucketName, s3ChunkSize, From 3ebdaf29b187984396f68c5c8544e5340a4de729 Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Wed, 20 Mar 2024 16:35:27 -0700 Subject: [PATCH 7/9] requested updates to notification.js --- src/lib/notification.js | 52 +++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/src/lib/notification.js b/src/lib/notification.js index 263dd4da..0d233e30 100644 --- a/src/lib/notification.js +++ b/src/lib/notification.js @@ -1,6 +1,4 @@ -import request from 'request'; import log from './logger.js'; -import { promisify } from 'util'; import { SNSClient, PublishCommand } from '@aws-sdk/client-sns'; const sns = new SNSClient(); @@ -23,22 +21,36 @@ async function publish(message, subject) { * @param {FetchReport} fetchReport * @param {Source[]} sources * @param {Object} argv + * @param {URL} apiURL + * @param {String} webhookKey */ -export function reportAndRecordFetch (fetchReport, sources, argv) { - return async (results) => { - fetchReport.results = results; - fetchReport.timeEnded = Date.now(); - fetchReport.errors = results.reduce((acc, {failures}) => { - Object.entries(failures).forEach(([key, count]) => { - acc[key] = (acc[key] || 0) + count; - }); - return acc; - }, {}); - - if (argv.dryrun) { - log.info(fetchReport); - log.info('Dry run ended.'); - return 0; - } - }; -} +export function reportAndRecordFetch (fetchReport, sources, env) { + return async (results) => { + fetchReport.results = results; + fetchReport.timeEnded = Date.now(); + fetchReport.errors = results.reduce((acc, {failures}) => { + Object.entries(failures).forEach(([key, count]) => { + acc[key] = (acc[key] || 0) + count; + }); + return acc; + }, {}); + + + const failures = fetchReport.results + .filter(r => !r.count); + + const successes = fetchReport.results + .filter(r => r.count > 0); + + failures.map(r => { + log.debug(r); + }); + log.info(`Dry run finished with ${successes.length} successes and ${failures.length} failures in ${(fetchReport.timeEnded - fetchReport.timeStarted)/1000} seconds`); + + if (!env.dryrun) { + await publish(fetchReport.results, 'fetcher/success'); + } + + return 0; + }; +} \ No newline at end of file From aa0d1cfe7adcaf9322f8c9600478d5d14157dae6 Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Wed, 20 Mar 2024 16:36:12 -0700 Subject: [PATCH 8/9] formatting --- src/lib/notification.js | 81 +++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/src/lib/notification.js b/src/lib/notification.js index 0d233e30..e345b150 100644 --- a/src/lib/notification.js +++ b/src/lib/notification.js @@ -4,15 +4,15 @@ import { SNSClient, PublishCommand } from '@aws-sdk/client-sns'; const sns = new SNSClient(); async function publish(message, subject) { - // the following just looks better in the log - if(process.env.TOPIC_ARN) { - const cmd = new PublishCommand({ - TopicArn: process.env.TOPIC_ARN, - Subject: subject, - Message: JSON.stringify(message), - }); - await sns.send(cmd); - } + // the following just looks better in the log + if (process.env.TOPIC_ARN) { + const cmd = new PublishCommand({ + TopicArn: process.env.TOPIC_ARN, + Subject: subject, + Message: JSON.stringify(message), + }); + await sns.send(cmd); + } } /** @@ -24,33 +24,36 @@ async function publish(message, subject) { * @param {URL} apiURL * @param {String} webhookKey */ -export function reportAndRecordFetch (fetchReport, sources, env) { - return async (results) => { - fetchReport.results = results; - fetchReport.timeEnded = Date.now(); - fetchReport.errors = results.reduce((acc, {failures}) => { - Object.entries(failures).forEach(([key, count]) => { - acc[key] = (acc[key] || 0) + count; - }); - return acc; - }, {}); - - - const failures = fetchReport.results - .filter(r => !r.count); - - const successes = fetchReport.results - .filter(r => r.count > 0); - - failures.map(r => { - log.debug(r); - }); - log.info(`Dry run finished with ${successes.length} successes and ${failures.length} failures in ${(fetchReport.timeEnded - fetchReport.timeStarted)/1000} seconds`); - - if (!env.dryrun) { - await publish(fetchReport.results, 'fetcher/success'); - } - - return 0; - }; -} \ No newline at end of file +export function reportAndRecordFetch(fetchReport, sources, env) { + return async (results) => { + fetchReport.results = results; + fetchReport.timeEnded = Date.now(); + fetchReport.errors = results.reduce((acc, { failures }) => { + Object.entries(failures).forEach(([key, count]) => { + acc[key] = (acc[key] || 0) + count; + }); + return acc; + }, {}); + + const failures = fetchReport.results.filter((r) => !r.count); + + const successes = fetchReport.results.filter((r) => r.count > 0); + + failures.map((r) => { + log.debug(r); + }); + log.info( + `Dry run finished with ${successes.length} successes and ${ + failures.length + } failures in ${ + (fetchReport.timeEnded - fetchReport.timeStarted) / 1000 + } seconds` + ); + + if (!env.dryrun) { + await publish(fetchReport.results, 'fetcher/success'); + } + + return 0; + }; +} From fa4ed5177a9028dfc45783d018554956635688a2 Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Mon, 8 Apr 2024 15:53:27 -0700 Subject: [PATCH 9/9] updates to notification.js --- src/lib/notification.js | 97 +++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/src/lib/notification.js b/src/lib/notification.js index e345b150..badb0238 100644 --- a/src/lib/notification.js +++ b/src/lib/notification.js @@ -1,18 +1,32 @@ +import request from 'request'; import log from './logger.js'; +import { promisify } from 'util'; import { SNSClient, PublishCommand } from '@aws-sdk/client-sns'; const sns = new SNSClient(); +/** +* Ping openaq-api to let it know cause fetching is complete +* @param {function} cb A function of form func(cause) called on completion +*/ +async function sendUpdatedWebhook (apiURL, webhookKey) { + var form = { + key: webhookKey, + action: 'DATABASE_UPDATED' + }; + return promisify(request.post)(apiURL, { form: form }); +} + async function publish(message, subject) { - // the following just looks better in the log - if (process.env.TOPIC_ARN) { - const cmd = new PublishCommand({ - TopicArn: process.env.TOPIC_ARN, - Subject: subject, - Message: JSON.stringify(message), - }); - await sns.send(cmd); - } + // the following just looks better in the log + if(process.env.TOPIC_ARN) { + const cmd = new PublishCommand({ + TopicArn: process.env.TOPIC_ARN, + Subject: subject, + Message: JSON.stringify(message), + }); + await sns.send(cmd); + } } /** @@ -24,36 +38,35 @@ async function publish(message, subject) { * @param {URL} apiURL * @param {String} webhookKey */ -export function reportAndRecordFetch(fetchReport, sources, env) { - return async (results) => { - fetchReport.results = results; - fetchReport.timeEnded = Date.now(); - fetchReport.errors = results.reduce((acc, { failures }) => { - Object.entries(failures).forEach(([key, count]) => { - acc[key] = (acc[key] || 0) + count; - }); - return acc; - }, {}); - - const failures = fetchReport.results.filter((r) => !r.count); - - const successes = fetchReport.results.filter((r) => r.count > 0); - - failures.map((r) => { - log.debug(r); - }); - log.info( - `Dry run finished with ${successes.length} successes and ${ - failures.length - } failures in ${ - (fetchReport.timeEnded - fetchReport.timeStarted) / 1000 - } seconds` - ); - - if (!env.dryrun) { - await publish(fetchReport.results, 'fetcher/success'); - } - - return 0; - }; -} +export function reportAndRecordFetch (fetchReport, sources, env, apiURL, webhookKey) { + return async (results) => { + fetchReport.results = results; + fetchReport.timeEnded = Date.now(); + fetchReport.errors = results.reduce((acc, {failures}) => { + Object.entries(failures).forEach(([key, count]) => { + acc[key] = (acc[key] || 0) + count; + }); + return acc; + }, {}); + + const failures = fetchReport.results + .filter(r => !r.count); + + const successes = fetchReport.results + .filter(r => r.count > 0); + + failures.map(r => { + log.debug(r); + }); + log.info(`Finished with ${successes.length} successes and ${failures.length} failures in ${(fetchReport.timeEnded - fetchReport.timeStarted)/1000} seconds`); + + if (!env.dryrun) { + await publish(fetchReport.results, 'fetcher/success'); + } else { + // for dev purposes + failures.map(r => console.warn(`No results`, r)); + fetchReport.results.map( r => log.debug(`${r.locations} locations from ${r.from} - ${r.to} | Parameters for ${r.sourceName}`, r.parameters)); + } + return 0; + }; +} \ No newline at end of file