diff --git a/.build_scripts/build.sh b/.build_scripts/build.sh new file mode 100644 index 00000000..bee762ca --- /dev/null +++ b/.build_scripts/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +if [ -f $HOME/docker/openaq_fetch.tar ] +then + echo "Loading cached worker image" + docker load < $HOME/docker/openaq_fetch.tar +fi + +touch local.env +docker-compose --project openaq build + +mkdir -p $HOME/docker +echo "Caching openaq_fetch docker image." +docker save openaq_fetch > $HOME/docker/openaq_fetch.tar diff --git a/.build_scripts/deploy.sh b/.build_scripts/deploy.sh new file mode 100644 index 00000000..ab6fd3b2 --- /dev/null +++ b/.build_scripts/deploy.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e + +if [[ $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_BRANCH == ${PRODUCTION_BRANCH} ]]; then + docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" + + echo "Pushing image: developmentseed/openaq-fetch:$TRAVIS_COMMIT" + docker tag openaq_fetch flasher/openaq-fetch:$TRAVIS_COMMIT + docker push flasher/openaq-fetch:$TRAVIS_COMMIT + echo "Also pushing as :latest" + docker tag openaq_fetch flasher/openaq-fetch:latest + docker push flasher/openaq-fetch:latest + + echo "Installing aws cli" + sudo pip install awscli + + echo "Running the update_task script" + sh .build_scripts/update-task.sh +else + echo "Not a push; nothing to do" +fi diff --git a/.build_scripts/ecs-task-definition.json b/.build_scripts/ecs-task-definition.json new file mode 100644 index 00000000..ee60a239 --- /dev/null +++ b/.build_scripts/ecs-task-definition.json @@ -0,0 +1,14 @@ +{ + "family": "openaq-fetch", + "containerDefinitions": [ + { + "name": "openaq-fetch", + "image": "flasher/openaq-fetch", + "memory": 400, + "command": [ + "npm", + "start" + ] + } + ] +} diff --git a/.build_scripts/entrypoint.sh b/.build_scripts/entrypoint.sh new file mode 100644 index 00000000..90b6faf9 --- /dev/null +++ b/.build_scripts/entrypoint.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +# Entrypoint for docker container, basically works as a stand-in for +# bashrc stuff, setting up the env and then executes whatever command is passed +# to it + +# Source the nvm script so we have access to npm and node +source $NVM_DIR/nvm.sh +# NOTE: use exec to make sure that npm receives SIGTERM, etc. +exec "$@" diff --git a/.build_scripts/insert-env.js b/.build_scripts/insert-env.js new file mode 100644 index 00000000..62b6aeb5 --- /dev/null +++ b/.build_scripts/insert-env.js @@ -0,0 +1,37 @@ +'use strict'; + +const input = '.build_scripts/ecs-task-definition.json'; +const output = 'ecs-task-generated.json'; +const envFile = 'local.env'; + +console.info('Inserting env variables into ECS task definition.'); + +let fs = require('fs'); + +// Load in intitial JSON +let obj = JSON.parse(fs.readFileSync(input, 'utf8')); + +// Add in env variables, split on newline and remove any extra empty things +// hanging around. +let envs = fs.readFileSync(envFile, 'utf8'); +let splitEnvs = []; +envs.split('\n').forEach(function (e) { + if (e) { + let idx = e.indexOf('='); + splitEnvs.push({ 'name': e.substr(0, idx), 'value': e.substr(idx + 1, e.length) }); + } +}); +obj['containerDefinitions'][0]['environment'] = splitEnvs; + +// Also set container version based on hash +let hash = 'latest'; +if (process.env.TRAVIS_COMMIT) { + hash = process.env.TRAVIS_COMMIT; +} else if (process.env.CURRENT_HASH) { + hash = process.env.CURRENT_HASH.split(':')[1]; +} +obj['containerDefinitions'][0]['image'] += ':' + hash; + +// Save to output file +fs.writeFileSync(output, JSON.stringify(obj)); +console.info('Inserted env variables into ECS task definition.'); diff --git a/.build_scripts/update-task.sh b/.build_scripts/update-task.sh new file mode 100644 index 00000000..56150373 --- /dev/null +++ b/.build_scripts/update-task.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +# Generate the appropriate command to use local env vars or a named AWS profile +# passed like sh .build_scripts/update_task.sh openaq (to be used locally) +aws="aws" +if [ ! -z "$1" ] + then + aws="aws --profile $1" +fi + +echo "Getting the revision of the old task" +# This should be updated to check for running revision, not necessarily latest revision +OLD_VERSION=$($aws ecs describe-task-definition --task-definition openaq-fetch | sed -n "/revision/p" | grep -o "[0-9]\+") +# Grab this so we're not trying to deploy latest, but rather the last good image +CURRENT_HASH=$($aws ecs describe-task-definition --task-definition openaq-fetch | grep -o "flasher/openaq-fetch:[a-zA-Z_0-9]\+") +export CURRENT_HASH=$CURRENT_HASH +echo "Current revision of ECS task is $OLD_VERSION" +echo "Current Docker image is $CURRENT_HASH" + +echo "Stopping the current task revision" +$aws ecs update-service --service openaq-fetch --task-definition openaq-fetch --desired-count 0 + +echo "Waiting for current task to stop" +$aws ecs wait services-stable --services openaq-fetch + +echo "Copying env variables from S3" +$aws s3 cp s3://openaq-env-variables/openaq-fetch/production.env local.env + +echo "Building new ECS task" +node .build_scripts/insert-env.js +$aws ecs register-task-definition --cli-input-json file://ecs-task-generated.json + +echo "Deploying 1 new ECS task " +$aws ecs update-service --service openaq-fetch --task-definition openaq-fetch --desired-count 1 + +echo "Waiting for new task to be scaled up" +$aws ecs wait services-stable --services openaq-fetch diff --git a/.gitignore b/.gitignore index b61a90e9..a60b876e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ npm-debug.log docs newrelic_agent.log dump +local.env +ecs-task-generated.json diff --git a/.travis.yml b/.travis.yml index 4e01d486..c3dccfb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,23 @@ +sudo: true language: node_js node_js: -- '0.12' +- '4.0' +env: + global: + - PRODUCTION_BRANCH=master +cache: + directories: + - "$HOME/docker" +services: +- docker +before_install: +- chmod +x ./.build_scripts/build.sh +- chmod +x ./.build_scripts/deploy.sh +after_success: +- ".build_scripts/build.sh" deploy: - provider: heroku - api_key: - secure: oFSFUawGi2M6NEh5eSLYGYVsGcpHGtVtP/c808eOsxpa5NCb2qxN5FqF2yZRdUjrfg2vftDLJLmUtbv/6DDc1nh+qGWEABAjmxzIXVH1tAJTBtLu0um6w+uQAXcakTdkJkbpcf5pXW/2pnS6EM0FTIVfS3Bw2O6mjeKx9sEWjcG5/5uZX6f/NtnHqQTRrIdbahDQXW4YCDBoUq4Osk/PtQJ2b7n0UEg+y72oMmZfUtfSXuc57jFOs9J49RP/rn/0M7RvcRcu7akUeP6eIXElYsJmT9cbQUvoMlcBlt5le89lE4hmsxuT9GeLAWU9uEfMvLvS3HBbFpBFZX8Qgk/NnT3QW4aq0FyFhwvgDN79c6yswfPHfJ5Urx1cf/DohBZFYLCQW/MeOHdNIHinh+UTGeO0QbRCfKEN1M89iqq4xiyf6lC+TbYsVEbxvIqRoSkldoqYnskXEKfYHbosY9QK81PF6OTeRcdzHu8cODC4e4rAWYfOoxvYVRDiub8blnCtlfOK6YiXAJMj3xhjbyv+NdyD6MMarHlpaEUx0TR3aMDaRAO3VDa+kqNrM5VQ87KN23W8dmMG3z3oO+8y9pfaSx+n9+khmUTQyG329LK47reP0lifKQkm3py1+qJoLrYWSVOCNSDvIBWj/c5aU7dek2jQ93NFOov9430mm46lQPQ= - app: openaq-fetch + provider: script + skip_cleanup: true + script: ".build_scripts/deploy.sh" on: - repo: openaq/openaq-fetch - branch: master + branch: ${PRODUCTION_BRANCH} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..940a0b82 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,59 @@ +FROM ubuntu:14.04 + +# Replace shell with bash so we can source files +RUN rm /bin/sh && ln -s /bin/bash /bin/sh + +# Set debconf to run non-interactively +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + +# Install base dependencies +RUN apt-get update && apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + git \ + libssl-dev \ + python \ + rsync \ + && rm -rf /var/lib/apt/lists/* + +# Install nvm with node and npm +# http://stackoverflow.com/questions/25899912/install-nvm-in-docker +ENV NVM_DIR /usr/local/nvm +ENV NODE_VERSION 4.0 +RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.26.0/install.sh | bash \ + && source $NVM_DIR/nvm.sh \ + && nvm install $NODE_VERSION \ + && nvm alias default $NODE_VERSION \ + && nvm use default +ENV PATH $NVM_BIN:$PATH + +# Go ahead and install nodemon for convenience while developing +RUN source $NVM_DIR/nvm.sh + +########################### +# App-specific stuff + +# mongo uses kerberos +RUN apt-get update && apt-get install -y libkrb5-dev + +# Install NPM dependencies. Do this first so that if package.json hasn't +# changed we don't have to re-run npm install during `docker build` +COPY package.json /app/package.json +WORKDIR /app +RUN source $NVM_DIR/nvm.sh; npm install +# Copy the app +COPY ["index.js", "newrelic.js", "/app/"] +COPY lib /app/lib/ +COPY test /app/test/ +COPY sources /app/sources/ +COPY adapters /app/adapters/ + +############################# +# entrypoint +# +RUN source $NVM_DIR/nvm.sh +ADD .build_scripts/entrypoint.sh / +RUN chmod +x /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/README.md b/README.md index 0aa1d854..357a6b05 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Overview This is the main data ingest pipeline for the [OpenAQ](https://openaq.org) project. -Starting with `fetch.js`, there is an ingest mechanism to gather global air quality measurements from a variety of sources. This is currently run every 10 minutes and saves all unique measurements to a database. +Starting with `index.js`, there is an ingest mechanism to gather global air quality measurements from a variety of sources. This is currently run every 10 minutes and saves all unique measurements to a database. [openaq-api](https://github.com/openaq/openaq-api) powers the API and more information on the data format can be found in [openaq-data-format](https://github.com/openaq/openaq-data-format). @@ -17,7 +17,7 @@ Install necessary Node.js packages by running Make sure you have MongoDB running locally and then you can get help with -`node fetch.js --help` +`node index.js --help` For the above to work, you will need to have certain environment variables set as in the table below @@ -29,12 +29,16 @@ For the above to work, you will need to have certain environment variables set a | SENDGRID_USERNAME | Email service username | not set | | API_URL | URL of openaq-api | http://localhost:3004/v1/webhooks | | WEBHOOK_KEY | Secret key to interact with openaq-api | '123' | +| FETCH_INTERVAL | How often to run fetch tasks | 10 minutes | ## Tests To confirm that everything is working as expected, you can run the tests with `npm test` +## Deployment +Deployment is handled automatically via Travis on the `master` branch and is deployed to Amazon's ECS. + ## Data Source Criteria This section lists the key criteria for air quality data aggregated onto the platform. A full explanation can be accessed here [https://medium.com/@openaq/where-does-openaq-data-come-from-a5cf9f3a5c85#.919hlx2by]. OpenAQ is an ever-evolving process that is shaped by its community: your feedback and questions are actively invited on the criteria listed in this section. diff --git a/adapters/agaar_mn.js b/adapters/agaar_mn.js index fae156ba..743c94cf 100644 --- a/adapters/agaar_mn.js +++ b/adapters/agaar_mn.js @@ -3,6 +3,7 @@ var request = require('request'); var _ = require('lodash'); var moment = require('moment-timezone'); +var log = require('../lib/logger'); exports.name = 'agaar_mn'; @@ -10,7 +11,7 @@ exports.fetchData = function (source, cb) { var finalURL = source.url; request(finalURL, function (err, res, body) { if (err || res.statusCode !== 200) { - console.error(err || res); + log.error(err || res); return cb({message: 'Failure to load data url.'}); } diff --git a/adapters/beijing.js b/adapters/beijing.js index f953dfe1..755109ee 100644 --- a/adapters/beijing.js +++ b/adapters/beijing.js @@ -3,6 +3,7 @@ var request = require('request'); var _ = require('lodash'); var moment = require('moment-timezone'); +var log = require('../lib/logger'); exports.name = 'beijing'; @@ -10,7 +11,7 @@ exports.fetchData = function (source, cb) { var finalURL = source.url + '?apitoken=' + process.env.INDIA_KIMONO_TOKEN; request(finalURL, function (err, res, body) { if (err || res.statusCode !== 200) { - console.error(err || res); + log.error(err || res); return cb({message: 'Failure to load data url.'}); } diff --git a/adapters/chile.js b/adapters/chile.js index 96dfda68..6c761fb1 100644 --- a/adapters/chile.js +++ b/adapters/chile.js @@ -27,7 +27,7 @@ exports.fetchData = function (source, cb) { async.parallel(tasks, function (err, results) { if (err) { - return console.log(err); + return cb({message: 'Failure to load data urls.'}); } // Wrap everything in a try/catch in case something goes wrong diff --git a/adapters/india.js b/adapters/india.js index 0e9a6664..5c72b007 100644 --- a/adapters/india.js +++ b/adapters/india.js @@ -4,6 +4,7 @@ var request = require('request'); var _ = require('lodash'); var moment = require('moment-timezone'); var utils = require('../lib/utils'); +var log = require('../lib/logger'); exports.name = 'india'; @@ -11,7 +12,7 @@ exports.fetchData = function (source, cb) { var finalURL = source.url + '?apitoken=' + process.env.INDIA_KIMONO_TOKEN; request(finalURL, function (err, res, body) { if (err || res.statusCode !== 200) { - console.error(err || res); + log.error(err || res); return cb({message: 'Failure to load data url.'}); } diff --git a/adapters/london.js b/adapters/london.js index 42ae480d..4518f66c 100644 --- a/adapters/london.js +++ b/adapters/london.js @@ -3,6 +3,7 @@ var request = require('request'); var _ = require('lodash'); var moment = require('moment-timezone'); +var log = require('../lib/logger'); exports.name = 'london'; @@ -10,7 +11,7 @@ exports.fetchData = function (source, cb) { var finalURL = source.url + '?apitoken=' + process.env.INDIA_KIMONO_TOKEN; request(finalURL, function (err, res, body) { if (err || res.statusCode !== 200) { - console.error(err || res); + log.error(err || res); return cb({message: 'Failure to load data url.'}); } diff --git a/adapters/netherlands.js b/adapters/netherlands.js index 2ea8cf11..4813a64a 100644 --- a/adapters/netherlands.js +++ b/adapters/netherlands.js @@ -5,6 +5,7 @@ var _ = require('lodash'); var cheerio = require('cheerio'); var async = require('async'); var moment = require('moment-timezone'); +var log = require('../lib/logger'); exports.name = 'netherlands'; @@ -12,7 +13,7 @@ exports.fetchData = function (source, cb) { var finalURL = source.url; request(finalURL, function (err, res, body) { if (err || res.statusCode !== 200) { - console.error(err || res); + log.error(err || res); return cb({message: 'Failure to load data url.'}); } @@ -46,7 +47,7 @@ exports.fetchData = function (source, cb) { async.parallel(tasks, function (err, results) { if (err) { - return console.error(err); + return cb({message: 'Failure to load data urls.'}); } var result = { diff --git a/adapters/nsw.js b/adapters/nsw.js index feee5701..562e973a 100644 --- a/adapters/nsw.js +++ b/adapters/nsw.js @@ -5,13 +5,14 @@ var _ = require('lodash'); var moment = require('moment-timezone'); var cheerio = require('cheerio'); var utils = require('../lib/utils'); +var log = require('../lib/logger'); exports.name = 'nsw'; exports.fetchData = function (source, cb) { request(source.url, function (err, res, body) { if (err || res.statusCode !== 200) { - console.error(err || res); + log.error(err || res); return cb({message: 'Failure to load data url.'}); } diff --git a/adapters/poland.js b/adapters/poland.js index 5fdb8b0a..879e6f73 100644 --- a/adapters/poland.js +++ b/adapters/poland.js @@ -38,7 +38,7 @@ exports.fetchData = function (source, cb) { async.series(tasks, function (err, results) { if (err) { - return console.error(err); + return cb({message: 'Failure to load data urls.'}); } // Wrap everything in a try/catch in case something goes wrong diff --git a/adapters/saopaulo.js b/adapters/saopaulo.js index fd37e1df..2f456a3d 100644 --- a/adapters/saopaulo.js +++ b/adapters/saopaulo.js @@ -6,6 +6,7 @@ var moment = require('moment-timezone'); var async = require('async'); var cheerio = require('cheerio'); var utils = require('../lib/utils'); +var log = require('../lib/logger'); exports.name = 'saopaulo'; @@ -41,7 +42,7 @@ exports.fetchData = function (source, cb) { async.parallelLimit(tasks, 4, function (err, results) { if (err) { - return console.error(err); + return cb({message: 'Failure to load data urls.'}); } // Wrap everything in a try/catch in case something goes wrong @@ -105,7 +106,7 @@ var formatData = function (results) { } else if (string.indexOf('ppm') !== -1) { return 'ppm'; } else { - console.warn('Unknown unit', string); + log.warn('Unknown unit', string); return undefined; } }; diff --git a/adapters/stateair.js b/adapters/stateair.js index 4e176085..20a91a47 100644 --- a/adapters/stateair.js +++ b/adapters/stateair.js @@ -4,13 +4,14 @@ var request = require('request'); var _ = require('lodash'); var moment = require('moment-timezone'); var cheerio = require('cheerio'); +var log = require('../lib/logger'); exports.name = 'stateair'; exports.fetchData = function (source, cb) { request(source.url, function (err, res, body) { if (err || res.statusCode !== 200) { - console.error(err || res); + log.error(err || res); return cb({message: 'Failure to load data url.'}); } diff --git a/adapters/texas.js b/adapters/texas.js index ff712ac2..4fd1683a 100644 --- a/adapters/texas.js +++ b/adapters/texas.js @@ -67,7 +67,7 @@ exports.fetchData = function (source, cb) { async.parallelLimit(tasks, 4, function (err, results) { if (err) { - return console.error(err); + return cb({message: 'Failure to load data urls.'}); } // Wrap everything in a try/catch in case something goes wrong diff --git a/app.json b/app.json deleted file mode 100644 index 49a68d94..00000000 --- a/app.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "OpenAQ Fetch", - "description": "A flexible tool to grab and serve global air pollution data, easily extendable to other data sources.", - "keywords": [ - "air pollution", - "node.js", - "javascript", - "hapi.js" - ], - "website": "https://openaq.org", - "repository": "https://github.com/openaq/openaq-api", - "logo": "https://avatars1.githubusercontent.com/u/11806583?v=3&s=200", - "env": { - "INDIA_KIMONO_TOKEN": { - "description": "Your Kimono API token to use if needed to query sources.", - "required": false - } - }, - "addons": [ - "newrelic", - "mongolab", - "sendgrid", - "scheduler" - ] -} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..c832e920 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,5 @@ +fetch: + build: . + volumes: + - .:/local + env_file: ./local.env diff --git a/fetch.js b/index.js similarity index 88% rename from fetch.js rename to index.js index 62637bb5..65b21c5c 100644 --- a/fetch.js +++ b/index.js @@ -21,6 +21,7 @@ var MongoClient = require('mongodb').MongoClient; var mailer = require('./lib/mailer'); var utils = require('./lib/utils'); var request = require('request'); +var log = require('./lib/logger'); var adapters = require('./adapters'); var sources = require('./sources'); @@ -28,6 +29,7 @@ var sources = require('./sources'); var dbURL = process.env.MONGOLAB_URI || 'mongodb://localhost:27017/openAQ'; var apiURL = process.env.API_URL || 'http://localhost:3004/v1/webhooks'; var webhookKey = process.env.WEBHOOK_KEY || '123'; +var fetchInterval = process.env.FETCH_INTERVAL || 10 * 60 * 1000; // Default to 10 minutes var measurementsCollection; // Flatten the sources into a single array, taking into account sources argument @@ -37,7 +39,7 @@ if (argv.source) { // Check here to make sure we have at least one valid source if (!sources) { - console.error('I\'m sorry Dave, I searched all known sources and can\'t ' + + log.error('I\'m sorry Dave, I searched all known sources and can\'t ' + 'find anything for', argv.source); process.exit(1); } @@ -75,7 +77,9 @@ var getAndSaveData = function (source) { return done(null, err); } + log.profile(source.name + ' fetch completed'); adapter.fetchData(source, function (err, data) { + log.profile(source.name + ' fetch completed'); // If we have an error, send an email to the contacts and stop if (err) { // Don't send an email if it's a dry run or noemail flag is set @@ -134,7 +138,7 @@ var getAndSaveData = function (source) { // Save or print depending on the state if (argv.dryrun) { - console.info(m); + log.info(JSON.stringify(m)); } else { bulk.insert(m); } @@ -166,31 +170,29 @@ var tasks = _.map(sources, function (source) { }); var runTasks = function (db) { + log.info('Running all fetch tasks.'); async.parallel(tasks, function (err, results) { if (err) { - console.error(err); + log.error(err); } else { if (!argv.dryrun) { - console.info('All data grabbed and saved.'); + log.info('All data grabbed and saved.'); } - console.info(results); - } - - // Close the database if it exists - if (db) { - db.close(); + results.forEach(function (r) { + log.info(r); + }); } // Send out the webhook to openaq-api since we're all done if (argv.dryrun) { - return console.info('Dryrun completed, have a good day!'); + return log.info('Dryrun completed, have a good day!'); } else { sendUpdatedWebhook(function (err) { if (err) { - console.error(err); + log.error(err); } - return console.info('Webhook posted, have a good day!'); + return log.info('Webhook posted, have a good day!'); }); } }); @@ -198,14 +200,14 @@ var runTasks = function (db) { // Branch here depending on whether this is a dryrun or not if (argv.dryrun) { - console.info('--- Dry run for Testing, nothing is saved to the database. ---'); + log.info('--- Dry run for Testing, nothing is saved to the database. ---'); runTasks(); } else { MongoClient.connect(dbURL, function (err, db) { if (err) { - return console.error(err); + return log.error(err); } - console.info('Connected to database.'); + log.info('Connected to database.'); // Get collection and ensure indexes measurementsCollection = db.collection('measurements'); @@ -248,10 +250,12 @@ if (argv.dryrun) { ], function (err, results) { if (err) { db.close(); - return console.error(err); + log.error(err); + process.exit(1); } - console.info('Indexes created and database ready to go.'); + log.info('Indexes created and database ready to go.'); runTasks(db); + setInterval(function () { runTasks(db); }, fetchInterval); }); }); } diff --git a/lib/logger.js b/lib/logger.js new file mode 100644 index 00000000..3f91eadc --- /dev/null +++ b/lib/logger.js @@ -0,0 +1,33 @@ +'use strict'; + +var winston = require('winston'); +require('winston-papertrail').Papertrail; +var os = require('os'); + +var logLevel = process.env.LOG_LEVEL || 'info'; + +var logger = new (winston.Logger)({ + transports: [ + new (winston.transports.Console)({ + timestamp: function () { + return new Date().toISOString(); + }, + colorize: true, + level: logLevel + }) + ] +}); + +// Add Papertrail logger if we have credentials +if (process.env.PAPERTRAIL_URL) { + logger.add(winston.transports.Papertrail, { + host: process.env.PAPERTRAIL_URL, + port: process.env.PAPERTRAIL_PORT, + hostname: process.env.PAPERTRAIL_HOSTNAME, + colorize: true, + program: os.hostname(), + level: logLevel + }); +} + +module.exports = logger; diff --git a/newrelic.js b/newrelic.js new file mode 100644 index 00000000..56b85b12 --- /dev/null +++ b/newrelic.js @@ -0,0 +1,24 @@ +/** + * New Relic agent configuration. + * + * See lib/config.defaults.js in the agent distribution for a more complete + * description of configuration variables and their potential values. + */ +exports.config = { + /** + * Array of application names. + */ + app_name: ['OpenAQ Fetch'], + /** + * Your New Relic license key. + */ + license_key: 'license key here', + logging: { + /** + * Level at which to log. 'trace' is most useful to New Relic when diagnosing + * issues with the agent, 'info' and higher will impose the least overhead on + * production applications. + */ + level: 'info' + } +}; diff --git a/package.json b/package.json index c93340fc..63b58168 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,9 @@ "repository": "https://github.com/openaq/openaq-fetch", "scripts": { "lint": "semistandard", - "test": "mocha test/**" + "test": "semistandard && mocha test/**", + "start": "node index.js --noemail", + "docker": "docker-compose --project openaq run --rm fetch" }, "author": "Joe Flasher", "license": "MIT", @@ -16,9 +18,12 @@ "moment": "^2.10.3", "moment-timezone": "^0.4.0", "mongodb": "^2.0.45", + "newrelic": "^1.24.0", "request": "^2.58.0", "require-dir": "^0.3.0", "sendgrid": "^1.9.0", + "winston": "^2.1.1", + "winston-papertrail": "^1.0.2", "yargs": "^3.25.0" }, "devDependencies": {