Using Docker for deployment to ECS

closes #47

.build_scripts/build.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+if [ -f $HOME/docker/openaq_fetch.tar ]
+then
+  echo "Loading cached worker image"
+  docker load < $HOME/docker/openaq_fetch.tar
+fi
+
+touch local.env
+docker-compose --project openaq build
+
+mkdir -p $HOME/docker
+echo "Caching openaq_fetch docker image."
+docker save openaq_fetch > $HOME/docker/openaq_fetch.tar

.build_scripts/deploy.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+if [[ $TRAVIS_PULL_REQUEST == "false" && $TRAVIS_BRANCH == ${PRODUCTION_BRANCH} ]]; then
+  docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
+
+  echo "Pushing image: developmentseed/openaq-fetch:$TRAVIS_COMMIT"
+  docker tag openaq_fetch flasher/openaq-fetch:$TRAVIS_COMMIT
+  docker push flasher/openaq-fetch:$TRAVIS_COMMIT
+  echo "Also pushing as :latest"
+  docker tag openaq_fetch flasher/openaq-fetch:latest
+  docker push flasher/openaq-fetch:latest
+
+  echo "Installing aws cli"
+  sudo pip install awscli
+
+  echo "Running the update_task script"
+  sh .build_scripts/update-task.sh
+else
+  echo "Not a push; nothing to do"
+fi

.build_scripts/ecs-task-definition.json

@@ -0,0 +1,14 @@
+{
+    "family": "openaq-fetch",
+    "containerDefinitions": [
+        {
+            "name": "openaq-fetch",
+            "image": "flasher/openaq-fetch",
+            "memory": 400,
+            "command": [
+                "npm",
+                "start"
+            ]
+        }
+    ]
+}

.build_scripts/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+# Entrypoint for docker container, basically works as a stand-in for
+# bashrc stuff, setting up the env and then executes whatever command is passed
+# to it
+
+# Source the nvm script so we have access to npm and node
+source $NVM_DIR/nvm.sh
+# NOTE: use exec to make sure that npm receives SIGTERM, etc.
+exec "$@"

.build_scripts/insert-env.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const input = '.build_scripts/ecs-task-definition.json';
+const output = 'ecs-task-generated.json';
+const envFile = 'local.env';
+
+console.info('Inserting env variables into ECS task definition.');
+
+let fs = require('fs');
+
+// Load in intitial JSON
+let obj = JSON.parse(fs.readFileSync(input, 'utf8'));
+
+// Add in env variables, split on newline and remove any extra empty things
+// hanging around.
+let envs = fs.readFileSync(envFile, 'utf8');
+let splitEnvs = [];
+envs.split('\n').forEach(function (e) {
+  if (e) {
+    let idx = e.indexOf('=');
+    splitEnvs.push({ 'name': e.substr(0, idx), 'value': e.substr(idx + 1, e.length) });
+  }
+});
+obj['containerDefinitions'][0]['environment'] = splitEnvs;
+
+// Also set container version based on hash
+let hash = 'latest';
+if (process.env.TRAVIS_COMMIT) {
+  hash = process.env.TRAVIS_COMMIT;
+} else if (process.env.CURRENT_HASH) {
+  hash = process.env.CURRENT_HASH.split(':')[1];
+}
+obj['containerDefinitions'][0]['image'] += ':' + hash;
+
+// Save to output file
+fs.writeFileSync(output, JSON.stringify(obj));
+console.info('Inserted env variables into ECS task definition.');

.build_scripts/update-task.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -e
+
+# Generate the appropriate command to use local env vars or a named AWS profile
+# passed like sh .build_scripts/update_task.sh openaq (to be used locally)
+aws="aws"
+if [ ! -z "$1" ]
+    then
+        aws="aws --profile $1"
+fi
+
+echo "Getting the revision of the old task"
+# This should be updated to check for running revision, not necessarily latest revision
+OLD_VERSION=$($aws ecs describe-task-definition --task-definition openaq-fetch | sed -n "/revision/p" | grep -o "[0-9]\+")
+# Grab this so we're not trying to deploy latest, but rather the last good image
+CURRENT_HASH=$($aws ecs describe-task-definition --task-definition openaq-fetch | grep -o "flasher/openaq-fetch:[a-zA-Z_0-9]\+")
+export CURRENT_HASH=$CURRENT_HASH
+echo "Current revision of ECS task is $OLD_VERSION"
+echo "Current Docker image is $CURRENT_HASH"
+
+echo "Stopping the current task revision"
+$aws ecs update-service --service openaq-fetch --task-definition openaq-fetch --desired-count 0
+
+echo "Waiting for current task to stop"
+$aws ecs wait services-stable --services openaq-fetch
+
+echo "Copying env variables from S3"
+$aws s3 cp s3://openaq-env-variables/openaq-fetch/production.env local.env
+
+echo "Building new ECS task"
+node .build_scripts/insert-env.js
+$aws ecs register-task-definition --cli-input-json file://ecs-task-generated.json
+
+echo "Deploying 1 new ECS task "
+$aws ecs update-service --service openaq-fetch --task-definition openaq-fetch --desired-count 1
+
+echo "Waiting for new task to be scaled up"
+$aws ecs wait services-stable --services openaq-fetch

.gitignore

@@ -4,3 +4,5 @@ npm-debug.log
 docs
 newrelic_agent.log
 dump
+local.env
+ecs-task-generated.json

.travis.yml

@@ -1,11 +1,23 @@
+sudo: true
 language: node_js
 node_js:
-- '0.12'
+- '4.0'
+env:
+  global:
+  - PRODUCTION_BRANCH=master
+cache:
+  directories:
+  - "$HOME/docker"
+services:
+- docker
+before_install:
+- chmod +x ./.build_scripts/build.sh
+- chmod +x ./.build_scripts/deploy.sh
+after_success:
+- ".build_scripts/build.sh"
 deploy:
-  provider: heroku
-  api_key:
-    secure: oFSFUawGi2M6NEh5eSLYGYVsGcpHGtVtP/c808eOsxpa5NCb2qxN5FqF2yZRdUjrfg2vftDLJLmUtbv/6DDc1nh+qGWEABAjmxzIXVH1tAJTBtLu0um6w+uQAXcakTdkJkbpcf5pXW/2pnS6EM0FTIVfS3Bw2O6mjeKx9sEWjcG5/5uZX6f/NtnHqQTRrIdbahDQXW4YCDBoUq4Osk/PtQJ2b7n0UEg+y72oMmZfUtfSXuc57jFOs9J49RP/rn/0M7RvcRcu7akUeP6eIXElYsJmT9cbQUvoMlcBlt5le89lE4hmsxuT9GeLAWU9uEfMvLvS3HBbFpBFZX8Qgk/NnT3QW4aq0FyFhwvgDN79c6yswfPHfJ5Urx1cf/DohBZFYLCQW/MeOHdNIHinh+UTGeO0QbRCfKEN1M89iqq4xiyf6lC+TbYsVEbxvIqRoSkldoqYnskXEKfYHbosY9QK81PF6OTeRcdzHu8cODC4e4rAWYfOoxvYVRDiub8blnCtlfOK6YiXAJMj3xhjbyv+NdyD6MMarHlpaEUx0TR3aMDaRAO3VDa+kqNrM5VQ87KN23W8dmMG3z3oO+8y9pfaSx+n9+khmUTQyG329LK47reP0lifKQkm3py1+qJoLrYWSVOCNSDvIBWj/c5aU7dek2jQ93NFOov9430mm46lQPQ=
-  app: openaq-fetch
+  provider: script
+  skip_cleanup: true
+  script: ".build_scripts/deploy.sh"
   on:
-    repo: openaq/openaq-fetch
-    branch: master
+    branch: ${PRODUCTION_BRANCH}

Dockerfile

@@ -0,0 +1,59 @@
+FROM ubuntu:14.04
+
+# Replace shell with bash so we can source files
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+
+# Set debconf to run non-interactively
+RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
+
+# Install base dependencies
+RUN apt-get update && apt-get install -y -q --no-install-recommends \
+        apt-transport-https \
+        build-essential \
+        ca-certificates \
+        curl \
+        git \
+        libssl-dev \
+        python \
+        rsync \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install nvm with node and npm
+# http://stackoverflow.com/questions/25899912/install-nvm-in-docker
+ENV NVM_DIR /usr/local/nvm
+ENV NODE_VERSION 4.0
+RUN curl https://raw.githubusercontent.com/creationix/nvm/v0.26.0/install.sh | bash \
+    && source $NVM_DIR/nvm.sh \
+    && nvm install $NODE_VERSION \
+    && nvm alias default $NODE_VERSION \
+    && nvm use default
+ENV PATH      $NVM_BIN:$PATH
+
+# Go ahead and install nodemon for convenience while developing
+RUN source $NVM_DIR/nvm.sh
+
+###########################
+# App-specific stuff
+
+# mongo uses kerberos
+RUN apt-get update && apt-get install -y libkrb5-dev
+
+# Install NPM dependencies. Do this first so that if package.json hasn't
+# changed we don't have to re-run npm install during `docker build`
+COPY package.json /app/package.json
+WORKDIR /app
+RUN source $NVM_DIR/nvm.sh; npm install
+# Copy the app
+COPY ["index.js", "newrelic.js", "/app/"]
+COPY lib /app/lib/
+COPY test /app/test/
+COPY sources /app/sources/
+COPY adapters /app/adapters/
+
+#############################
+# entrypoint
+#
+RUN source $NVM_DIR/nvm.sh
+ADD .build_scripts/entrypoint.sh /
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

README.md

@@ -4,7 +4,7 @@
 ## Overview
 This is the main data ingest pipeline for the [OpenAQ](https://openaq.org) project.
 
-Starting with `fetch.js`, there is an ingest mechanism to gather global air quality measurements from a variety of sources. This is currently run every 10 minutes and saves all unique measurements to a database.
+Starting with `index.js`, there is an ingest mechanism to gather global air quality measurements from a variety of sources. This is currently run every 10 minutes and saves all unique measurements to a database.
 
 [openaq-api](https://github.com/openaq/openaq-api) powers the API and more information on the data format can be found in [openaq-data-format](https://github.com/openaq/openaq-data-format).
 
@@ -17,7 +17,7 @@ Install necessary Node.js packages by running
 
 Make sure you have MongoDB running locally and then you can get help with
 
-`node fetch.js --help`
+`node index.js --help`
 
 For the above to work, you will need to have certain environment variables set as in the table below
 
@@ -29,12 +29,16 @@ For the above to work, you will need to have certain environment variables set a
 | SENDGRID_USERNAME | Email service username | not set |
 | API_URL | URL of openaq-api | http://localhost:3004/v1/webhooks |
 | WEBHOOK_KEY | Secret key to interact with openaq-api | '123' |
+| FETCH_INTERVAL | How often to run fetch tasks | 10 minutes |
 
 ## Tests
 To confirm that everything is working as expected, you can run the tests with
 
 `npm test`
 
+## Deployment
+Deployment is handled automatically via Travis on the `master` branch and is deployed to Amazon's ECS.
+
 ## Data Source Criteria
 This section lists the key criteria for air quality data aggregated onto the platform. A full explanation can be accessed here [https://medium.com/@openaq/where-does-openaq-data-come-from-a5cf9f3a5c85#.919hlx2by]. OpenAQ is an ever-evolving process that is shaped by its community: your feedback and questions are actively invited on the criteria listed in this section. 
 

adapters/agaar_mn.js

@@ -3,14 +3,15 @@
 var request = require('request');
 var _ = require('lodash');
 var moment = require('moment-timezone');
+var log = require('../lib/logger');
 
 exports.name = 'agaar_mn';
 
 exports.fetchData = function (source, cb) {
   var finalURL = source.url;
   request(finalURL, function (err, res, body) {
     if (err || res.statusCode !== 200) {
-      console.error(err || res);
+      log.error(err || res);
       return cb({message: 'Failure to load data url.'});
     }
 

adapters/beijing.js

@@ -3,14 +3,15 @@
 var request = require('request');
 var _ = require('lodash');
 var moment = require('moment-timezone');
+var log = require('../lib/logger');
 
 exports.name = 'beijing';
 
 exports.fetchData = function (source, cb) {
   var finalURL = source.url + '?apitoken=' + process.env.INDIA_KIMONO_TOKEN;
   request(finalURL, function (err, res, body) {
     if (err || res.statusCode !== 200) {
-      console.error(err || res);
+      log.error(err || res);
       return cb({message: 'Failure to load data url.'});
     }
 

adapters/chile.js

@@ -27,7 +27,7 @@ exports.fetchData = function (source, cb) {
 
   async.parallel(tasks, function (err, results) {
     if (err) {
-      return console.log(err);
+      return cb({message: 'Failure to load data urls.'});
     }
 
     // Wrap everything in a try/catch in case something goes wrong

adapters/india.js

@@ -4,14 +4,15 @@ var request = require('request');
 var _ = require('lodash');
 var moment = require('moment-timezone');
 var utils = require('../lib/utils');
+var log = require('../lib/logger');
 
 exports.name = 'india';
 
 exports.fetchData = function (source, cb) {
   var finalURL = source.url + '?apitoken=' + process.env.INDIA_KIMONO_TOKEN;
   request(finalURL, function (err, res, body) {
     if (err || res.statusCode !== 200) {
-      console.error(err || res);
+      log.error(err || res);
       return cb({message: 'Failure to load data url.'});
     }
 

adapters/london.js

@@ -3,14 +3,15 @@
 var request = require('request');
 var _ = require('lodash');
 var moment = require('moment-timezone');
+var log = require('../lib/logger');
 
 exports.name = 'london';
 
 exports.fetchData = function (source, cb) {
   var finalURL = source.url + '?apitoken=' + process.env.INDIA_KIMONO_TOKEN;
   request(finalURL, function (err, res, body) {
     if (err || res.statusCode !== 200) {
-      console.error(err || res);
+      log.error(err || res);
       return cb({message: 'Failure to load data url.'});
     }
 

adapters/netherlands.js

@@ -5,14 +5,15 @@ var _ = require('lodash');
 var cheerio = require('cheerio');
 var async = require('async');
 var moment = require('moment-timezone');
+var log = require('../lib/logger');
 
 exports.name = 'netherlands';
 
 exports.fetchData = function (source, cb) {
   var finalURL = source.url;
   request(finalURL, function (err, res, body) {
     if (err || res.statusCode !== 200) {
-      console.error(err || res);
+      log.error(err || res);
       return cb({message: 'Failure to load data url.'});
     }
 
@@ -46,7 +47,7 @@ exports.fetchData = function (source, cb) {
 
     async.parallel(tasks, function (err, results) {
       if (err) {
-        return console.error(err);
+        return cb({message: 'Failure to load data urls.'});
       }
 
       var result = {

adapters/nsw.js

@@ -5,13 +5,14 @@ var _ = require('lodash');
 var moment = require('moment-timezone');
 var cheerio = require('cheerio');
 var utils = require('../lib/utils');
+var log = require('../lib/logger');
 
 exports.name = 'nsw';
 
 exports.fetchData = function (source, cb) {
   request(source.url, function (err, res, body) {
     if (err || res.statusCode !== 200) {
-      console.error(err || res);
+      log.error(err || res);
       return cb({message: 'Failure to load data url.'});
     }