Merge pull request #51 from HackIllinois/prod-tweaks

Production Tweaks

.dockerignore

@@ -0,0 +1,14 @@
+.gitignore
+.dockerignore
+.nodemonignore
+.env
+*.md
+Dockerfile
+!README.md
+
+config/*
+!config/prod.config
+
+node_modules
+test
+temp

.gitignore

@@ -17,4 +17,4 @@ temp
 /.idea
 
 *.config
-
+.env

.nodemonignore

@@ -0,0 +1,12 @@
+database/**
+resources/**
+node_modules/**
+temp/**
+
+.gitignore
+.nodemonignore
+.env
+.DS_STORE
+package.json
+CONTRIBUTING.md
+README.md

Dockerfile

@@ -0,0 +1,28 @@
+FROM node:4.6.2
+
+# ports
+EXPOSE 8080
+
+# install flyway command line tool (and java...)
+WORKDIR /tmp
+RUN  wget https://repo1.maven.org/maven2/org/flywaydb/flyway-commandline/4.0.3/flyway-commandline-4.0.3-linux-x64.tar.gz \
+		&& tar -xzf flyway-commandline-4.0.3-linux-x64.tar.gz \
+		&& mv flyway-4.0.3 /opt/flyway-4.0.3 \
+		&& ln -s /opt/flyway-4.0.3/jre/bin/java /usr/local/bin/java \
+		&& ln -s /opt/flyway-4.0.3/flyway /usr/local/bin/flyway \
+		&& rm -rf /tmp/flyway-* \
+
+# set up the api's working directory
+RUN  mkdir /usr/local/api
+WORKDIR /usr/local/api
+
+# install dependencies
+# (improves build time when no additional modules were added)
+COPY ./package.json ./package.json
+RUN  npm install
+
+# copy the rest of the api into the container
+COPY . .
+
+# launch the api
+CMD  ["npm", "run", "prod"]

README.md

@@ -21,11 +21,11 @@ code to this repository.
 
 #### Node.js Version
 
-Our application is deployed with Node.js v4.4.3. It is recommended that you install
-this version exactly, although using a different release version (v4.4.x) will
+Our application is deployed with Node.js v4.6.2. It is recommended that you install
+this version, although using a different release version (v4.6.x) will
 probably work too.
 
-The production-ready version of Node.js can be downloaded from [here](https://nodejs.org/dist/v4.4.3/) or from your favorite package manager.
+The production-ready version of Node.js can be downloaded from [here](https://nodejs.org/dist/v4.6.2/).
 
 #### MySQL Version
 
@@ -47,45 +47,36 @@ We have provided templates in the directory already with all available configura
 These templates are named `{ENV}.config.template`. You should copy these templates into
 files named `{ENV}.config` and fill them with sensible values; these values can be raw
 values or existing environment variables. Note that changes to the templates are
-commited to the project codebase, but changes to any `*.config` files are ignored.
+committed to the project codebase, but changes to any `*.config` files are ignored.
 
 A list of configuration keys is provided below:
 
 | Key | Possible Values | Purpose |
 | --- | --------------- | ------- |
-| NODE_ENV | 'production' or 'development' | Determines how environment should be configured |
-| PROFILE | Any string | Presents an externally-meaningful identifier |
-| HACKILLINOIS_SECRET | Any string | Sets the master secret (required on production) |
-| HACKILLINOIS_PORT | Any valid port number | Overrides default port (8080) |
-| HACKILLINOIS_SUPERUSER_EMAIL | Any valid email | Overrides the default superuser email ('[email protected]') |
-| HACKILLINOIS_SUPERUSER_PASSWORD | Any string | Overrides the default superuser password ('ABCD1234!') |
+| NODE_ENV | 'production', 'development', 'testing' | Determines how environment should be configured |
+| AWS | 0 or 1 | Whether or not to use AWS |
+| HACKILLINOIS_ID | Any string | Sets the identifier used to represent this instance |
+| HACKILLINOIS_SECRET | Any string | Sets the master secret |
+| HACKILLINOIS_PORT | Any valid port number | Sets the port |
+| HACKILLINOIS_SUPERUSER_EMAIL | Any valid email | Sets the default superuser email |
+| HACKILLINOIS_SUPERUSER_PASSWORD | Any string | Sets the default superuser password |
 | HACKILLINOIS_MAIL_KEY | Any string | Sets the mail service API key |
-| DB_NAME |  Any valid MySQL schema name | Overrides default name (hackillinois) |
-| DB_USERNAME | Any string | Overrides default MySQL username ('root') |
-| DB_PASSWORD | Any string | Overrides default MySQL password ('') |
-| DB_HOSTNAME | Any valid URI | Overrides default MySQL host ('127.0.0.1') |
-| DB_PORT | Any valid port number | Overrides default MySQL port (3306) |
+| DB_NAME |  Any valid MySQL schema name | Sets database name |
+| DB_USERNAME | Any string | Sets MySQL username |
+| DB_PASSWORD | Any string | Sets MySQL password |
+| DB_HOSTNAME | Any valid URI | Sets MySQL host |
+| DB_PORT | Any valid port number | Sets MySQL port |
 
 Additionally, an [AWS shared credentials file](http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/node-configuring.html)
 can be made available with configuration options for those systems under the profile
-identified by the `PROFILE` configuration key. We do not handle AWS-specific configuration
-options in our configuration files.
+identified by `hackillinois-api`. Be sure to set the `AWS` configuration key to `1` and
+add the key `AWS_PROFILE` to your configuration with your corresponding profile name.
 
 #### Considerations
 
-Not all configuration options must be set during development (although all options should
+Not all configuration options must be set during development (but all options _should_
 be set in production). When certain keys are left empty, the API determines whether
-or not it can use a local alternative or a default value at startup. Here are some
-considerations that will help you determine which keys to set as you develop:
-
-* Anyone contributing to a feature that involves email transmissions
-will need to set the `HACKILLINOIS_MAIL_KEY` to a valid SparkPost API key.
-* Anyone contributing to a feature that involves any AWS products will need to set
-up an AWS shared credentials file (see above)
-
-Note that this API is targeted for hosting via AWS, so any AWS-specific settings
-(e.g. those in IAM roles) are used by this API before settings in any environment
-variables or other credentials files.
+or not it can use a local alternative or a default value at startup.
 
 ## Installation
 
@@ -119,21 +110,34 @@ contributors should be familiar with.
 ## Usage
 
 A local API instance can be created on port 8080 via the following commands,
-executed from the root of the project directory.
+executed from the root of the project directory. Note that in development, you must
+to install the process manager `nodemon` globally via `[sudo] npm install -g nodemon`.
 
 To run the API for development:
 ```
 npm run dev
 ```
 
-To run the API in production:
+Note that `node` must and `nodemon` must be on your path and that the configuration
+for the target environment must be present in the `config` directory. The server will
+restart automatically when changes are made. To stop the API, simply type `Control-C`.
+
+For production, we build a Docker image and deploy this to a container ecosystem. You can
+find the Dockerfile in the project root directory.
+
+## Logging
+
+The API access logs are available in `/temp/logs/api.log` as well as on the console
+during development. In production, the logs are periodically pushed to AWS CloudWatch
+(and are not accessible until they are pushed). Note that the logs (excluding those on
+the console) are optimized for searchability, not readability.
+
+You can clean up the file logs during development by running
+
 ```
-npm run prod
+npm run clean-logs
 ```
 
-Use `Control-C` to kill the server. Note that `node` must be on your path and that
-the configuration for the target environment must be present in the `config` directory.
-
 ## Documentation
 
 All documentation is available on the [project wiki](https://github.com/HackIllinois/api-2017/wiki).

api.js

@@ -1,4 +1,5 @@
 var express = require('express');
+var requid = require('cuid');
 
 var config = require('./api/config');
 var database = require('./api/database');
@@ -10,6 +11,7 @@ config.cwd = process.__dirname;
 
 var instance = express();
 instance.disable('x-powered-by');
+instance.use(function (req, res, next) { req.id = requid(); next(); });
 
 var api = require('./api/');
 instance.use('/v1', api.v1);

api/config.js

@@ -3,74 +3,42 @@
 // NOTE: all durations are expressed using notation understood by the
 // `ms` NPM module. These durations must be converted before they are used.
 
-var logger = require('./logging');
-
 const DEVELOPMENT_IDENTIFIER = 'development';
 const PRODUCTION_IDENTIFIER = 'production';
-const TEST_ENVIRONMENT = 'test';
-
-var environment = process.env.NODE_ENV;
-
-var secret = process.env.HACKILLINOIS_SECRET || undefined;
-var superuserEmail = process.env.HACKILLINOIS_SUPERUSER_EMAIL || undefined;
-var superuserPassword = process.env.HACKILLINOIS_SUPERUSER_PASSWORD || undefined;
-var mailApiKey = process.env.HACKILLINOIS_MAIL_KEY || undefined;
-var isTest = environment === TEST_ENVIRONMENT;
-
-var isDevelopment = environment === DEVELOPMENT_IDENTIFIER;
-var isProduction = environment === PRODUCTION_IDENTIFIER;
-
-if (!(isProduction || isDevelopment || isTest)) {
-	logger.error("an environment was not provided");
-	logger.error("set NODE_ENV to '%s', '%s', %s",
-		TEST_ENVIRONMENT, DEVELOPMENT_IDENTIFIER, PRODUCTION_IDENTIFIER);
-
-	process.exit(1);
-}
-
-if (!superuserEmail) {
-	logger.error("set configuration key 'HACKILLINOIS_SUPERUSER_EMAIL' to the desired admin email");
-	process.exit(1);
-}
-
-if (!superuserPassword) {
-	logger.error("set configuration key 'HACKILLINOIS_SUPERUSER_PASSWORD' to a secure, random string");
-	process.exit(1);
-}
-
-if (!secret) {
-	logger.error("set configuration key 'HACKILLINOIS_SECRET' to a secure, random string");
-	process.exit(1);
-}
-
-if (isProduction && !mailApiKey) {
-	logger.error("set configuration key 'HACKILLINOIS_MAIL_KEY' to the mailing provider's API key");
-	process.exit(1);
-}
+const TEST_IDENTIFIER = 'test';
 
 var config = {};
-config.auth = {};
+config.auth = { };
+config.aws = { defaults: {} };
 config.database = {};
 config.database.primary = { pool: {} };
 config.mail = {};
+config.logs = {};
 config.storage = {};
 config.superuser = {};
 config.token = { expiration: {} };
 
-config.isProduction = isProduction;
-config.isDevelopment = isDevelopment;
-config.isTest = isTest;
-config.secret = secret;
+config.environment = process.env.NODE_ENV;
+config.isProduction = (config.environment === PRODUCTION_IDENTIFIER);
+config.isDevelopment = (config.environment === DEVELOPMENT_IDENTIFIER);
+config.isTest = (config.environment === TEST_IDENTIFIER);
+config.id = process.env.HACKILLINOIS_ID;
+config.secret = process.env.HACKILLINOIS_SECRET;
 config.port = process.env.HACKILLINOIS_PORT;
-config.profile = process.env.PROFILE;
 
-config.superuser.email = superuserEmail;
-config.superuser.password = superuserPassword;
+config.superuser.email = process.env.HACKILLINOIS_SUPERUSER_EMAIL;
+config.superuser.password = process.env.HACKILLINOIS_SUPERUSER_PASSWORD;
 
 config.auth.secret = config.secret;
 config.auth.header = 'Authorization';
 config.auth.expiration = '7d';
 
+var sharedAWSCreds = new (require('aws-sdk').SharedIniFileCredentials)();
+config.aws.enabled = (process.env.AWS && !!parseInt(process.env.AWS));
+config.aws.defaults.credentials = (!!sharedAWSCreds.accessKeyId) ? sharedAWSCreds : undefined;
+config.aws.defaults.region = 'us-east-1';
+config.aws.defaults.sslEnabled = true;
+
 config.token.expiration.DEFAULT = '7d';
 config.token.expiration.AUTH = config.token.expiration.DEFAULT;
 
@@ -83,13 +51,33 @@ config.database.primary.pool.min = 0;
 config.database.primary.pool.max = 7500;
 config.database.primary.pool.idleTimeout = '5s';
 
-config.mail.key = mailApiKey;
+config.mail.key = process.env.HACKILLINOIS_MAIL_KEY;
 config.mail.sinkhole = '.sink.sparkpostmail.com';
 config.mail.whitelistedDomains = ['@hackillinois.org'];
 config.mail.whitelistedLists = ['test'];
 
-config.storage.bucketExtension = (isDevelopment) ? '-development' : '-2017';
-
-logger.info("prepared environment for %s", environment);
+config.logs.streamPrefix = 'instances';
+config.logs.groupName = (!config.isProduction) ? 'api-dev' : 'api';
+
+config.storage.bucketExtension = (!config.isProduction) ? '-development' : '-2017';
+
+var exit = true;
+if (!(config.isProduction || config.isDevelopment || config.isTest)) {
+	console.error("error: set NODE_ENV to '%s', '%s', or '%s'", PRODUCTION_IDENTIFIER, DEVELOPMENT_IDENTIFIER, TEST_IDENTIFIER);
+} else if (!config.superuser.email) {
+	console.error("error: set configuration key 'HACKILLINOIS_SUPERUSER_EMAIL' to the desired admin email");
+} else if (!config.superuser.password) {
+	console.error("error: set configuration key 'HACKILLINOIS_SUPERUSER_PASSWORD' to a secure, random string");
+} else if (!config.secret) {
+	console.error("error: set configuration key 'HACKILLINOIS_SECRET' to a secure, random string");
+} else if (config.isProduction && !config.mail.key) {
+	console.error("error: set configuration key 'HACKILLINOIS_MAIL_KEY' to the mailing provider's API key");
+} else {
+	exit = false;
+}
+if (exit) {
+	console.error("fatal: environment incomplete. shutting down...");
+	process.exit();
+}
 
 module.exports = config;

api/database.js

@@ -1,8 +1,6 @@
-var logger = require('./logging');
 var config = require('./config');
+var logger = require('./logging');
 
-var Bookshelf = require('bookshelf');
-var Knex = require('knex');
 var milliseconds = require('ms');
 
 var KNEX_CONFIG = {
@@ -22,10 +20,12 @@ var KNEX_CONFIG = {
 };
 
 function DatabaseManager() {
-	this._knex = Knex(KNEX_CONFIG);
+	this._knex = require('knex')(KNEX_CONFIG);
 
-	this._bookshelf = Bookshelf(this._knex);
-	this._bookshelf.plugin('pagination')
+	this._bookshelf = require('bookshelf')(this._knex);
+	this._bookshelf.plugin('pagination');
+
+	logger.info("connected to database as %s", config.database.primary.user);
 }
 
 DatabaseManager.prototype.constructor = DatabaseManager;
@@ -37,9 +37,4 @@ DatabaseManager.prototype.connection = function () {
 	return this._knex;
 };
 
-if (!config.isTest) {
-	// do not say this when the datastore is mocked
-	logger.info("connected to database as %s", config.database.primary.user);
-}
-
 module.exports = new DatabaseManager();

api/files.js

@@ -14,11 +14,22 @@ var time = require('./v1/utils/time');
 // NOTE all paths are relative to the root of the project directory
 const DIRECTORY_SEPARATOR = '/';
 const TEMP_DIRECTORY = './temp/';
+const LOG_DIRECTORY = TEMP_DIRECTORY + 'logs/';
 const MAIL_DIRECTORY = TEMP_DIRECTORY + 'mail/';
 const STORAGE_DIRECTORY = TEMP_DIRECTORY + 'storage/';
 
 const META_EXTENSION = '.meta';
 const META_SEPARATOR = '^|';
+const LOG_FILENAME = 'api.log';
+
+/**
+ * Sets up the log file for local development
+ * @return {String}		the path to the log file
+ */
+module.exports.initializeLogfile = function () {
+	mkdirp.sync(LOG_DIRECTORY);
+	return LOG_DIRECTORY + LOG_FILENAME;
+};
 
 /**
  * Writes a new mail entry to the temp directory. When called in non-development