Merge pull request #66 from JarvusInnovations/develop

Release: emergence v1.1.0

.dockerignore

@@ -0,0 +1,7 @@
+**/node_modules
+.eslintrc.json
+.gitignore
+.npmignore
+cloud-config.yaml
+Dockerfile
+run-dev-container

.npmignore

@@ -0,0 +1,6 @@
+.dockerignore
+.eslintrc.json
+.gitignore
+cloud-config.yaml
+Dockerfile
+run-dev-container

Dockerfile

@@ -1,5 +1,6 @@
 FROM ubuntu:16.04
 
+
 # initialize .ssh directory
 RUN mkdir -p ~/.ssh \
     && chmod 700 ~/.ssh \
@@ -23,9 +24,6 @@ RUN export DEBIAN_FRONTEND=noninteractive \
         mysql-client \
         mysql-server \
         nginx \
-        nodejs \
-        nodejs-legacy \
-        npm \
         openssh-server \
         postfix \
         python \
@@ -38,8 +36,12 @@ RUN export DEBIAN_FRONTEND=noninteractive \
         tmux \
         vim \
     && add-apt-repository -y ppa:ondrej/php \
-    && apt-get update \
+    && add-apt-repository ppa:certbot/certbot \
+    && curl -sL https://deb.nodesource.com/setup_10.x | bash \
+    # && apt-get update \ # above calls apt-get update
     && apt-get install -y --allow-unauthenticated --no-install-recommends \
+        certbot python3-pyasn1 \
+        nodejs \
         php-apcu \
         php5.6-cli \
         php5.6-curl \
@@ -66,14 +68,19 @@ RUN service nginx stop \
 
 
 # install Habitat client and packages for emergence
-RUN curl -s https://raw.githubusercontent.com/habitat-sh/habitat/master/components/hab/install.sh | sudo bash
-RUN hab pkg install jarvus/sencha-cmd/5.1.3.61/20170606195324 jarvus/underscore \
+RUN curl -s https://raw.githubusercontent.com/habitat-sh/habitat/master/components/hab/install.sh | bash
+RUN hab pkg install jarvus/sencha-cmd/6.5.2.15 jarvus/underscore \
     && hab pkg binlink jarvus/sencha-cmd sencha \
     && hab pkg binlink jarvus/underscore underscore
 
 
+# install helpful administrative commands
+RUN npm install -g htpasswd
+
+
 # install emergence
-RUN npm install -g emergence
+COPY . /src
+RUN npm install -g /src
 
 
 # setup and expose emergence

bin/certbot-auth

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+log () {
+    echo "emergence-certbot-auth: $@"
+}
+
+die () {
+    echo >&2 "emergence-certbot-auth: $@"
+    exit 1
+}
+
+
+# check usage
+[ -n "${CERTBOT_DOMAIN}" ] || die "required environment variable missing: \$CERTBOT_DOMAIN"
+[ -n "${CERTBOT_VALIDATION}" ] || die "required environment variable missing: \$CERTBOT_VALIDATION"
+[ -n "${CERTBOT_TOKEN}" ] || die "required environment variable missing: \$CERTBOT_TOKEN"
+
+
+# determine location of this script
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+
+# resolve site handle for domain
+SITE_HANDLE="$(${DIR}/resolve-site ${CERTBOT_DOMAIN})"
+[ -n "${SITE_HANDLE}" ] || die "could not resolve domain '${CERTBOT_DOMAIN}' under /emergence/sites/*/site.json"
+
+
+# write auth file
+log "writing ${SITE_HANDLE}/${CERTBOT_TOKEN}"
+echo "${CERTBOT_VALIDATION}" | "${DIR}/write-file" "${SITE_HANDLE}" "site-root/.well-known/acme-challenge/${CERTBOT_TOKEN}" || die "failed to write token to VFS"

bin/certbot-cleanup

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+log () {
+    echo "emergence-certbot-cleanup: $@"
+}
+
+die () {
+    echo >&2 "emergence-certbot-cleanup: $@"
+    exit 1
+}
+
+
+# check usage
+[ -n "${CERTBOT_DOMAIN}" ] || die "required environment variable missing: \$CERTBOT_DOMAIN"
+[ -n "${CERTBOT_TOKEN}" ] || die "required environment variable missing: \$CERTBOT_TOKEN"
+
+
+# determine location of this script
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+
+# resolve site handle for domain
+SITE_HANDLE="$(${DIR}/resolve-site ${CERTBOT_DOMAIN})"
+[ -n "${SITE_HANDLE}" ] || die "could not resolve domain '${CERTBOT_DOMAIN}' under /emergence/sites/*/site.json"
+
+
+# delete auth file
+log "deleting ${SITE_HANDLE}/${CERTBOT_TOKEN}"
+"${DIR}/delete-file" "${SITE_HANDLE}" "site-root/.well-known/acme-challenge/${CERTBOT_TOKEN}" || die "failed to delete token from VFS"

bin/delete-file

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+die () {
+    echo >&2 "$@"
+    exit 1
+}
+
+
+# check usage
+[ -n "$1" ] && [ -n "$2" ] || die "Usage: emergence-delete-file <site-handle> <site-file-path>"
+
+
+# determine location of this script
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+
+# execute php script
+cat <<'END_OF_PHP' | "${DIR}/shell" "$1" --stdin "$2"
+<?php
+
+$node = Site::resolvePath($argv[1]);
+
+if (!$node) {
+    error_log('emergence-delete-file: cannot remove \''.$argv[1].'\': No such file');
+    exit(1);
+}
+
+try {
+    $node->delete();
+    exit(0);
+} catch (Exception $e) {
+    error_log('emergence-delete-file: failed to delete file from VFS: '.$e->getMessage());
+    exit(1);
+}
+
+END_OF_PHP
+
+

bin/read-file

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+die () {
+    echo >&2 "$@"
+    exit 1
+}
+
+
+# check usage
+[ -n "$1" ] && [ -n "$2" ] || die "Usage: emergence-read-file <site-handle> <site-file-path>"
+
+
+# determine location of this script
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+
+# execute php script
+cat <<'END_OF_PHP' | "${DIR}/shell" "$1" --stdin "$2"
+<?php
+
+$node = Site::resolvePath($argv[1]);
+
+if (!$node) {
+    error_log('emergence-read-file: cannot read \''.$argv[1].'\': No such file');
+    exit(1);
+}
+
+readfile($node->RealPath);
+exit(0);
+
+END_OF_PHP
+
+

bin/resolve-site

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+die () {
+    echo >&2 "emergence-resolve-site: $@"
+    exit 1
+}
+
+
+# check usage
+[ -n "$1" ] || die "Usage: emergence-resolve-site <hostname>"
+[ -d "/emergence/sites" ] || die "/emergence/sites not found"
+
+
+# determine location of this script
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+
+# determine path to underscore
+UNDERSCORE="$DIR/../node_modules/.bin/underscore"
+
+
+# search sites
+pushd /emergence/sites > /dev/null
+for SITE_DIR in `find . -maxdepth 1 ! -path . -type d`; do
+    SITE_HANDLE="$(basename ${SITE_DIR})"
+    SITE_HOSTNAMES="$(sudo cat ${SITE_DIR}/site.json | $UNDERSCORE process '(data.hostnames||[]).concat([data.primary_hostname]).filter(x=>x)' --outfmt text)"
+
+    for SITE_HOSTNAME in $SITE_HOSTNAMES; do
+        if [[ "${1}" == $SITE_HOSTNAME ]]; then # NOT MATCHING WILDCARD
+            echo "${SITE_HANDLE}"
+            exit 0
+        fi
+    done
+done
+popd > /dev/null
+
+
+# return failure if no match found
+exit 1

bin/shell

@@ -57,6 +57,10 @@ fi
 
 # execute interactive shell
 TERM=$TERM sudo -E -u www-data -g www-data php -d auto_prepend_file=$autoPrependScript -d apc.enable_cli=on -d memory_limit=-1 $INPUT_ARGS
+SCRIPT_STATUS=$?
 
 # clean up
 rm $autoPrependScript;
+
+# relay exit status of php script
+exit $SCRIPT_STATUS

bin/write-file

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+die () {
+    echo >&2 "$@"
+    exit 1
+}
+
+
+# check usage
+[ -n "$1" ] && [ -n "$2" ] || die "Usage: emergence-write-file <site-handle> <site-file-path> [host-file-path]"
+
+
+# determine location of this script
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+
+
+
+# read stdin to tmp
+if [ -z "${3}" ]; then
+    # determine path to underscore
+    UNDERSCORE="$DIR/../node_modules/.bin/underscore"
+
+    # read application group name
+    APP_GROUP="$(sudo cat /emergence/config.json | $UNDERSCORE extract --outfmt text group)"
+    INPUT_FILE="$(mktemp)"
+    chgrp "${APP_GROUP}" "${INPUT_FILE}"
+    chmod g+r "${INPUT_FILE}"
+
+    cat - > "${INPUT_FILE}"
+elif [ ! -f "${3}" ]; then
+    die "Input file ${3} not found"
+else
+    INPUT_FILE="${3}"
+fi
+
+
+# execute php script
+cat <<'END_OF_PHP' | exec "${DIR}/shell" "${1}" --stdin "${2}" "${INPUT_FILE}"
+<?php
+
+try {
+    Emergence_FS::importFile($argv[2], $argv[1]);
+    exit(0);
+} catch (Exception $e) {
+    error_log('emergence-write-file: failed to delete file from VFS: '.$e->getMessage());
+    exit(1);
+}
+
+END_OF_PHP
+
+

kernel-lib/server.js

@@ -7,7 +7,8 @@ var http = require('http'),
     url = require('url'),
     static = require('node-static'),
     events = require('events'),
-    nodeCleanup = require('node-cleanup');
+    nodeCleanup = require('node-cleanup'),
+    httpAuth = require('http-auth');
 
 
 exports.createServer = function (paths, options) {
@@ -41,9 +42,9 @@ util.inherits(Server, events.EventEmitter);
 
 Server.prototype.start = function () {
     // create authenticator
-    this.httpAuth = require('http-auth')({
-        authRealm: 'Emergence Node Management',
-        authFile: '/emergence/admins.htpasswd'
+    const basicAuth = httpAuth.basic({
+        realm: 'Emergence Node Management',
+        file: '/emergence/admins.htpasswd'
     });
 
     // create static fileserver
@@ -58,7 +59,7 @@ Server.prototype.start = function () {
 
         this.webProtocol = 'https';
     } else {
-        this.webServer = http.createServer(this.handleWebRequest.bind(this)).listen(this.options.port, this.options.host);
+        this.webServer = http.createServer(basicAuth, this.handleRequest.bind(this)).listen(this.options.port, this.options.host);
 
         this.webProtocol = 'http';
     }
@@ -72,14 +73,6 @@ Server.prototype.start = function () {
     console.log('Management server listening on '+this.webProtocol+'://'+this.options.host+':'+this.options.port);
 };
 
-Server.prototype.handleWebRequest = function (request, response) {
-    var me = this;
-
-    me.httpAuth.apply(request, response, function () {
-        me.handleRequest(request, response);
-    });
-};
-
 Server.prototype.handleRequest = function (request, response) {
     var me = this;