From 5b4f4c6b8e24c17b1874d0c670700a01e485a064 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Mon, 1 May 2017 12:35:54 -0700 Subject: [PATCH 01/23] Create vault token for policy to read secret --- .../templates/registry_policy.tmpl | 3 +++ init.sh | 1 + lib/dock.sh | 21 +++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 consul-resources/templates/registry_policy.tmpl diff --git a/consul-resources/templates/registry_policy.tmpl b/consul-resources/templates/registry_policy.tmpl new file mode 100644 index 0000000..eeb9539 --- /dev/null +++ b/consul-resources/templates/registry_policy.tmpl @@ -0,0 +1,3 @@ +path "secret/organization/{{bpid}}" { + policy = "read" +} diff --git a/init.sh b/init.sh index 66c551e..b20cf02 100644 --- a/init.sh +++ b/init.sh @@ -53,6 +53,7 @@ main() { aws::get_org_id dock::set_hostname dock::set_config_org + dock::store_vault_token container::start # rules must be run after docker has started iptables::run_rules diff --git a/lib/dock.sh b/lib/dock.sh index 99074b8..348bffe 100644 --- a/lib/dock.sh +++ b/lib/dock.sh @@ -10,6 +10,7 @@ # @module dock source "${DOCK_INIT_BASE}/lib/cert.sh" +source "${DOCK_INIT_BASE}/lib/consul.sh" source "${DOCK_INIT_BASE}/lib/util/backoff.sh" source "${DOCK_INIT_BASE}/lib/util/log.sh" source "${DOCK_INIT_BASE}/lib/util/rollbar.sh" @@ -20,6 +21,26 @@ dock::set_config_org() { echo DOCKER_OPTS=\"\$DOCKER_OPTS --label org="${ORG_ID}"\" >> /etc/default/docker } +# creates a token for a specific policy +dock::store_vault_token() { + # export VAULT_ADDR="/* new host */" + log::info "Storing vault token for private registry key" + local NODE_ENV=$(consul::get node/env) + local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" + VAULT_TOKEN=$(cat "${token_path}"/auth-token) + vault auth ${VAULT_TOKEN} + POLICY=$(vault policies | grep "^${ORG_ID}\b") + if [[ $POLICY ]]; then + log::info "Policy found for $ORG_ID, generating token" + vault token-create -policy=${ORG_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token + else + log::info "Creating new policy and token for $ORG_ID" + sed "s/{{bpid}}/${ORG_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" + vault policy-write ${ORG_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" + vault token-create -policy=${ORG_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token + fi +} + # adds org to hostname dock::set_hostname() { log::info "Adding organization id in hostname" From e25f4bc1ad33e3eb61b53f444ffdb9b5a3688552 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Mon, 1 May 2017 15:17:58 -0700 Subject: [PATCH 02/23] Reuse AWS creds for bp tag --- .../templates/get-org-tag.sh.ctmpl | 24 +---- init.sh | 1 + lib/aws.sh | 95 +++++++++++-------- lib/dock.sh | 11 +-- 4 files changed, 62 insertions(+), 69 deletions(-) diff --git a/consul-resources/templates/get-org-tag.sh.ctmpl b/consul-resources/templates/get-org-tag.sh.ctmpl index ecc8d7e..9644056 100644 --- a/consul-resources/templates/get-org-tag.sh.ctmpl +++ b/consul-resources/templates/get-org-tag.sh.ctmpl @@ -4,26 +4,6 @@ set -e # WARNING: do not echo anything except ORG ID here {{ with vault "aws_1h/creds/dock-init" }} -AWS_ACCESS_KEY="{{ .Data.access_key }}" -AWS_SECRET_KEY="{{ .Data.secret_key }}" +export AWS_ACCESS_KEY="{{ .Data.access_key }}" +export AWS_SECRET_KEY="{{ .Data.secret_key }}" {{ end }} - -EC2_HOME=/usr/local/ec2 -export EC2_HOME - -JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/jre -export JAVA_HOME - -instance_id=$(ec2-metadata -i | awk '{print $2}') - -# Note: this only works for us-.{4}-\d -region=$(ec2-metadata --availability-zone | awk '{ where = match($2, /us\-.+\-[1|2]/); print substr($2, where, 9); }') - -/usr/local/ec2/bin/ec2-describe-tags \ - --aws-access-key="$AWS_ACCESS_KEY" \ - --aws-secret-key="$AWS_SECRET_KEY" \ - --filter "resource-type=instance" \ - --filter "resource-id=$instance_id" \ - --filter "key=org" \ - --region "$region" \ - | awk '{print $5}' diff --git a/init.sh b/init.sh index b20cf02..ad8c65a 100644 --- a/init.sh +++ b/init.sh @@ -51,6 +51,7 @@ main() { consul::configure_consul_template dock::generate_certs aws::get_org_id + aws::get_poppa_id dock::set_hostname dock::set_config_org dock::store_vault_token diff --git a/lib/aws.sh b/lib/aws.sh index 56ee0de..2a4408e 100644 --- a/lib/aws.sh +++ b/lib/aws.sh @@ -8,33 +8,26 @@ source "${DOCK_INIT_BASE}/lib/util/halter.sh" # @author Ryan Sandor Richards # @module aws -# Backoff routine that attempts to fetch the dock's org id from EC2 tags -aws::fetch_org_id_from_tags() { - local attempt=${1} +# get aws creds for these scripts... +aws::get_aws_creds() { + # Generate the org-tag fetching script + rollbar::fatal_trap \ + "Dock-Init: Failed to Render Org Script" \ + "Consule-Template was unable to realize the given template." - log::info 'Attempting to get org id...' - data='{"attempt":'"${attempt}"'}' + ORG_SCRIPT=$DOCK_INIT_BASE/util/get-org-id.sh + + local config="$DOCK_INIT_BASE/consul-resources/template-config.hcl" + local template="$DOCK_INIT_BASE" + template+="/consul-resources/templates/get-org-tag.sh.ctmpl:$ORG_SCRIPT" + + consul-template -config="${config}" -once -template="${template}" - rollbar::warning_trap \ - "Dock-Init: Cannot Fetch Org" \ - "Attempting to get the Org Tag from AWS and failing." \ - "$data" - ORG_ID=$(bash "$ORG_SCRIPT") - log::trace "Script Output: $ORG_ID" rollbar::clear_trap + # give amazon a chance to get the auth + sleep 5 - if [[ "$ORG_ID" != "" ]]; then - # Assume first value in host_tags comma separated list is org ID... - ORG_ID=$(echo "$ORG_ID" | cut -d, -f 1) - export ORG_ID - return 0 - else - # report the attempt to rollbar, since we don't want this to always fail - rollbar::report_warning \ - "Dock-Init: Failed to Fetch Org" \ - "Org Script returned an empty string. Retrying." - return 1 - fi + source "${DOCK_INIT_BASE}/util/get-org-id.sh" } # Fetches the org tags from EC2 and sets it to the `ORG_ID` environment variable @@ -46,26 +39,13 @@ aws::get_org_id() { "Dock-Init: Failed to Render Org Script" \ "Consule-Template was unable to realize the given template." if [ -z ${AWS_ACCESS_KEY+x} ] || [ -z ${AWS_SECRET_KEY+x} ]; then - ORG_SCRIPT=$DOCK_INIT_BASE/util/get-org-id.sh - - local config="$DOCK_INIT_BASE/consul-resources/template-config.hcl" - local template="$DOCK_INIT_BASE" - template+="/consul-resources/templates/get-org-tag.sh.ctmpl:$ORG_SCRIPT" - - consul-template -config="${config}" -once -template="${template}" - - rollbar::clear_trap - - # give amazon a chance to get the auth - sleep 5 - - # Attempt to fetch the org id from the tags via the fetch script - backoff aws::fetch_org_id_from_tags - else - log::info "Taking aws creds from system" - backoff aws::get_org_id_onprem + aws::get_aws_creds fi + log::info "access key $AWS_ACCESS_KEY" + log::info "secret key $AWS_SECRET_KEY" + backoff aws::get_org_id_onprem + if [[ "$ORG_ID" == "" ]]; then # this will print an error, so that's good rollbar::report_error \ @@ -110,3 +90,36 @@ aws::get_org_id_onprem() { export ORG_ID } + +# Fetches the poppa tags from EC2 and sets it to the `POPPA_ID` environment variable +aws::get_poppa_id() { + log::info "Setting Poppa ID $AWS_ACCESS_KEY $AWS_SECRET_KEY" + + # Generate the org-tag fetching script + rollbar::fatal_trap \ + "Dock-Init: Failed to Render Org Script" \ + "Consule-Template was unable to realize the given template." + EC2_HOME=/usr/local/ec2 + export EC2_HOME + + JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/jre + export JAVA_HOME + + local instance_id=$(ec2-metadata -i | awk '{print $2}') + + # Note: this only works for us-.{4}-\d + local region=$(ec2-metadata --availability-zone | awk '{ where = match($2, /us\-.+\-[1|2]/); print substr($2, where, 9); }') + + POPPA_ID=$(bash /usr/local/ec2/bin/ec2-describe-tags \ + --aws-access-key="${AWS_ACCESS_KEY}" \ + --aws-secret-key="${AWS_SECRET_KEY}" \ + --filter "resource-type=instance" \ + --filter "resource-id=${instance_id}" \ + --filter "key=big-poppa" \ + --region "${region}" \ + | awk '{print $5}') + + export POPPA_ID + + log::info "Got POPPA_ID: $POPPA_ID" +} diff --git a/lib/dock.sh b/lib/dock.sh index 348bffe..57a2f8d 100644 --- a/lib/dock.sh +++ b/lib/dock.sh @@ -29,16 +29,15 @@ dock::store_vault_token() { local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" VAULT_TOKEN=$(cat "${token_path}"/auth-token) vault auth ${VAULT_TOKEN} - POLICY=$(vault policies | grep "^${ORG_ID}\b") + POLICY=$(vault policies | grep "^${POPPA_ID}\b") if [[ $POLICY ]]; then - log::info "Policy found for $ORG_ID, generating token" - vault token-create -policy=${ORG_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token + log::info "Policy found for $POPPA_ID, generating token" else log::info "Creating new policy and token for $ORG_ID" - sed "s/{{bpid}}/${ORG_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" - vault policy-write ${ORG_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" - vault token-create -policy=${ORG_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token + sed "s/{{bpid}}/${POPPA_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" + vault policy-write ${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" fi + vault token-create -policy=${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token } # adds org to hostname From 7274e6af1add9ed1babaca117499176e97bd752c Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Mon, 1 May 2017 15:54:26 -0700 Subject: [PATCH 03/23] Add logging, move token-create to vault --- init.sh | 2 +- lib/aws.sh | 4 +--- lib/dock.sh | 20 -------------------- lib/vault.sh | 22 ++++++++++++++++++++++ 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/init.sh b/init.sh index ad8c65a..e5d8a39 100644 --- a/init.sh +++ b/init.sh @@ -54,7 +54,7 @@ main() { aws::get_poppa_id dock::set_hostname dock::set_config_org - dock::store_vault_token + vault::store_private_registry_token container::start # rules must be run after docker has started iptables::run_rules diff --git a/lib/aws.sh b/lib/aws.sh index 2a4408e..b49e8ed 100644 --- a/lib/aws.sh +++ b/lib/aws.sh @@ -42,8 +42,6 @@ aws::get_org_id() { aws::get_aws_creds fi - log::info "access key $AWS_ACCESS_KEY" - log::info "secret key $AWS_SECRET_KEY" backoff aws::get_org_id_onprem if [[ "$ORG_ID" == "" ]]; then @@ -93,7 +91,7 @@ aws::get_org_id_onprem() { # Fetches the poppa tags from EC2 and sets it to the `POPPA_ID` environment variable aws::get_poppa_id() { - log::info "Setting Poppa ID $AWS_ACCESS_KEY $AWS_SECRET_KEY" + log::info "Setting Poppa ID" # Generate the org-tag fetching script rollbar::fatal_trap \ diff --git a/lib/dock.sh b/lib/dock.sh index 57a2f8d..99074b8 100644 --- a/lib/dock.sh +++ b/lib/dock.sh @@ -10,7 +10,6 @@ # @module dock source "${DOCK_INIT_BASE}/lib/cert.sh" -source "${DOCK_INIT_BASE}/lib/consul.sh" source "${DOCK_INIT_BASE}/lib/util/backoff.sh" source "${DOCK_INIT_BASE}/lib/util/log.sh" source "${DOCK_INIT_BASE}/lib/util/rollbar.sh" @@ -21,25 +20,6 @@ dock::set_config_org() { echo DOCKER_OPTS=\"\$DOCKER_OPTS --label org="${ORG_ID}"\" >> /etc/default/docker } -# creates a token for a specific policy -dock::store_vault_token() { - # export VAULT_ADDR="/* new host */" - log::info "Storing vault token for private registry key" - local NODE_ENV=$(consul::get node/env) - local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" - VAULT_TOKEN=$(cat "${token_path}"/auth-token) - vault auth ${VAULT_TOKEN} - POLICY=$(vault policies | grep "^${POPPA_ID}\b") - if [[ $POLICY ]]; then - log::info "Policy found for $POPPA_ID, generating token" - else - log::info "Creating new policy and token for $ORG_ID" - sed "s/{{bpid}}/${POPPA_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" - vault policy-write ${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" - fi - vault token-create -policy=${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token -} - # adds org to hostname dock::set_hostname() { log::info "Adding organization id in hostname" diff --git a/lib/vault.sh b/lib/vault.sh index 74aff74..e0e33ed 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -4,6 +4,7 @@ # @author Anandkumar Patel # @module vault +source "${DOCK_INIT_BASE}/lib/consul.sh" source "${DOCK_INIT_BASE}/lib/util/log.sh" source "${DOCK_INIT_BASE}/lib/util/rollbar.sh" @@ -52,3 +53,24 @@ vault::set_s3_keys() { export S3_SECRET_KEY rollbar::clear_trap } + +# creates a token for a specific policy +vault::store_private_registry_token() { + rollbar::fatal_trap \ + "Dock-Init: Cannot create private registry token" \ + "Attempting to create private registry token. ${OUTPUT}" + # export VAULT_ADDR="/* new host */" + log::info "Storing vault token for private registry key" + local NODE_ENV=$(consul::get node/env) + export VAULT_ADDR="http://${VAULT_HOSTNAME}:${VAULT_PORT}" + POLICY=$(vault policies | grep "^${POPPA_ID}\b") + log::info "checked vault policies" + if [[ $POLICY ]]; then + log::info "Policy found for $POPPA_ID, generating token" + else + log::info "Creating new policy and token for $POPPA_ID" + sed "s/{{bpid}}/${POPPA_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" + vault policy-write ${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" + fi + vault token-create -policy=${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token +} From 93739681ff3c86a676bbae17c0619a52091e756e Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Mon, 1 May 2017 16:12:23 -0700 Subject: [PATCH 04/23] Add backoff --- lib/aws.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/aws.sh b/lib/aws.sh index b49e8ed..4d806bd 100644 --- a/lib/aws.sh +++ b/lib/aws.sh @@ -39,7 +39,7 @@ aws::get_org_id() { "Dock-Init: Failed to Render Org Script" \ "Consule-Template was unable to realize the given template." if [ -z ${AWS_ACCESS_KEY+x} ] || [ -z ${AWS_SECRET_KEY+x} ]; then - aws::get_aws_creds + backoff aws::get_aws_creds fi backoff aws::get_org_id_onprem From 111ac32bae34715aa4778da9326fc60c27ff4aaf Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Mon, 1 May 2017 16:34:34 -0700 Subject: [PATCH 05/23] Left comments for future customization --- lib/vault.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/vault.sh b/lib/vault.sh index e0e33ed..da52d40 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -56,15 +56,15 @@ vault::set_s3_keys() { # creates a token for a specific policy vault::store_private_registry_token() { - rollbar::fatal_trap \ - "Dock-Init: Cannot create private registry token" \ - "Attempting to create private registry token. ${OUTPUT}" - # export VAULT_ADDR="/* new host */" log::info "Storing vault token for private registry key" local NODE_ENV=$(consul::get node/env) + # this will pull from the vault currently running (our vault) export VAULT_ADDR="http://${VAULT_HOSTNAME}:${VAULT_PORT}" - POLICY=$(vault policies | grep "^${POPPA_ID}\b") - log::info "checked vault policies" + # this might also be needed if we use a different root token + + # VAULT_TOKEN=$(cat "${token_path}"/auth-token) + # vault auth ${VAULT_TOKEN} vault auth ${VAULT_TOKEN} + local POLICY=$(vault policies | grep "^${POPPA_ID}\b") if [[ $POLICY ]]; then log::info "Policy found for $POPPA_ID, generating token" else @@ -72,5 +72,6 @@ vault::store_private_registry_token() { sed "s/{{bpid}}/${POPPA_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" vault policy-write ${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" fi + # need to set the final directory for the token here vault token-create -policy=${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token } From 209fa809c9f316946bfbf7de10d577803ae7b2bc Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Mon, 1 May 2017 17:12:38 -0700 Subject: [PATCH 06/23] Combine function vars... --- init.sh | 3 +-- lib/aws.sh | 64 +++++++++++++++++++++------------------------------- lib/vault.sh | 2 +- 3 files changed, 28 insertions(+), 41 deletions(-) diff --git a/init.sh b/init.sh index e5d8a39..1822767 100644 --- a/init.sh +++ b/init.sh @@ -50,8 +50,7 @@ main() { consul::get_environment consul::configure_consul_template dock::generate_certs - aws::get_org_id - aws::get_poppa_id + aws::get_org_tags dock::set_hostname dock::set_config_org vault::store_private_registry_token diff --git a/lib/aws.sh b/lib/aws.sh index 4d806bd..125c65d 100644 --- a/lib/aws.sh +++ b/lib/aws.sh @@ -31,7 +31,7 @@ aws::get_aws_creds() { } # Fetches the org tags from EC2 and sets it to the `ORG_ID` environment variable -aws::get_org_id() { +aws::get_org_tags() { log::info "Setting Github Org ID" # Generate the org-tag fetching script @@ -42,8 +42,18 @@ aws::get_org_id() { backoff aws::get_aws_creds fi - backoff aws::get_org_id_onprem + EC2_HOME=/usr/local/ec2 + export EC2_HOME + + JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/jre + export JAVA_HOME + + export INSTANCE_ID=$(ec2-metadata -i | awk '{print $2}') + # Note: this only works for us-.{4}-\d + export REGION=$(ec2-metadata --availability-zone | awk '{ where = match($2, /us\-.+\-[1|2]/); print substr($2, where, 9); }') + backoff aws::fetch_org_id + backoff aws::fetch_poppa_id if [[ "$ORG_ID" == "" ]]; then # this will print an error, so that's good rollbar::report_error \ @@ -54,9 +64,10 @@ aws::get_org_id() { fi log::info "Got Org ID: $ORG_ID" + log::info "Got Poppa ID: $POPPA_ID" } -aws::get_org_id_onprem() { +aws::fetch_org_id() { local attempt=${1} log::info 'Attempting to get org id on prem' data='{"attempt":'"${attempt}"'}' @@ -66,58 +77,35 @@ aws::get_org_id_onprem() { "Attempting to get the Org Tag from AWS and failing." \ "$data" - EC2_HOME=/usr/local/ec2 - export EC2_HOME - - JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/jre - export JAVA_HOME - - local instance_id=$(ec2-metadata -i | awk '{print $2}') - - # Note: this only works for us-.{4}-\d - local region=$(ec2-metadata --availability-zone | awk '{ where = match($2, /us\-.+\-[1|2]/); print substr($2, where, 9); }') - ORG_ID=$(bash /usr/local/ec2/bin/ec2-describe-tags \ --aws-access-key="${AWS_ACCESS_KEY}" \ --aws-secret-key="${AWS_SECRET_KEY}" \ --filter "resource-type=instance" \ - --filter "resource-id=${instance_id}" \ + --filter "resource-id=${INSTANCE_ID}" \ --filter "key=org" \ - --region "${region}" \ + --region "${REGION}" \ | awk '{print $5}') export ORG_ID } # Fetches the poppa tags from EC2 and sets it to the `POPPA_ID` environment variable -aws::get_poppa_id() { +aws::fetch_poppa_id() { log::info "Setting Poppa ID" # Generate the org-tag fetching script rollbar::fatal_trap \ "Dock-Init: Failed to Render Org Script" \ "Consule-Template was unable to realize the given template." - EC2_HOME=/usr/local/ec2 - export EC2_HOME - - JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/jre - export JAVA_HOME - - local instance_id=$(ec2-metadata -i | awk '{print $2}') - # Note: this only works for us-.{4}-\d - local region=$(ec2-metadata --availability-zone | awk '{ where = match($2, /us\-.+\-[1|2]/); print substr($2, where, 9); }') - - POPPA_ID=$(bash /usr/local/ec2/bin/ec2-describe-tags \ - --aws-access-key="${AWS_ACCESS_KEY}" \ - --aws-secret-key="${AWS_SECRET_KEY}" \ - --filter "resource-type=instance" \ - --filter "resource-id=${instance_id}" \ - --filter "key=big-poppa" \ - --region "${region}" \ - | awk '{print $5}') - - export POPPA_ID + POPPA_ID=$(bash /usr/local/ec2/bin/ec2-describe-tags \ + --aws-access-key="${AWS_ACCESS_KEY}" \ + --aws-secret-key="${AWS_SECRET_KEY}" \ + --filter "resource-type=instance" \ + --filter "resource-id=${INSTANCE_ID}" \ + --filter "key=big-poppa" \ + --region "${REGION}" \ + | awk '{print $5}') - log::info "Got POPPA_ID: $POPPA_ID" + export POPPA_ID } diff --git a/lib/vault.sh b/lib/vault.sh index da52d40..b3fd2ea 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -73,5 +73,5 @@ vault::store_private_registry_token() { vault policy-write ${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" fi # need to set the final directory for the token here - vault token-create -policy=${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 {print $1 }' > /opt/runnable/dock-init/private-token + vault token-create -policy=${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/private-token } From 81dd4f011ec5942229bb8d79cbbca7631e041584 Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Wed, 10 May 2017 11:20:07 -0400 Subject: [PATCH 07/23] connect to user vault --- lib/vault.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/vault.sh b/lib/vault.sh index b3fd2ea..3674f2c 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -59,7 +59,7 @@ vault::store_private_registry_token() { log::info "Storing vault token for private registry key" local NODE_ENV=$(consul::get node/env) # this will pull from the vault currently running (our vault) - export VAULT_ADDR="http://${VAULT_HOSTNAME}:${VAULT_PORT}" + export VAULT_ADDR="http://${USER_VAULT_HOSTNAME}:${USER_VAULT_PORT}" # this might also be needed if we use a different root token # VAULT_TOKEN=$(cat "${token_path}"/auth-token) From 2250bf38f164f17de3e27b2a7ad6b4277a9e55b9 Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Wed, 10 May 2017 11:21:08 -0400 Subject: [PATCH 08/23] renamed policy --- lib/vault.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/vault.sh b/lib/vault.sh index 3674f2c..6c719f8 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -64,14 +64,14 @@ vault::store_private_registry_token() { # VAULT_TOKEN=$(cat "${token_path}"/auth-token) # vault auth ${VAULT_TOKEN} vault auth ${VAULT_TOKEN} - local POLICY=$(vault policies | grep "^${POPPA_ID}\b") + local POLICY=$(vault policies | grep "^dock-${POPPA_ID}\b") if [[ $POLICY ]]; then log::info "Policy found for $POPPA_ID, generating token" else log::info "Creating new policy and token for $POPPA_ID" sed "s/{{bpid}}/${POPPA_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" - vault policy-write ${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" + vault policy-write dock-${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" fi # need to set the final directory for the token here - vault token-create -policy=${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/private-token + vault token-create -policy=dock-${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/private-token } From 12e84007b9f0ec3c1e2fc80221891e636af1abfc Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Wed, 10 May 2017 11:25:55 -0400 Subject: [PATCH 09/23] envs --- init.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/init.sh b/init.sh index 1822767..749a787 100644 --- a/init.sh +++ b/init.sh @@ -33,6 +33,19 @@ else export VAULT_HOSTNAME fi +if [ -z "${USER_VAULT_PORT+x}" ]; then + export USER_VAULT_PORT=8200 +else + export USER_VAULT_PORT +fi + +if [ -z "${USER_VAULT_HOSTNAME+x}" ]; then + export USER_VAULT_HOSTNAME=$USER_VAULT_HOSTNAME +else + export USER_VAULT_HOSTNAME +fi + + export DOCKER_NETWORK=172.17.0.0/16 source "${DOCK_INIT_BASE}/lib/consul.sh" From 6144f95308cc778a5d5110779b6623a1b44f1237 Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Wed, 10 May 2017 11:27:07 -0400 Subject: [PATCH 10/23] uncommented --- lib/vault.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/vault.sh b/lib/vault.sh index 6c719f8..43e3c9d 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -62,8 +62,8 @@ vault::store_private_registry_token() { export VAULT_ADDR="http://${USER_VAULT_HOSTNAME}:${USER_VAULT_PORT}" # this might also be needed if we use a different root token - # VAULT_TOKEN=$(cat "${token_path}"/auth-token) - # vault auth ${VAULT_TOKEN} vault auth ${VAULT_TOKEN} + VAULT_TOKEN=$(cat "${token_path}"/auth-token) + vault auth ${VAULT_TOKEN} local POLICY=$(vault policies | grep "^dock-${POPPA_ID}\b") if [[ $POLICY ]]; then log::info "Policy found for $POPPA_ID, generating token" From d31b231aac51a65f452e1284ff68652fac0ca09f Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Fri, 12 May 2017 15:36:40 -0700 Subject: [PATCH 11/23] minors --- init.sh | 1 + lib/consul.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/init.sh b/init.sh index 749a787..56cc4dc 100644 --- a/init.sh +++ b/init.sh @@ -51,6 +51,7 @@ export DOCKER_NETWORK=172.17.0.0/16 source "${DOCK_INIT_BASE}/lib/consul.sh" source "${DOCK_INIT_BASE}/lib/aws.sh" source "${DOCK_INIT_BASE}/lib/dock.sh" +source "${DOCK_INIT_BASE}/lib/vault.sh" source "${DOCK_INIT_BASE}/lib/container.sh" source "${DOCK_INIT_BASE}/lib/iptables.sh" source "${DOCK_INIT_BASE}/lib/cleanup.sh" diff --git a/lib/consul.sh b/lib/consul.sh index 948c713..89e9590 100644 --- a/lib/consul.sh +++ b/lib/consul.sh @@ -25,7 +25,7 @@ consul::connect() { backoff consul::connect_backoff } -# Echos a value from consul foer the given keypath +# Echos a value from consul for the given keypath # @param $1 keypath Keypath for the value to get from consul consul::get() { # Strip leading slashes so it works with both '/my/path' and 'my/path' From 19d0ac031d2658dced61a4ea1dfc372135f3cc1a Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Fri, 12 May 2017 15:46:32 -0700 Subject: [PATCH 12/23] work on dock init, applying feedback --- lib/consul.sh | 4 ++++ lib/vault.sh | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/consul.sh b/lib/consul.sh index 89e9590..79b0e96 100644 --- a/lib/consul.sh +++ b/lib/consul.sh @@ -60,6 +60,10 @@ consul::configure_consul_template() { log::info "$token_path" VAULT_TOKEN=$(cat "${token_path}"/auth-token) export VAULT_TOKEN + local user_token_path="${DOCK_INIT_BASE}/consul-resources/user-vault/${NODE_ENV}" + log::info "$user_token_path" + USER_VAULT_TOKEN=$(cat "${user_token_path}"/auth-token) + export USER_VAULT_TOKEN local template="$DOCK_INIT_BASE/consul-resources/templates/" template+="template-config.hcl.ctmpl" diff --git a/lib/vault.sh b/lib/vault.sh index 43e3c9d..2bc3c00 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -62,8 +62,8 @@ vault::store_private_registry_token() { export VAULT_ADDR="http://${USER_VAULT_HOSTNAME}:${USER_VAULT_PORT}" # this might also be needed if we use a different root token - VAULT_TOKEN=$(cat "${token_path}"/auth-token) - vault auth ${VAULT_TOKEN} + USER_VAULT_TOKEN=$(cat "${user_token_path}"/auth-token) + vault auth ${USER_VAULT_TOKEN} local POLICY=$(vault policies | grep "^dock-${POPPA_ID}\b") if [[ $POLICY ]]; then log::info "Policy found for $POPPA_ID, generating token" @@ -73,5 +73,5 @@ vault::store_private_registry_token() { vault policy-write dock-${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" fi # need to set the final directory for the token here - vault token-create -policy=dock-${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/private-token + vault write -f auth/token/create/organizations-readonly -policy=dock-${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/private-token } From e82679a573ac891d20290db126788ad401ee8bee Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Fri, 12 May 2017 15:52:15 -0700 Subject: [PATCH 13/23] minors --- lib/cleanup.sh | 1 + lib/consul.sh | 5 ++--- lib/vault.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 15d98d4..8d5a4d7 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -11,6 +11,7 @@ cleanup::exit_trap() { "${CERT_PATH}"/pass \ "${DOCK_INIT_BASE}"/consul-resources/template-config.hcl \ "${DOCK_INIT_BASE}"/consul-resources/vault/**/auth-token \ + "${DOCK_INIT_BASE}"/consul-resources/vault/**/user-vault-auth-token \ "${DOCK_INIT_BASE}"/consul-resources/vault/**/token-* \ "${DOCK_INIT_BASE}"/key/rollbar.token fi diff --git a/lib/consul.sh b/lib/consul.sh index 79b0e96..f66f37f 100644 --- a/lib/consul.sh +++ b/lib/consul.sh @@ -60,9 +60,8 @@ consul::configure_consul_template() { log::info "$token_path" VAULT_TOKEN=$(cat "${token_path}"/auth-token) export VAULT_TOKEN - local user_token_path="${DOCK_INIT_BASE}/consul-resources/user-vault/${NODE_ENV}" - log::info "$user_token_path" - USER_VAULT_TOKEN=$(cat "${user_token_path}"/auth-token) + + USER_VAULT_TOKEN=$(cat "${user_token_path}"/user-vault-auth-token) export USER_VAULT_TOKEN local template="$DOCK_INIT_BASE/consul-resources/templates/" diff --git a/lib/vault.sh b/lib/vault.sh index 2bc3c00..eb8e426 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -62,7 +62,7 @@ vault::store_private_registry_token() { export VAULT_ADDR="http://${USER_VAULT_HOSTNAME}:${USER_VAULT_PORT}" # this might also be needed if we use a different root token - USER_VAULT_TOKEN=$(cat "${user_token_path}"/auth-token) + USER_VAULT_TOKEN=$(cat "${user_token_path}"/user-vault-auth-token) vault auth ${USER_VAULT_TOKEN} local POLICY=$(vault policies | grep "^dock-${POPPA_ID}\b") if [[ $POLICY ]]; then From daeb6e79a3d399139bd8e2c2f18b935698fa63f0 Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Fri, 12 May 2017 15:57:00 -0700 Subject: [PATCH 14/23] working on tests --- lib/consul.sh | 4 ++-- test/consul.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/consul.sh b/lib/consul.sh index f66f37f..005b719 100644 --- a/lib/consul.sh +++ b/lib/consul.sh @@ -58,8 +58,8 @@ consul::configure_consul_template() { local NODE_ENV=$(consul::get node/env) local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" log::info "$token_path" - VAULT_TOKEN=$(cat "${token_path}"/auth-token) - export VAULT_TOKEN + # VAULT_TOKEN=$(cat "${token_path}"/auth-token) + # export VAULT_TOKEN USER_VAULT_TOKEN=$(cat "${user_token_path}"/user-vault-auth-token) export USER_VAULT_TOKEN diff --git a/test/consul.sh b/test/consul.sh index 8b41586..1d9db03 100644 --- a/test/consul.sh +++ b/test/consul.sh @@ -29,7 +29,7 @@ describe 'consul.sh' it 'should read in the vault token' consul::configure_consul_template - cat::called_with "${DOCK_INIT_BASE}/consul-resources/vault/TEST-NODE-ENV/auth-token" + cat::called_with "${DOCK_INIT_BASE}/consul-resources/vault/TEST-NODE-ENV/user-vault-auth-token" end it 'generate the consul-template configuration' From 2b446a5c30ca61e41e0fb7b6913403293b60a9cb Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Fri, 12 May 2017 16:06:48 -0700 Subject: [PATCH 15/23] working on tests --- lib/consul.sh | 6 +++--- lib/vault.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/consul.sh b/lib/consul.sh index 005b719..8cf19ee 100644 --- a/lib/consul.sh +++ b/lib/consul.sh @@ -58,10 +58,10 @@ consul::configure_consul_template() { local NODE_ENV=$(consul::get node/env) local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" log::info "$token_path" - # VAULT_TOKEN=$(cat "${token_path}"/auth-token) - # export VAULT_TOKEN + VAULT_TOKEN=$(cat "${token_path}"/auth-token) + export VAULT_TOKEN - USER_VAULT_TOKEN=$(cat "${user_token_path}"/user-vault-auth-token) + USER_VAULT_TOKEN=$(cat "${token_path}"/user-vault-auth-token) export USER_VAULT_TOKEN local template="$DOCK_INIT_BASE/consul-resources/templates/" diff --git a/lib/vault.sh b/lib/vault.sh index eb8e426..fe874e3 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -62,7 +62,7 @@ vault::store_private_registry_token() { export VAULT_ADDR="http://${USER_VAULT_HOSTNAME}:${USER_VAULT_PORT}" # this might also be needed if we use a different root token - USER_VAULT_TOKEN=$(cat "${user_token_path}"/user-vault-auth-token) + USER_VAULT_TOKEN=$(cat "${token_path}"/user-vault-auth-token) vault auth ${USER_VAULT_TOKEN} local POLICY=$(vault policies | grep "^dock-${POPPA_ID}\b") if [[ $POLICY ]]; then From a1aeebe540073e94676e106a49b64042ac04ce38 Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Mon, 15 May 2017 13:27:18 -0700 Subject: [PATCH 16/23] minor fix --- lib/vault.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/vault.sh b/lib/vault.sh index fe874e3..35e669b 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -58,6 +58,9 @@ vault::set_s3_keys() { vault::store_private_registry_token() { log::info "Storing vault token for private registry key" local NODE_ENV=$(consul::get node/env) + local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" + log::info "$token_path" + # this will pull from the vault currently running (our vault) export VAULT_ADDR="http://${USER_VAULT_HOSTNAME}:${USER_VAULT_PORT}" # this might also be needed if we use a different root token From dcc8a8bcd79ab72b8e8132a022e3e7d213e31331 Mon Sep 17 00:00:00 2001 From: Anton Podviaznikov Date: Mon, 15 May 2017 13:56:16 -0700 Subject: [PATCH 17/23] formatting --- lib/consul.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/consul.sh b/lib/consul.sh index 8cf19ee..d6c44fd 100644 --- a/lib/consul.sh +++ b/lib/consul.sh @@ -54,23 +54,23 @@ consul::configure_consul_template() { "Consul-Template was unable to realize the config template." # expose VAULT_TOKEN for consul-template config - if [ -z ${AWS_ACCESS_KEY+x} ] || [ -z ${AWS_SECRET_KEY+x} ]; then - local NODE_ENV=$(consul::get node/env) - local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" - log::info "$token_path" - VAULT_TOKEN=$(cat "${token_path}"/auth-token) - export VAULT_TOKEN + if [ -z ${AWS_ACCESS_KEY+x} ] || [ -z ${AWS_SECRET_KEY+x} ]; then + local NODE_ENV=$(consul::get node/env) + local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" + log::info "$token_path" + VAULT_TOKEN=$(cat "${token_path}"/auth-token) + export VAULT_TOKEN - USER_VAULT_TOKEN=$(cat "${token_path}"/user-vault-auth-token) - export USER_VAULT_TOKEN + USER_VAULT_TOKEN=$(cat "${token_path}"/user-vault-auth-token) + export USER_VAULT_TOKEN - local template="$DOCK_INIT_BASE/consul-resources/templates/" - template+="template-config.hcl.ctmpl" - template+=":$DOCK_INIT_BASE/consul-resources/template-config.hcl" + local template="$DOCK_INIT_BASE/consul-resources/templates/" + template+="template-config.hcl.ctmpl" + template+=":$DOCK_INIT_BASE/consul-resources/template-config.hcl" - consul-template -once -template="$template" - else - log::info "AWS access key and secret already created, skipping template creation" - fi + consul-template -once -template="$template" + else + log::info "AWS access key and secret already created, skipping template creation" + fi rollbar::clear_trap } From 352dcfde84c17a9bbb5cd0f13155c56868881544 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Tue, 16 May 2017 15:26:48 -0700 Subject: [PATCH 18/23] Create policy and token --- .../templates/registry_policy.tmpl | 2 +- lib/vault.sh | 21 +++++++------------ 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/consul-resources/templates/registry_policy.tmpl b/consul-resources/templates/registry_policy.tmpl index eeb9539..08c5950 100644 --- a/consul-resources/templates/registry_policy.tmpl +++ b/consul-resources/templates/registry_policy.tmpl @@ -1,3 +1,3 @@ -path "secret/organization/{{bpid}}" { +path "secret/organization/dock-{{bpid}}" { policy = "read" } diff --git a/lib/vault.sh b/lib/vault.sh index 35e669b..35effcf 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -54,27 +54,22 @@ vault::set_s3_keys() { rollbar::clear_trap } -# creates a token for a specific policy +# creates a token for a the organizations-readonly policy vault::store_private_registry_token() { log::info "Storing vault token for private registry key" local NODE_ENV=$(consul::get node/env) local token_path="${DOCK_INIT_BASE}/consul-resources/vault/${NODE_ENV}" - log::info "$token_path" - + unset VAULT_TOKEN # this will pull from the vault currently running (our vault) export VAULT_ADDR="http://${USER_VAULT_HOSTNAME}:${USER_VAULT_PORT}" # this might also be needed if we use a different root token USER_VAULT_TOKEN=$(cat "${token_path}"/user-vault-auth-token) vault auth ${USER_VAULT_TOKEN} - local POLICY=$(vault policies | grep "^dock-${POPPA_ID}\b") - if [[ $POLICY ]]; then - log::info "Policy found for $POPPA_ID, generating token" - else - log::info "Creating new policy and token for $POPPA_ID" - sed "s/{{bpid}}/${POPPA_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" - vault policy-write dock-${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" - fi - # need to set the final directory for the token here - vault write -f auth/token/create/organizations-readonly -policy=dock-${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/private-token + log::info "Creating new policy and token for dock-$POPPA_ID" + sed "s/{{bpid}}/${POPPA_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" + vault policy-write dock-${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" + vault token-create -policy=dock-${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/private-token + VAULT_TOKEN=$(cat "${token_path}"/auth-token) + export VAULT_TOKEN } From 5c48dcf9f6e9d62d5629962a81a55ec623057c83 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Wed, 17 May 2017 10:39:32 -0700 Subject: [PATCH 19/23] Fix to make consistent with starlord secret path --- consul-resources/templates/registry_policy.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consul-resources/templates/registry_policy.tmpl b/consul-resources/templates/registry_policy.tmpl index 08c5950..8ffbab4 100644 --- a/consul-resources/templates/registry_policy.tmpl +++ b/consul-resources/templates/registry_policy.tmpl @@ -1,3 +1,3 @@ -path "secret/organization/dock-{{bpid}}" { +path "secret/organization/{{bpid}}/*" { policy = "read" } From b1a9e1c5ad6a4c844cd2a957ec79c51cf9c9423c Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Wed, 17 May 2017 15:36:31 -0700 Subject: [PATCH 20/23] Updated tag id and private token filename --- lib/aws.sh | 2 +- lib/vault.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/aws.sh b/lib/aws.sh index 125c65d..0909ddf 100644 --- a/lib/aws.sh +++ b/lib/aws.sh @@ -103,7 +103,7 @@ aws::fetch_poppa_id() { --aws-secret-key="${AWS_SECRET_KEY}" \ --filter "resource-type=instance" \ --filter "resource-id=${INSTANCE_ID}" \ - --filter "key=big-poppa" \ + --filter "key=big-poppa-org-id" \ --region "${REGION}" \ | awk '{print $5}') diff --git a/lib/vault.sh b/lib/vault.sh index 35effcf..79cc341 100644 --- a/lib/vault.sh +++ b/lib/vault.sh @@ -69,7 +69,7 @@ vault::store_private_registry_token() { log::info "Creating new policy and token for dock-$POPPA_ID" sed "s/{{bpid}}/${POPPA_ID}/g" "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.tmpl" > "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" vault policy-write dock-${POPPA_ID} "${DOCK_INIT_BASE}/consul-resources/templates/registry_policy.hcl" - vault token-create -policy=dock-${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/private-token + vault token-create -policy=dock-${POPPA_ID} | awk '/token/ { print $2 }' | awk 'NR==1 { print $1 }' > /opt/runnable/dock-init/user-private-registry-token VAULT_TOKEN=$(cat "${token_path}"/auth-token) export VAULT_TOKEN } From a02ecaaae23ec7612a3fe22a09a8a8a8f18e5180 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Thu, 18 May 2017 12:14:24 -0700 Subject: [PATCH 21/23] Rename script --- .gitignore | 2 +- .../{get-org-tag.sh.ctmpl => get-aws-creds.sh.ctmpl} | 0 lib/aws.sh | 6 +++--- 3 files changed, 4 insertions(+), 4 deletions(-) rename consul-resources/templates/{get-org-tag.sh.ctmpl => get-aws-creds.sh.ctmpl} (100%) diff --git a/.gitignore b/.gitignore index 6d8de77..74616e6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ npm-debug.log consul-resources/template-config.hcl consul-resources/vault/vault.hcl hosts-registry.txt -util/get-org-id.sh +util/get-aws-creds.sh consul-resources/vault/**/auth-token consul-resources/vault/**/token-01 consul-resources/vault/**/token-02 diff --git a/consul-resources/templates/get-org-tag.sh.ctmpl b/consul-resources/templates/get-aws-creds.sh.ctmpl similarity index 100% rename from consul-resources/templates/get-org-tag.sh.ctmpl rename to consul-resources/templates/get-aws-creds.sh.ctmpl diff --git a/lib/aws.sh b/lib/aws.sh index 0909ddf..7631a08 100644 --- a/lib/aws.sh +++ b/lib/aws.sh @@ -15,11 +15,11 @@ aws::get_aws_creds() { "Dock-Init: Failed to Render Org Script" \ "Consule-Template was unable to realize the given template." - ORG_SCRIPT=$DOCK_INIT_BASE/util/get-org-id.sh + ORG_SCRIPT=$DOCK_INIT_BASE/util/get-aws-creds.sh local config="$DOCK_INIT_BASE/consul-resources/template-config.hcl" local template="$DOCK_INIT_BASE" - template+="/consul-resources/templates/get-org-tag.sh.ctmpl:$ORG_SCRIPT" + template+="/consul-resources/templates/get-aws-creds.sh.ctmpl:$ORG_SCRIPT" consul-template -config="${config}" -once -template="${template}" @@ -27,7 +27,7 @@ aws::get_aws_creds() { # give amazon a chance to get the auth sleep 5 - source "${DOCK_INIT_BASE}/util/get-org-id.sh" + source "${DOCK_INIT_BASE}/util/get-aws-creds.sh" } # Fetches the org tags from EC2 and sets it to the `ORG_ID` environment variable From 489bc435d3dda5e276035e501a143e2cc4ae0d1a Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Thu, 18 May 2017 12:19:10 -0700 Subject: [PATCH 22/23] Rename tag to runnable-org-id --- lib/aws.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/aws.sh b/lib/aws.sh index 7631a08..2ea7a22 100644 --- a/lib/aws.sh +++ b/lib/aws.sh @@ -103,7 +103,7 @@ aws::fetch_poppa_id() { --aws-secret-key="${AWS_SECRET_KEY}" \ --filter "resource-type=instance" \ --filter "resource-id=${INSTANCE_ID}" \ - --filter "key=big-poppa-org-id" \ + --filter "key=runnable-org-id" \ --region "${REGION}" \ | awk '{print $5}') From 545469f9743f436ecd0dc391c4dae9995c7a45c8 Mon Sep 17 00:00:00 2001 From: Henry Mollman Date: Thu, 18 May 2017 14:37:34 -0700 Subject: [PATCH 23/23] PR comments --- consul-resources/templates/get-aws-creds.sh.ctmpl | 2 -- init.sh | 2 +- lib/aws.sh | 2 +- lib/cleanup.sh | 1 + 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/consul-resources/templates/get-aws-creds.sh.ctmpl b/consul-resources/templates/get-aws-creds.sh.ctmpl index 9644056..05b2db2 100644 --- a/consul-resources/templates/get-aws-creds.sh.ctmpl +++ b/consul-resources/templates/get-aws-creds.sh.ctmpl @@ -1,8 +1,6 @@ #!/bin/bash set -e -# WARNING: do not echo anything except ORG ID here - {{ with vault "aws_1h/creds/dock-init" }} export AWS_ACCESS_KEY="{{ .Data.access_key }}" export AWS_SECRET_KEY="{{ .Data.secret_key }}" diff --git a/init.sh b/init.sh index 56cc4dc..3161821 100644 --- a/init.sh +++ b/init.sh @@ -64,7 +64,7 @@ main() { consul::get_environment consul::configure_consul_template dock::generate_certs - aws::get_org_tags + aws::get_org_ids dock::set_hostname dock::set_config_org vault::store_private_registry_token diff --git a/lib/aws.sh b/lib/aws.sh index 2ea7a22..09ab734 100644 --- a/lib/aws.sh +++ b/lib/aws.sh @@ -31,7 +31,7 @@ aws::get_aws_creds() { } # Fetches the org tags from EC2 and sets it to the `ORG_ID` environment variable -aws::get_org_tags() { +aws::get_org_ids() { log::info "Setting Github Org ID" # Generate the org-tag fetching script diff --git a/lib/cleanup.sh b/lib/cleanup.sh index 8d5a4d7..12abd17 100644 --- a/lib/cleanup.sh +++ b/lib/cleanup.sh @@ -13,6 +13,7 @@ cleanup::exit_trap() { "${DOCK_INIT_BASE}"/consul-resources/vault/**/auth-token \ "${DOCK_INIT_BASE}"/consul-resources/vault/**/user-vault-auth-token \ "${DOCK_INIT_BASE}"/consul-resources/vault/**/token-* \ + "${DOCK_INIT_BASE}"/util/get-aws-creds.sh \ "${DOCK_INIT_BASE}"/key/rollbar.token fi }