diff --git a/ansible/base.yml b/ansible/base.yml index 6ee8aead..d0bc4734 100644 --- a/ansible/base.yml +++ b/ansible/base.yml @@ -1,4 +1,6 @@ --- +- hosts: registry + - hosts: localhost connection: local tasks: diff --git a/ansible/default-hosts/docks.js b/ansible/default-hosts/docks.js new file mode 100755 index 00000000..b2804306 --- /dev/null +++ b/ansible/default-hosts/docks.js @@ -0,0 +1,75 @@ +#!/usr/bin/env node + +'use strict'; + +var aws = require('aws-sdk'); +var ec2 = new aws.EC2({ + accessKeyId: '${AWS_ACCESS_KEY_ID_1}', + secretAccessKey: '${AWS_SECRET_ACCESS_KEY_1}', + region: '${AWS_REGION}' +}); + +var params = { + Filters: [ + // Only search for docks in the cluster security group + { + Name: 'instance.group-id', + Values: ['${AWS_DOCK_SG}'] // This script is the same for all environments + }, + // Only fetch instances that are tagged as docks + { + Name: 'tag:role', + Values: ['dock'] + }, + // Only fetch running instances + { + Name: 'instance-state-name', + Values: ['running'] + } + ] +}; + +ec2.describeInstances(params, function (err, data) { + if (err) { + console.error("An error occurred: ", err); + process.exit(1); + } + + // Get a set of instances from the describe response + var instances = []; + data.Reservations.forEach(function (res) { + res.Instances.forEach(function (instance) { + instances.push(instance); + }); + }); + + // Map the instances to their private ip addresses + // NOTE This will work locally because of the wilcard ssh proxy in the config + var hosts = instances.map(function (instance) { + return instance.PrivateIpAddress; + }); + + var hostVars = {}; + instances.forEach(function (instance) { + for (var i = 0; i < instance.Tags.length; i++) { + if (instance.Tags[i].Key === 'org') { + hostVars[instance.PrivateIpAddress] = { + host_tags: instance.Tags[i].Value + ',build,run' + }; + } + } + }); + + // Output the resulting JSON + // NOTE http://docs.ansible.com/ansible/developing_inventory.html + console.log(JSON.stringify( + { + docks: { + hosts: hosts + }, + _meta : { + hostvars : hostVars + } + } + )); +}); diff --git a/ansible/default-hosts/hosts b/ansible/default-hosts/hosts new file mode 100644 index 00000000..2ef19a71 --- /dev/null +++ b/ansible/default-hosts/hosts @@ -0,0 +1,159 @@ +[bastion] +dafault-bastion + +[hipache] +default-main httpsCheckForBackend80=false prependIncomingPort=true subDomainDepth=4 + +[userland] +default-main + +[mongodb] +default-main + +[api_group:children] +worker +api +socket-server + +[api] +default-main + +[big-poppa] +default-main + +[cream] +default-main + +[consul] +default-main + +[docker-listener] +default-main + +[vault] +default-main + +[worker] +default-main + +[navi] +default-main + +[ingress] +default-main + +[link] +default-main + +[mongo-navi] +default-main + +[charon] +default-main + +[khronos] +default-main + +[optimus] +default-main + +[detention] +default-main + +[palantiri] +default-main + +[rabbitmq] +default-main + +[web] +default-main + +[redis] +default-main + +[redis-slave] +default-main + +[sauron] +default-main + +[shiva] +default-main + +[socket-server] +default-main + +[socket-server-proxy] +default-main + +[registry] +default-main + +[swarm-manager] +default-main + +[metis] +default-main + +[drake] +default-main + +[pheidi] +default-main + +[github-varnish] +default-main + +[single-host-proxy] +default-main + +[docks] + +[dock] + +[prometheus] +default-main + +[bear-clone:children] +api +bastion +big-poppa +charon +consul +cream +dock +docker-listener +docks +drake +hipache +ingress +khronos +metis +mongodb +navi +optimus +pheidi +prometheus +rabbitmq +redis +redis-slave +registry +sauron +shiva +single-host-proxy +socket-server +socket-server-proxy +swarm-manager +userland +web +worker + +[local] +127.0.0.1 + +[ec2] +local + +[targets] +localhost ansible_connection=local bastion_name=default-bastion diff --git a/ansible/default-hosts/variables b/ansible/default-hosts/variables new file mode 100644 index 00000000..a5128c5d --- /dev/null +++ b/ansible/default-hosts/variables @@ -0,0 +1,134 @@ +[api_group:vars] +api_aws_access_key_id=${AWS_ACCESS_KEY_ID_1} +api_aws_secret_access_key=${AWS_SECRET_ACCESS_KEY_1} +api_github_client_id=${GITHUB_CLIEND_ID} +api_github_client_secret=${GITHUB_CLIENT_SECRET} +api_github_deploy_keys_bucket=runnable.deploykeys.${ENV} +api_mongo_auth=${MONGO_USERNAME}:${MONGO_PASSWORD} +api_mongo_database=${ENV} +api_mongo_replset_name=${ENV}-rs0 +api_s3_context_bucket=runnable.context.resources.${ENV} + +[big-poppa:vars] +big_poppa_pg_pass=${POSTGRES_PASSWORD} +big_poppa_pg_host=${POSTGRES_HOST}:${POSTGRES_PORT} +big_poppa_pg_port=${POSTGRES_PORT} +big_poppa_pg_user=big_poppa +big_poppa_github_token=${GITHUB_ACCESS_TOKEN} +big_poppa_mongo_auth=${MONGO_USERNAME}:${MONGO_PASSWORD} +big_poppa_mongo_database=${MONGO_DATABASE} +big_poppa_mongo_replset_name=${MONGO_DATABASE}-rs0 +big_poppa_pg_pool_min=10 +big_poppa_pg_pool_max=20 + +[cream:vars] +cream_hello_runnable_github_token=${GITHUB_ACCESS_TOKEN} +cream_stripe_secret_key=${STRIPE_SECRET_KEY} +cream_stripe_publishable_key=${STRIPE_PUBLISHABLE_KEY} + +[docks:vars] +docker_config=docks +docks_rollbar_key=${ROLLBAR_TOKEN_DOCKS} + +[dock:vars] +docks_rollbar_key=${ROLLBAR_TOKEN_DOCKS} + +[drake:vars] +drake_port=80 + +[khronos:vars] +khronos_mongo_auth=${MONGO_USER}:${MONGO_PASSWORD} +khronos_mongo_database=${MONGO_DATABASE} +khronos_mongo_replset_name=${MONGO_DATABASE} + +[metis:vars] + +[navi:vars] +navi_cookie_secret=${COOKIE_SECRET} +_navi_proxy_port=65100 +_navi_proxy_ssl_port=65101 + +[optimus:vars] +optimus_aws_access_id=${AWS_ACCESS_KEY_ID_1} +optimus_aws_secret_id=${AWS_SECRET_ACCESS_KEY_1} +optimus_github_deploy_keys_bucket=runnable.deploykeys.${ENV} + +[palantiri:vars] + +[pheidi:vars] +pheidi_mongo_auth=${MONGO_USER}:${MONGO_PASSWORD} +pheidi_mongo_database=${MONGO_DATABASE} +pheidi_mongo_replset_name=${MONGO_DATABASE} +pheidi_runnabot_tokens=${GITHUB_ACCESS_TOKEN} + +[sauron:vars] + +[registry:vars] +registry_s3_access_key=${AWS_ACCESS_KEY_ID_1} +registry_s3_secret_key=${AWS_SECRET_ACCESS_KEY_1} +registry_s3_bucket=runnableimages.${ENV} +registry_s3_region=${AWS_REGION} + +[shiva:vars] +aws_access_key_id=${AWS_ACCESS_KEY_ID_1} +aws_secret_access_key=${AWS_ACCESS_KEY_ID_1} +shiva_aws_region=${AWS_REGION} +shiva_dock_security_groups=${AWS_DOCK_SG} +shiva_ssh_key_name=${AWS_SSH_KEY_NAME} +shiva_aws_instance_image_id=${AWS_DOCK_AMI_ID} +shiva_aws_instance_image_name=${AWS_DOCK_AMI_NAME} +shiva_aws_instance_type=t2.medium +shiva_dock_pool_asg_name=${ENV}-asg-dock-pool +shiva_aws_launch_configuration_name=${ENV}-lc-${AWS_LC_VERSION} +shiva_aws_auto_scaling_group_subnets=${AWS_ASG_SUBNET} +shiva_aws_auto_scaling_group_max=29 +shiva_aws_auto_scaling_group_prefix=asg-${ENV}- + +[swarm-manager:vars] +aws_access_key=${AWS_ACCESS_KEY_ID_1} +aws_secret_key=${AWS_SECRET_ACCESS_KEY_1} +environment_name=${ENV} + +[vault:vars] +vault_hello_runnable_github_token=${GITHUB_ACCESS_TOKEN_HELLO_RUNNABLE} +vault_aws_access_key_id=${AWS_ACCESS_KEY_ID_1} +vault_aws_secret_key=${AWS_SECRET_ACCESS_KEY_1} +vault_aws_region=${AWS_REGION} +vault_root_token=${LOCAL_VAULT_ROOT_TOKEN} +vault_unseal_tokens={'one':'${LOCAL_VAULT_TOKEN_1}', 'two': '${LOCAL_VAULT_TOKEN_2}', 'three': '${LOCAL_VAULT_TOKEN_3}', 'four': '${LOCAL_VAULT_TOKEN_4}', 'five': '${LOCAL_VAULT_TOKEN_5}'} +_vault_port=65240 +_vault_ssl_port=65241 + +[${ENV}:vars] +bastion_sshd_port=60709 +datadog_tags=env:${ENV} +datadog_mongodb_user=datadog +datadog_mongodb_pwd= +domain=${DOMAIN} +mongo_port=27017 +node_env=${ENV} +pg_user=astral +pg_pass=${POSTGRES_PASSWORD} +pg_host=${POSTGRES_HOST}:${POSTGRES_PORT} +rabbit_password=${RABBIT_PASSWORD} +rabbit_username=${RABBIT_USERNAME} +_registry_port=65001 +_consul_api_port=65200 +_consul_https_port=65201 +_swarm_master_port=65250 +user_content_domain=${USER_CONTENT_DOMAIN} +max_navi_port=65000 +_redis_port=65075 +_redis_tls_port=65076 +api_hello_runnable_github_token=${GITHUB_ACCESS_TOKEN_HELLO_RUNNABLE} +vault_auth_token=${REMOTE_VAULT_ROOT_TOKEN} +vault_token_01=${REMOTE_VAULT_TOKEN_1} +vault_token_02=${REMOTE_VAULT_TOKEN_2} +vault_token_03=${REMOTE_VAULT_TOKEN_3} +vault_token_04=${REMOTE_VAULT_TOKEN_4} +vault_token_05=${REMOTE_VAULT_TOKEN_5} +github_domain=api.github.com +is_github_enterprise=false +github_protocol=https +proxy_container_image=runnable/sticky-nginx +proxy_container_image_version=v1.8.1 diff --git a/ansible/dock.yml b/ansible/dock.yml index 9eea4fb6..aa9b7b6a 100644 --- a/ansible/dock.yml +++ b/ansible/dock.yml @@ -11,12 +11,13 @@ name={{ dock }} groups=dock +- include: image-builder.yml git_branch="v4.2.3" - include: charon.yml git_branch="v4.0.0" -- include: dock-init.yml git_branch="v10.1.0" +- include: dock-init.yml git_branch="v10.1.1" - include: krain.yml git_branch="v0.3.0" - hosts: "{{ dock }}" - tasks: + tasks: - name: remove datadog agent become: true apt: @@ -29,3 +30,4 @@ - { role: install-ssm } - { role: dock-images } - { role: docks-psad } + diff --git a/ansible/grizzly-hosts/docks.js b/ansible/grizzly-hosts/docks.js new file mode 100755 index 00000000..5bb4c07e --- /dev/null +++ b/ansible/grizzly-hosts/docks.js @@ -0,0 +1,75 @@ +#!/usr/bin/env node + +'use strict'; + +var aws = require('aws-sdk'); +var ec2 = new aws.EC2({ + accessKeyId: 'AKIAIS2HMUM2REGVTVIQ', + secretAccessKey: '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', + region: 'us-west-2' +}); + +var params = { + Filters: [ + // Only search for docks in the cluster security group + { + Name: 'instance.group-id', + Values: ['sg-ec0da194'] // This script is the same for all environments + }, + // Only fetch instances that are tagged as docks + { + Name: 'tag:role', + Values: ['dock'] + }, + // Only fetch running instances + { + Name: 'instance-state-name', + Values: ['running'] + } + ] +}; + +ec2.describeInstances(params, function (err, data) { + if (err) { + console.error("An error occurred: ", err); + process.exit(1); + } + + // Get a set of instances from the describe response + var instances = []; + data.Reservations.forEach(function (res) { + res.Instances.forEach(function (instance) { + instances.push(instance); + }); + }); + + // Map the instances to their private ip addresses + // NOTE This will work locally because of the wilcard ssh proxy in the config + var hosts = instances.map(function (instance) { + return instance.PrivateIpAddress; + }); + + var hostVars = {}; + instances.forEach(function (instance) { + for (var i = 0; i < instance.Tags.length; i++) { + if (instance.Tags[i].Key === 'org') { + hostVars[instance.PrivateIpAddress] = { + host_tags: instance.Tags[i].Value + ',build,run' + }; + } + } + }); + + // Output the resulting JSON + // NOTE http://docs.ansible.com/ansible/developing_inventory.html + console.log(JSON.stringify( + { + docks: { + hosts: hosts + }, + _meta : { + hostvars : hostVars + } + } + )); +}); diff --git a/ansible/grizzly-hosts/hosts b/ansible/grizzly-hosts/hosts new file mode 100644 index 00000000..0c4f119b --- /dev/null +++ b/ansible/grizzly-hosts/hosts @@ -0,0 +1,159 @@ +[bastion] +gamma-bastion + +[hipache] +gamma-grizzly-main httpsCheckForBackend80=false prependIncomingPort=true subDomainDepth=4 + +[userland] +gamma-grizzly-main + +[mongodb] +gamma-grizzly-main + +[api_group:children] +worker +api +socket-server + +[api] +gamma-grizzly-main + +[big-poppa] +gamma-grizzly-main + +[cream] +gamma-grizzly-main + +[consul] +gamma-grizzly-main + +[docker-listener] +gamma-grizzly-main + +[vault] +gamma-grizzly-main + +[worker] +gamma-grizzly-main + +[navi] +gamma-grizzly-main + +[ingress] +gamma-grizzly-main + +[link] +gamma-grizzly-main + +[mongo-navi] +gamma-grizzly-main + +[charon] +gamma-grizzly-main + +[khronos] +gamma-grizzly-main + +[optimus] +gamma-grizzly-main + +[detention] +gamma-grizzly-main + +[palantiri] +gamma-grizzly-main + +[rabbitmq] +gamma-grizzly-main + +[web] +gamma-grizzly-main + +[redis] +gamma-grizzly-main + +[redis-slave] +gamma-grizzly-main + +[sauron] +gamma-grizzly-main + +[shiva] +gamma-grizzly-main + +[socket-server] +gamma-grizzly-main + +[socket-server-proxy] +gamma-grizzly-main + +[registry] +gamma-grizzly-main + +[swarm-manager] +gamma-grizzly-main + +[metis] +gamma-grizzly-main + +[drake] +gamma-grizzly-main + +[pheidi] +gamma-grizzly-main + +[github-varnish] +gamma-grizzly-main + +[single-host-proxy] +gamma-grizzly-main + +[docks] + +[dock] + +[prometheus] +gamma-grizzly-main + +[grizzly:children] +api +bastion +big-poppa +charon +consul +cream +dock +docker-listener +docks +drake +hipache +ingress +khronos +metis +mongodb +navi +optimus +pheidi +prometheus +rabbitmq +redis +redis-slave +registry +sauron +shiva +single-host-proxy +socket-server +socket-server-proxy +swarm-manager +userland +web +worker + +[local] +127.0.0.1 + +[ec2] +local + +[targets] +localhost ansible_connection=local bastion_name=gamma-bastion diff --git a/ansible/grizzly-hosts/variables b/ansible/grizzly-hosts/variables new file mode 100644 index 00000000..f98833f9 --- /dev/null +++ b/ansible/grizzly-hosts/variables @@ -0,0 +1,150 @@ +[api_group:vars] +api_aws_access_key_id=AKIAIS2HMUM2REGVTVIQ +api_aws_secret_access_key=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 +api_github_client_id=fb6620cd05b411759d15 +api_github_client_secret=8a7ff50364ce874865cebabae3d61697801ec950 +api_github_deploy_keys_bucket=runnable.deploykeys.grizzly +api_mongo_auth=b3f65941632347b2941ef9cf0d6a1fd2:0d3ab8d34f4e4b5a82ea7bc10ebaab5b +api_mongo_database=grizzly +api_mongo_replset_name=grizzly-rs0 +api_s3_context_bucket=runnable.context.resources.grizzly +api_number_of_containers=1 + +[big-poppa:vars] +big_poppa_pg_pass=790e5214041d4ff4b0dadbb63cfcc66d +big_poppa_pg_host=grizzly.cnksgdqarobf.us-west-2.rds.amazonaws.com:5432 +big_poppa_pg_port=5432 +big_poppa_pg_user=e1058667bd5f4e50 +big_poppa_github_token=5346739a35753d289b9ef52bd68328ad36897091 +big_poppa_mongo_auth=b3f65941632347b2941ef9cf0d6a1fd2:0d3ab8d34f4e4b5a82ea7bc10ebaab5b +big_poppa_mongo_database=grizzly +big_poppa_mongo_replset_name=grizzly-rs0 +big_poppa_pg_pool_min=10 +big_poppa_pg_pool_max=20 + +[cream:vars] +cream_hello_runnable_github_token=7d97ecc61565ab2170a285060bc31628ba5c3443 +cream_stripe_secret_key=sk_test_4De8Zdkfcyb29swkMmjZUMRh +cream_stripe_publishable_key=pk_test_sHr5tQaPtgwiE2cpW6dQkzi8 + +[docks:vars] +docker_config=docks +docks_rollbar_key=${ROLLBAR_TOKEN_DOCKS} + +[dock:vars] +docks_rollbar_key=${ROLLBAR_TOKEN_DOCKS} + +[drake:vars] +drake_port=80 + +[khronos:vars] +khronos_mongo_auth=b3f65941632347b2941ef9cf0d6a1fd2:0d3ab8d34f4e4b5a82ea7bc10ebaab5b +khronos_mongo_database=grizzly +khronos_mongo_replset_name=grizzly + +[metis:vars] + +[link:vars] +link_hello_runnable_github_token=7d97ecc61565ab2170a285060bc31628ba5c3443 + +[navi:vars] +navi_cookie_secret=17578a96-8610-4db0-83ea-7b4491e8c1a5 +_navi_proxy_port=65100 +_navi_proxy_ssl_port=65101 +navi_number_of_containers=1 + +[optimus:vars] +optimus_aws_access_id=AKIAIS2HMUM2REGVTVIQ +optimus_aws_secret_id=k7L6Ljvl46ThhZ6ed3VeN6lRG83p3kR/1QXVDYUA +optimus_github_deploy_keys_bucket=runnable.deploykeys.grizzly + +[palantiri:vars] + +[pheidi:vars] +pheidi_mongo_auth=b3f65941632347b2941ef9cf0d6a1fd2:0d3ab8d34f4e4b5a82ea7bc10ebaab5b +pheidi_mongo_database=grizzly +pheidi_mongo_replset_name=grizzly +pheidi_runnabot_tokens=5346739a35753d289b9ef52bd68328ad36897091 + +[sauron:vars] + +[registry:vars] +registry_s3_access_key=AKIAIS2HMUM2REGVTVIQ +registry_s3_secret_key=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 +registry_s3_bucket=runnableimages.grizzly +registry_s3_region=us-west-2 + +[shiva:vars] +aws_access_key_id=AKIAIS2HMUM2REGVTVIQ +aws_secret_access_key=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 +shiva_aws_region=us-west-2 +shiva_dock_security_groups=sg-ec0da194 +shiva_ssh_key_name=gamma-key +shiva_aws_instance_image_id=ami-74ee6a14 +shiva_aws_instance_image_name=grizzly-dock-2.0.15 +shiva_aws_instance_type=t2.medium +shiva_dock_pool_asg_name=grizzly-asg-dock-pool +shiva_aws_launch_configuration_name=grizzly-dock-2.0.16 +shiva_aws_auto_scaling_group_subnets=subnet-d485c1a3 +shiva_aws_auto_scaling_group_max=29 +shiva_aws_auto_scaling_group_prefix=asg-grizzly- + +[swarm-manager:vars] +aws_access_key=AKIAIS2HMUM2REGVTVIQ +aws_secret_key=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 +environment_name=grizzly + +[vault:vars] +vault_hello_runnable_github_token=7d97ecc61565ab2170a285060bc31628ba5c3443 +vault_aws_access_key_id=AKIAIS2HMUM2REGVTVIQ +vault_aws_secret_key=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 +vault_aws_region=us-west-2 +vault_root_token=1ca30253-c3d3-9a90-5399-31844b339fce +vault_unseal_tokens={'one':'d8a1906386b875439b744211699362625f07f4d3ade1376b9e2f9ab62347f54d01', 'two': '6e6ff3355f7be3ad1da095a3b8334e7a6c994694774a4545f53bfba99f5c92ee02', 'three': '400830bae75f5663f66abba8b5faa62e867c98173ed484ee321f5165c6b8ff9903', 'four': 'e008fc18d4c3d5671cf027f9f2252513448af38add5d23dcfe909b9cecf6e16104', 'five': 'ce6f3f976ce760a9f73a09f2ffeccd47ae6f2d0994c3e27739b43150b5128c1605'} +_vault_port=65240 +_vault_ssl_port=65241 + +[web:vars] +web_intercom_id=xs5g95pd +web_sift_public_key=eea9746dff +web_aws_bucket_region=us-west-2 +marketing_aws_region=us-west-2 + +[grizzly:vars] +bastion_sshd_port=60709 +datadog_tags=env:grizzly +datadog_mongodb_user=datadog +datadog_mongodb_pwd= +domain=runnablecloud.com +mongo_port=27017 +node_env=grizzly +pg_user=e1058667bd5f4e50 +pg_pass=790e5214041d4ff4b0dadbb63cfcc66d +pg_host=grizzly.cnksgdqarobf.us-west-2.rds.amazonaws.com:5432 +rabbit_password=dkW4nGBvTM+iV+7yXNnXLE1XJ0WpKk84nuDt1TJS1V4= +rabbit_username=8NjWopyYRKClypDP6WuQXit3CKVhgksvm82ScQuL1ms= +_registry_port=65001 +_consul_api_port=65200 +_consul_https_port=65201 +_swarm_master_port=65250 +user_content_domain=runnabae.com +max_navi_port=65000 +_redis_port=65075 +_redis_tls_port=65076 +api_hello_runnable_github_token=7d97ecc61565ab2170a285060bc31628ba5c3443 +vault_auth_token=fd79064e-06e6-321e-0161-18f3d01fe5f9 +vault_token_01=b7da720ffe3cac65316f658c8cee1c3ce24d90a20d2d3d9de7f53505ab40a8e004 +vault_token_02=c806906837aadaeba9f74d9138bf8e0fe8012d521a605eb38d247bf11188fd4e02 +vault_token_03=e7d4bd5eb4edbc3b8c6ac7cac993b7af68d5b1af7ffa54f83b54bdd107a4dca103 +vault_token_04=98085f397d7bcab514f2efd77dc2259c62990c5f68b737d65185f325bd6c890f05 +vault_token_05=e3bb5ca08b0fa927dc6e113db39dac0c1750bad979b4e632bbffb1226c99c69001 +github_domain=api.github.com +is_github_enterprise=false +github_protocol=https +proxy_container_image=runnable/sticky-nginx +proxy_container_image_version=v1.8.1 +charon_api_token=7d97ecc61565ab2170a285060bc31628ba5c3443 +swarm_version=v1.2.3-0 +github_api_url=https://api.github.com +github_domain=github.com +github_protocol=https diff --git a/ansible/group_vars/alpha-api-base.yml b/ansible/group_vars/alpha-api-base.yml index d01c68fd..75cdb85a 100644 --- a/ansible/group_vars/alpha-api-base.yml +++ b/ansible/group_vars/alpha-api-base.yml @@ -51,7 +51,7 @@ api_base_container_envs: >- -e GITHUB_PROTOCOL=http -e HELLO_RUNNABLE_GITHUB_TOKEN={{ api_hello_runnable_github_token }} -e KRAIN_PORT={{ krain_port }} - -e MIXPANEL_APP_ID={{ api_mixpanel_app_id }} + {% if api_mixpanel_app_id is defined %} -e MIXPANEL_APP_ID={{ api_mixpanel_app_id }} {% endif %} -e MONGO_REPLSET_NAME={{ api_mongo_replset_name }} -e MONGO=mongodb://{{ api_mongo_auth }}@{{ mongo_hosts }}/{{ api_mongo_database }} -e NAVI_HOST=http://{{ navi_host_address }}:{{ navi_http_port }} diff --git a/ansible/group_vars/alpha-big-poppa-http.yml b/ansible/group_vars/alpha-big-poppa-http.yml index 0942ea03..8097e79a 100644 --- a/ansible/group_vars/alpha-big-poppa-http.yml +++ b/ansible/group_vars/alpha-big-poppa-http.yml @@ -12,8 +12,10 @@ dockerfile_enviroment: [ ] dockerfile_pre_install_commands: [ + 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main"> /etc/apt/sources.list.d/pgdg.list', + 'wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -', "apt-get update", - "apt-get install postgresql-client=9.3+154ubuntu1 -y", + "apt-get install postgresql-client-9.4 -y", "echo '//registry.npmjs.org/:_authToken=${NPM_TOKEN}' > .npmrc" ] diff --git a/ansible/group_vars/alpha-big-poppa-worker.yml b/ansible/group_vars/alpha-big-poppa-worker.yml index 13cf253c..5bcf1be1 100644 --- a/ansible/group_vars/alpha-big-poppa-worker.yml +++ b/ansible/group_vars/alpha-big-poppa-worker.yml @@ -11,8 +11,10 @@ dockerfile_enviroment: [ ] dockerfile_pre_install_commands: [ + 'echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main"> /etc/apt/sources.list.d/pgdg.list', + 'wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -', "apt-get update", - "apt-get install postgresql-client=9.3+154ubuntu1 -y", + "apt-get install postgresql-client-9.4 -y", "echo '//registry.npmjs.org/:_authToken=${NPM_TOKEN}' > .npmrc" ] diff --git a/ansible/group_vars/alpha-consul-template-updater.yml b/ansible/group_vars/alpha-consul-template-updater.yml index 648aeb38..ace0bd57 100644 --- a/ansible/group_vars/alpha-consul-template-updater.yml +++ b/ansible/group_vars/alpha-consul-template-updater.yml @@ -1,4 +1,5 @@ name: consul-template-updater-{{ proxy_service_name }} +kill_container_name: consul-template-updater-{{ proxy_service_name }} # container_kill_start settings container_image: avthart/consul-template diff --git a/ansible/group_vars/alpha-consul.yml b/ansible/group_vars/alpha-consul.yml index b94c19c5..2a9e2c18 100644 --- a/ansible/group_vars/alpha-consul.yml +++ b/ansible/group_vars/alpha-consul.yml @@ -40,3 +40,7 @@ consul_seed: value: "{{ api_hostname }}" - key: api/url value: "{{ api_url }}" + - key: s3/bucket + value: "{{ registry_s3_bucket }}" + - key: s3/region + value: "{{ registry_s3_region }}" diff --git a/ansible/group_vars/alpha-detention.yml b/ansible/group_vars/alpha-detention.yml index cb1cbf12..a6d91102 100644 --- a/ansible/group_vars/alpha-detention.yml +++ b/ansible/group_vars/alpha-detention.yml @@ -12,6 +12,8 @@ redis_key: "frontend:{{ detention_hostname }}" is_redis_update_required: 'yes' dockerfile_post_install_commands: [ + "wget -nv https://github.com/eSlider/sassc-binaries/raw/develop/dist/sassc -O /usr/bin/sass", + "chmod +x /usr/bin/sass", "npm run grunt" ] diff --git a/ansible/group_vars/alpha-lets-encrypt-certs-generation.yml b/ansible/group_vars/alpha-lets-encrypt-certs-generation.yml new file mode 100644 index 00000000..6b405052 --- /dev/null +++ b/ansible/group_vars/alpha-lets-encrypt-certs-generation.yml @@ -0,0 +1,21 @@ +--- +name: nginx + +# used by consul template updater +target_container_name: nginx +target_updater_file_path: /etc/nginx/sites-enabled + +# used by container_kill_start +container_image: "{{ name }}" +container_tag: "1.10" + +restart_policy: always + +container_run_opts: > + -d + --name "{{ name }}" + -p 80:80 + -v {{ target_updater_file_path }}:/etc/nginx/sites-enabled + -v /etc/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + -v /var/www/html:/var/www/html + -v /etc/ssl/certs/{{ domain }}:/etc/ssl/certs/{{ domain }}:ro diff --git a/ansible/group_vars/alpha-link.yml b/ansible/group_vars/alpha-link.yml index 1d8e2d22..62f37d2c 100644 --- a/ansible/group_vars/alpha-link.yml +++ b/ansible/group_vars/alpha-link.yml @@ -10,7 +10,7 @@ container_envs: > -e API_URL={{ api_url }} -e DATADOG_HOST={{ datadog_host_address }} -e DATADOG_PORT={{ datadog_port }} - -e HELLO_RUNNABLE_GITHUB_TOKEN=5d8f7029d3d6941b0fc62a7eb8c605d8e0bc7c29 + -e HELLO_RUNNABLE_GITHUB_TOKEN={{ link_hello_runnable_github_token }} -e MONGO=mongodb://{{ navi_mongo_host_address }}:{{ navi_mongo_port }}/{{ navi_mongo_database }} -e NODE_ENV={{ node_env }} -e RABBITMQ_HOSTNAME={{ rabbit_host_address }} diff --git a/ansible/group_vars/alpha-marketing.yml b/ansible/group_vars/alpha-marketing.yml index 94e57d3b..f82f756c 100644 --- a/ansible/group_vars/alpha-marketing.yml +++ b/ansible/group_vars/alpha-marketing.yml @@ -11,6 +11,7 @@ dockerfile_enviroment: [ "API_URL https://{{ api_hostname }}", "AWS_ACCESS_KEY {{ aws_access_key }}", "AWS_SECRET_KEY {{ aws_secret_key }}", + "AWS_REGION {{ marketing_aws_region | default('us-standard') }}", "ANGULAR_URL {{ angular_url }}", "AWS_BUCKET {{ domain }}", "NODE_ENV {{ node_env }}" diff --git a/ansible/group_vars/alpha-metis.yml b/ansible/group_vars/alpha-metis.yml index fe371c36..a0a4761f 100644 --- a/ansible/group_vars/alpha-metis.yml +++ b/ansible/group_vars/alpha-metis.yml @@ -27,7 +27,7 @@ container_envs: > -e REDIS_CACERT={{ redis_ca_cert_path }} -e REDIS_HOST={{ redis_host_address }} -e REDIS_PORT={{ redis_tls_port }} - -e REGISTRY_HOST={{ registry_host }} + -e REGISTRY_HOST={{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }} -e ROLLBAR_KEY={{ metis_rollbar_key }} -e DOCKER_PORT={{ docker_port }} diff --git a/ansible/group_vars/alpha-mongo-navi.yml b/ansible/group_vars/alpha-mongo-navi.yml index 9d9cf42d..538d1c54 100644 --- a/ansible/group_vars/alpha-mongo-navi.yml +++ b/ansible/group_vars/alpha-mongo-navi.yml @@ -5,9 +5,9 @@ db_path: /opt/mongodb/db # container_kill_start settings container_image: mongo -container_tag: latest +container_tag: 3.2.4 container_run_opts: > -h {{ name }} -d - -p 27017:27017 + -p {{ navi_mongo_port }}:27017 -v {{ db_path }}:/data/db:rw diff --git a/ansible/group_vars/alpha-mongo.yml b/ansible/group_vars/alpha-mongo.yml index 06227dc4..a575c695 100644 --- a/ansible/group_vars/alpha-mongo.yml +++ b/ansible/group_vars/alpha-mongo.yml @@ -5,8 +5,8 @@ db_path: /opt/mongodb/db # container_kill_start settings container_image: mongo -container_tag: latest -container_run_opts: -d --name mongodb -p 27017:27017 -v {{ db_path }}:/data/db:rw +container_tag: 2.6.11 +container_run_opts: -d --name mongodb -p {{ mongo_port }}:27017 -v {{ db_path }}:/data/db:rw # do not monitor docker-daemon for mongo, as there is none no_datadog_docker_monitoring: true diff --git a/ansible/group_vars/alpha-redis.yml b/ansible/group_vars/alpha-redis.yml index 807f32a1..b970e844 100644 --- a/ansible/group_vars/alpha-redis.yml +++ b/ansible/group_vars/alpha-redis.yml @@ -5,6 +5,6 @@ db_path: "/opt/redis/db" # container_kill_start settings container_image: redis container_tag: "latest" -container_run_opts: "-d --name redis -v {{ db_path }}:/data -p {{ redis_port }}:{{ redis_port }}" +container_run_opts: "-d --name redis -v {{ db_path }}:/data -p {{ redis_port }}:6379" hosted_ports: [ "{{ redis_port }}" ] diff --git a/ansible/group_vars/alpha-shiva.yml b/ansible/group_vars/alpha-shiva.yml index 24f3daa5..244d5b39 100644 --- a/ansible/group_vars/alpha-shiva.yml +++ b/ansible/group_vars/alpha-shiva.yml @@ -24,10 +24,10 @@ container_envs: > -e REDIS_CACERT={{ redis_ca_cert_path }} -e REDIS_PORT={{ redis_tls_port }} -e REDIS_IPADDRESS={{ redis_host_address }} - -e REGISTRY_HOST={{ registry_host }} + -e REGISTRY_HOST={{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }} {% if shiva_rollbar_token is defined %} -e ROLLBAR_KEY={{ shiva_rollbar_token }} {% endif %} -e DOCKER_PORT={{ docker_port }} - {% if shiva_consult_hostname is defined %} -e CONSUL_HOSTNAME={{ shiva_consult_hostname }} {% endif %} + {% if shiva_consult_hostname is defined %} -e CONSUL_HOSTNAME={{ hostvars[groups['dock'][0]]['ansible_default_ipv4']['address'] }} {% endif %} {% if shiva_aws_region is defined %} -e AWS_REGION={{ shiva_aws_region }} {% endif %} {% if shiva_dock_security_groups is defined %} -e AWS_DOCK_SECURITY_GROUPS={{ shiva_dock_security_groups }} {% endif %} {% if shiva_ssh_key_name is defined %} -e AWS_SSH_KEY_NAME={{ shiva_ssh_key_name }} {% endif %} diff --git a/ansible/group_vars/alpha-swarm-daemon.yml b/ansible/group_vars/alpha-swarm-daemon.yml index 643d73f3..61dc5f71 100644 --- a/ansible/group_vars/alpha-swarm-daemon.yml +++ b/ansible/group_vars/alpha-swarm-daemon.yml @@ -1,8 +1,8 @@ name: "swarm-deamon" # container_kill_start settings -container_image: swarm -container_tag: 1.2.3 +container_image: runnable/swarm +container_tag: "{{ swarm_version }}" container_run_opts: > --name {{ swarm_container_name }} diff --git a/ansible/group_vars/alpha-swarm-manager.yml b/ansible/group_vars/alpha-swarm-manager.yml index 84e0ef8b..9ed91bf1 100644 --- a/ansible/group_vars/alpha-swarm-manager.yml +++ b/ansible/group_vars/alpha-swarm-manager.yml @@ -1,8 +1,8 @@ name: "swarm-manager" # container_kill_start settings -container_image: swarm -container_tag: 1.2.3-0 +container_image: runnable/swarm +container_tag: "{{ swarm_version }}" memory_hard_limit: 10g diff --git a/ansible/group_vars/alpha-web.yml b/ansible/group_vars/alpha-web.yml index be7bf832..5139b787 100644 --- a/ansible/group_vars/alpha-web.yml +++ b/ansible/group_vars/alpha-web.yml @@ -14,6 +14,7 @@ dockerfile_enviroment: [ "MIXPANEL_PROXY_URL {{ mixpanel_proxy_url }}", "AWS_ACCESS_KEY {{ aws_access_key }}", "AWS_BUCKET app.{{ domain }}", + "AWS_REGION {{ web_aws_bucket_region | default('us-standard') }}", "AWS_SECRET_KEY {{ aws_secret_key }}", "INTERCOM_APP_ID {{ web_intercom_id }}", "MARKETING_URL {{ marketing_url }}", diff --git a/ansible/lets-encrypt-certs-generation.yml b/ansible/lets-encrypt-certs-generation.yml new file mode 100644 index 00000000..1459fc02 --- /dev/null +++ b/ansible/lets-encrypt-certs-generation.yml @@ -0,0 +1,8 @@ +--- +- hosts: userland + vars_files: + - group_vars/alpha-lets-encrypt-certs-generation.yml + roles: + - role: lets-encrypt-certs-generation + + - role: container_kill_start diff --git a/ansible/roles/base_ubuntu/tasks/main.yml b/ansible/roles/base_ubuntu/tasks/main.yml index ee624a85..33f5ba32 100644 --- a/ansible/roles/base_ubuntu/tasks/main.yml +++ b/ansible/roles/base_ubuntu/tasks/main.yml @@ -4,6 +4,6 @@ when: dock is not defined lineinfile: dest=/etc/hosts - line="{{ registry_host }} registry.runnable.com" + line="{{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }} registry.runnable.com" state=present regexp=".+ registry\.runnable\.com" diff --git a/ansible/roles/builder/tasks/main.yml b/ansible/roles/builder/tasks/main.yml index b747286c..27064685 100644 --- a/ansible/roles/builder/tasks/main.yml +++ b/ansible/roles/builder/tasks/main.yml @@ -50,6 +50,19 @@ src: "{{ dockerfile }}" dest: "{{ build_dir }}/{{ name }}" +- name: copy secrets into build dir + tags: [ deploy ] + become: true + copy: + src=../docker-files/base/{{ item }} + dest={{ build_dir }}/{{ name }} + owner=ubuntu + group=ubuntu + mode=0700 + with_items: + - id_rsa + - known_hosts + - name: build docker image and tag tags: deploy become: yes diff --git a/ansible/roles/builder/templates/basic_node/Dockerfile b/ansible/roles/builder/templates/basic_node/Dockerfile index 594ddb5f..a8eadc03 100644 --- a/ansible/roles/builder/templates/basic_node/Dockerfile +++ b/ansible/roles/builder/templates/basic_node/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.runnable.com/runnable/{{ base_dockerfile }}:latest +FROM node:latest {% if hosted_ports is defined %} # Expose port to Host @@ -16,12 +16,18 @@ ENV {{ env }} RUN npm install -g n@2.1.0 RUN n {{ node_version }} && npm install -g npm@{{ npm_version }} +WORKDIR /{{ name }} {% if dockerfile_pre_install_commands is defined %} {% for command in dockerfile_pre_install_commands %} RUN {{ command }} {% endfor %} {% endif %} +RUN mkdir /root/.ssh/ +ADD id_rsa /root/.ssh/id_rsa +ADD known_hosts /root/.ssh/known_hosts +RUN chmod 600 /root/.ssh/id_rsa + # Add package.json from the current build context (`.` is the repo) second ADD ./repo/package.json /{{ name }}/package.json diff --git a/ansible/roles/consul-services/tasks/main.yml b/ansible/roles/consul-services/tasks/main.yml index 26458199..dc76654b 100644 --- a/ansible/roles/consul-services/tasks/main.yml +++ b/ansible/roles/consul-services/tasks/main.yml @@ -29,7 +29,7 @@ tags: ['master'] port: '{{ redis_port }}' - name: 'registry' - host_address: '{{ registry_host }}' + host_address: "{{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }}" tags: ['master'] port: '{{ registry_port }}' diff --git a/ansible/roles/container_kill_start/files/findTagRunning.sh b/ansible/roles/container_kill_start/files/findTagRunning.sh index 27a7243e..3c977cb2 100755 --- a/ansible/roles/container_kill_start/files/findTagRunning.sh +++ b/ansible/roles/container_kill_start/files/findTagRunning.sh @@ -5,7 +5,6 @@ CONTAINER_NAME="$2" CONTAINERS=`docker ps -a | grep -v '^CONTAINER' | awk '{print $1}'` if [ "" = "${CONTAINERS}" ] ; then - echo "this is 0" exit 0 else for container in ${CONTAINERS} ; do diff --git a/ansible/roles/container_kill_start/tasks/main.yml b/ansible/roles/container_kill_start/tasks/main.yml index 0b6e4aa6..75a37f69 100644 --- a/ansible/roles/container_kill_start/tasks/main.yml +++ b/ansible/roles/container_kill_start/tasks/main.yml @@ -20,7 +20,7 @@ - name: look for running containers running {{ container_image }} tags: deploy become: true - script: findTagRunning.sh {{ container_image }} + script: findTagRunning.sh {{ container_image }} {{ kill_container_name | default('') }} register: old_containers_id changed_when: old_containers_id.stdout | length > 4 diff --git a/ansible/roles/datadog/tasks/main.yml b/ansible/roles/datadog/tasks/main.yml index 9c717f04..1d4613dc 100644 --- a/ansible/roles/datadog/tasks/main.yml +++ b/ansible/roles/datadog/tasks/main.yml @@ -68,7 +68,7 @@ become: true when: has_dd_integration is defined template: - src="{{ name }}.yaml.j2" + src="roles/datadog/templates/{{ name }}.yaml.j2" dest="/etc/dd-agent/conf.d/{{ name }}.yaml" notify: restart datadog-agent diff --git a/ansible/roles/datadog/templates/mongo.yaml.j2 b/ansible/roles/datadog/templates/mongodb.yaml.j2 similarity index 70% rename from ansible/roles/datadog/templates/mongo.yaml.j2 rename to ansible/roles/datadog/templates/mongodb.yaml.j2 index 03a1b1b2..21bcc99b 100644 --- a/ansible/roles/datadog/templates/mongo.yaml.j2 +++ b/ansible/roles/datadog/templates/mongodb.yaml.j2 @@ -1,4 +1,4 @@ init_config: instances: - - server: mongodb://{{ datadog_mongodb_user }}:{{ datadog_mongodb_pwd }}@localhost:27017 + - server: mongodb://{{ datadog_mongodb_user }}:{{ datadog_mongodb_pwd }}@localhost:{{ mongo_port }} diff --git a/ansible/roles/dock-images/tasks/main.yml b/ansible/roles/dock-images/tasks/main.yml index 1c284621..75487a68 100644 --- a/ansible/roles/dock-images/tasks/main.yml +++ b/ansible/roles/dock-images/tasks/main.yml @@ -17,8 +17,7 @@ become: true command: docker pull {{ item }} with_items: - - "registry.runnable.com/runnable/image-builder:v4.2.3" - - "swarm:1.2.5" + - "runnable/swarm:{{ swarm_version }}" - "registry:2.3.1" - "google/cadvisor:v0.24.1" - "prom/node-exporter:v0.13.0" diff --git a/ansible/roles/docker_client/README.md b/ansible/roles/docker_client/README.md index 3b95b530..f6bf908d 100644 --- a/ansible/roles/docker_client/README.md +++ b/ansible/roles/docker_client/README.md @@ -7,7 +7,7 @@ Ansible Role to Install Docker Client Certs on Ubuntu Creating new docker client certs: 1. cd into this dir ```cd ``` 2. ensure you have ca-key.pem here `roles/docker_client/ca-key.pem` -3. run cert generator ```sudo ./scripts/genClientCert.sh ``` +3. run cert generator ```sudo ./scripts/genClientCert.sh ``` ## Author Information diff --git a/ansible/roles/docker_client/scripts/genClientCert.sh b/ansible/roles/docker_client/scripts/genClientCert.sh index a4737355..1d951211 100755 --- a/ansible/roles/docker_client/scripts/genClientCert.sh +++ b/ansible/roles/docker_client/scripts/genClientCert.sh @@ -7,10 +7,12 @@ fi CLIENT=./files/certs/$1 echo 'WARN: hard coded alpha-api-old gamma-services and beta-services for SWARM' -# if [[ $2 = '' ]]; then -# echo 'script requires a client ip address' -# exit 1 -# fi +if [[ $2 = '' ]]; then + echo 'script requires a client ip address' + exit 1 +fi + +MAIN_HOST_IP_ADDRESS=$2 mkdir $CLIENT @@ -28,7 +30,8 @@ openssl req \ chmod 400 "$CLIENT/client.csr" echo extendedKeyUsage=clientAuth,serverAuth > "$CLIENT/extfile.cnf" -echo subjectAltName=IP:10.4.0.148,IP:10.8.4.40,IP:10.12.12.136,IP:10.8.5.63,IP:10.8.6.59,IP:10.4.6.251,IP:127.0.0.1,DNS:localhost,DNS:swarm-staging-codenow.runnableapp.com >> "$CLIENT/extfile.cnf" +echo subjectAltName=IP:127.0.0.1,DNS:localhost >> "$CLIENT/extfile.cnf" +echo subjectAltName=IP:${MAIN_HOST_IP_ADDRESS},IP:10.4.0.148,IP:10.8.4.40,IP:10.12.12.136,IP:10.8.5.63,IP:10.8.6.59,IP:10.4.6.251,IP:127.0.0.1,DNS:localhost,DNS:swarm-staging-codenow.runnableapp.com >> "$CLIENT/extfile.cnf" # generate cert for client openssl x509 \ diff --git a/ansible/roles/image-builder/tasks/main.yml b/ansible/roles/image-builder/tasks/main.yml index ac71aa5e..affd810b 100644 --- a/ansible/roles/image-builder/tasks/main.yml +++ b/ansible/roles/image-builder/tasks/main.yml @@ -8,9 +8,3 @@ - name: build the image-builder tags: deploy command: sudo docker build --no-cache --tag="registry.runnable.com/{{ image_builder_docker_namespace }}:{{ git_branch }}" /opt/runnable/image-builder - -- name: push image-builder - tags: deploy - run_once: true - command: sudo docker push "registry.runnable.com/{{ image_builder_docker_namespace }}:{{ git_branch }}" - when: dock is not defined diff --git a/ansible/roles/lets-encrypt-certs-generation/tasks/main.yml b/ansible/roles/lets-encrypt-certs-generation/tasks/main.yml new file mode 100644 index 00000000..a0195190 --- /dev/null +++ b/ansible/roles/lets-encrypt-certs-generation/tasks/main.yml @@ -0,0 +1,35 @@ +--- +- name: make sure cert directory is in place + tags: [ configure_proxy, certs ] + become: true + file: + dest: /etc/ssl/certs/{{ domain }} + state: directory + +- name: make sure nginx directory is in place + tags: [ configure_proxy, configure_files ] + become: true + file: + dest: /etc/nginx + state: directory + +- name: put nginx configuration in place + tags: [ configure_proxy, configure_files ] + become: yes + template: + src: proxy-nginx.conf + dest: /etc/nginx/nginx.conf + +- name: assert nginx sites-enabled directory + tags: [ configure_proxy, configure_files ] + become: yes + file: + state: directory + dest: /etc/nginx/sites-enabled + +- name: put lets-encrypt conf in place + tags: [ configure_proxy, configure_files ] + become: yes + template: + src: default.conf + dest: /etc/nginx/sites-enabled/ diff --git a/ansible/roles/lets-encrypt-certs-generation/templates/default b/ansible/roles/lets-encrypt-certs-generation/templates/default new file mode 100644 index 00000000..47263454 --- /dev/null +++ b/ansible/roles/lets-encrypt-certs-generation/templates/default @@ -0,0 +1,5 @@ +server { + listen [::]:80 default_server; + server_name _; + return 200 "Its alive"; +} diff --git a/ansible/roles/lets-encrypt-certs-generation/templates/default.conf b/ansible/roles/lets-encrypt-certs-generation/templates/default.conf new file mode 100644 index 00000000..54554361 --- /dev/null +++ b/ansible/roles/lets-encrypt-certs-generation/templates/default.conf @@ -0,0 +1,14 @@ +server { + listen 80; + server_name {{ domain }} *.{{ domain }}; + index index.html index.htm; + root /var/www/html; + + location ~ /.well-known { + allow all; + } + + location /test { + return 200 "Its alive"; + } +} diff --git a/ansible/roles/lets-encrypt-certs-generation/templates/proxy-nginx.conf b/ansible/roles/lets-encrypt-certs-generation/templates/proxy-nginx.conf new file mode 100644 index 00000000..dc663d03 --- /dev/null +++ b/ansible/roles/lets-encrypt-certs-generation/templates/proxy-nginx.conf @@ -0,0 +1,29 @@ +user www-data; +worker_processes 4; +pid /run/nginx.pid; + +events { + worker_connections 5000; +} + +http { + ## + # Basic Settings + ## + tcp_nodelay on; + keepalive_timeout 65; + server_tokens off; + + ## + # Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + ## + # Virtual Host Configs + ## + + include /etc/nginx/sites-enabled/*; +} diff --git a/ansible/roles/local-vault/templates/vault.hcl.j2 b/ansible/roles/local-vault/templates/vault.hcl.j2 index 49b5bbf5..959b956d 100644 --- a/ansible/roles/local-vault/templates/vault.hcl.j2 +++ b/ansible/roles/local-vault/templates/vault.hcl.j2 @@ -1,8 +1,8 @@ disable_mlock = true backend "s3" { - bucket = "runnable.vault.bear-clone" - access_key = "{{ vault_aws_access_key }}" + bucket = "runnable.vault.{{ environment_name }}" + access_key = "{{ vault_aws_access_key_id }}" secret_key = "{{ vault_aws_secret_key }}" region = "us-west-2" } diff --git a/ansible/roles/mongo-server/tasks/main.yml b/ansible/roles/mongo-server/tasks/main.yml index 9368e698..aedba96d 100644 --- a/ansible/roles/mongo-server/tasks/main.yml +++ b/ansible/roles/mongo-server/tasks/main.yml @@ -1,4 +1,20 @@ --- +- name: assert necessary groups + tags: [ tls ] + become: true + group: + name: mongodb + state: present + +- name: assert necessary users + tags: [ tls ] + become: true + user: + name: mongodb + shell: /bin/bash + groups: mongodb + append: yes + - name: make folder for certificates tags: [ tls ] become: true diff --git a/ansible/roles/runnable-domain-proxy/templates/registry.tmpl b/ansible/roles/runnable-domain-proxy/templates/registry.tmpl index c0500a16..769850b8 100644 --- a/ansible/roles/runnable-domain-proxy/templates/registry.tmpl +++ b/ansible/roles/runnable-domain-proxy/templates/registry.tmpl @@ -1,5 +1,5 @@ upstream docker-registry { - server {{ registry_host }}:{{ registry_port }}; + server {{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }}:{{ registry_port }}; } server { @@ -13,7 +13,7 @@ server { location / { auth_basic off; - proxy_pass http://{{ registry_host }}:{{ registry_port }}; + proxy_pass http://{{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }}:{{ registry_port }}; proxy_set_header Host $http_host; # required for docker client's sake proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP } @@ -25,7 +25,7 @@ server { return 404; } - proxy_pass http://{{ registry_host }}:{{ registry_port }}; + proxy_pass http://{{ hostvars[groups['registry'][0]]['ansible_default_ipv4']['address'] }}:{{ registry_port }}; proxy_set_header Host $http_host; # required for docker client's sake proxy_set_header X-Real-IP $remote_addr; # pass on real client's IP proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/ansible/single-host-part-1-setup.yml b/ansible/single-host-part-1-setup.yml new file mode 100644 index 00000000..d088bc78 --- /dev/null +++ b/ansible/single-host-part-1-setup.yml @@ -0,0 +1,4 @@ +# Initial values (Only run the first time) +- include: consul-values.yml -e write_values="true" # Only run the first time +- include: consul-services.yml # Only run the first time +- include: vault-values.yml -e write_values="true" diff --git a/ansible/single-host-part-1.yml b/ansible/single-host-part-1.yml new file mode 100644 index 00000000..cf21cd4e --- /dev/null +++ b/ansible/single-host-part-1.yml @@ -0,0 +1,3 @@ +## Service Discovery: +- include: consul.yml +- include: vault.yml diff --git a/ansible/single-host-part-2.yml b/ansible/single-host-part-2.yml new file mode 100644 index 00000000..e8b1a51f --- /dev/null +++ b/ansible/single-host-part-2.yml @@ -0,0 +1,6 @@ +# Databases +- include: mongo.yml +- include: rabbitmq.yml +- include: redis.yml +- include: redis-tls.yml # Only used by navi and shiva +- include: registry.yml diff --git a/ansible/single-host-part-3.yml b/ansible/single-host-part-3.yml new file mode 100644 index 00000000..7122e344 --- /dev/null +++ b/ansible/single-host-part-3.yml @@ -0,0 +1,28 @@ +# Docks Services +# - include: swarm-manager.yml +# - include: palantiri.yml git_branch="{{ palantiri_branch }}" -t deploy +# - include: sauron.yml git_branch="{{ sauron_branch }}" -t deploy +# - include: shiva.yml git_branch="{{ astral_branch }}" -t deploy +# - include: khronos.yml git_branch="{{ khronos_branch }}" -t deploy +# - include: docker-listener.yml git_branch="{{ docker_listener_branch }}" -t deploy + +## Proxies +# - include: registrator-api.yml # Only one of these is needed, so registrator-navi is not needed +# - include: single-host-proxy.yml # API depends on NGINX to be running +# - include: github-varnish.yml git_branch="{{ github_varnish_branch }}" -t deploy + +# Main +# - include: big-poppa.yml git_branch="{{ big_poppa_branch }}" -t deploy +# - include: api.yml git_branch="{{ api_branch }}" -t deploy +# - include: drake.yml git_branch="{{ drake_branch }}" -t deploy +# - include: cream.yml git_branch="{{ cream_branch }}" -t deploy # CREAM fails if big-poppa or API is down +# - include: web.yml git_branch="{{ angular_branch }}" -t deploy # fucked + +# Networking services +# - include: detention.yml git_branch="{{ detention_branch }}" -t deploy +- include: link.yml git_branch="{{ link_branch }}" -t deploy +- include: navi.yml git_branch="{{ navi_branch }}" -t deploy # Connects to Redis over tls port + +# Other +- include: optimus.yml git_branch="{{ optimus_branch }}" -t deploy +- include: pheidi.yml git_branch="{{ pheidi_branch }}" -t deploy diff --git a/ansible/single-host.yml b/ansible/single-host.yml index 1888688c..351eabb0 100644 --- a/ansible/single-host.yml +++ b/ansible/single-host.yml @@ -1,49 +1,3 @@ -## configure security group policy -- include: sg_configure.yml - -## Install Datadog Agent -# - include: datadog.yml - -## begin with databases: -- include: consul.yml -- include: vault.yml - -# Initial values (Only run the first time) -- include: consul-values.yml -e write_values="true" # Only run the first time -- include: consul-services.yml # Only run the first time -- include: vault-values.yml -e write_values="true" - -# Databases -- include: rabbitmq.yml -- include: redis.yml -- include: redis-tls.yml # Only used by navi and shiva -- include: registry.yml - -# Docks Services -- include: swarm-manager.yml -- include: palantiri.yml git_branch="{{ palantiri_branch }}" -t deploy -- include: sauron.yml git_branch="{{ sauron_branch }}" -t deploy -- include: shiva.yml git_branch="{{ astral_branch }}" -t deploy -- include: khronos.yml git_branch="{{ khronos_branch }}" -t deploy -- include: docker-listener.yml git_branch="{{ docker_listener_branch }}" -t deploy - -## Proxies -- include: registrator-api.yml # Only one of these is neededi, so registrator-navi is not needed -- include: single-host-proxy.yml # API depends on NGINX to be running -- include: github-varnish.yml git_branch="{{ github_varnish_branch }}" -t deploy - -# Main -- include: big-poppa.yml git_branch="{{ big_poppa_branch }}" -t deploy -- include: api.yml git_branch="{{ api_branch }}" -t deploy -- include: cream.yml git_branch="{{ cream_branch }}" -t deploy # CREAM fails if big-poppa or API is down -- include: web.yml git_branch="{{ angular_branch }}" -t deploy # fucked - -# Networking services -- include: detention.yml git_branch="{{ detention_branch }}" -t deploy -- include: link.yml git_branch="{{ link_branch }}" -t deploy -- include: navi.yml git_branch="{{ navi_branch }}" -t deploy # Connects to Redis over tls port - -# Other -- include: optimus.yml git_branch="{{ optimus_branch }}" -t deploy -- include: drake.yml git_branch="{{ drake_branch }}" -t deploy -- include: pheidi.yml git_branch="{{ pheidi_branch }}" -t deploy +- include: single-host-part-1.yml +- include: single-host-part-2.yml +- include: single-host-part-3.yml diff --git a/ansible/vault.yml b/ansible/vault.yml index c0c5a4f2..7e1bb165 100644 --- a/ansible/vault.yml +++ b/ansible/vault.yml @@ -9,6 +9,10 @@ - { role: container_kill_start } tasks: + - name: make sure httplib2 is installed + become: true + apt: package=python-httplib2 state=present + - name: get seal status tags: [ deploy ] uri: diff --git a/ssh/config b/ssh/config index 9686e824..ebad9b5f 100644 --- a/ssh/config +++ b/ssh/config @@ -123,6 +123,9 @@ Host gamma-bear-clone-main Host gamma-bear-clone-dock ProxyCommand ssh -q ubuntu@gamma-bastion nc 10.248.1.58 22 +Host gamma-grizzly-main + ProxyCommand ssh -q ubuntu@gamma-bastion nc 10.4.0.26 22 + ################################################################################ # Delta ################################################################################