packer files for load balancer

.gitignore

@@ -6,3 +6,4 @@
 *.tfplan
 packer_cache
 packer/build
+*.pem

packer/lb-ubuntu-14-04_amd64-amis.json

@@ -0,0 +1,66 @@
+{
+  "variables": {
+    "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}",
+    "aws_secret_key": "{{env `AWS_ACCESS_KEY`}}",
+    "consul_version": "0.5.0",
+    "consul_template_version": "0.7.0",
+    "weave_version": "latest_release",
+    "build_version": "{{ timestamp }}"
+  },
+  "builders": [{
+    "type": "amazon-ebs",
+    "ami_name": "lb-ubuntu-14.04_amd64_{{user `build_version`}}",
+    "ami_description": "Ubuntu 14.04 LTS, Docker, Consul {{user `consul_version`}}, Consul Template {{user `consul_template_version`}} and Weave {{user `weave_version`}}.",
+    "access_key": "{{user `aws_access_key`}}",
+    "secret_key": "{{user `aws_secret_key`}}",
+    "region": "eu-west-1",
+    "source_ami": "ami-394ecc4e",
+    "instance_type": "m1.medium",
+    "ami_groups": "all",
+    "ssh_username": "ubuntu",
+    "ssh_timeout": "10m",
+    "ami_regions": ["eu-west-1"]
+  }],
+  "provisioners": [
+    {
+      "type": "file",
+      "source": "scripts/common/templates",
+      "destination": "/tmp"
+    },
+    {
+      "type": "file",
+      "source": "scripts/ubuntu/upstart",
+      "destination": "/tmp"
+    },
+    {
+      "type": "shell",
+      "environment_vars": [
+        "CONSUL_VERSION={{user `consul_version`}}",
+        "CONSUL_TEMPLATE_VERSION={{user `consul_template_version`}}",
+        "WEAVE_VERSION={{user `weave_version`}}"
+      ],
+      "scripts": [
+        "scripts/ubuntu/base.sh",
+        "scripts/common/sshd.sh",
+        "scripts/ubuntu/install_docker.sh",
+        "scripts/common/install_consul.sh",
+        "scripts/common/install_consul_template.sh",
+        "scripts/ubuntu/install_haproxy.sh",
+        "scripts/common/install_weave.sh"
+      ],
+      "execute_command": "{{ .Vars }} sudo  -E -S sh '{{ .Path }}'"
+    }
+  ],
+  "push": {
+    "name": "capgemini/lb-ubuntu-14-04"
+  },
+  "post-processors": [{
+    "type": "atlas",
+    "token": "{{env `ATLAS_TOKEN`}}",
+    "artifact": "capgemini/lb_ubuntu-14.04_amd64",
+    "artifact_type": "aws.ami",
+    "metadata": {
+      "created_at": "{{timestamp}}"
+    }
+  }]
+}

packer/lb-ubuntu-14-04_amd64.json

@@ -0,0 +1,84 @@
+{
+  "variables": {
+    "ssh_name": "vagrant",
+    "ssh_pass": "vagrant",
+    "hostname": "ubuntu",
+    "consul_version": "0.5.0",
+    "consul_template_version": "0.7.0",
+    "weave_version": "latest_release",
+    "build_version": "{{ timestamp }}"
+  },
+  "builders": [{
+    "type": "virtualbox-iso",
+    "guest_os_type": "Ubuntu_64",
+    "headless": true,
+    "vm_name": "lb_ubuntu-14.04_amd64_virtualbox_{{user `build_version`}}",
+    "disk_size": 40960,
+    "vboxmanage": [
+      [ "modifyvm", "{{.Name}}", "--memory", "2048" ],
+      [ "modifyvm", "{{.Name}}", "--cpus",   "2"    ]
+    ],
+    "iso_url": "http://releases.ubuntu.com/14.04.2/ubuntu-14.04.2-server-amd64.iso",
+    "iso_checksum": "83aabd8dcf1e8f469f3c72fff2375195",
+    "iso_checksum_type": "md5",
+    "http_directory" : "./http/ubuntu-14.04",
+    "http_port_min" : 9001,
+    "http_port_max" : 9001,
+    "boot_command" : [
+      "<esc><esc><enter><wait>",
+      "/install/vmlinuz noapic ",
+      "preseed/url=http://{{ .HTTPIP }}:{{ .HTTPPort }}/preseed.cfg ",
+      "debian-installer=en_US auto locale=en_US kbd-chooser/method=us ",
+      "hostname={{user `hostname`}} ",
+      "fb=false debconf/frontend=noninteractive ",
+      "keyboard-configuration/modelcode=SKIP keyboard-configuration/layout=USA ",
+      "keyboard-configuration/variant=USA console-setup/ask_detect=false ",
+      "initrd=/install/initrd.gz -- <enter>"
+    ],
+    "ssh_username": "{{user `ssh_name`}}",
+    "ssh_password": "{{user `ssh_pass`}}",
+    "ssh_wait_timeout": "60m",
+    "shutdown_command": "echo '{{user `ssh_pass`}}' | sudo -S shutdown -P now",
+    "output_directory": "build/ubuntu-14.04_amd64_virtualbox",
+    "format": "ova"
+  }],
+  "provisioners": [
+    {
+      "type": "file",
+      "source": "scripts/common/templates",
+      "destination": "/tmp"
+    },
+    {
+      "type": "file",
+      "source": "scripts/ubuntu/upstart",
+      "destination": "/tmp"
+    },
+    {
+      "type": "shell",
+      "environment_vars": [
+        "CONSUL_VERSION={{user `consul_version`}}",
+        "CONSUL_TEMPLATE_VERSION={{user `consul_template_version`}}",
+        "WEAVE_VERSION={{user `weave_version`}}"
+      ],
+      "scripts": [
+        "scripts/ubuntu/base.sh",
+        "scripts/ubuntu/virtualbox/install-dkms.sh",
+        "scripts/common/virtualbox/install_vbox_guest_additions.sh",
+        "scripts/common/virtualbox/sudoers.sh",
+        "scripts/common/virtualbox/vagrant_ssh.sh",
+        "scripts/common/sshd.sh",
+        "scripts/ubuntu/install_docker.sh",
+        "scripts/common/install_consul.sh",
+        "scripts/common/install_consul_template.sh",
+        "scripts/ubuntu/install_haproxy.sh",
+        "scripts/common/install_weave.sh"
+      ],
+      "execute_command": "echo '{{user `ssh_pass`}}' | {{ .Vars }} sudo  -E -S sh '{{ .Path }}'"
+    }
+  ],
+  "post-processors": [{
+    "type": "vagrant",
+    "output": "build/ubuntu-14.04_amd64_{{.Provider}}/lb_ubuntu-14.04_amd64_{{.Provider}}_{{user `build_version`}}.box",
+    "keep_input_artifact": true
+  }]
+}

packer/scripts/common/install_consul_template.sh

@@ -0,0 +1,8 @@
+# install consul template
+wget https://github.com/hashicorp/consul-template/releases/download/v${CONSUL_TEMPLATE_VERSION}/consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.tar.gz
+tar xzf consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64.tar.gz
+sudo mv consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64/consul-template /usr/bin
+sudo rmdir consul-template_${CONSUL_TEMPLATE_VERSION}_linux_amd64
+
+# consul template upstart for haproxy
+sudo cp /tmp/upstart/consul_template.conf /etc/init/consul_template.conf

packer/scripts/common/templates/haproxy.cfg

@@ -0,0 +1,18 @@
+global
+    maxconn 4
+    log 127.0.0.1 local0 notice
+    user haproxy
+    group haproxy
+
+defaults
+    log global
+    retries 2
+    timeout connect 3000
+    timeout server 5000
+    timeout client 5000
+
+listen stats *:1936
+    mode http
+    stats enable
+    stats uri /haproxy?stats
+    stats hide-version

packer/scripts/common/templates/haproxy.ctmpl

@@ -0,0 +1,46 @@
+global
+    chroot  /var/lib/haproxy
+    daemon
+    group  root
+    log  127.0.0.1 local0
+    maxconn  8000
+    pidfile  /var/run/haproxy.pid
+    stats  socket /var/lib/haproxy/stats
+    user  root
+    # Descrease the size of the rewrite buffer so that
+    # there is more room to handle large (>8K) headers. See the HAProxy
+    # manual around tune.bufsize and tune.maxrewrite.
+    tune.maxrewrite 4096
+
+
+defaults
+    log global
+    retries 2
+    timeout connect 3000
+    timeout server 5000
+    timeout client 5000
+
+defaults
+    log     global
+    maxconn 8000
+    option  redispatch
+    option  http-server-close
+    retries 3
+    stats   enable
+    timeout http-request 10s
+    timeout queue 1m
+    timeout connect 10s
+    timeout client 1m
+    timeout server 1m
+    timeout check 10s
+    option forwardfor except 127.0.0.1
+    mode http
+
+listen stats *:1936
+    mode http
+    stats enable
+    stats uri /haproxy?stats
+    stats hide-version
+
+frontend inbound
+    bind :80 name http

packer/scripts/ubuntu/base.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-sudo apt-get update -y
+#sudo apt-get update -y
 sudo apt-get upgrade -y
 sudo apt-get update -y
 
-sudo apt-get -y install git curl libcurl3 default-jre-headless unzip wget python-setuptools python-protobuf
+sudo apt-get -y install git curl libcurl3 default-jre-headless unzip wget python-setuptools python-protobuf cgroup-bin

packer/scripts/ubuntu/install_haproxy.sh

@@ -0,0 +1,14 @@
+# install HAproxy
+sudo apt-get install -y haproxy
+sudo chmod a+w /etc/rsyslog.conf
+echo '$ModLoad imudp' >> /etc/rsyslog.conf
+echo '$UDPServerAddress 127.0.0.1' >> /etc/rsyslog.conf
+echo '$UDPServerRun 514' >> /etc/rsyslog.conf
+sudo service rsyslog restart
+sup cp /tmp/templates/haproxy.cfg /etc/haproxy/haproxy.cfg
+
+# eve upstart
+sudo cp /tmp/upstart/haproxy.conf /etc/init/haproxy.conf
+
+# consul config
+echo '{"service": {"name": "haproxy", "tags": ["haproxy"]}}' > /etc/consul.d/haproxy.json

packer/scripts/ubuntu/upstart/consul_template.conf

@@ -0,0 +1,14 @@
+description "Consul Template"
+
+start on vagrant-ready or runlevel [2345]
+stop on runlevel [!2345]
+
+respawn
+
+console log
+
+script
+  exec /usr/bin/consul-template \
+   -consul 127.0.0.1:8500 \
+   -template "/tmp/templates/haproxy.ctmpl:/etc/haproxy/haproxy.cfg:service haproxy restart" >> /var/log/ctemplate.log 2>&1
+end script

packer/scripts/ubuntu/upstart/haproxy.conf

@@ -0,0 +1,20 @@
+description "Properly handle haproxy"
+
+start on startup
+
+env PID_PATH=/var/run/haproxy.pid
+env BIN_PATH=/usr/sbin/haproxy
+env CONF_PATH=/etc/haproxy/haproxy.cfg
+
+script
+exec /bin/bash <<EOF
+  $BIN_PATH -f $CONF_PATH -D -p $PID_PATH
+
+  trap "$BIN_PATH -f $CONF_PATH -p $PID_PATH -sf \\\$(cat $PID_PATH)" SIGHUP
+  trap "kill -TERM \\\$(cat $PID_PATH) && exit 0" SIGTERM SIGINT
+
+  while true; do # Iterate to keep job running.
+    sleep 1 # Don't sleep to long as signals will not be handled during sleep.
+  done
+EOF
+end script

terraform/aws/aws-vpc.tf

@@ -4,6 +4,10 @@ provider "aws" {
   region = "${var.region}"
 }
 
+provider "atlas" {
+  token = "${var.atlas_token}"
+}
+
 /* Define our vpc */
 resource "aws_vpc" "default" {
   cidr_block           = "${var.vpc_cidr_block}"

terraform/aws/loadbalancer.tf

@@ -0,0 +1,43 @@
+/* Loabalancer server */
+resource "atlas_artifact" "loadbalancer" {
+  name = "${var.atlas_artifact.loadbalancer}"
+  type = "aws.ami"
+}
+
+resource "aws_instance" "loadbalancer" {
+  instance_type     = "${var.instance_type.loadbalancer}"
+  ami               = "${atlas_artifact.loadbalancer.metadata_full.region-eu-west-1}"
+  count             = "${var.loadbalancer}"
+  subnet_id         = "${aws_subnet.public.id}"
+  security_groups   = ["${aws_security_group.default.id}", "${aws_security_group.loadbalancer.id}"]
+  depends_on        = ["aws_internet_gateway.public"]
+  key_name          = "${var.key_name}"
+  source_dest_check = false
+  tags = {
+    Name = "capgemini-mesos-loadbalancer"
+  }
+  connection {
+    user       = "ubuntu"
+    key_file   = "${var.key_file}"
+    host        = "${aws_instance.loadbalancer.public_ip}"
+    script_path = "/tmp/${element(aws_instance.loadbalancer.*.id, count.index)}.sh"
+  }
+  provisioner "file" {
+    source      = "${path.module}/scripts/common.sh"
+    destination = "/tmp/${element(aws_instance.loadbalancer.*.id, count.index)}-00common.sh"
+  }
+  provisioner "file" {
+    source      = "${path.module}/scripts/setup-loadbalancer.sh"
+    destination = "/tmp/${element(aws_instance.loadbalancer.*.id, count.index)}-01setup-loadbalancer.sh"
+  }
+  provisioner "remote-exec" {
+    inline = [
+      "echo main ${element(aws_instance.loadbalancer.*.private_ip, count.index)} ${element(aws_instance.loadbalancer.*.private_dns, count.index)} ${var.atlas_token} ${var.atlas_infrastructure} | cat /tmp/${element(aws_instance.loadbalancer.*.id, count.index)}-*.sh - | bash"
+    ]
+  }
+}
+
+resource "aws_eip" "loadbalancer" {
+  instance = "${aws_instance.loadbalancer.id}"
+  vpc = true
+}

terraform/aws/outputs.tf

@@ -1,3 +1,7 @@
 output "nat.ip" {
   value = "${aws_instance.nat.public_ip}"
 }
+
+output "loadbalancer.ip" {
+  value = "${aws_instance.loadbalancer.public_ip}"
+}

terraform/aws/scripts/setup-loadbalancer.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+main() {
+  local node=$(ec2_hostname "$1")
+  local private_dns="$2"
+  local atlas_token="$3"
+  local atlas_infrastructure="$4"
+  wait_ssh_ready "$node"
+
+  set_consul_atlas "$node" "$atlas_token" "$atlas_infrastructure"
+  register_service "$node" consul
+  register_service "$node" docker
+
+}

terraform/aws/scripts/setup-slave.sh

@@ -14,4 +14,5 @@ main() {
   register_service "$node" docker
   register_service "$node" consul
   register_service "$node" dnsmasq
+  register_service "$node" haproxy
 }