corekube-rackconnect-v3.yaml

heat_template_version: 2013-05-23

description: >
  Deploy Kubernetes on a CoreOS cluster that operates on
  Rackspace's RackConnect v3
  Maintainer - Cedric Hauber <hauber.c@gmail.com> | Github: @cedbossneo

parameters:
 kubernetes_minion_count:
   description: Number of CoreOS machines to deploy as Kubernetes Minion
   type: number
   default: 3
   constraints:
   - range:
       min: 1
       max: 12
     description: Must be between 2 and 12 servers.
 keyname:
   type: string
   description: Name of keypair to be used for compute instance
 flavor:
   type: string
   default: 4 GB General Purpose v1
   constraints:
   - allowed_values:
     - 4 GB General Purpose v1
     - 8 GB General Purpose v1
     description: |
       Must be a valid Rackspace Cloud Server flavor for the region you have
       selected to deploy into.
 rackconnect-network:
   type: string
   description: UUID of RackConnect v3 network
 coreos_image:
   type: string
   description: Rackspace Cloud Servers CoreOS Stable (633.1.0) UUID
   default: "a41d8b7c-d41b-4369-a180-0734f3b1cd8c"
 git_command:
   type: string
   description: Git repo checkout command
   default: "/usr/bin/git clone https://github.com/metral/overlord ; /usr/bin/git -C overlord checkout -qf 96574056d0cc7ad8e52228df76374cb3537af8ba"
 flannel_url:
   type: string
   description: Flannel (0.4.0) Binary URL
   default: "http://a26736f09be9784cb06f-8d30bd32035bc1ac3e9215d2d6eedfc5.r53.cf1.rackcdn.com/flanneld"
 discovery_net_interface:
   type: string
   description: Host network interface for which the Discovery node will operate over
   default: "eth1"
 kubernetes_net_interface:
   type: string
   description: Host network interface for which Kubernetes' overlay will operate over
   default: "eth1"

resources:

 coreos_cluster_uuid:
   type: OS::Heat::RandomString

 discovery:
   type: OS::Nova::Server
   properties:
     key_name: { get_param: keyname }
     image: { get_param: coreos_image }
     flavor: { get_param: flavor }
     name: "discovery"
     user_data_format: RAW
     networks:
     - uuid: { get_param: rackconnect-network}
     - uuid: "11111111-1111-1111-1111-111111111111"
     config_drive: "true"
     user_data:
       str_replace:
         template: |
           #cloud-config
           write_files:
             - path: /run/get_discovery_interface_ip.sh
               permissions: '0755'
               content: |
                 #!/bin/bash
                 # Get's the IP of the interface that discovery will be
                 # accessible over
                 DISCOVERY_IF=%discovery_net_interface%
                 /usr/bin/ip -4 addr show $DISCOVERY_IF | /usr/bin/awk '/inet/ {print $2}' | /usr/bin/cut -d/ -f1 > /run/IP
                 /usr/bin/sed -i 's/^/IP=/' /run/IP
           coreos:
             update:
               group: stable
               reboot-strategy: off
             units:
               - name: private-discovery-setup.service
                 command: start
                 content: |
                   [Unit]
                   After=network-online.target
                   Requires=network-online.target

                   [Service]
                   ExecStart=/usr/bin/bash /run/get_discovery_interface_ip.sh
               - name: private-discovery.service
                 command: start
                 content: |
                   [Unit]
                   After=network-online.target private-discovery-setup.service
                   Requires=network-online.target private-discovery-setup.service

                   [Service]
                   EnvironmentFile=/run/IP
                   RestartSec=5s
                   ExecStartPre=/usr/bin/docker pull quay.io/coreos/etcd:v2.0.9
                   ExecStart=/usr/bin/docker run -d --name discovery \
                     -p 2379:2379 -p 2380:2380 \
                     -v /usr/share/ca-certificates/:/etc/ssl/certs \
                     --net host quay.io/coreos/etcd:v2.0.9 -name discovery \
                     -initial-advertise-peer-urls http://${IP}:2380,http://${IP}:7001 \
                     -listen-peer-urls http://${IP}:2380,http://${IP}:7001 \
                     -initial-cluster discovery=http://${IP}:2380,discovery=http://${IP}:7001 \
                     -advertise-client-urls http://${IP}:2379,http://${IP}:4001 \
                     -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001
         params:
           "%discovery_net_interface%": { get_param: discovery_net_interface }

 overlord:
   type: OS::Nova::Server
   properties:
     key_name: { get_param: keyname }
     image: { get_param: coreos_image }
     flavor: { get_param: flavor }
     name: "overlord"
     user_data_format: RAW
     config_drive: "true"
     networks:
     - uuid: { get_param: rackconnect-network}
     - uuid: "11111111-1111-1111-1111-111111111111"
     user_data:
       str_replace:
         template: |
           #cloud-config
           coreos:
             etcd:
               name: overlord
               discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
               addr: $private_ipv4:4001
               peer-addr: $private_ipv4:7001
             update:
               group: stable
               reboot-strategy: off
             units:
               - name: etcd.service
                 command: start
               - name: fleet.socket
                 command: start
                 content: |
                   [Socket]
                   # Talk to the API over a Unix domain socket (default)
                   ListenStream=/var/run/fleet.sock

                   # Talk to the API over an exposed port
                   ListenStream=10001
                   Service=fleet-local.service

                   [Install]
                   WantedBy=sockets.target
               - name: fleet-local.service
                 command: start
                 content: |
                   # fleet-local is kicked off by fleet.socket after API port
                   # is opened
                   [Unit]
                   Description=fleet-local
                   Wants=etcd.service
                   After=etcd.service

                   [Service]
                   Environment=FLEET_PUBLIC_IP=$private_ipv4
                   Environment=FLEET_METADATA=kubernetes_role=overlord
                   ExecStart=/usr/bin/fleet
                   Restart=always
                   RestartSec=10s
               - name: overlord.service
                 command: start
                 content: |
                   # Overlord / logic layer service to deploy kubernetes to
                   # the cluster
                   [Unit]
                   After=network-online.target
                   Requires=network-online.target

                   [Service]
                   WorkingDirectory=/root
                   Environment="DIR=overlord"
                   ExecStartPre=/usr/bin/rm -rf $DIR
                   ExecStartPre=%git_command%
                   ExecStart=/usr/bin/bash ${DIR}/build_run.sh
         params:
           "%discovery%": { get_attr: [discovery, first_address] }
           "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
           "%git_command%": { get_param: git_command }

 kubernetes_master:
   type: OS::Nova::Server
   properties:
     key_name: { get_param: keyname }
     image: { get_param: coreos_image }
     flavor: { get_param: flavor }
     name: "kubernetes_master"
     networks:
     - uuid: { get_param: rackconnect-network}
     - uuid: "11111111-1111-1111-1111-111111111111"
     user_data_format: RAW
     config_drive: "true"
     user_data:
       str_replace:
         template: |
           #cloud-config
           write_files:
             - path: /run/get_discovery_ip_port.sh
               permissions: '0755'
               content: |
                 #!/bin/bash
                 # Sets up environment file with the discovery node's IP &
                 # port so # that in Overlode's template
                 # master-apiserver@.service it can be passed
                 # in as an argument
                 /usr/bin/cat /run/systemd/system/etcd.service.d/20-cloudinit.conf | /usr/bin/grep -i discovery | /usr/bin/cut -f3 -d"=" | /usr/bin/awk -F '/v' '{print $1}' > /run/discovery_ip_port
                 /usr/bin/sed -i 's/^/DISCOVERY_IP_PORT=/' /run/discovery_ip_port
           coreos:
             etcd:
               name: kubernetes_master
               discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
               addr: $private_ipv4:4001
               peer-addr: $private_ipv4:7001
             fleet:
               public-ip: $private_ipv4
               metadata: kubernetes_role=master
             update:
               group: stable
               reboot-strategy: off
             units:
               - name: etcd.service
                 command: start
               - name: fleet.service
                 command: start
               - name: get_discovery_ip_port.service
                 runtime: true
                 command: start
                 content: |
                   # Runs get_discovery_ip_port.sh to have discovery IP &
                   # port ready for consumption by overlord when creating
                   # the Kubernetes' master-api@.service template
                   [Unit]
                   After=network-online.target
                   Requires=network-online.target

                   [Service]
                   ExecStart=/usr/bin/bash /run/get_discovery_ip_port.sh
               - name: flannel-install.service
                 command: start
                 content: |
                   # Installs flannel
                   [Unit]
                   After=network-online.target
                   Requires=network-online.target

                   [Service]
                   Type=oneshot
                   RemainAfterExit=yes
                   ExecStart=/usr/bin/wget -N -P /opt/bin %flannel_url%
                   ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
               - name: flannel.service
                 command: start
                 content: |
                   # Configures & starts flannel
                   [Unit]
                   After=network-online.target etcd.service flannel-install.service
                   Requires=network-online.target etcd.service flannel-install.service

                   [Service]
                   ExecStartPre=/usr/bin/etcdctl mk /coreos.com/network/config '{"Network":"10.244.0.0/15", "Backend": {"Type": "vxlan"}}'
                   ExecStart=/opt/bin/flanneld -iface=%kubernetes_net_interface%
                   Restart=always
                   RestartSec=5s
               - name: flannel-env.path
                 command: start
                 content: |
                   # Ensures flannel env vars are set to use with Docker
                   [Path]
                   PathExists=/run/flannel/subnet.env
                   Unit=docker.service
               - name: docker.service
                 command: start
                 content: |
                   # Starts new docker server that uses flannel
                   [Unit]
                   After=flannel-env.path network-online.target flannel.service
                   Requires=flannel-env.path network-online.target flannel.service
                   Description=Docker Application Container Engine

                   [Service]
                   EnvironmentFile=/run/flannel/subnet.env
                   ExecStartPre=/bin/mount --make-rprivate /
                   ExecStartPre=/usr/bin/systemctl kill docker.service
                   ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}

                   [Install]
                   WantedBy=multi-user.target
         params:
           "%discovery%": { get_attr: [discovery, first_address] }
           "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
           "%flannel_url%": { get_param: flannel_url }
           "%kubernetes_net_interface%": { get_param: kubernetes_net_interface }

 kubernetes_minions:
   type: "OS::Heat::ResourceGroup"
   properties:
     count: { get_param: kubernetes_minion_count }
     resource_def:
       type: OS::Nova::Server
       properties:
         key_name: { get_param: keyname }
         image: { get_param: coreos_image }
         flavor: { get_param: flavor }
         name: kubernetes_minion_%index%
         networks:
         - uuid: { get_param: rackconnect-network}
         - uuid: "11111111-1111-1111-1111-111111111111"
         user_data_format: RAW
         config_drive: "true"
         user_data:
           str_replace:
             template: |
               #cloud-config
               coreos:
                 etcd:
                   name: kubernetes_minion_%index%
                   discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
                   addr: $private_ipv4:4001
                   peer-addr: $private_ipv4:7001
                 fleet:
                   public-ip: $private_ipv4
                   metadata: kubernetes_role=minion
                 update:
                   group: stable
                   reboot-strategy: off
                 units:
                   - name: etcd.service
                     command: start
                   - name: fleet.service
                     command: start
                   - name: flannel-install.service
                     command: start
                     content: |
                       # Installs flannel
                       [Unit]
                       After=network-online.target
                       Requires=network-online.target

                       [Service]
                       Type=oneshot
                       RemainAfterExit=yes
                       ExecStart=/usr/bin/wget -N -P /opt/bin %flannel_url%
                       ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                   - name: flannel.service
                     command: start
                     content: |
                       # Configures & starts flannel
                       [Unit]
                       After=etcd.service flannel-install.service
                       Requires=etcd.service flannel-install.service

                       [Service]
                       ExecStart=/opt/bin/flanneld -iface=%kubernetes_net_interface%
                       Restart=always
                       RestartSec=5s
                   - name: flannel-env.path
                     command: start
                     content: |
                       # Ensures flannel env vars are set to use with Docker
                       [Path]
                       PathExists=/run/flannel/subnet.env
                       Unit=docker.service
                   - name: docker.service
                     command: start
                     content: |
                       # Starts new docker server that uses flannel
                       [Unit]
                       After=flannel-env.path network-online.target flannel.service
                       Requires=flannel-env.path network-online.target flannel.service
                       Description=Docker Application Container Engine

                       [Service]
                       EnvironmentFile=/run/flannel/subnet.env
                       ExecStartPre=/bin/mount --make-rprivate /
                       ExecStartPre=/usr/bin/systemctl kill docker.service
                       ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}

                       [Install]
                       WantedBy=multi-user.target
             params:
               "%discovery%": { get_attr: [discovery, first_address] }
               "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
               "%flannel_url%": { get_param: flannel_url }
               "%kubernetes_net_interface%": { get_param: kubernetes_net_interface }

outputs:
 discovery_ip:
   value: { get_attr: [ discovery, accessIPv4 ] }
   description: The IP of the Discovery
 overlord_ip:
   value: { get_attr: [ overlord, accessIPv4 ] }
   description: The IP of the Overlord
 master_ip:
   value: { get_attr: [ kubernetes_master, accessIPv4 ] }
   description: The IP of the Kubernetes Master
 minion_ips:
   value: { get_attr: [ kubernetes_minions, accessIPv4 ] }
   description: The IP of the Kubernetes Minions