corekube-openstack.yaml

 heat_template_version: 2013-05-23

 description: >
   Deploy Kubernetes on a CoreOS cluster that operates on an
   OpenStack Powered Cloud (Openstack Kilo | Rackspace Private Cloud)
   Maintainer - Mike Metral <metral@gmail.com> | Github: @metral

 parameters:
   kubernetes_minion_count:
     description: Number of CoreOS machines to deploy as Kubernetes Minion
     type: number
     default: 3
     constraints:
     - range:
         min: 1
         max: 12
       description: Must be between 2 and 12 servers.
   keyname:
     type: string
     description: Name of keypair to be used for compute instance
   flavor:
     type: string
     default: m1.medium
     constraints:
     - allowed_values:
       - m1.medium
       - m1.large
       - m1.xlarge
       description: |
         Must be a valid flavor
   coreos_image:
     type: string
     description: Rackspace Cloud Servers CoreOS Stable (808.0.0) UUID
     default: "CoreOS"
   git_command:
     type: string
     description: Git repo checkout command
     default: "/usr/bin/git clone https://github.com/metral/overlord ; /usr/bin/git -C overlord checkout -qf 199028526132ff9491d26c00b2fd9b9b9d7239a8"
   flannel_url:
     type: string
     description: Flannel (0.5.5) Binary URL
     default: "http://0afbabe78c1b1691fad4-21af6827a69c3ad910ccdb6595a59d19.r49.cf1.rackcdn.com/flanneld"
   private_discovery_etcd_version:
     type: string
     description: Version of etcd used for Private Discovery node
     default: "2.2.2"
   discovery_net_interface:
     type: string
     description: Host network interface for which the Discovery node will operate over
     default: "eth0"
   kubernetes_net_interface:
     type: string
     description: Host network interface for which Kubernetes' overlay will operate over
     default: "eth0"
   timeout:
     description: Wait condition timeout seconds
     type: number
     default: 600
   floating-network-id:
     type: string
     label: Floating Network ID
     description: UUID of the external network. The private network created by this stack will route to this network. Any floating IP addresses needed by this stack will also route to this network. 

 resources:

   coreos_cluster_uuid:
     type: OS::Heat::RandomString

   stack_security:
     type: OS::Neutron::SecurityGroup
     properties:
       name: "corekube"
       rules:
         - remote_ip_prefix: 0.0.0.0/0
           protocol: tcp
           port_range_min: 0
           port_range_max: 65535
         - remote_ip_prefix: 0.0.0.0/0
           protocol: udp
           port_range_min: 0
           port_range_max: 65535
         - remote_ip_prefix: 0.0.0.0/0
           protocol: icmp

   corekube_network:
     type: OS::Neutron::ProviderNet
     properties:
       name: "corekube"
       network_type: "vlan"
       physical_network: "vlan"

   corekube_subnet:
     type: OS::Neutron::Subnet
     properties:
       name: "corekube_subnet"
       network_id:
         get_resource: corekube_network
       cidr: "10.10.10.0/24"
       gateway_ip: "10.10.10.1"
       allocation_pools: [{"start": "10.10.10.2", "end": "10.10.10.254"}]
       dns_nameservers:
         - 8.8.8.8
         - 8.8.4.4

   corekube_router:
     type: OS::Neutron::Router
     properties:
       name: "corekube_router"
       external_gateway_info:
         enable_snat: true
         network:
           get_param: floating-network-id

   corekube_router_interface:
     type: OS::Neutron::RouterInterface
     properties:
       router_id:
         get_resource: corekube_router
       subnet_id:
         get_resource: corekube_subnet

   discovery_wait_handle:
     type: OS::Heat::WaitConditionHandle

   discovery_wait_condition:
     type: OS::Heat::WaitCondition
     properties:
       handle: { get_resource: discovery_wait_handle }
       timeout: { get_param: timeout }

   discovery_floatingip:
     type: OS::Neutron::FloatingIP
     properties:
       floating_network_id:
         get_param: floating-network-id

   discovery_port:
     type: OS::Neutron::Port
     properties:
       security_groups:
         - get_resource: stack_security
       network_id:
         get_resource: corekube_network

   discovery_floatingip_association:
     type: OS::Neutron::FloatingIPAssociation
     properties:
       floatingip_id:
         get_resource: discovery_floatingip
       port_id:
         get_resource: discovery_port

   discovery:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image }
       flavor: { get_param: flavor }
       name: "discovery"
       networks:
         - port:
             get_resource: discovery_port
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             write_files:
               - path: /run/get_discovery_interface_ip.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   # Get's the IP of the interface that discovery will be
                   # accessible over
                   DISCOVERY_IF=%discovery_net_interface%
                   /usr/bin/ip -4 addr show $DISCOVERY_IF | /usr/bin/awk '/inet/ {print $2}' | /usr/bin/cut -d/ -f1 > /run/IP
                   /usr/bin/sed -i 's/^/IP=/' /run/IP
               - path: /run/wcnotify.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   %wc_notify% --data-binary '{"status": "SUCCESS"}'
             coreos:
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: private-discovery-setup.service
                   command: start
                   content: |
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     Type=forking
                     ExecStart=/usr/bin/bash /run/get_discovery_interface_ip.sh
                 - name: private-discovery.service
                   command: start
                   content: |
                     [Unit]
                     After=network-online.target private-discovery-setup.service
                     Requires=network-online.target private-discovery-setup.service

                     [Service]
                     EnvironmentFile=/run/IP
                     Restart=on-failure
                     RestartSec=5s
                     ExecStartPre=/usr/bin/docker pull quay.io/coreos/etcd:v%private_discovery_etcd_version%
                     ExecStart=/usr/bin/docker run -d --name discovery \
                       -p 2379:2379 -p 2380:2380 \
                       -v /usr/share/ca-certificates/:/etc/ssl/certs \
                       --net host quay.io/coreos/etcd:v%private_discovery_etcd_version% -name discovery \
                       -initial-advertise-peer-urls http://${IP}:2380,http://${IP}:7001 \
                       -listen-peer-urls http://${IP}:2380,http://${IP}:7001 \
                       -initial-cluster discovery=http://${IP}:2380,discovery=http://${IP}:7001 \
                       -advertise-client-urls http://${IP}:2379,http://${IP}:4001 \
                       -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001
                 - name: private-discovery-done.service
                   command: start
                   content: |
                     # Notifiy wait condition
                     [Unit]
                     After=network-online.target private-discovery.service
                     Requires=network-online.target private-discovery.service

                     [Service]
                     ExecStart=/run/wcnotify.sh
           params:
             "%wc_notify%": { get_attr: [discovery_wait_handle, curl_cli] }
             "%discovery_net_interface%": { get_param: discovery_net_interface }
             "%private_discovery_etcd_version%": { get_param: private_discovery_etcd_version}

   overlord_wait_handle:
     type: OS::Heat::WaitConditionHandle

   overlord_wait_condition:
     type: OS::Heat::WaitCondition
     properties:
       handle: {get_resource: overlord_wait_handle}
       timeout: {get_param: timeout}

   overlord_floatingip:
     type: OS::Neutron::FloatingIP
     properties:
       floating_network_id:
         get_param: floating-network-id

   overlord_port:
     type: OS::Neutron::Port
     properties:
       security_groups:
         - get_resource: stack_security
       network_id:
         get_resource: corekube_network

   overlord_floatingip_association:
     type: OS::Neutron::FloatingIPAssociation
     properties:
       floatingip_id:
         get_resource: overlord_floatingip
       port_id:
         get_resource: overlord_port

   overlord:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image }
       flavor: { get_param: flavor }
       name: "overlord"
       networks:
         - port:
             get_resource: overlord_port
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             write_files:
               - path: /run/wcnotify.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   %wc_notify% --data-binary '{"status": "SUCCESS"}'
               - path: /run/setup_etcd_cloudinit_conf.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash

                   ENV="/etc/environment"

                   # Test for RW access to $1
                   touch $ENV
                   if [ $? -ne 0 ]; then
                       echo exiting, unable to modify: $ENV
                       exit 1
                   fi

                   # Setup environment target
                   sed -i -e '/^COREOS_PUBLIC_IPV4=/d' \
                       -e '/^COREOS_PRIVATE_IPV4=/d' \
                       "${ENV}"

                   # We spin loop until the the IP addresses are set
                   function get_ip () {
                       IF=$1
                       IP=
                       while [ 1 ]; do
                           IP=$(ifconfig $IF | awk '/inet / {print $2}')
                           if [ "$IP" != "" ]; then
                               break
                           fi
                           sleep .1
                       done
                       echo $IP
                   }

                   # Echo results of IP queries to environment file as soon as network interfaces
                   # get assigned IPs
                   echo COREOS_PUBLIC_IPV4=$(get_ip eth0) >> $ENV # Also assigned to same IP
                   echo COREOS_PRIVATE_IPV4=$(get_ip eth0) >> $ENV #eno1 should be changed to your device name
                   A=`cat /etc/environment | grep COREOS_PRIVATE_IPV4 | cut -f2 -d "="`
                   sed -i "s#=:#=${A}:#g" /run/systemd/system/etcd.service.d/20-cloudinit.conf
                   systemctl daemon-reload
             coreos:
               etcd:
                 name: overlord
                 discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
                 addr: $private_ipv4:4001
                 peer-addr: $private_ipv4:7001
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: setup-etcd-cloudinit-conf.service
                   command: start
                   runtime: true
                   content: |
                     [Unit]
                     Description=Setup environment with private (and public) IP addresses

                     [Service]
                     Type=oneshot
                     RemainAfterExit=yes
                     ExecStart=/run/setup_etcd_cloudinit_conf.sh
                 - name: etcd.service 
                   command: start
                 - name: fleet.socket
                   command: start
                   content: |
                     [Socket]
                     # Talk to the API over a Unix domain socket (default)
                     ListenStream=/var/run/fleet.sock

                     # Talk to the API over an exposed port
                     ListenStream=10001
                     Service=fleet-local.service

                     [Install]
                     WantedBy=sockets.target
                 - name: fleet-local.service
                   command: start
                   content: |
                     # fleet-local is kicked off by fleet.socket after API port
                     # is opened
                     [Unit]
                     Description=fleet-local
                     Wants=etcd.service
                     Requires=etcd.service

                     [Service]
                     Environment=FLEET_PUBLIC_IP=$private_ipv4
                     Environment=FLEET_METADATA=kubernetes_role=overlord
                     ExecStart=/usr/bin/fleet
                     Restart=always
                     RestartSec=10s
                 - name: overlord.service
                   command: start
                   content: |
                     # Overlord / logic layer service to deploy kubernetes to
                     # the cluster
                     [Unit]
                     After=network-online.target etcd.service
                     Requires=network-online.target etcd.service

                     [Service]
                     WorkingDirectory=/root
                     Environment="DIR=overlord"
                     ExecStartPre=/usr/bin/rm -rf $DIR
                     ExecStartPre=%git_command%
                     ExecStart=/usr/bin/bash ${DIR}/build_run.sh
                 - name: overlord-done.service
                   command: start
                   content: |
                     # Notifiy wait condition
                     [Unit]
                     After=network-online.target overlord.service
                     Requires=network-online.target overlord.service

                     [Service]
                     ExecStart=/run/wcnotify.sh
           params:
             "%discovery%": { get_attr: [discovery, networks, corekube, 0] }
             "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
             "%git_command%": { get_param: git_command }
             "%wc_notify%": { get_attr: [overlord_wait_handle, curl_cli] }

   kubernetes_master_floatingip:
     type: OS::Neutron::FloatingIP
     properties:
       floating_network_id:
         get_param: floating-network-id

   kubernetes_master_port:
     type: OS::Neutron::Port
     properties:
       security_groups:
         - get_resource: stack_security
       network_id:
         get_resource: corekube_network

   kubernetes_master_floatingip_association:
     type: OS::Neutron::FloatingIPAssociation
     properties:
       floatingip_id:
         get_resource: kubernetes_master_floatingip
       port_id:
         get_resource: kubernetes_master_port

   kubernetes_master:
     type: OS::Nova::Server
     properties:
       key_name: { get_param: keyname }
       image: { get_param: coreos_image }
       flavor: { get_param: flavor }
       name: "kubernetes_master"
       networks:
         - port:
             get_resource: kubernetes_master_port
       user_data_format: RAW
       config_drive: "true"
       user_data:
         str_replace:
           template: |
             #cloud-config
             write_files:
               - path: /run/setup_etcd_cloudinit_conf.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash

                   ENV="/etc/environment"

                   # Test for RW access to $1
                   touch $ENV
                   if [ $? -ne 0 ]; then
                       echo exiting, unable to modify: $ENV
                       exit 1
                   fi

                   # Setup environment target
                   sed -i -e '/^COREOS_PUBLIC_IPV4=/d' \
                       -e '/^COREOS_PRIVATE_IPV4=/d' \
                       "${ENV}"

                   # We spin loop until the the IP addresses are set
                   function get_ip () {
                       IF=$1
                       IP=
                       while [ 1 ]; do
                           IP=$(ifconfig $IF | awk '/inet / {print $2}')
                           if [ "$IP" != "" ]; then
                               break
                           fi
                           sleep .1
                       done
                       echo $IP
                   }

                   # Echo results of IP queries to environment file as soon as network interfaces
                   # get assigned IPs
                   echo COREOS_PUBLIC_IPV4=$(get_ip eth0) >> $ENV # Also assigned to same IP
                   echo COREOS_PRIVATE_IPV4=$(get_ip eth0) >> $ENV #eno1 should be changed to your device name
                   A=`cat /etc/environment | grep COREOS_PRIVATE_IPV4 | cut -f2 -d "="`
                   sed -i "s#=:#=${A}:#g" /run/systemd/system/etcd.service.d/20-cloudinit.conf
                   systemctl daemon-reload
               - path: /run/get_discovery_ip_port.sh
                 permissions: '0755'
                 content: |
                   #!/bin/bash
                   # Sets up environment file with the discovery node's IP &
                   # port so # that in Overlode's template 
                   # master-apiserver@.service it can be passed 
                   # in as an argument 
                   /usr/bin/cat /run/systemd/system/etcd.service.d/20-cloudinit.conf | /usr/bin/grep -i discovery | /usr/bin/cut -f3 -d"=" | /usr/bin/awk -F '/v' '{print $1}' > /run/discovery_ip_port
                   /usr/bin/sed -i 's/^/DISCOVERY_IP_PORT=/' /run/discovery_ip_port
             coreos:
               etcd:
                 name: kubernetes_master
                 discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
                 addr: $private_ipv4:4001
                 peer-addr: $private_ipv4:7001
               fleet:
                 public-ip: $private_ipv4
                 metadata: kubernetes_role=master
               update:
                 group: stable
                 reboot-strategy: off
               units:
                 - name: setup-etcd-cloudinit-conf.service
                   command: start
                   runtime: true
                   content: |
                     [Unit]
                     Description=Setup environment with private (and public) IP addresses

                     [Service]
                     Type=oneshot
                     RemainAfterExit=yes
                     ExecStart=/run/setup_etcd_cloudinit_conf.sh
                 - name: etcd.service
                   command: start
                 - name: fleet.service
                   command: start
                 - name: get_discovery_ip_port.service
                   command: start
                   runtime: true
                   content: |
                     # Runs get_discovery_ip_port.sh to have discovery IP &
                     # port ready for consumption by overlord when creating
                     # the Kubernetes' master-api@.service template
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     ExecStart=/usr/bin/bash /run/get_discovery_ip_port.sh
                 - name: flannel-install.service
                   command: start
                   content: |
                     # Installs flannel
                     [Unit]
                     After=network-online.target
                     Requires=network-online.target

                     [Service]
                     Type=oneshot
                     RemainAfterExit=yes
                     ExecStart=/usr/bin/wget -N -P /opt/bin %flannel_url%
                     ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                 - name: flannel.service
                   command: start
                   content: |
                     # Configures & starts flannel
                     [Unit]
                     After=network-online.target etcd.service flannel-install.service
                     Requires=network-online.target etcd.service flannel-install.service

                     [Service]
                     ExecStartPre=/usr/bin/etcdctl --endpoint=%discovery%:4001 mk /coreos.com/network/config '{"Network":"10.244.0.0/15", "Backend": {"Type": "vxlan"}}'
                     ExecStart=/opt/bin/flanneld -etcd-endpoints=http://%discovery%:4001 -iface=%kubernetes_net_interface%
                     Restart=always
                     RestartSec=5s
                 - name: flannel-env.path
                   command: start
                   content: |
                     # Ensures flannel env vars are set to use with Docker
                     [Path]
                     PathExists=/run/flannel/subnet.env
                     Unit=docker.service
                 - name: docker.service
                   command: start
                   content: |
                     # Starts new docker server that uses flannel 
                     [Unit]
                     After=flannel-env.path network-online.target flannel.service
                     Requires=flannel-env.path network-online.target flannel.service
                     Description=Docker Application Container Engine

                     [Service]
                     EnvironmentFile=/run/flannel/subnet.env
                     ExecStartPre=/bin/mount --make-rprivate /
                     ExecStartPre=/usr/bin/systemctl kill docker.service
                     ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}

                     [Install]
                     WantedBy=multi-user.target
                 - name: generate-k8s-certs.service
                   command: start
                   content: |
                     [Unit]
                     Description=Generate Kubernetes API Server certificates
                     ConditionPathExists=!/srv/kubernetes/.certs.lock
                     Requires=network-online.target
                     After=network-online.target

                     [Service]
                     Type=oneshot
                     EnvironmentFile=/etc/environment
                     ExecStartPre=-/usr/sbin/groupadd -r kube-cert
                     ExecStartPre=/usr/bin/wget -q -N -P /opt/bin https://gist.githubusercontent.com/metral/0731466d7b35d4020dfd/raw/2fb0c5858e5eaefdf6b45b9492f1d6c75a7815b0/make-ca-cert.sh
                     ExecStartPre=/usr/bin/chmod u=rwx,go= /opt/bin/make-ca-cert.sh
                     ExecStart=/opt/bin/make-ca-cert.sh ${COREOS_PRIVATE_IPV4} IP:10.1.0.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.freeletics.local
                     ExecStart=/opt/bin/make-ca-cert.sh ${COREOS_PRIVATE_IPV4} IP:10.1.0.1,IP:127.0.0.1,DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.freeletics.local,DNS:localhost
                     RemainAfterExit=true
           params:
             "%discovery%": { get_attr: [discovery, networks, corekube, 0] }
             "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
             "%flannel_url%": { get_param: flannel_url }
             "%kubernetes_net_interface%": { get_param: kubernetes_net_interface }

   kubernetes_minions:
     type: "OS::Heat::ResourceGroup"
     properties:
       count: { get_param: kubernetes_minion_count }
       resource_def:
         type: OS::Nova::Server
         properties:
           key_name: { get_param: keyname }
           image: { get_param: coreos_image }
           flavor: { get_param: flavor }
           name: kubernetes_minion_%index%
           networks:
            - network: { get_resource: corekube_network }
           security_groups:
             - get_resource: stack_security
           user_data_format: RAW
           config_drive: "true"
           user_data:
             str_replace:
               template: |
                 #cloud-config
                 write_files:
                   - path: /run/setup_etcd_cloudinit_conf.sh
                     permissions: '0755'
                     content: |
                       #!/bin/bash

                       ENV="/etc/environment"

                       # Test for RW access to $1
                       touch $ENV
                       if [ $? -ne 0 ]; then
                           echo exiting, unable to modify: $ENV
                           exit 1
                       fi

                       # Setup environment target
                       sed -i -e '/^COREOS_PUBLIC_IPV4=/d' \
                           -e '/^COREOS_PRIVATE_IPV4=/d' \
                           "${ENV}"

                       # We spin loop until the the IP addresses are set
                       function get_ip () {
                           IF=$1
                           IP=
                           while [ 1 ]; do
                               IP=$(ifconfig $IF | awk '/inet / {print $2}')
                               if [ "$IP" != "" ]; then
                                   break
                               fi
                               sleep .1
                           done
                           echo $IP
                       }

                       # Echo results of IP queries to environment file as soon as network interfaces
                       # get assigned IPs
                       echo COREOS_PUBLIC_IPV4=$(get_ip eth0) >> $ENV # Also assigned to same IP
                       echo COREOS_PRIVATE_IPV4=$(get_ip eth0) >> $ENV #eno1 should be changed to your device name
                       A=`cat /etc/environment | grep COREOS_PRIVATE_IPV4 | cut -f2 -d "="`
                       sed -i "s#=:#=${A}:#g" /run/systemd/system/etcd.service.d/20-cloudinit.conf
                       systemctl daemon-reload
                 coreos:
                   etcd:
                     name: kubernetes_minion_%index%
                     discovery: http://%discovery%:2379/v2/keys/discovery/%uuid%
                     addr: $private_ipv4:4001
                     peer-addr: $private_ipv4:7001
                   fleet:
                     public-ip: $private_ipv4
                     metadata: kubernetes_role=minion
                   update:
                     group: stable
                     reboot-strategy: off
                   units:
                     - name: setup-etcd-cloudinit-conf.service
                       command: start
                       runtime: true
                       content: |
                         [Unit]
                         Description=Setup environment with private (and public) IP addresses

                         [Service]
                         Type=oneshot
                         RemainAfterExit=yes
                         ExecStart=/run/setup_etcd_cloudinit_conf.sh
                     - name: etcd.service
                       command: start
                     - name: fleet.service
                       command: start
                     - name: flannel-install.service
                       command: start
                       content: |
                         # Installs flannel
                         [Unit]
                         After=network-online.target
                         Requires=network-online.target

                         [Service]
                         Type=oneshot
                         RemainAfterExit=yes
                         ExecStart=/usr/bin/wget -N -P /opt/bin %flannel_url%
                         ExecStart=/usr/bin/chmod +x /opt/bin/flanneld
                     - name: flannel.service
                       command: start
                       content: |
                         # Configures & starts flannel
                         [Unit]
                         After=etcd.service flannel-install.service
                         Requires=etcd.service flannel-install.service

                         [Service]
                         ExecStart=/opt/bin/flanneld -etcd-endpoints=http://%discovery%:4001 -iface=%kubernetes_net_interface%
                         Restart=always
                         RestartSec=5s
                     - name: flannel-env.path
                       command: start
                       content: |
                         # Ensures flannel env vars are set to use with Docker
                         [Path]
                         PathExists=/run/flannel/subnet.env
                         Unit=docker.service
                     - name: docker.service
                       command: start
                       content: |
                         # Starts new docker server that uses flannel
                         [Unit]
                         After=flannel-env.path network-online.target flannel.service
                         Requires=flannel-env.path network-online.target flannel.service
                         Description=Docker Application Container Engine

                         [Service]
                         EnvironmentFile=/run/flannel/subnet.env
                         ExecStartPre=/bin/mount --make-rprivate /
                         ExecStartPre=/usr/bin/systemctl kill docker.service
                         ExecStart=/usr/bin/docker -d --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}

                         [Install]
                         WantedBy=multi-user.target
               params:
                 "%discovery%": { get_attr: [discovery, networks, corekube, 0] }
                 "%uuid%": { get_attr: [coreos_cluster_uuid, value] }
                 "%flannel_url%": { get_param: flannel_url }
                 "%kubernetes_net_interface%": { get_param: kubernetes_net_interface }

 outputs:
   discovery_ip:
     value: { get_attr: [ discovery, first_address ] }
     description: The IP of the Discovery
   overlord_ip:
     value: { get_attr: [ overlord, first_address ] }
     description: The IP of the Overlord
   master_ip:
     value: { get_attr: [ kubernetes_master, first_address ] }
     description: The IP of the Kubernetes Master
   minion_ips:
     value: { get_attr: [ kubernetes_minions, first_address ] }
     description: The IP of the Kubernetes Minions