Move to template-based configuration of devops environments

To create a devops virtual environment (networks, volumes, nodes),
now can be used YAML template desctibed in [1].

How it can be used:

* New approach, environment will be created using the data from
  full_config object:

  environment.create_environment(full_config)

* Old-style approach (deprecated), in this case at first will be
  created a 'full_config' object using shell environment variables,
  and then called environment.create_environment(full_config):

  environment.describe_environment(boot_from='cdrom'):

* Override old-style approach, in this case the YAML config will
  be read from the file located in DEVOPS_SETTINGS_TEMPLATE shell
  environment variable to the 'full_config' object and passed to
  create_environment(full_config); creating config in old-style
  will be skipped:

  export DEVOPS_SETTINGS_TEMPLATE=/tmp/some_template_file.yaml

- add create_envoronment() method that creates environment from
  provided config.
- environment.describe() method is temporary disabled until new
  database schema be implemented, where all necessary details could
  be passed from define() to desctibe() via database objects.
- add template generator for keep legacy behaviour of the
  method describe_environment()
- Environment.define() is deprecated and left for back compatibility
  with older fuel-qa branches
- Some objects in template require the extended database schema,
  so in this patch there are some workarounds to use templates with
  original schema (particular data from address_pools and
  l2_network_devices are stored in the Network model)

[1] - https://review.openstack.org/#/c/240170/

Change-Id: Iad081bee94c4d3b404eda57b7c1c31011b837ba7
Implements: blueprint template-based-virtual-devops-environments

devops/helpers/templates.py

@@ -0,0 +1,343 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import yaml
+
+from devops.helpers.helpers import _get_file_size
+
+
+def yaml_template_load(config_file):
+    def yaml_include(loader, node):
+        file_name = os.path.join(os.path.dirname(loader.name), node.value)
+        if not os.path.isfile(file_name):
+            raise ValueError(
+                "Cannot load the environment template {0} : include file {1} "
+                "doesn't exist.".format(config_file, file_name))
+        with file(file_name) as inputfile:
+            return yaml.load(inputfile)
+
+    def yaml_get_env_variable(loader, node):
+        if not node.value.strip():
+            raise ValueError("Environment variable is required after {tag} in "
+                             "{filename}".format(tag=node.tag,
+                                                 filename=loader.name))
+        node_value = node.value.split(',', 1)
+        # Get the name of environment variable
+        env_variable = node_value[0].strip()
+
+        # Get the default value for environment variable if it exists in config
+        if len(node_value) > 1:
+            default_val = node_value[1].strip()
+        else:
+            default_val = None
+
+        value = os.environ.get(env_variable, default_val)
+        if value is None:
+            raise ValueError("Environment variable {var} is not set from shell"
+                             " environment! No default value provided in file "
+                             "{filename}".format(var=env_variable,
+                                                 filename=loader.name))
+
+        return yaml.load(value)
+
+    if not os.path.isfile(config_file):
+        raise ValueError(
+            "Cannot load the environment template {0} : file "
+            "doesn't exist.".format(config_file))
+
+    yaml.add_constructor("!include", yaml_include)
+    yaml.add_constructor("!os_env", yaml_get_env_variable)
+
+    return yaml.load(open(config_file))
+
+
+def get_devops_config(filename):
+    """Read the YAML file and create a 'full_config' object.
+
+    :param filename: path to a file that contains a YAML template
+        :rtype: dict
+    """
+    import devops
+    config_file = os.path.join(os.path.dirname(devops.__file__),
+                               'templates', filename)
+    return yaml_template_load(config_file)
+
+
+def create_admin_config(admin_vcpu, admin_memory, admin_sysvolume_capacity,
+                        admin_iso_path, boot_from, interfaceorder,
+                        networks_bonding=None,
+                        networks_bondinginterfaces=None):
+
+    if networks_bonding:
+        # Translate a dict of lists {net_name: [eth0, eth1],} into a new dict:
+        # {eth0: net_name, eth1: net_name,}
+        ifaces = {
+            label: iname
+            for iname in networks_bondinginterfaces
+            for label in networks_bondinginterfaces[iname]
+        }
+        admin_interfaces = [
+            {
+                'label': label,
+                'l2_network_device': ifaces[label]
+            } for label in sorted(ifaces.keys())
+        ]
+    else:
+        admin_interfaces = [
+            {
+                'label': 'eth' + str(n),
+                'l2_network_device': iname
+            } for n, iname in enumerate(interfaceorder)
+        ]
+
+    if boot_from == 'usb':
+        boot_device_order = ['hd']
+        iso_device = 'disk'
+        iso_bus = 'usb'
+    else:  # boot_from == 'cdrom':
+        boot_device_order = ['hd', 'cdrom']
+        iso_device = 'cdrom'
+        iso_bus = 'ide'
+
+    admin_config = {
+        'name': 'admin',  # Custom name of VM for Fuel admin node
+        'role': 'fuel_master',  # Fixed role for (Fuel admin) node properties
+        'params': {
+            'vcpu': admin_vcpu,
+            'memory': admin_memory,
+            'boot': boot_device_order,
+            'volumes': [
+                {
+                    'name': 'system',
+                    'capacity': admin_sysvolume_capacity,
+                    'format': 'qcow2',
+                },
+                {
+                    'name': 'iso',
+                    'source_image': admin_iso_path,
+                    'capacity': _get_file_size(admin_iso_path),
+                    'format': 'raw',
+                    'device': iso_device,
+                    'bus': iso_bus,
+                },
+            ],
+            'interfaces': admin_interfaces,
+        },
+    }
+    return admin_config
+
+
+def create_slave_config(slave_name, slave_role, slave_vcpu, slave_memory,
+                        slave_volume_capacity, interfaceorder,
+                        use_all_disks=None,
+                        networks_multiplenetworks=None,
+                        networks_nodegroups=None,
+                        networks_bonding=None,
+                        networks_bondinginterfaces=None):
+
+    if networks_multiplenetworks:
+        nodegroups_idx = 1 - int(slave_name[-2:]) % 2
+        slave_interfaces = [
+            {
+                'label': 'eth' + str(n),
+                'l2_network_device': iname
+            } for n, iname in enumerate(
+                networks_nodegroups[nodegroups_idx]['pools'])
+        ]
+    elif networks_bonding:
+        # Translate a dict of lists {net_name: [eth0, eth1],} into a new dict:
+        # {eth0: net_name, eth1: net_name,}
+        ifaces = {
+            label: iname
+            for iname in networks_bondinginterfaces
+            for label in networks_bondinginterfaces[iname]
+        }
+        slave_interfaces = [
+            {
+                'label': label,
+                'l2_network_device': ifaces[label]
+            } for label in sorted(ifaces.keys())
+        ]
+    else:
+        slave_interfaces = [
+            {
+                'label': 'eth' + str(n),
+                'l2_network_device': iname
+            } for n, iname in enumerate(interfaceorder)
+        ]
+
+    slave_config = {
+        'name': slave_name,
+        'role': slave_role,
+        'params': {
+            'vcpu': slave_vcpu,
+            'memory': slave_memory,
+            'boot': ['network', 'hd'],
+            'volumes': [
+                {
+                    'name': 'system',
+                    'capacity': slave_volume_capacity,
+                },
+            ],
+            'interfaces': slave_interfaces,
+        },
+    }
+    if use_all_disks:
+        slave_config['params']['volumes'].extend([
+            {
+                'name': 'cinder',
+                'capacity': slave_volume_capacity
+            },
+            {
+                'name': 'swift',
+                'capacity': slave_volume_capacity
+            }
+        ])
+    return slave_config
+
+
+def create_address_pools(interfaceorder, networks_pools):
+    address_pools = {
+        iname: {
+            'net': ':'.join(networks_pools[iname])
+        } for iname in interfaceorder
+    }
+    return address_pools
+
+
+def create_l2_network_devices(interfaceorder,
+                              networks_dhcp,
+                              networks_forwarding):
+    l2_network_devices = {
+        iname: {
+            'address_pool': iname,
+            'dhcp': networks_dhcp[iname],
+            'forward': {
+                'mode': networks_forwarding[iname],
+            }
+        } for iname in interfaceorder
+    }
+    return l2_network_devices
+
+
+def create_devops_config(boot_from,
+                         env_name,
+                         admin_vcpu,
+                         admin_memory,
+                         admin_sysvolume_capacity,
+                         admin_iso_path,
+                         nodes_count,
+                         slave_vcpu,
+                         slave_memory,
+                         slave_volume_capacity,
+                         use_all_disks,
+                         ironic_nodes_count,
+                         networks_bonding,
+                         networks_bondinginterfaces,
+                         networks_multiplenetworks,
+                         networks_nodegroups,
+                         networks_interfaceorder,
+                         networks_pools,
+                         networks_forwarding,
+                         networks_dhcp):
+    """Creates devops config object
+
+    This method is used for backward compatibility with old-style
+    environment creation, where most of environment parameters were
+    passed with shell environment variables.
+
+    See models/environment.py and settings.py for details about
+    input parameters structure.
+    """
+
+    # If bonding enabled, then a different interfaces order is provided.
+    if networks_bonding:
+        interfaceorder = networks_bondinginterfaces.keys()
+    else:
+        interfaceorder = networks_interfaceorder
+
+    # Create address pools object
+    address_pools = create_address_pools(interfaceorder, networks_pools)
+
+    # Create network devices object
+    l2_network_devices = create_l2_network_devices(interfaceorder,
+                                                   networks_dhcp,
+                                                   networks_forwarding)
+    # Create admin and slave nodes
+    config_nodes = []
+
+    admin_config = create_admin_config(
+        admin_vcpu=admin_vcpu,
+        admin_memory=admin_memory,
+        admin_sysvolume_capacity=admin_sysvolume_capacity,
+        admin_iso_path=admin_iso_path,
+        boot_from=boot_from,
+        interfaceorder=interfaceorder,
+        networks_bonding=networks_bonding,
+        networks_bondinginterfaces=networks_bondinginterfaces)
+
+    config_nodes.append(admin_config)
+
+    for slave_n in range(1, nodes_count):
+        slave_name = 'slave-{0:0>2}'.format(slave_n)
+
+        slave_config = create_slave_config(
+            slave_name=slave_name,
+            slave_role='fuel_slave',
+            slave_vcpu=slave_vcpu,
+            slave_memory=slave_memory,
+            slave_volume_capacity=slave_volume_capacity,
+            interfaceorder=interfaceorder,
+            use_all_disks=use_all_disks,
+            networks_multiplenetworks=networks_multiplenetworks,
+            networks_nodegroups=networks_nodegroups,
+            networks_bonding=networks_bonding,
+            networks_bondinginterfaces=networks_bondinginterfaces)
+
+        config_nodes.append(slave_config)
+
+    for ironic_n in range(1, ironic_nodes_count + 1):
+        ironic_name = 'ironic-slave-{0:0>2}'.format(ironic_n)
+
+        ironic_config = create_slave_config(
+            slave_name=ironic_name,
+            slave_role='fuel_slave',
+            slave_vcpu=slave_vcpu,
+            slave_memory=slave_memory,
+            slave_volume_capacity=slave_volume_capacity,
+            interfaceorder=['ironic'],
+            use_all_disks=None)
+
+        config_nodes.append(ironic_config)
+
+    config = {
+        'template':
+            {
+                'devops_settings':
+                    {
+                        'env_name': env_name,
+                        'address_pools': address_pools,
+                        'groups': [
+                            {
+                                'name': 'rack-01',
+                                'l2_network_devices': l2_network_devices,
+                                'nodes': config_nodes,
+                            },
+                        ]
+                    }
+            }
+    }
+
+    return config