From 6cebd461917c8eedf13537d1f9cbe92dd97c18d8 Mon Sep 17 00:00:00 2001 From: micafer Date: Fri, 15 Sep 2017 13:36:40 +0200 Subject: [PATCH 1/5] Fix #408 --- IM/ConfManager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/IM/ConfManager.py b/IM/ConfManager.py index a07825b74..813d16fd7 100644 --- a/IM/ConfManager.py +++ b/IM/ConfManager.py @@ -123,6 +123,8 @@ def check_vm_ips(self, timeout=Config.WAIT_RUNNING_VM_TIMEOUT): for vm in self.inf.get_vm_list(): if vm.hasPublicNet(): ip = vm.getPublicIP() + if not ip: + ip = vm.getPrivateIP() else: ip = vm.getPrivateIP() if not ip: @@ -140,6 +142,8 @@ def check_vm_ips(self, timeout=Config.WAIT_RUNNING_VM_TIMEOUT): if vm.hasPublicNet(): ip = vm.getPublicIP() + if not ip: + ip = vm.getPrivateIP() else: ip = vm.getPrivateIP() if not ip: From 34edeb96b44427d492af1386461b213867dd9190 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Sep 2017 08:35:50 +0200 Subject: [PATCH 2/5] Add log messages --- IM/connectors/OCCI.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/IM/connectors/OCCI.py b/IM/connectors/OCCI.py index 09582c569..45bed8f0a 100644 --- a/IM/connectors/OCCI.py +++ b/IM/connectors/OCCI.py @@ -1201,17 +1201,19 @@ def appdb_call(path): else: return None - @staticmethod - def get_site_id(site_name): + def get_site_id(self, site_name): data = OCCICloudConnector.appdb_call('/rest/1.0/va_providers') if data: for site in data['appdb:appdb']['virtualization:provider']: if site_name == site['provider:name'] and site['@in_production'] == "true": return site['@id'] + else: + self.log_warn("No data returned from EGI AppDB.") + + self.log_warn("No site ID returned from EGI AppDB for site: %s." % site_name) return None - @staticmethod - def get_image_id_and_site_url(site_id, image_name, vo_name=None): + def get_image_id_and_site_url(self, site_id, image_name, vo_name=None): data = OCCICloudConnector.appdb_call('/rest/1.0/va_providers/%s' % site_id) if data: site_url = data['appdb:appdb']['virtualization:provider']["provider:endpoint_url"] @@ -1224,6 +1226,10 @@ def get_image_id_and_site_url(site_id, image_name, vo_name=None): return parts[1], site_url else: return image_basename, site_url + else: + self.log_warn("No data returned from EGI AppDB.") + + self.log_warn("No image ID returned from EGI AppDB for image: %s/%s." % (site_id, image_name)) return '', '' From 0ee49c288e070be88d49fb474282c39accb4647b Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Sep 2017 10:56:07 +0200 Subject: [PATCH 3/5] Fix # 410 --- IM/ansible_utils/ansible_launcher.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/IM/ansible_utils/ansible_launcher.py b/IM/ansible_utils/ansible_launcher.py index 24784490f..a6b0b2499 100755 --- a/IM/ansible_utils/ansible_launcher.py +++ b/IM/ansible_utils/ansible_launcher.py @@ -38,7 +38,12 @@ else: from ansible.cli import CLI from ansible.parsing.dataloader import DataLoader - from ansible.vars import VariableManager + try: + # for Ansible version 2.4.0 or higher + from ansible.vars.manager import VariableManager + except: + # for Ansible version 2.3.2 or lower + from ansible.vars import VariableManager import ansible.inventory from .ansible_executor_v2 import IMPlaybookExecutor From 5c3f44b9f7996776a100b6499aebbc0bd0b7e801 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Sep 2017 13:40:41 +0200 Subject: [PATCH 4/5] Fix #410 --- IM/ansible_utils/ansible_executor_v2.py | 12 ++-- IM/ansible_utils/ansible_launcher.py | 74 ++++++++++++++++++------- 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/IM/ansible_utils/ansible_executor_v2.py b/IM/ansible_utils/ansible_executor_v2.py index 7c56bb921..75702d655 100644 --- a/IM/ansible_utils/ansible_executor_v2.py +++ b/IM/ansible_utils/ansible_executor_v2.py @@ -374,8 +374,6 @@ def run(self): self._inventory.set_playbook_basedir( os.path.dirname(playbook_path)) - i = 1 - # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', pb) @@ -393,8 +391,12 @@ def run(self): # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the # original object. - all_vars = self._variable_manager.get_vars( - loader=self._loader, play=play) + try: + # for Ansible version 2.3.2 or lower + all_vars = self._variable_manager.get_vars(loader=self._loader, play=play) + except TypeError: + # for Ansible version 2.4.0 or higher + all_vars = self._variable_manager.get_vars(play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) @@ -442,8 +444,6 @@ def run(self): if result not in (0, 3): break - i = i + 1 # per play - self._tqm.send_callback( 'v2_playbook_on_stats', self._tqm._stats) diff --git a/IM/ansible_utils/ansible_launcher.py b/IM/ansible_utils/ansible_launcher.py index a6b0b2499..fbee4a916 100755 --- a/IM/ansible_utils/ansible_launcher.py +++ b/IM/ansible_utils/ansible_launcher.py @@ -25,6 +25,7 @@ import subprocess import signal import logging +from distutils.version import LooseVersion from ansible import errors from ansible import __version__ as ansible_version @@ -41,9 +42,11 @@ try: # for Ansible version 2.4.0 or higher from ansible.vars.manager import VariableManager - except: + from ansible.inventory.manager import InventoryManager + except ImportError: # for Ansible version 2.3.2 or lower from ansible.vars import VariableManager + from ansible.inventory import Inventory import ansible.inventory from .ansible_executor_v2 import IMPlaybookExecutor @@ -136,6 +139,54 @@ def run(self): display("ERROR: %s" % e, output=self.output) self.result.put((0, (1, []), output)) + def get_play_prereqs(self, options): + if LooseVersion(ansible_version) >= LooseVersion("2.4.0"): + # for Ansible version 2.4.0 or higher + return self.get_play_prereqs_2_4(options) + else: + # for Ansible version 2.3.2 or lower + return self.get_play_prereqs_2(options) + + def get_play_prereqs_2(self, options): + loader = DataLoader() + + if self.vault_pass: + loader.set_vault_password(self.vault_pass) + + variable_manager = VariableManager() + variable_manager.extra_vars = self.extra_vars + + # Add this to avoid the Ansible bug: no host vars as host is not in inventory + # In version 2.0.1 it must be fixed + ansible.inventory.HOSTS_PATTERNS_CACHE = {} + + inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=options.inventory) + variable_manager.set_inventory(inventory) + + if self.host: + inventory.subset(self.host) + # let inventory know which playbooks are using so it can know the + # basedirs + inventory.set_playbook_basedir(os.path.dirname(self.playbook_file)) + + return loader, inventory, variable_manager + + def get_play_prereqs_2_4(self, options): + loader = DataLoader() + + if self.vault_pass: + loader.set_vault_password(self.vault_pass) + + # create the inventory, and filter it based on the subset specified (if any) + inventory = InventoryManager(loader=loader, sources=options.inventory) + + # create the variable manager, which will be shared throughout + # the code, ensuring a consistent view of global variables + variable_manager = VariableManager(loader=loader, inventory=inventory) + variable_manager.extra_vars = self.extra_vars + + return loader, inventory, variable_manager + def launch_playbook_v2(self): ''' run ansible-playbook operations v2.X''' # create parser for CLI options @@ -176,31 +227,12 @@ def launch_playbook_v2(self): raise errors.AnsibleError( "the playbook: %s does not appear to be a file" % self.playbook_file) - variable_manager = VariableManager() - variable_manager.extra_vars = self.extra_vars - if self.inventory_file: options.inventory = self.inventory_file options.forks = self.threads - loader = DataLoader() - # Add this to avoid the Ansible bug: no host vars as host is not in inventory - # In version 2.0.1 it must be fixed - ansible.inventory.HOSTS_PATTERNS_CACHE = {} - - if self.vault_pass: - loader.set_vault_password(self.vault_pass) - - inventory = ansible.inventory.Inventory( - loader=loader, variable_manager=variable_manager, host_list=options.inventory) - variable_manager.set_inventory(inventory) - - if self.host: - inventory.subset(self.host) - # let inventory know which playbooks are using so it can know the - # basedirs - inventory.set_playbook_basedir(os.path.dirname(self.playbook_file)) + loader, inventory, variable_manager = self.get_play_prereqs(options) num_retries = 0 return_code = 4 From 277351231b036e94929aa1391143cfac80310ef7 Mon Sep 17 00:00:00 2001 From: micafer Date: Tue, 19 Sep 2017 14:11:55 +0200 Subject: [PATCH 5/5] Fix #410 --- IM/ansible_utils/ansible_executor_v2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/IM/ansible_utils/ansible_executor_v2.py b/IM/ansible_utils/ansible_executor_v2.py index 75702d655..b2c5117b3 100644 --- a/IM/ansible_utils/ansible_executor_v2.py +++ b/IM/ansible_utils/ansible_executor_v2.py @@ -371,8 +371,6 @@ def run(self): for playbook_path in self._playbooks: pb = Playbook.load( playbook_path, variable_manager=self._variable_manager, loader=self._loader) - self._inventory.set_playbook_basedir( - os.path.dirname(playbook_path)) # make sure the tqm has callbacks loaded self._tqm.load_callbacks()