Merge pull request #1635 from grycap/virtualenv

Virtualenv

.github/workflows/main.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python 3.
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.12'
 
       - name: Install dependencies
         run: python -m pip install tox

IM/ConfManager.py

@@ -56,6 +56,7 @@
 from IM.recipe import Recipe
 from IM.config import Config
 from radl.radl import system, contextualize_item
+from IM.CtxtAgentBase import CtxtAgentBase
 
 
 class ConfManager(LoggerMixin, threading.Thread):
@@ -397,7 +398,8 @@ def launch_ctxt_agent(self, vm, tasks):
                     vault_password = vm.info.systems[0].getValue("vault.password")
                     if vault_password:
                         vault_export = "export VAULT_PASS='%s' && " % vault_password
-                    (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + "python3 " + Config.REMOTE_CONF_DIR +
+                    (pid, _, _) = ssh.execute("nohup sh -c \"" + vault_export + CtxtAgentBase.VENV_DIR +
+                                              "/bin/python3 " + Config.REMOTE_CONF_DIR +
                                               "/" + str(self.inf.id) + "/" + ctxt_agent_command +
                                               Config.REMOTE_CONF_DIR + "/" + str(self.inf.id) + "/" +
                                               "/general_info.cfg " + remote_dir + "/" + os.path.basename(conf_file) +

IM/CtxtAgentBase.py

@@ -37,6 +37,7 @@ class CtxtAgentBase:
     # the ConfManager
     PLAYBOOK_RETRIES = 1
     INTERNAL_PLAYBOOK_RETRIES = 1
+    VENV_DIR = "/var/tmp/.ansible"  # nosec
 
     def __init__(self, conf_data_filename):
         self.logger = None
@@ -508,7 +509,7 @@ def install_ansible_roles(self, general_conf_data, playbook):
                 task["name"] = "Create YAML file to install the collections with ansible-galaxy"
                 yaml_data[0]['tasks'].append(task)
 
-                task = {"command": "ansible-galaxy collection install -c -r %s" % filename}
+                task = {"command": self.VENV_DIR + "/bin/ansible-galaxy collection install -c -r %s" % filename}
                 task["name"] = "Install galaxy collections"
                 task["become"] = "yes"
                 task["register"] = "collections_install"
@@ -562,7 +563,7 @@ def install_ansible_roles(self, general_conf_data, playbook):
                 task["name"] = "Create YAML file to install the roles with ansible-galaxy"
                 yaml_data[0]['tasks'].append(task)
 
-                task = {"command": "ansible-galaxy install -c -r %s" % filename}
+                task = {"command": self.VENV_DIR + "/bin/ansible-galaxy install -c -r %s" % filename}
                 task["name"] = "Install galaxy roles"
                 task["become"] = "yes"
                 task["register"] = "roles_install"

IM/connectors/OSCAR.py

@@ -134,7 +134,7 @@ def _get_service_json(radl_system):
         if radl_system.getValue("name"):
             service["name"] = radl_system.getValue("name")
         if radl_system.getValue("memory.size"):
-            service["memory"] = "%dMi" % radl_system.getFeature('memory.size').getValue('M')
+            service["memory"] = "%dMi" % radl_system.getFeature('memory.size').getValue('Mi')
         if radl_system.getValue("cpu.count"):
             service["cpu"] = "%g" % radl_system.getValue("cpu.count")
         if radl_system.getValue("gpu.count"):
@@ -260,7 +260,7 @@ def update_system_info_from_service_info(self, system, service_info):
                               conflict="other", missing="other")
         if "memory" in service_info and service_info["memory"]:
             memory = self.convert_memory_unit(service_info["memory"], "Mi")
-            system.addFeature(Feature("memory.size", "=", memory, "M"),
+            system.addFeature(Feature("memory.size", "=", memory, "Mi"),
                               conflict="other", missing="other")
         if "script" in service_info and service_info["script"]:
             system.addFeature(Feature("script", "=", service_info["script"]),

contextualization/ansible_install.sh

@@ -39,6 +39,10 @@ distribution_id() {
     echo ${RETVAL}
 }
 
+# Create a symbolic link to python3 in case of not venv created
+ls /var/tmp/.ansible/bin/ || mkdir -p /var/tmp/.ansible/bin/
+ls /var/tmp/.ansible/bin/python3 || ln -s /usr/bin/python3 /var/tmp/.ansible/bin/python3
+
 if [ $(which ansible-playbook) ]; then
     echo "Ansible installed. Do not install."
 else

contextualization/conf-ansible.yml

@@ -6,6 +6,7 @@
   vars:
     # Ansible specific Version or "latest"
     ANSIBLE_VERSION: 4.10.0
+    VENV_PATH: /var/tmp/.ansible
   tasks:
   ############## To avoid some issues with cloud-init and unattended upgrades ###############
     - name: Avoid unattended upgrades
@@ -99,15 +100,15 @@
   ################### Install Ansible/pip requisites #########################
 
     - name: Debian/Ubuntu install requisites with apt
-      apt: name=python3-pip,wget,python3-setuptools,python3-psutil,sshpass,openssh-client,unzip install_recommends=no
+      apt: name=python3-pip,wget,python3-setuptools,sshpass,openssh-client,unzip install_recommends=no
       when: ansible_os_family == "Debian"
 
     - name: Yum install requisites RH 7/8 or Fedora
-      command: yum install -y python3-pip python3-setuptools python3-psutil sshpass openssh-clients
+      command: yum install -y python3-pip python3-setuptools sshpass openssh-clients
       when: ansible_os_family == "RedHat"
 
     - name: Zypper install requirements Suse
-      zypper: name=python3-pip,python3-setuptools,python3-psutil,wget,python3-cryptography state=present
+      zypper: name=python3-pip,python3-setuptools,wget,python3-cryptography state=present
       when: ansible_os_family == "Suse"
 
     - name: Install python-setuptools
@@ -116,49 +117,65 @@
 
   ######################################### Use pip to enable to set the version #############################################
 
+    - name: Set extra_args var
+      set_fact:
+        extra_args: ''
+
+    - name: Set extra_args var in py3.11
+      set_fact:
+        extra_args: --break-system-packages
+      when: ansible_python_version is version('3.11', '>=')
+
+    - name: Install virtualenv with pip
+      pip:
+        name: virtualenv
+        executable: pip3
+        extra_args: "{{ extra_args }}"
+
     # Version over 21 does not work with python 3.6 or older
     - name: Upgrade pip in py3.6-
       pip:
         name: pip>18.0,<21.0
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
         # in some old distros we need to trust in the pypi to avoid SSL errors
         extra_args: --trusted-host files.pythonhosted.org --trusted-host pypi.org --trusted-host pypi.python.org
       when: ansible_python_version is version('3.7', '<')
 
     - name: Upgrade pip in py3.7-py3.8
       pip:
         name: pip>20.0
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
       when: ansible_python_version is version('3.7', '>=') and ansible_python_version is version('3.9', '<')
 
     - name: Upgrade pip in py3.9-py3.10
       pip:
         name: pip>=22.0
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
       when: ansible_python_version is version('3.9', '>=') and ansible_python_version is version('3.11', '<')
 
     # Version 66 (#2497) fails
     - name: Upgrade setuptools with Pip in py3.11-
       pip:
         name: setuptools<66.0.0
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
       when: ansible_python_version is version('3.11', '<')
 
     - name: Set extra_args var
       set_fact:
         extra_args: --prefer-binary
 
-    - name: Set extra_args var in py3.11
-      set_fact:
-        extra_args: --prefer-binary --break-system-packages
-      when: ansible_python_version is version('3.11', '>=')
+    - name: Install psutil
+      pip:
+        name: psutil
+        virtualenv: "{{ VENV_PATH }}"
+        extra_args: "{{ extra_args }}"
 
     - name: Install cryptography & pyOpenSSL in py3.11-
       pip:
         name:
           - cryptography>36.0.0,<39.0.0
           - pyOpenSSL>20.0,<22.1.0
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
         extra_args: "{{ extra_args }}"
       when: ansible_python_version is version('3.11', '<')
 
@@ -167,7 +184,7 @@
         name: 
           - cryptography>36.0.0
           - pyOpenSSL>20.0
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
         extra_args: "{{ extra_args }}"
       when: ansible_python_version is version('3.11', '>=')
 
@@ -178,7 +195,7 @@
           - pyyaml
           - paramiko>=2.9.5
           - packaging
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
         extra_args: "{{ extra_args }}"
 
     - name: Set Ansible newer version for python 3.8+
@@ -201,14 +218,14 @@
       pip:
         name: ansible
         version: "{{ ANSIBLE_VERSION }}"
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
         extra_args: "{{ extra_args }}"
       when: ANSIBLE_VERSION != "latest"
 
     - name: Install latest ansible version with Pip
       pip:
         name: ansible
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
         extra_args: "{{ extra_args }}"
       when: ANSIBLE_VERSION == "latest"
 
@@ -219,13 +236,13 @@
         name:
           - jmespath
           - scp
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
         extra_args: "{{ extra_args }}"
 
     - name: Install pywinrm with Pip
       pip:
         name: pywinrm
-        executable: pip3
+        virtualenv: "{{ VENV_PATH }}"
         extra_args: "{{ extra_args }}"
       ignore_errors: yes
 

contextualization/ctxt_agent_dist.py

@@ -162,7 +162,8 @@ def LaunchRemoteAgent(self, vm, vault_pass, pk_file, changed_pass_ok):
         vm_dir = os.path.abspath(os.path.dirname(self.vm_conf_data_filename))
         remote_dir = os.path.abspath(os.path.dirname(self.conf_data_filename))
         try:
-            (pid, _, _) = ssh_client.execute(vault_export + "nohup python3 " + remote_dir + "/ctxt_agent_dist.py " +
+            (pid, _, _) = ssh_client.execute(vault_export + "nohup " + CtxtAgentBase.VENV_DIR + "/bin/python3 " +
+                                             remote_dir + "/ctxt_agent_dist.py " +
                                              self.conf_data_filename + " " + self.vm_conf_data_filename +
                                              " 1 > " + vm_dir + "/stdout 2> " + vm_dir +
                                              "/stderr < /dev/null & echo -n $!")

requirements-tests.txt

@@ -1,11 +1,10 @@
-ansible >= 2.4
-ansible-base
+ansible == 8.7.0
 paramiko >= 1.14
 PyYAML
 cheroot
 boto3
 apache-libcloud >= 3.3.1
-RADL >= 1.3.3
+RADL >= 1.3.4
 flask
 werkzeug
 netaddr

test/integration/TestIM.py

@@ -154,7 +154,7 @@ def test_11_create(self):
         self.__class__.inf_id = inf_id
 
         all_configured = self.wait_inf_state(
-            inf_id, VirtualMachine.CONFIGURED, 2400)
+            inf_id, VirtualMachine.CONFIGURED, 2700)
         self.assertTrue(
             all_configured, msg="ERROR waiting the infrastructure to be configured (timeout).")
 

test/unit/REST.py

@@ -228,8 +228,8 @@ def test_CreateInfrastructure(self, get_infrastructure, CreateInfrastructure):
                                data=read_file_as_bytes("../files/test_simple.json"))
         self.assertEqual(res.json, {"one": {"cloudType": "OpenNebula",
                                             "cloudEndpoint": "http://ramses.i3m.upv.es:2633",
-                                            "compute": [{"cpuCores": 1, "memoryInMegabytes": 1024},
-                                                        {"cpuCores": 1, "memoryInMegabytes": 1024}], "storage": []}})
+                                            "compute": [{"cpuCores": 1, "memoryInMegabytes": 1074},
+                                                        {"cpuCores": 1, "memoryInMegabytes": 1074}], "storage": []}})
 
         headers["Content-Type"] = "application/json"
         CreateInfrastructure.side_effect = InvaliddUserException()

test/unit/connectors/Fogbow.py

@@ -300,7 +300,7 @@ def test_30_updateVMInfo(self, sleep, requests):
         self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
         self.assertEqual(vm.info.systems[0].getValue("net_interface.1.ip"), "10.0.0.1")
         self.assertEqual(vm.info.systems[0].getValue("net_interface.0.ip"), "8.8.8.8")
-        self.assertEqual(vm.info.systems[0].getValue("memory.size"), 1073741824)
+        self.assertEqual(vm.info.systems[0].getValue("memory.size"), 1024000000)
         self.assertEqual(vm.info.systems[0].getValue("disk.1.device"), "/dev/sdb")
 
         data = json.loads(requests.call_args_list[1][1]["data"])

test/unit/connectors/GCE.py

@@ -129,7 +129,7 @@ def test_10_concrete(self, get_driver):
 
         concrete = gce_cloud.concreteSystem(radl_system, auth)
         self.assertEqual(len(concrete), 1)
-        self.assertEqual(concrete[0].getValue("memory.size"), 2147483648)
+        self.assertEqual(concrete[0].getValue("memory.size"), 2048000000)
         self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
 
     @patch('libcloud.compute.drivers.gce.GCENodeDriver')

test/unit/connectors/Kubernetes.py

@@ -188,7 +188,7 @@ def test_20_launch(self, save_data, requests):
                          'labels': {'name': 'test-1'}},
             "spec": {
                 "accessModes": ["ReadWriteOnce"],
-                "resources": {"requests": {"storage": 10737418240}},
+                "resources": {"requests": {"storage": 10000000000}},
             },
         }
         self.assertEqual(requests.call_args_list[1][0][1],
@@ -237,8 +237,8 @@ def test_20_launch(self, save_data, requests):
                         "imagePullPolicy": "Always",
                         "ports": [{"containerPort": 8080, "protocol": "TCP"}],
                         "resources": {
-                            "limits": {"cpu": "1", "memory": "536870912"},
-                            "requests": {"cpu": "1", "memory": "536870912"},
+                            "limits": {"cpu": "1", "memory": "512000000"},
+                            "requests": {"cpu": "1", "memory": "512000000"},
                         },
                         "env": [{"name": "var", "value": "some_val"},
                                 {"name": "var2", "value": "some,val2"}],

test/unit/connectors/OCCI.py

@@ -327,7 +327,7 @@ def test_30_updateVMInfo(self, get_keystone_uri, requests):
         self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
 
         memory = vm.info.systems[0].getValue("memory.size")
-        self.assertEqual(memory, 1824522240)
+        self.assertEqual(memory, 1740000000)
 
     @patch('requests.request')
     @patch('IM.connectors.OCCI.KeyStoneAuth.get_keystone_uri')

test/unit/connectors/OSCAR.py

@@ -124,7 +124,7 @@ def test_20_launch(self, save_data, requests):
         radl_data = """
             system test (
                 name = 'plants' and
-                memory.size = 2G and
+                memory.size = 2GI and
                 cpu.count = 1.0 and
                 cpu.sgx = 1 and
                 gpu.count = 1 and
@@ -288,7 +288,7 @@ def test_55_alter(self, requests):
         new_radl_data = """
             system test (
             cpu.count>=2 and
-            memory.size>=4G
+            memory.size>=4GI
             )"""
         new_radl = radl_parse.parse_radl(new_radl_data)
 
@@ -306,7 +306,7 @@ def test_55_alter(self, requests):
         self.assertTrue(success, msg="ERROR: modifying VM info.")
         self.assertNotIn("ERROR", self.log.getvalue(), msg="ERROR found in log: %s" % self.log.getvalue())
         self.assertEqual(new_vm.info.systems[0].getValue("cpu.count"), 2)
-        self.assertEqual(new_vm.info.systems[0].getFeature("memory.size").getValue("M"), 4096)
+        self.assertEqual(new_vm.info.systems[0].getFeature("memory.size").getValue("M"), 4295)
         self.assertEqual(requests.call_args_list[0][0][0], "PUT")
         self.assertEqual(requests.call_args_list[0][0][1], "http://oscar.com:80/system/services/fname")
         self.assertEqual(json.loads(requests.call_args_list[0][1]['data']), {'memory': '4096Mi', 'cpu': '2'})

test/unit/connectors/OpenNebula.py

@@ -144,7 +144,7 @@ def test_20_launch(self, save_data, getONEVersion, server_proxy):
             OS = [ ARCH = "x86_64" ]
 
             DISK = [ IMAGE_ID = "1" ]
- DISK = [ SAVE = no, TYPE = fs , FORMAT = qcow2, SIZE = 1024, TARGET = hdb ]
+ DISK = [ SAVE = no, TYPE = fs , FORMAT = qcow2, SIZE = 1000, TARGET = hdb ]
 
 
             SCHED_REQUIREMENTS = "CLUSTER_ID=\\"0\\""\n"""

test/unit/test_ctxt_agent.py

@@ -277,7 +277,8 @@ def test_95_install_ansible_roles(self):
         copy_content = yaml_data[0]['tasks'][1]['copy'][pos + 9:-2]
         self.assertEqual(copy_content, "[{src: ansible_role}, {name: hadoop, src: "
                          "'git+https://github.com/micafer/ansible-role-hadoop'}]")
-        self.assertEqual(yaml_data[0]['tasks'][2]['command'][:47], "ansible-galaxy install -c -r /tmp/galaxy_roles_")
+        self.assertEqual(yaml_data[0]['tasks'][2]['command'][:69],
+                         ctxt_agent.VENV_DIR + "/bin/ansible-galaxy install -c -r /tmp/galaxy_roles_")
 
         os.unlink(res)
 
@@ -300,8 +301,8 @@ def test_99_install_ansible_collections(self):
         pos = yaml_data[0]['tasks'][0]['copy'].find('content="')
         copy_content = yaml_data[0]['tasks'][0]['copy'][pos + 9:-2]
         self.assertEqual(copy_content, "{collections: [{name: ns.collection, version: '1.0'}]}")
-        self.assertEqual(yaml_data[0]['tasks'][1]['command'][:64],
-                         "ansible-galaxy collection install -c -r /tmp/galaxy_collections_")
+        self.assertEqual(yaml_data[0]['tasks'][1]['command'][:86],
+                         ctxt_agent.VENV_DIR + "/bin/ansible-galaxy collection install -c -r /tmp/galaxy_collections_")
 
         os.unlink(res)
 

test/unit/test_im_logic.py

@@ -1552,8 +1552,8 @@ def test_estimate_resources(self):
             'cloud0': {
                 'cloudType': 'Dummy',
                 'cloudEndpoint': 'http://server.com:80/path',
-                'compute': [{'cpuCores': 2, 'memoryInMegabytes': 4096, 'diskSizeInGigabytes': 40},
-                            {'cpuCores': 1, 'memoryInMegabytes': 2048, 'diskSizeInGigabytes': 10}],
+                'compute': [{'cpuCores': 2, 'memoryInMegabytes': 4000, 'diskSizeInGigabytes': 40},
+                            {'cpuCores': 1, 'memoryInMegabytes': 2000, 'diskSizeInGigabytes': 10}],
                 'storage': [{'sizeInGigabytes': 100}]
             }})