From 9788d656bde0775bba543a6cc31358387ca98277 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 27 Feb 2024 08:52:47 +0100 Subject: [PATCH 1/8] Remove old node_filter --- IM/tosca/Tosca.py | 96 +-------------------------------------- examples/tosca.yml | 15 +----- test/files/tosca_long.yml | 16 ------- 3 files changed, 2 insertions(+), 125 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index b5adbc261..6775973aa 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -1525,8 +1525,7 @@ def _final_function_result(self, func, node, inf_info=None): def _find_host_node(self, node, nodetemplates, base_root_type="tosca.nodes.Compute"): """ - Select the node to host each node, using the node requirements - In most of the cases the are directly specified, otherwise "node_filter" is used + Select the node to host each node """ # check for a HosteOn relation @@ -1543,99 +1542,6 @@ def _find_host_node(self, node, nodetemplates, base_root_type="tosca.nodes.Compu else: return self._find_host_node(n, nodetemplates) - # There are no direct HostedOn node - # check node_filter requirements - if node.requirements and base_root_type == "tosca.nodes.Compute": - for requires in node.requirements: - if 'host' in requires: - value = requires.get('host') - if isinstance(value, dict): - if 'node_filter' in value: - node_filter = value.get('node_filter') - return self._get_compute_from_node_filter(node_filter, nodetemplates) - - return None - - def _node_fulfill_filter(self, node, node_filter): - """ - Check if a node fulfills the features of a node filter - """ - - # Get node properties - node_props = {} - for cap_type in ['os', 'host']: - if node.get_capability(cap_type): - for prop in node.get_capability(cap_type).get_properties_objects(): - if prop.value is not None: - unit = None - value = self._final_function_result(prop.value, node) - if prop.name in ['disk_size', 'mem_size']: - value, unit = Tosca._get_size_and_unit(value) - node_props[prop.name] = (value, unit) - - filter_props = {} - # Get node_filter properties - for elem in node_filter: - if isinstance(elem, dict): - for cap_type in ['os', 'host']: - if cap_type in elem: - for p in elem.get(cap_type).get('properties'): - p_name = list(p.keys())[0] - p_value = list(p.values())[0] - if isinstance(p_value, dict): - filter_props[p_name] = (list(p_value.keys())[0], - list(p_value.values())[0]) - else: - filter_props[p_name] = ("equal", p_value) - - operator_map = { - 'equal': operator.eq, - 'greater_than': operator.gt, - 'greater_or_equal': operator.ge, - 'less_than': operator.lt, - 'less_or_equal': operator.le - } - - # Compare the properties - for name, value in filter_props.items(): - op, filter_value = value - if name in ['disk_size', 'mem_size']: - filter_value, _ = Tosca._get_size_and_unit(filter_value) - - if name in node_props: - node_value, _ = node_props[name] - conv_operator = operator_map.get(op, None) - if conv_operator: - comparation = conv_operator(node_value, filter_value) - else: - if op == "in_range": - comparation = node_value >= filter_value[0] and node_value <= filter_value[1] - elif op == "valid_values": - comparation = node_value in filter_value - else: - Tosca.logger.warning("Logical operator %s not supported." % op) - - if not comparation: - return False - else: - # if this property is not specified in the node, return False - # TODO: we must think about default values - return False - - return True - - def _get_compute_from_node_filter(self, node_filter, nodetemplates): - """ - Select the first node that fulfills the specified "node_filter" - """ - - for node in nodetemplates: - root_type = Tosca._get_root_parent_type(node).type - - if root_type == "tosca.nodes.Compute": - if self._node_fulfill_filter(node, node_filter.get('capabilities')): - return node - return None @staticmethod diff --git a/examples/tosca.yml b/examples/tosca.yml index ea0f9689a..b1634772a 100644 --- a/examples/tosca.yml +++ b/examples/tosca.yml @@ -64,20 +64,7 @@ topology_template: properties: root_password: { get_input: mysql_root_password } requirements: - - host: - node_filter: - capabilities: - # Constraints for selecting “host” (Container Capability) - - host: - properties: - - num_cpus: { in_range: [1,4] } - - mem_size: { greater_or_equal: 1 GB } - # Constraints for selecting “os” (OperatingSystem Capability) - - os: - properties: - - architecture: { equal: x86_64 } - - type: linux - - distribution: ubuntu + - host: db_server db_server: type: tosca.nodes.Compute diff --git a/test/files/tosca_long.yml b/test/files/tosca_long.yml index acf3e31ff..d4af58c53 100644 --- a/test/files/tosca_long.yml +++ b/test/files/tosca_long.yml @@ -198,22 +198,6 @@ topology_template: size: 10 GB type: ssd - mysql: - type: tosca.nodes.DBMS - requirements: - - host: - node_filter: - capabilities: - # Constraints for selecting "host" (Container Capability) - - host: - properties: - - num_cpus: { in_range: [ 1, 4 ] } - - mem_size: { greater_or_equal: 2 GB } - # Constraints for selecting "os" (OperatingSystem Capability) - - os: - properties: - - type: linux - outputs: galaxy_url: value: { concat: [ 'http://', get_attribute: [ lrms_server, public_address, 0 ], ':8080' ] } From 8dadaacd29508630c3a961e03829852d4353cc8a Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 27 Feb 2024 09:01:56 +0100 Subject: [PATCH 2/8] Improve code --- IM/tosca/Tosca.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 6775973aa..b167edefd 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -1523,24 +1523,26 @@ def _final_function_result(self, func, node, inf_info=None): # TODO: resolve function values related with run-time values as IM # or ansible variables - def _find_host_node(self, node, nodetemplates, base_root_type="tosca.nodes.Compute"): + def _find_host_node(self, node, nodetemplates, base_root_type="tosca.nodes.Compute", node_type=None): """ Select the node to host each node """ - - # check for a HosteOn relation - root_type = Tosca._get_root_parent_type(node).type - if root_type == base_root_type: + if node_type and node.type == node_type: return node + else: + root_type = Tosca._get_root_parent_type(node).type + if root_type == base_root_type: + return node if node.requirements: for r, n in node.relationships.items(): + # check for a HosteOn relation if Tosca._is_derived_from(r, r.HOSTEDON) or Tosca._is_derived_from(r, r.BINDSTO): root_type = Tosca._get_root_parent_type(n).type if root_type == base_root_type: return n else: - return self._find_host_node(n, nodetemplates) + return self._find_host_node(n, nodetemplates, base_root_type, node_type) return None @@ -2182,7 +2184,7 @@ def _gen_k8s_system(self, node, nodetemplates): variables += "%s=%s" % (k, v) res.setValue("environment.variables", variables) - runtime = self._find_host_node(node, nodetemplates, base_root_type="tosca.nodes.SoftwareComponent") + runtime = self._find_host_node(node, nodetemplates, node_type="tosca.nodes.Container.Runtime.Docker") if runtime: # Get the properties of the runtime From 55b110dbea252b5258fd05aa285ee2142454b905 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 27 Feb 2024 09:59:20 +0100 Subject: [PATCH 3/8] Get operation in abstract compute --- IM/tosca/Tosca.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index b167edefd..7ac90a653 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -2,7 +2,6 @@ import logging import yaml import copy -import operator import requests_cache import json import re @@ -1697,6 +1696,24 @@ def _add_ansible_roles(self, node, nodetemplates, system): feature = Feature('disk.0.applications', 'contains', app_features) system.addFeature(feature) + def _get_operator(self, value): + """ + Get the operator for a value + """ + operator_map = { + 'equal': '=', + 'greater_than': '>', + 'greater_or_equal': '>=', + 'less_than': '<', + 'less_or_equal': '<=' + } + + if isinstance(value, dict) and len(value) == 1: + operator = operator_map.get(list(value.keys())[0]) + else: + operator = None + return operator + def _gen_system(self, node, nodetemplates): """ Take a node of type "Compute" and get the RADL.system to represent it @@ -1775,10 +1792,12 @@ def _gen_system(self, node, nodetemplates): elif prop.name in ["preemtible_instance", "sgx"]: value = 'yes' if value else 'no' - if isinstance(value, float) or isinstance(value, int): - operator = ">=" - else: - operator = "=" + operator = self._get_operator(prop.value) + if not operator: + if isinstance(value, float) or isinstance(value, int): + operator = ">=" + else: + operator = "=" feature = Feature(name, operator, value, unit) res.addFeature(feature) From 8740be2f7457386a5f8f3c1eb59fb47cbe087f00 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 27 Feb 2024 10:22:40 +0100 Subject: [PATCH 4/8] Get operation in abstract compute --- IM/tosca/Tosca.py | 9 ++++++--- test/files/tosca_long.yml | 1 + test/unit/Tosca.py | 1 + 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/IM/tosca/Tosca.py b/IM/tosca/Tosca.py index 7ac90a653..e3dcf6bdd 100644 --- a/IM/tosca/Tosca.py +++ b/IM/tosca/Tosca.py @@ -1710,9 +1710,10 @@ def _get_operator(self, value): if isinstance(value, dict) and len(value) == 1: operator = operator_map.get(list(value.keys())[0]) + value = list(value.values())[0] else: operator = None - return operator + return operator, value def _gen_system(self, node, nodetemplates): """ @@ -1792,8 +1793,10 @@ def _gen_system(self, node, nodetemplates): elif prop.name in ["preemtible_instance", "sgx"]: value = 'yes' if value else 'no' - operator = self._get_operator(prop.value) - if not operator: + operator, new_value = self._get_operator(value) + if operator: + value = new_value + else: if isinstance(value, float) or isinstance(value, int): operator = ">=" else: diff --git a/test/files/tosca_long.yml b/test/files/tosca_long.yml index d4af58c53..595dea71d 100644 --- a/test/files/tosca_long.yml +++ b/test/files/tosca_long.yml @@ -182,6 +182,7 @@ topology_template: properties: # host Operating System image properties type: linux + architecture: { equal: x86_64 } #distribution: scientific #version: 6.6 requirements: diff --git a/test/unit/Tosca.py b/test/unit/Tosca.py index 9c383a36f..a69b7e898 100755 --- a/test/unit/Tosca.py +++ b/test/unit/Tosca.py @@ -77,6 +77,7 @@ def test_tosca_to_radl(self): lrms_wn = radl.get_system_by_name('lrms_wn') self.assertEqual(lrms_wn.getValue('memory.size'), 2000000000) + self.assertEqual(lrms_wn.getValue('cpu.arch'), 'x86_64') lrms_server = radl.get_system_by_name('lrms_server') self.assertEqual(lrms_server.getValue('instance_name'), 'myslurmserver') self.assertEqual(lrms_server.getValue('memory.size'), 1000000000) From ef05073973e797b81ad7a8c68b20d86e8092bce6 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 27 Feb 2024 16:04:16 +0100 Subject: [PATCH 5/8] Add SQAaaS job --- .github/workflows/sqaaas.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/sqaaas.yaml diff --git a/.github/workflows/sqaaas.yaml b/.github/workflows/sqaaas.yaml new file mode 100644 index 000000000..b8e9435af --- /dev/null +++ b/.github/workflows/sqaaas.yaml @@ -0,0 +1,23 @@ +name: Test IM + +on: + push: + branches: ["master"] + pull_request: + branches: ["master"] + +jobs: + sqaaas_job: + runs-on: ubuntu-latest + name: Job that triggers SQAaaS platform + steps: + - name: SQAaaS action step + uses: eosc-synergy/sqaaas-assessment-action@v2 + with: + qc_uni_steps: unit-test + + - name: unit-test + uses: actions/sqaaas-step@v1 + with: + name: unit-test + tool: tox \ No newline at end of file From efbcf1fb568450e019c7773ba494efd5943fab97 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 27 Feb 2024 16:08:45 +0100 Subject: [PATCH 6/8] Add SQAaaS job --- .github/workflows/sqaaas.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/sqaaas.yaml b/.github/workflows/sqaaas.yaml index b8e9435af..c0f8a01e8 100644 --- a/.github/workflows/sqaaas.yaml +++ b/.github/workflows/sqaaas.yaml @@ -13,11 +13,3 @@ jobs: steps: - name: SQAaaS action step uses: eosc-synergy/sqaaas-assessment-action@v2 - with: - qc_uni_steps: unit-test - - - name: unit-test - uses: actions/sqaaas-step@v1 - with: - name: unit-test - tool: tox \ No newline at end of file From be6c97ba9a3f191669e55b7d281c04c11c9d8a7c Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 27 Feb 2024 16:10:55 +0100 Subject: [PATCH 7/8] Add SQAaaS job --- .github/workflows/sqaaas.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sqaaas.yaml b/.github/workflows/sqaaas.yaml index c0f8a01e8..54285b128 100644 --- a/.github/workflows/sqaaas.yaml +++ b/.github/workflows/sqaaas.yaml @@ -12,4 +12,4 @@ jobs: name: Job that triggers SQAaaS platform steps: - name: SQAaaS action step - uses: eosc-synergy/sqaaas-assessment-action@v2 + uses: eosc-synergy/sqaaas-assessment-action@v1 From c918cf174af2220d1a42508a3b2c5cd53c45bd0b Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Tue, 27 Feb 2024 16:11:57 +0100 Subject: [PATCH 8/8] Del SQAaaS job --- .github/workflows/sqaaas.yaml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 .github/workflows/sqaaas.yaml diff --git a/.github/workflows/sqaaas.yaml b/.github/workflows/sqaaas.yaml deleted file mode 100644 index 54285b128..000000000 --- a/.github/workflows/sqaaas.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: Test IM - -on: - push: - branches: ["master"] - pull_request: - branches: ["master"] - -jobs: - sqaaas_job: - runs-on: ubuntu-latest - name: Job that triggers SQAaaS platform - steps: - - name: SQAaaS action step - uses: eosc-synergy/sqaaas-assessment-action@v1