From a434cced38f66c15b96cb7ce08b178bbd03dc510 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Fri, 17 May 2024 11:58:18 -0400 Subject: [PATCH] Update cluster state management (#227) * Fix function return types and dictionary key checks * Update pre-commit Github actions * Update cluster state management, i.e. present, absent, started, restarted, and stopped Signed-off-by: Webster Mudge --- .github/workflows/pre-commit.yml | 6 +- plugins/modules/cluster.py | 35 ++- .../{test_cluster.py => test_base_cluster.py} | 168 --------------- .../modules/cluster/test_basic_cluster.py | 201 ++++++++++++++++++ .../modules/cluster/test_compute_cluster.py | 99 +++++++++ .../cluster/test_experience_cluster.py | 83 ++++++++ 6 files changed, 399 insertions(+), 193 deletions(-) rename tests/unit/plugins/modules/cluster/{test_cluster.py => test_base_cluster.py} (72%) create mode 100644 tests/unit/plugins/modules/cluster/test_basic_cluster.py create mode 100644 tests/unit/plugins/modules/cluster/test_compute_cluster.py create mode 100644 tests/unit/plugins/modules/cluster/test_experience_cluster.py diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 840bb2e1..1b6688dc 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: pre-commit +name: Execute Precommit Linting and Checks on: pull_request: @@ -23,6 +23,6 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 - uses: pre-commit/action@v3.0.1 diff --git a/plugins/modules/cluster.py b/plugins/modules/cluster.py index e24c148f..af811078 100644 --- a/plugins/modules/cluster.py +++ b/plugins/modules/cluster.py @@ -61,7 +61,6 @@ } DOCUMENTATION = r""" ---- module: cluster short_description: Manage the lifecycle and state of a cluster description: @@ -77,7 +76,6 @@ """ EXAMPLES = r""" ---- - name: Create an ECS cluster cloudera.cluster.cluster: host: example.cloudera.com @@ -119,7 +117,6 @@ """ RETURN = r""" ---- cloudera_manager: description: Details about Cloudera Manager Cluster type: dict @@ -276,21 +273,18 @@ def process(self): elif self.state == "absent": # Delete cluster - refresh = False - - # TODO Check for status when deleting - # if existing and existing.entity_status == "": - # self.wait_for_active_cmd(cluster_api, self.cluster_name) - # elif existing: if existing: self.changed = True if not self.module.check_mode: - self.cluster_api.delete_cluster(cluster_name=self.name) - self.wait_for_active_cmd(self.name) + if existing.entity_status != "STOPPED": + stop = self.cluster_api.stop_command(cluster_name=self.name) + self.wait_command(stop, polling=self.timeout, delay=self.delay) + + delete = self.cluster_api.delete_cluster(cluster_name=self.name) + self.wait_command(delete, polling=self.timeout, delay=self.delay) elif self.state == "started": - # TODO NONE seems to be fresh cluster, never run before # Already started if existing and existing.entity_status == "GOOD_HEALTH": refresh = False @@ -312,11 +306,11 @@ def process(self): # If newly created or created by not yet initialize if not existing or existing.entity_status == "NONE": first_run = self.cluster_api.first_run(cluster_name=self.name) - self.wait_for_composite_cmd(first_run.id) + self.wait_command(first_run) # Start the existing and previously initialized cluster else: start = self.cluster_api.start_command(cluster_name=self.name) - self.wait_for_composite_cmd(start.id) + self.wait_command(start, polling=self.timeout, delay=self.delay) if self.state == "stopped": # Already stopped @@ -339,7 +333,7 @@ def process(self): self.changed = True if not self.module.check_mode: stop = self.cluster_api.stop_command(cluster_name=self.name) - self.wait_for_composite_cmd(stop.id) + self.wait_command(stop, polling=self.timeout, delay=self.delay) if self.state == "restarted": # Start underway @@ -357,7 +351,7 @@ def process(self): self.changed = True if not self.module.check_mode: restart = self.cluster_api.restart_command(cluster_name=self.name) - self.wait_for_composite_cmd(restart.id) + self.wait_command(restart, polling=self.timeout, delay=self.delay) if refresh: # Retrieve the updated cluster details @@ -547,7 +541,6 @@ def create_cluster_from_parameters(self): timeout=self.timeout, ) parcel.activate() - # Apply host templates for ht, refs in template_map.items(): self.host_template_api.apply_host_template( @@ -674,10 +667,10 @@ def create_cluster_from_parameters(self): if self.auto_assign: self.cluster_api.auto_assign_roles(cluster_name=self.name) - def marshal_service(self, options: str) -> ApiService: + def marshal_service(self, options: dict) -> ApiService: service = ApiService(name=options["name"], type=options["type"]) - if "display_name" in options: + if options["display_name"]: service.display_name = options["display_name"] # Service-wide configuration @@ -741,9 +734,7 @@ def marshal_hostrefs(self, hosts: dict) -> list[ApiHostRef]: ) return results - def find_base_role_group_name( - self, service_type: str, role_type: str - ) -> ApiRoleConfigGroup: + def find_base_role_group_name(self, service_type: str, role_type: str) -> str: rcgs = [ rcg for s in self.service_api.read_services(cluster_name=self.name).items diff --git a/tests/unit/plugins/modules/cluster/test_cluster.py b/tests/unit/plugins/modules/cluster/test_base_cluster.py similarity index 72% rename from tests/unit/plugins/modules/cluster/test_cluster.py rename to tests/unit/plugins/modules/cluster/test_base_cluster.py index b13c3043..09b501e7 100644 --- a/tests/unit/plugins/modules/cluster/test_cluster.py +++ b/tests/unit/plugins/modules/cluster/test_base_cluster.py @@ -365,97 +365,6 @@ def test_present_base_host_role_overrides(conn, module_args): LOG.info(str(e.value.cloudera_manager)) -def test_present_basic_cluster(conn, module_args): - args = """ - name: Basic_Cluster - cluster_version: "7.1.9-1.cdh7.1.9.p0.44702451" - type: BASE_CLUSTER - state: present - services: - - name: core-settings-0 - type: CORE_SETTINGS - display_name: CORE_SETTINGS_TEST - - name: zookeeper-0 - type: ZOOKEEPER - display_name: ZK_TEST - config: - zookeeper_datadir_autocreate: yes - - name: hdfs-0 - type: HDFS - display_name: HDFS_TEST - config: - zookeeper_service: zookeeper-0 - core_connector: core-settings-0 - role_groups: - - type: DATANODE - config: - dfs_data_dir_list: /dfs/dn - - type: NAMENODE - config: - dfs_name_dir_list: /dfs/nn - - type: SECONDARYNAMENODE - config: - fs_checkpoint_dir_list: /dfs/snn - - name: yarn-0 - type: YARN - display_name: YARN_TEST - config: - hdfs_service: hdfs-0 - zookeeper_service: zookeeper-0 - role_groups: - - type: RESOURCEMANAGER - config: - yarn_scheduler_maximum_allocation_mb: 4096 - yarn_scheduler_maximum_allocation_vcores: 4 - - type: NODEMANAGER - config: - yarn_nodemanager_resource_memory_mb: 4096 - yarn_nodemanager_resource_cpu_vcores: 4 - yarn_nodemanager_local_dirs: /tmp/nm - yarn_nodemanager_log_dirs: /var/log/nm - - type: GATEWAY - config: - mapred_submit_replication: 3 - mapred_reduce_tasks: 6 - host_templates: - - name: Master1 - role_groups: - - service: HDFS - type: NAMENODE - - service: HDFS - type: SECONDARYNAMENODE - - service: YARN - type: RESOURCEMANAGER - - service: YARN - type: JOBHISTORY - - name: Worker - role_groups: - - service: HDFS - type: DATANODE - - service: YARN - type: NODEMANAGER - - service: ZOOKEEPER - type: SERVER - parcels: - CDH: "7.1.9-1.cdh7.1.9.p0.44702451" - hosts: - - name: test10-worker-free-01.cldr.internal - host_template: Master1 - - name: test10-worker-free-02.cldr.internal - host_template: Worker - - name: test10-worker-free-03.cldr.internal - host_template: Worker - """ - conn.update(yaml.safe_load(args)) - module_args(conn) - - with pytest.raises(AnsibleExitJson) as e: - cluster.main() - - LOG.info(str(e.value.cloudera_manager)) - - -@pytest.mark.skip(reason="Not yet implemented") def test_started_base(conn, module_args): conn.update( name="PVC-Base", @@ -470,7 +379,6 @@ def test_started_base(conn, module_args): LOG.info(str(e.value.cloudera_manager)) -@pytest.mark.skip(reason="Not yet implemented") def test_restarted_base(conn, module_args): conn.update( name="PVC-Base", @@ -485,12 +393,10 @@ def test_restarted_base(conn, module_args): LOG.info(str(e.value.cloudera_manager)) -@pytest.mark.skip(reason="Not yet implemented") def test_stopped_base(conn, module_args): conn.update( name="PVC-Base", cluster_version="7.1.9", # "1.5.1-b626.p0.42068229", - # type="COMPUTE_CLUSTER", state="stopped", ) module_args(conn) @@ -498,11 +404,9 @@ def test_stopped_base(conn, module_args): with pytest.raises(AnsibleExitJson) as e: cluster.main() - # LOG.info(str(e.value)) LOG.info(str(e.value.cloudera_manager)) -@pytest.mark.skip(reason="Not yet implemented") def test_absent_base(conn, module_args): conn.update( name="Example_Base", @@ -516,78 +420,6 @@ def test_absent_base(conn, module_args): LOG.info(str(e.value.cloudera_manager)) -def test_present_compute_minimum(conn, module_args): - conn.update( - name="Example_Compute", - cluster_version="7.1.9", - contexts=["SDX"], - state="present", - ) - module_args(conn) - - with pytest.raises(AnsibleExitJson) as e: - cluster.main() - - LOG.info(str(e.value.cloudera_manager)) - - -@pytest.mark.skip(reason="Not yet implemented") -def test_started_compute_minimum(conn, module_args): - conn.update( - name="Example_Compute", - cluster_version="7.1.9", - contexts=["SDX"], - state="started", - ) - module_args(conn) - - with pytest.raises(AnsibleExitJson) as e: - cluster.main() - - LOG.info(str(e.value.cloudera_manager)) - - -def test_absent_compute(conn, module_args): - conn.update( - name="Example_Compute", - state="absent", - ) - module_args(conn) - - with pytest.raises(AnsibleExitJson) as e: - cluster.main() - - LOG.info(str(e.value.cloudera_manager)) - - -def test_present_experience_minimum(conn, module_args): - conn.update( - name="Example_Experience", - cluster_version="1.5.3", - type="EXPERIENCE_CLUSTER", - state="present", - ) - module_args(conn) - - with pytest.raises(AnsibleExitJson) as e: - cluster.main() - - LOG.info(str(e.value.cloudera_manager)) - - -def test_absent_experience(conn, module_args): - conn.update( - name="Example_Experience", - state="absent", - ) - module_args(conn) - - with pytest.raises(AnsibleExitJson) as e: - cluster.main() - - LOG.info(str(e.value.cloudera_manager)) - - def test_pytest_cluster_with_template(module_args): module_args( { diff --git a/tests/unit/plugins/modules/cluster/test_basic_cluster.py b/tests/unit/plugins/modules/cluster/test_basic_cluster.py new file mode 100644 index 00000000..a6b23623 --- /dev/null +++ b/tests/unit/plugins/modules/cluster/test_basic_cluster.py @@ -0,0 +1,201 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import os +import pytest +import yaml + +from ansible_collections.cloudera.cluster.plugins.modules import cluster +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) + +LOG = logging.getLogger(__name__) + + +@pytest.fixture +def conn(): + conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) + + if os.getenv("CM_HOST", None): + conn.update(host=os.getenv("CM_HOST")) + + if os.getenv("CM_PORT", None): + conn.update(port=os.getenv("CM_PORT")) + + if os.getenv("CM_ENDPOINT", None): + conn.update(url=os.getenv("CM_ENDPOINT")) + + if os.getenv("CM_PROXY", None): + conn.update(proxy=os.getenv("CM_PROXY")) + + return { + **conn, + "verify_tls": "no", + "debug": "no", + } + + +def test_present(conn, module_args): + args = """ + name: Basic_Cluster + cluster_version: "7.1.9-1.cdh7.1.9.p0.44702451" + type: BASE_CLUSTER + state: present + services: + - name: core-settings-0 + type: CORE_SETTINGS + display_name: CORE_SETTINGS_TEST + - name: zookeeper-0 + type: ZOOKEEPER + display_name: ZK_TEST + config: + zookeeper_datadir_autocreate: yes + - name: hdfs-0 + type: HDFS + display_name: HDFS_TEST + config: + zookeeper_service: zookeeper-0 + core_connector: core-settings-0 + role_groups: + - type: DATANODE + config: + dfs_data_dir_list: /dfs/dn + - type: NAMENODE + config: + dfs_name_dir_list: /dfs/nn + - type: SECONDARYNAMENODE + config: + fs_checkpoint_dir_list: /dfs/snn + - name: yarn-0 + type: YARN + display_name: YARN_TEST + config: + hdfs_service: hdfs-0 + zookeeper_service: zookeeper-0 + role_groups: + - type: RESOURCEMANAGER + config: + yarn_scheduler_maximum_allocation_mb: 4096 + yarn_scheduler_maximum_allocation_vcores: 4 + - type: NODEMANAGER + config: + yarn_nodemanager_resource_memory_mb: 4096 + yarn_nodemanager_resource_cpu_vcores: 4 + yarn_nodemanager_local_dirs: /tmp/nm + yarn_nodemanager_log_dirs: /var/log/nm + - type: GATEWAY + config: + mapred_submit_replication: 3 + mapred_reduce_tasks: 6 + host_templates: + - name: Master1 + role_groups: + - service: HDFS + type: NAMENODE + - service: HDFS + type: SECONDARYNAMENODE + - service: YARN + type: RESOURCEMANAGER + - service: YARN + type: JOBHISTORY + - name: Worker + role_groups: + - service: HDFS + type: DATANODE + - service: YARN + type: NODEMANAGER + - service: ZOOKEEPER + type: SERVER + parcels: + CDH: "7.1.9-1.cdh7.1.9.p0.44702451" + hosts: + - name: test10-worker-free-01.cldr.internal + host_template: Master1 + - name: test10-worker-free-02.cldr.internal + host_template: Worker + - name: test10-worker-free-03.cldr.internal + host_template: Worker + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_started(conn, module_args): + args = """ + name: Basic_Cluster + state: started + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_restarted(conn, module_args): + args = """ + name: Basic_Cluster + state: restarted + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_stopped(conn, module_args): + args = """ + name: Basic_Cluster + state: stopped + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_absent(conn, module_args): + args = """ + name: Basic_Cluster + state: absent + """ + conn.update(yaml.safe_load(args)) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) diff --git a/tests/unit/plugins/modules/cluster/test_compute_cluster.py b/tests/unit/plugins/modules/cluster/test_compute_cluster.py new file mode 100644 index 00000000..21300490 --- /dev/null +++ b/tests/unit/plugins/modules/cluster/test_compute_cluster.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import os +import pytest +import yaml + +from ansible_collections.cloudera.cluster.plugins.modules import cluster +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) + +LOG = logging.getLogger(__name__) + + +@pytest.fixture +def conn(): + conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) + + if os.getenv("CM_HOST", None): + conn.update(host=os.getenv("CM_HOST")) + + if os.getenv("CM_PORT", None): + conn.update(port=os.getenv("CM_PORT")) + + if os.getenv("CM_ENDPOINT", None): + conn.update(url=os.getenv("CM_ENDPOINT")) + + if os.getenv("CM_PROXY", None): + conn.update(proxy=os.getenv("CM_PROXY")) + + return { + **conn, + "verify_tls": "no", + "debug": "no", + } + + +def test_present_compute_minimum(conn, module_args): + conn.update( + name="Example_Compute", + cluster_version="7.1.9", + contexts=["SDX"], + state="present", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +@pytest.mark.skip(reason="Not yet implemented") +def test_started_compute_minimum(conn, module_args): + conn.update( + name="Example_Compute", + cluster_version="7.1.9", + contexts=["SDX"], + state="started", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_absent_compute(conn, module_args): + conn.update( + name="Example_Compute", + state="absent", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) diff --git a/tests/unit/plugins/modules/cluster/test_experience_cluster.py b/tests/unit/plugins/modules/cluster/test_experience_cluster.py new file mode 100644 index 00000000..8e9f7087 --- /dev/null +++ b/tests/unit/plugins/modules/cluster/test_experience_cluster.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import, division, print_function + +__metaclass__ = type + +import logging +import os +import pytest +import yaml + +from ansible_collections.cloudera.cluster.plugins.modules import cluster +from ansible_collections.cloudera.cluster.tests.unit import ( + AnsibleExitJson, + AnsibleFailJson, +) + +LOG = logging.getLogger(__name__) + + +@pytest.fixture +def conn(): + conn = dict(username=os.getenv("CM_USERNAME"), password=os.getenv("CM_PASSWORD")) + + if os.getenv("CM_HOST", None): + conn.update(host=os.getenv("CM_HOST")) + + if os.getenv("CM_PORT", None): + conn.update(port=os.getenv("CM_PORT")) + + if os.getenv("CM_ENDPOINT", None): + conn.update(url=os.getenv("CM_ENDPOINT")) + + if os.getenv("CM_PROXY", None): + conn.update(proxy=os.getenv("CM_PROXY")) + + return { + **conn, + "verify_tls": "no", + "debug": "no", + } + + +def test_present_experience_minimum(conn, module_args): + conn.update( + name="Example_Experience", + cluster_version="1.5.3", + type="EXPERIENCE_CLUSTER", + state="present", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager)) + + +def test_absent_experience(conn, module_args): + conn.update( + name="Example_Experience", + state="absent", + ) + module_args(conn) + + with pytest.raises(AnsibleExitJson) as e: + cluster.main() + + LOG.info(str(e.value.cloudera_manager))