Merge pull request #545 from inspec/nm/update-dataproc-clusters

CHEF-7369 Updated dataproc_cluster resource

docs/resources/google_dataproc_cluster.md

@@ -192,6 +192,18 @@ Properties that can be accessed from the `google_dataproc_cluster` resource:
 
   * `region`: The region in which the cluster and associated nodes will be created in.
 
+  * `project_id`: The Google Cloud Platform project ID that the cluster belongs to.
+
+  * `virtual_cluster_config`: Optional. The virtual cluster config is used when creating a Dataproc cluster that does not directly control the underlying compute resources, for example, when creating a Dataproc-on-GKE cluster (https://cloud.google.com/dataproc/docs/guides/dpgke/dataproc-gke-overview). Dataproc may set default values, and values may change when clusters are updated. Exactly one of config or virtual_cluster_config must be specified.
+
+  * `status`: Output only. Cluster status.
+
+  * `status_history`: Output only. The previous cluster status.
+
+  * `cluster_uuid`: Output only. A cluster UUID (Unique Universal Identifier). Dataproc generates this value when it creates the cluster.
+
+  * `metrics`: Output only. Contains cluster daemon metrics such as HDFS and YARN stats.Beta Feature: This report is available for testing purposes only. It may be changed before final release.
+
 
 ## GCP Permissions
 

docs/resources/google_dataproc_clusters.md

@@ -26,6 +26,12 @@ See [google_dataproc_cluster.md](google_dataproc_cluster.md) for more detailed i
   * `labels`: an array of `google_dataproc_cluster` labels
   * `configs`: an array of `google_dataproc_cluster` config
   * `regions`: an array of `google_dataproc_cluster` region
+  * `project_ids`: an array of `google_dataproc_cluster` project_id
+  * `virtual_cluster_configs`: an array of `google_dataproc_cluster` virtual_cluster_config
+  * `statuses`: an array of `google_dataproc_cluster` status
+  * `status_histories`: an array of `google_dataproc_cluster` status_history
+  * `cluster_uuids`: an array of `google_dataproc_cluster` cluster_uuid
+  * `metrics`: an array of `google_dataproc_cluster` metrics
 
 ## Filter Criteria
 This resource supports all of the above properties as filter criteria, which can be used

libraries/google_dataproc_cluster.rb

@@ -42,6 +42,12 @@ class DataprocCluster < GcpResourceBase
   attr_reader :labels
   attr_reader :config
   attr_reader :region
+  attr_reader :project_id
+  attr_reader :virtual_cluster_config
+  attr_reader :status
+  attr_reader :status_history
+  attr_reader :cluster_uuid
+  attr_reader :metrics
 
   def initialize(params)
     super(params.merge({ use_http_transport: true }))
@@ -55,6 +61,12 @@ def parse
     @labels = @fetched['labels']
     @config = GoogleInSpec::Dataproc::Property::ClusterConfig.new(@fetched['config'], to_s)
     @region = @fetched['region']
+    @project_id = @fetched['projectId']
+    @virtual_cluster_config = @fetched['virtualClusterConfig']
+    @status = @fetched['status']
+    @status_history = @fetched['statusHistory']
+    @cluster_uuid = @fetched['clusterUuid']
+    @metrics = @fetched['metrics']
   end
 
   def exists?

libraries/google_dataproc_clusters.rb

@@ -27,6 +27,12 @@ class DataprocClusters < GcpResourceBase
   filter_table_config.add(:labels, field: :labels)
   filter_table_config.add(:configs, field: :config)
   filter_table_config.add(:regions, field: :region)
+  filter_table_config.add(:project_ids, field: :project_id)
+  filter_table_config.add(:virtual_cluster_configs, field: :virtual_cluster_config)
+  filter_table_config.add(:statuses, field: :status)
+  filter_table_config.add(:status_histories, field: :status_history)
+  filter_table_config.add(:cluster_uuids, field: :cluster_uuid)
+  filter_table_config.add(:metrics, field: :metrics)
 
   filter_table_config.connect(self, :table)
 
@@ -70,6 +76,12 @@ def transformers
       'labels' => ->(obj) { [:labels, obj['labels']] },
       'config' => ->(obj) { [:config, GoogleInSpec::Dataproc::Property::ClusterConfig.new(obj['config'], to_s)] },
       'region' => ->(obj) { [:region, obj['region']] },
+      'projectId' => ->(obj) { [:project_id, obj['projectId']] },
+      'virtualClusterConfig' => ->(obj) { [:virtual_cluster_config, obj['virtualClusterConfig']] },
+      'status' => ->(obj) { [:status, obj['status']] },
+      'statusHistory' => ->(obj) { [:status_history, obj['statusHistory']] },
+      'clusterUuid' => ->(obj) { [:cluster_uuid, obj['clusterUuid']] },
+      'metrics' => ->(obj) { [:metrics, obj['metrics']] },
     }
   end
 

test/integration/verify/controls/google_dataproc_cluster.rb

@@ -57,6 +57,8 @@
     its('config.master_config.machine_type_uri') { should match dataproc_cluster['config']['master_config']['machine_type'] }
     its('config.worker_config.machine_type_uri') { should match dataproc_cluster['config']['worker_config']['machine_type'] }
     its('config.software_config.properties') { should include(dataproc_cluster['config']['software_config']['prop_key'] => dataproc_cluster['config']['software_config']['prop_value']) }
+    its('project_id') { should eq gcp_project_id }
+    its('cluster_uuid') { should eq "test-uuid-67ff7c8e-558a-45ad-97c7" }
   end
 
   describe google_dataproc_cluster(project: gcp_project_id, region: gcp_location, cluster_name: 'nonexistent') do

test/integration/verify/controls/google_dataproc_clusters.rb

@@ -52,5 +52,7 @@
   describe google_dataproc_clusters(project: gcp_project_id, region: gcp_location) do
     its('count') { should be >= 1 }
     its('cluster_names') { should include dataproc_cluster['name'] }
+    its('project_ids') { should include gcp_project_id }
+    its('cluster_uuids') { should include "test-uuid-67ff7c8e-558a-45ad-97c7" }
   end
 end