diff --git a/docs/resources/google_dataproc_cluster.md b/docs/resources/google_dataproc_cluster.md index 2177903c7..66f710e5b 100644 --- a/docs/resources/google_dataproc_cluster.md +++ b/docs/resources/google_dataproc_cluster.md @@ -192,6 +192,18 @@ Properties that can be accessed from the `google_dataproc_cluster` resource: * `region`: The region in which the cluster and associated nodes will be created in. + * `project_id`: The Google Cloud Platform project ID that the cluster belongs to. + + * `virtual_cluster_config`: Optional. The virtual cluster config is used when creating a Dataproc cluster that does not directly control the underlying compute resources, for example, when creating a Dataproc-on-GKE cluster (https://cloud.google.com/dataproc/docs/guides/dpgke/dataproc-gke-overview). Dataproc may set default values, and values may change when clusters are updated. Exactly one of config or virtual_cluster_config must be specified. + + * `status`: Output only. Cluster status. + + * `status_history`: Output only. The previous cluster status. + + * `cluster_uuid`: Output only. A cluster UUID (Unique Universal Identifier). Dataproc generates this value when it creates the cluster. + + * `metrics`: Output only. Contains cluster daemon metrics such as HDFS and YARN stats.Beta Feature: This report is available for testing purposes only. It may be changed before final release. + ## GCP Permissions diff --git a/docs/resources/google_dataproc_clusters.md b/docs/resources/google_dataproc_clusters.md index 39371b642..8bf6bd25d 100644 --- a/docs/resources/google_dataproc_clusters.md +++ b/docs/resources/google_dataproc_clusters.md @@ -26,6 +26,12 @@ See [google_dataproc_cluster.md](google_dataproc_cluster.md) for more detailed i * `labels`: an array of `google_dataproc_cluster` labels * `configs`: an array of `google_dataproc_cluster` config * `regions`: an array of `google_dataproc_cluster` region + * `project_ids`: an array of `google_dataproc_cluster` project_id + * `virtual_cluster_configs`: an array of `google_dataproc_cluster` virtual_cluster_config + * `statuses`: an array of `google_dataproc_cluster` status + * `status_histories`: an array of `google_dataproc_cluster` status_history + * `cluster_uuids`: an array of `google_dataproc_cluster` cluster_uuid + * `metrics`: an array of `google_dataproc_cluster` metrics ## Filter Criteria This resource supports all of the above properties as filter criteria, which can be used diff --git a/libraries/google_dataproc_cluster.rb b/libraries/google_dataproc_cluster.rb index fd594a470..46a363af2 100644 --- a/libraries/google_dataproc_cluster.rb +++ b/libraries/google_dataproc_cluster.rb @@ -42,6 +42,12 @@ class DataprocCluster < GcpResourceBase attr_reader :labels attr_reader :config attr_reader :region + attr_reader :project_id + attr_reader :virtual_cluster_config + attr_reader :status + attr_reader :status_history + attr_reader :cluster_uuid + attr_reader :metrics def initialize(params) super(params.merge({ use_http_transport: true })) @@ -55,6 +61,12 @@ def parse @labels = @fetched['labels'] @config = GoogleInSpec::Dataproc::Property::ClusterConfig.new(@fetched['config'], to_s) @region = @fetched['region'] + @project_id = @fetched['projectId'] + @virtual_cluster_config = @fetched['virtualClusterConfig'] + @status = @fetched['status'] + @status_history = @fetched['statusHistory'] + @cluster_uuid = @fetched['clusterUuid'] + @metrics = @fetched['metrics'] end def exists? diff --git a/libraries/google_dataproc_clusters.rb b/libraries/google_dataproc_clusters.rb index 213d0e434..f15c8a25f 100644 --- a/libraries/google_dataproc_clusters.rb +++ b/libraries/google_dataproc_clusters.rb @@ -27,6 +27,12 @@ class DataprocClusters < GcpResourceBase filter_table_config.add(:labels, field: :labels) filter_table_config.add(:configs, field: :config) filter_table_config.add(:regions, field: :region) + filter_table_config.add(:project_ids, field: :project_id) + filter_table_config.add(:virtual_cluster_configs, field: :virtual_cluster_config) + filter_table_config.add(:statuses, field: :status) + filter_table_config.add(:status_histories, field: :status_history) + filter_table_config.add(:cluster_uuids, field: :cluster_uuid) + filter_table_config.add(:metrics, field: :metrics) filter_table_config.connect(self, :table) @@ -70,6 +76,12 @@ def transformers 'labels' => ->(obj) { [:labels, obj['labels']] }, 'config' => ->(obj) { [:config, GoogleInSpec::Dataproc::Property::ClusterConfig.new(obj['config'], to_s)] }, 'region' => ->(obj) { [:region, obj['region']] }, + 'projectId' => ->(obj) { [:project_id, obj['projectId']] }, + 'virtualClusterConfig' => ->(obj) { [:virtual_cluster_config, obj['virtualClusterConfig']] }, + 'status' => ->(obj) { [:status, obj['status']] }, + 'statusHistory' => ->(obj) { [:status_history, obj['statusHistory']] }, + 'clusterUuid' => ->(obj) { [:cluster_uuid, obj['clusterUuid']] }, + 'metrics' => ->(obj) { [:metrics, obj['metrics']] }, } end diff --git a/test/integration/verify/controls/google_dataproc_cluster.rb b/test/integration/verify/controls/google_dataproc_cluster.rb index b0d3e0a4c..ec6535e61 100644 --- a/test/integration/verify/controls/google_dataproc_cluster.rb +++ b/test/integration/verify/controls/google_dataproc_cluster.rb @@ -57,6 +57,8 @@ its('config.master_config.machine_type_uri') { should match dataproc_cluster['config']['master_config']['machine_type'] } its('config.worker_config.machine_type_uri') { should match dataproc_cluster['config']['worker_config']['machine_type'] } its('config.software_config.properties') { should include(dataproc_cluster['config']['software_config']['prop_key'] => dataproc_cluster['config']['software_config']['prop_value']) } + its('project_id') { should eq gcp_project_id } + its('cluster_uuid') { should eq "test-uuid-67ff7c8e-558a-45ad-97c7" } end describe google_dataproc_cluster(project: gcp_project_id, region: gcp_location, cluster_name: 'nonexistent') do diff --git a/test/integration/verify/controls/google_dataproc_clusters.rb b/test/integration/verify/controls/google_dataproc_clusters.rb index 37fa4d2c0..05984a7fe 100644 --- a/test/integration/verify/controls/google_dataproc_clusters.rb +++ b/test/integration/verify/controls/google_dataproc_clusters.rb @@ -52,5 +52,7 @@ describe google_dataproc_clusters(project: gcp_project_id, region: gcp_location) do its('count') { should be >= 1 } its('cluster_names') { should include dataproc_cluster['name'] } + its('project_ids') { should include gcp_project_id } + its('cluster_uuids') { should include "test-uuid-67ff7c8e-558a-45ad-97c7" } end end