Prevent application from crashing due to Collector errors (#11)

Signed-off-by: Tom M G <[email protected]>
7onn · Apr 30, 2022 · ca684b7 · ca684b7
1 parent 8b00774
commit ca684b7
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 4 deletions.
diff --git a/collector/dataproc_is_cluster_running.go b/collector/dataproc_is_cluster_running.go
@@ -60,6 +60,8 @@ func (e *DataprocIsClusterRunningCollector) Update(ch chan<- prometheus.Metric)
 			regionalDataprocClusters, err := e.service.Projects.Regions.Clusters.List(e.project, region).Do()
 			if err != nil {
 				level.Error(e.logger).Log("msg", fmt.Sprintf("Failure when querying Dataproc Clusters in %s at %s", e.project, region), "err", err)
+				wgRegions.Done()
+				return
 			}
 
 			for _, cluster := range regionalDataprocClusters.Clusters {

diff --git a/collector/gce_is_disk_attached.go b/collector/gce_is_disk_attached.go
@@ -71,11 +71,12 @@ func (e *GCEIsDiskAttachedCollector) Update(ch chan<- prometheus.Metric) error {
 
 		for _, z := range r.Zones {
 			zone := GetGCPZoneFromURL(e.logger, z)
-			ch := make(chan struct{})
-			go func(ch chan struct{}) {
+			go func(ch chan<- prometheus.Metric) {
 				regionalDisks, err := e.service.Disks.List(e.project, zone).Do()
 				if err != nil {
 					level.Error(e.logger).Log("msg", fmt.Sprintf("error requesting machine disks for project %s in zone %s", e.project, zone), "err", err)
+					wgZones.Done()
+					return
 				}
 				disks = append(disks, regionalDisks.Items...)
 				wgZones.Done()

diff --git a/collector/gce_is_machine_running.go b/collector/gce_is_machine_running.go
@@ -71,11 +71,12 @@ func (e *GCEIsMachineRunningCollector) Update(ch chan<- prometheus.Metric) error
 
 		for _, z := range r.Zones {
 			zone := GetGCPZoneFromURL(e.logger, z)
-			ch := make(chan struct{})
-			go func(ch chan struct{}) {
+			go func(ch chan<- prometheus.Metric) {
 				regionalInstances, err := e.service.Instances.List(e.project, zone).Do()
 				if err != nil {
 					level.Error(e.logger).Log("msg", fmt.Sprintf("error requesting machines for project %s in zone %s", e.project, zone), "err", err)
+					wgZones.Done()
+					return
 				}
 				vms = append(vms, regionalInstances.Items...)
 				wgZones.Done()

diff --git a/collector/gce_is_old_snapshot.go b/collector/gce_is_old_snapshot.go
@@ -59,6 +59,7 @@ func (e *GCEIsOldSnapshotCollector) Update(ch chan<- prometheus.Metric) error {
 	snapshots, err := e.service.Snapshots.List(e.project).Do()
 	if err != nil {
 		level.Error(e.logger).Log("msg", fmt.Sprintf("error requesting disk snapshots for project %s", e.project), "err", err)
+		return err
 	}
 
 	sort.Slice(snapshots.Items, func(i, j int) bool {