Move AwsExporterMetrics

This is required to restore the metrics count for RDS requests. Since the describe methods where move to the *All Methods, we can't reliable call the number of counts in the rds.go anymore. So move the exporter to the awsclient package.

main.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/app-sre/aws-resource-exporter/pkg"
+	"github.com/app-sre/aws-resource-exporter/pkg/awsclient"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/sts"
@@ -124,7 +125,7 @@ func run() int {
 	level.Info(logger).Log("msg", "Starting"+namespace, "version", version.Info())
 	level.Info(logger).Log("msg", "Build context", version.BuildContext())
 
-	pkg.AwsExporterMetrics = pkg.NewExporterMetrics()
+	awsclient.AwsExporterMetrics = awsclient.NewExporterMetrics(namespace)
 
 	var configFile string
 	if path := os.Getenv("AWS_RESOURCE_EXPORTER_CONFIG_FILE"); path != "" {
@@ -137,7 +138,7 @@ func run() int {
 		level.Error(logger).Log("msg", "Could not load configuration file", "err", err)
 		return 1
 	}
-	collectors := append(cs, pkg.AwsExporterMetrics)
+	collectors := append(cs, awsclient.AwsExporterMetrics)
 	prometheus.MustRegister(
 		collectors...,
 	)

pkg/awsclient/awsclient.go

@@ -68,12 +68,13 @@ func (c *awsClient) DescribeDBLogFilesAll(ctx context.Context, instanceId string
 
 	var logOutPuts []*rds.DescribeDBLogFilesOutput
 	err := c.DescribeDBLogFilesPagesWithContext(ctx, input, func(ddlo *rds.DescribeDBLogFilesOutput, b bool) bool {
-		// AwsExporterMetrics.IncrementRequests()
+		AwsExporterMetrics.IncrementRequests()
 		logOutPuts = append(logOutPuts, ddlo)
 		return true
 	})
 
 	if err != nil {
+		AwsExporterMetrics.IncrementErrors()
 		return nil, err
 	}
 
@@ -85,11 +86,13 @@ func (c *awsClient) DescribePendingMaintenanceActionsAll(ctx context.Context) ([
 
 	var instancesPendMaintActionsData []*rds.ResourcePendingMaintenanceActions
 	err := c.DescribePendingMaintenanceActionsPagesWithContext(ctx, describePendingMaintInput, func(dpm *rds.DescribePendingMaintenanceActionsOutput, b bool) bool {
+		AwsExporterMetrics.IncrementRequests()
 		instancesPendMaintActionsData = append(instancesPendMaintActionsData, dpm.PendingMaintenanceActions...)
 		return true
 	})
 
 	if err != nil {
+		AwsExporterMetrics.IncrementErrors()
 		return nil, err
 	}
 
@@ -101,10 +104,12 @@ func (c *awsClient) DescribeDBInstancesAll(ctx context.Context) ([]*rds.DBInstan
 
 	var instances []*rds.DBInstance
 	err := c.DescribeDBInstancesPagesWithContext(ctx, input, func(ddo *rds.DescribeDBInstancesOutput, b bool) bool {
+		AwsExporterMetrics.IncrementRequests()
 		instances = append(instances, ddo.DBInstances...)
 		return true
 	})
 	if err != nil {
+		AwsExporterMetrics.IncrementErrors()
 		return nil, err
 	}
 	return instances, nil

pkg/exporter.go → pkg/awsclient/exporter.go

@@ -1,4 +1,4 @@
-package pkg
+package awsclient
 
 import (
 	"sync"
@@ -23,7 +23,7 @@ type ExporterMetrics struct {
 }
 
 // NewExporterMetrics creates a new exporter metrics instance
-func NewExporterMetrics() *ExporterMetrics {
+func NewExporterMetrics(namespace string) *ExporterMetrics {
 	return &ExporterMetrics{
 		APIRequests: prometheus.NewDesc(
 			prometheus.BuildFQName(namespace, "", "apirequests"),

pkg/ec2.go

@@ -82,14 +82,14 @@ func (e *EC2Exporter) collectInRegion(sess *session.Session, logger log.Logger,
 	quota, err := getQuotaValueWithContext(aws, ec2ServiceCode, transitGatewayPerAccountQuotaCode, ctx)
 	if err != nil {
 		level.Error(logger).Log("msg", "Could not retrieve Transit Gateway quota", "error", err.Error())
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 
 	gateways, err := getAllTransitGatewaysWithContext(aws, ctx)
 	if err != nil {
 		level.Error(logger).Log("msg", "Could not retrieve Transit Gateway quota", "error", err.Error())
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 

pkg/rds.go

@@ -348,7 +348,6 @@ func (e *RDSExporter) requestRDSLogMetrics(ctx context.Context, sessionIndex int
 	logOutPuts, err := e.svcs[sessionIndex].DescribeDBLogFilesAll(ctx, instanceId)
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to DescribeDBLogFiles failed", "region", e.getRegion(sessionIndex), "instance", &instanceId, "err", err)
-		AwsExporterMetrics.IncrementErrors()
 		return nil, err
 	}
 
@@ -525,7 +524,6 @@ func (e *RDSExporter) CollectLoop() {
 			instances, err := e.svcs[i].DescribeDBInstancesAll(ctx)
 			if err != nil {
 				level.Error(e.logger).Log("msg", "Call to DescribeDBInstances failed", "region", *e.sessions[i].Config.Region, "err", err)
-				AwsExporterMetrics.IncrementErrors()
 			}
 
 			wg := sync.WaitGroup{}

pkg/route53.go

@@ -81,7 +81,7 @@ func (e *Route53Exporter) getRecordsPerHostedZoneMetrics(client *route53.Route53
 
 			if err != nil {
 				errChan <- fmt.Errorf("Could not get Limits for hosted zone with ID '%s' and name '%s'. Error was: %s", *hostedZone.Id, *hostedZone.Name, err.Error())
-				AwsExporterMetrics.IncrementErrors()
+				awsclient.AwsExporterMetrics.IncrementErrors()
 				return
 			}
 			level.Info(e.logger).Log("msg", fmt.Sprintf("Currently at hosted zone: %d / %d", i, len(hostedZones)))
@@ -126,19 +126,19 @@ func (e *Route53Exporter) CollectLoop() {
 		level.Info(e.logger).Log("msg", "Got all zones")
 		if err != nil {
 			level.Error(e.logger).Log("msg", "Could not retrieve the list of hosted zones", "error", err.Error())
-			AwsExporterMetrics.IncrementErrors()
+			awsclient.AwsExporterMetrics.IncrementErrors()
 		}
 
 		err = e.getHostedZonesPerAccountMetrics(client, hostedZones, ctx)
 		if err != nil {
 			level.Error(e.logger).Log("msg", "Could not get limits for hosted zone", "error", err.Error())
-			AwsExporterMetrics.IncrementErrors()
+			awsclient.AwsExporterMetrics.IncrementErrors()
 		}
 
 		errs := e.getRecordsPerHostedZoneMetrics(route53Svc, hostedZones, ctx)
 		for _, err = range errs {
 			level.Error(e.logger).Log("msg", "Could not get limits for hosted zone", "error", err.Error())
-			AwsExporterMetrics.IncrementErrors()
+			awsclient.AwsExporterMetrics.IncrementErrors()
 		}
 
 		level.Info(e.logger).Log("msg", "Route53 metrics Updated")

pkg/vpc.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/app-sre/aws-resource-exporter/pkg/awsclient"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -160,7 +161,7 @@ func (e *VPCExporter) collectVpcsPerRegionQuota(client *servicequotas.ServiceQuo
 	quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_VPCS_PER_REGION)
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to VpcsPerRegion ServiceQuota failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	e.cache.AddMetric(prometheus.MustNewConstMetric(e.VpcsPerRegionQuota, prometheus.GaugeValue, quota, region))
@@ -172,7 +173,7 @@ func (e *VPCExporter) collectVpcsPerRegionUsage(ec2Svc *ec2.EC2, region string)
 	describeVpcsOutput, err := ec2Svc.DescribeVpcsWithContext(ctx, &ec2.DescribeVpcsInput{})
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to DescribeVpcs failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	usage := len(describeVpcsOutput.Vpcs)
@@ -183,7 +184,7 @@ func (e *VPCExporter) collectSubnetsPerVpcQuota(client *servicequotas.ServiceQuo
 	quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_SUBNETS_PER_VPC)
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to SubnetsPerVpc ServiceQuota failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	e.cache.AddMetric(prometheus.MustNewConstMetric(e.SubnetsPerVpcQuota, prometheus.GaugeValue, quota, region))
@@ -200,7 +201,7 @@ func (e *VPCExporter) collectSubnetsPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2, r
 	})
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to DescribeSubnets failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	usage := len(describeSubnetsOutput.Subnets)
@@ -211,7 +212,7 @@ func (e *VPCExporter) collectRoutesPerRouteTableQuota(client *servicequotas.Serv
 	quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_ROUTES_PER_ROUTE_TABLE)
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to RoutesPerRouteTable ServiceQuota failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	e.cache.AddMetric(prometheus.MustNewConstMetric(e.RoutesPerRouteTableQuota, prometheus.GaugeValue, quota, region))
@@ -225,7 +226,7 @@ func (e *VPCExporter) collectRoutesPerRouteTableUsage(rtb *ec2.RouteTable, ec2Sv
 	})
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	quota := len(descRouteTableOutput.RouteTables)
@@ -236,7 +237,7 @@ func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcQuota(client *servicequo
 	quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_INTERFACE_VPC_ENDPOINTS_PER_VPC)
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to InterfaceVpcEndpointsPerVpc ServiceQuota failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	e.cache.AddMetric(prometheus.MustNewConstMetric(e.InterfaceVpcEndpointsPerVpcQuota, prometheus.GaugeValue, quota, region))
@@ -253,7 +254,7 @@ func (e *VPCExporter) collectInterfaceVpcEndpointsPerVpcUsage(vpc *ec2.Vpc, ec2S
 	})
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to DescribeVpcEndpoints failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	quota := len(descVpcEndpoints.VpcEndpoints)
@@ -264,7 +265,7 @@ func (e *VPCExporter) collectRoutesTablesPerVpcQuota(client *servicequotas.Servi
 	quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_ROUTE_TABLES_PER_VPC)
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to RoutesTablesPerVpc ServiceQuota failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	e.cache.AddMetric(prometheus.MustNewConstMetric(e.RouteTablesPerVpcQuota, prometheus.GaugeValue, quota, region))
@@ -281,7 +282,7 @@ func (e *VPCExporter) collectRoutesTablesPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.E
 	})
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to DescribeRouteTables failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	quota := len(descRouteTables.RouteTables)
@@ -292,7 +293,7 @@ func (e *VPCExporter) collectIPv4BlocksPerVpcQuota(client *servicequotas.Service
 	quota, err := e.GetQuotaValue(client, SERVICE_CODE_VPC, QUOTA_IPV4_BLOCKS_PER_VPC)
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to IPv4BlocksPerVpc ServiceQuota failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	e.cache.AddMetric(prometheus.MustNewConstMetric(e.IPv4BlocksPerVpcQuota, prometheus.GaugeValue, quota, region))
@@ -306,7 +307,7 @@ func (e *VPCExporter) collectIPv4BlocksPerVpcUsage(vpc *ec2.Vpc, ec2Svc *ec2.EC2
 	})
 	if err != nil {
 		level.Error(e.logger).Log("msg", "Call to DescribeVpcs failed", "region", region, "err", err)
-		AwsExporterMetrics.IncrementErrors()
+		awsclient.AwsExporterMetrics.IncrementErrors()
 		return
 	}
 	if len(descVpcs.Vpcs) != 1 {