diff --git a/component/prometheus/exporter/azure/azure.go b/component/prometheus/exporter/azure/azure.go index 95bcd5c1c26f..fa51a1ac01b0 100644 --- a/component/prometheus/exporter/azure/azure.go +++ b/component/prometheus/exporter/azure/azure.go @@ -36,6 +36,7 @@ type Arguments struct { MetricHelpTemplate string `river:"metric_help_template,attr,optional"` AzureCloudEnvironment string `river:"azure_cloud_environment,attr,optional"` ValidateDimensions bool `river:"validate_dimensions,attr,optional"` + Regions []string `river:"regions,attr,optional"` } var DefaultArguments = Arguments{ @@ -78,5 +79,6 @@ func (a *Arguments) Convert() *azure_exporter.Config { MetricHelpTemplate: a.MetricHelpTemplate, AzureCloudEnvironment: a.AzureCloudEnvironment, ValidateDimensions: a.ValidateDimensions, + Regions: a.Regions, } } diff --git a/converter/internal/staticconvert/internal/build/azure_exporter.go b/converter/internal/staticconvert/internal/build/azure_exporter.go index 5c2688eb76d3..b51b36103d44 100644 --- a/converter/internal/staticconvert/internal/build/azure_exporter.go +++ b/converter/internal/staticconvert/internal/build/azure_exporter.go @@ -26,5 +26,6 @@ func toAzureExporter(config *azure_exporter.Config) *azure.Arguments { MetricHelpTemplate: config.MetricHelpTemplate, AzureCloudEnvironment: config.AzureCloudEnvironment, ValidateDimensions: config.ValidateDimensions, + Regions: config.Regions, } } diff --git a/docs/sources/flow/reference/components/prometheus.exporter.azure.md b/docs/sources/flow/reference/components/prometheus.exporter.azure.md index 9363067fd0a7..ef98bc727533 100644 --- a/docs/sources/flow/reference/components/prometheus.exporter.azure.md +++ b/docs/sources/flow/reference/components/prometheus.exporter.azure.md @@ -11,19 +11,28 @@ title: prometheus.exporter.azure # prometheus.exporter.azure -The `prometheus.exporter.azure` component embeds [`azure-metrics-exporter`](https://github.com/webdevops/azure-metrics-exporter) to collect metrics from [Azure Monitor](https://azure.microsoft.com/en-us/products/monitor). The exporter uses [Azure Resource Graph](https://azure.microsoft.com/en-us/get-started/azure-portal/resource-graph/#overview) queries to identify resources for gathering metrics. +The `prometheus.exporter.azure` component embeds [`azure-metrics-exporter`](https://github.com/webdevops/azure-metrics-exporter) to collect metrics from [Azure Monitor](https://azure.microsoft.com/en-us/products/monitor). The exporter supports all metrics defined by Azure Monitor. You can find the complete list of available metrics in the [Azure Monitor documentation](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported). Metrics for this integration are exposed with the template `azure_{type}_{metric}_{aggregation}_{unit}` by default. As an example, the Egress metric for BlobService would be exported as `azure_microsoft_storage_storageaccounts_blobservices_egress_total_bytes`. +The exporter offers two options for gathering metrics, +1. (Default) Use an [Azure Resource Graph](https://azure.microsoft.com/en-us/get-started/azure-portal/resource-graph/#overview) query to identify resources for gathering metrics + 1. This will make 1 API call per resource identified + 1. Subscriptions with a reasonable amount of resources are liable to hit the [12000 requests per hour rate limit](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling#subscription-and-tenant-limits) azure enforces +1. Set the regions to gather metrics from and get metrics for all resources across those regions + 1. This will make an API call per subscription reducing the number of API calls dramatically + 1. This approach does not work with all resource types and Azure's does not document which resource types do/do not work + 1. A resource type which is not support will produce errors which look like `Resource type: microsoft.containerservice/managedclusters not enabled for Cross Resource metrics` + ## Authentication {{< param "PRODUCT_NAME" >}} must be running in an environment with access to Azure. The exporter uses the Azure SDK for go and supports [authentication](https://learn.microsoft.com/en-us/azure/developer/go/azure-sdk-authentication?tabs=bash#2-authenticate-with-azure). The account used by {{< param "PRODUCT_NAME" >}} needs: -- [Read access to the resources that will be queried by Resource Graph](https://learn.microsoft.com/en-us/azure/governance/resource-graph/overview#permissions-in-azure-resource-graph) +- When using an Azure Resoure Graph query, [read access to the resources that will be queried by Resource Graph](https://learn.microsoft.com/en-us/azure/governance/resource-graph/overview#permissions-in-azure-resource-graph) - Permissions to call the [Microsoft.Insights Metrics API](https://learn.microsoft.com/en-us/rest/api/monitor/metrics/list) which should be the `Microsoft.Insights/Metrics/Read` permission ## Usage @@ -51,23 +60,26 @@ prometheus.exporter.azure LABEL { You can use the following arguments to configure the exporter's behavior. Omitted fields take their default values. -| Name | Type | Description | Default | Required | -| ----------------------------- | -------------- | -------------------------------------------------------------------- | ----------------------------------------------------------------------------- | -------- | -| `subscriptions` | `list(string)` | List of subscriptions to scrap metrics from. | | yes | -| `resource_type` | `string` | The Azure Resource Type to scrape metrics for. | | yes | -| `metrics` | `list(string)` | The metrics to scrape from resources. | | yes | -| `resource_graph_query_filter` | `string` | The [Kusto query][] filter to apply when searching for resources. | | no | -| `metric_aggregations` | `list(string)` | Aggregations to apply for the metrics produced. | | no | -| `timespan` | `string` | [ISO8601 Duration][] over which the metrics are being queried. | `"PT1M"` (1 minute) | no | -| `included_dimensions` | `list(string)` | List of dimensions to include on the final metrics. | | no | -| `included_resource_tags` | `list(string)` | List of resource tags to include on the final metrics. | `["owner"]` | no | -| `metric_namespace` | `string` | Namespace for `resource_type` which have multiple levels of metrics. | | no | -| `azure_cloud_environment` | `string` | Name of the cloud environment to connect to. | `"azurecloud"` | no | -| `metric_name_template` | `string` | Metric template used to expose the metrics. | `"azure_{type}_{metric}_{aggregation}_{unit}"` | no | -| `metric_help_template` | `string` | Description of the metric. | `"Azure metric {metric} for {type} with aggregation {aggregation} as {unit}"` | no | +| Name | Type | Description | Default | Required | +|-------------------------------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------|----------| +| `subscriptions` | `list(string)` | List of subscriptions to scrap metrics from. | | yes | +| `resource_type` | `string` | The Azure Resource Type to scrape metrics for. | | yes | +| `metrics` | `list(string)` | The metrics to scrape from resources. | | yes | +| `resource_graph_query_filter` | `string` | The [Kusto query][] filter to apply when searching for resources. Cannot be used if `regions` is set. | | no | +| `regions` | `list(string)` | The list of regions for gathering metrics and enables gathering metrics for all resources in the subscription. Cannot be used if `resource_graph_query_filter` is set. | | no | +| `metric_aggregations` | `list(string)` | Aggregations to apply for the metrics produced. | | no | +| `timespan` | `string` | [ISO8601 Duration][] over which the metrics are being queried. | `"PT1M"` (1 minute) | no | +| `included_dimensions` | `list(string)` | List of dimensions to include on the final metrics. | | no | +| `included_resource_tags` | `list(string)` | List of resource tags to include on the final metrics. | `["owner"]` | no | +| `metric_namespace` | `string` | Namespace for `resource_type` which have multiple levels of metrics. | | no | +| `azure_cloud_environment` | `string` | Name of the cloud environment to connect to. | `"azurecloud"` | no | +| `metric_name_template` | `string` | Metric template used to expose the metrics. | `"azure_{type}_{metric}_{aggregation}_{unit}"` | no | +| `metric_help_template` | `string` | Description of the metric. | `"Azure metric {metric} for {type} with aggregation {aggregation} as {unit}"` | no | The list of available `resource_type` values and their corresponding `metrics` can be found in [Azure Monitor essentials][]. +The list of available `regions` to your subscription can be found by running the azure CLI command `az account list-locations --query '[].name'`. + The `resource_graph_query_filter` can be embedded into a template query of the form `Resources | where type =~ "" | project id, tags`. Valid values for `metric_aggregations` are `minimum`, `maximum`, `average`, `total`, and `count`. If no aggregation is specified, the value is retrieved from the metric. For example, the aggregation value of the metric `Availability` in [Microsoft.ClassicStorage/storageAccounts](https://learn.microsoft.com/en-us/azure/azure-monitor/reference/supported-metrics/microsoft-classicstorage-storageaccounts-metrics) is `average`. @@ -107,6 +119,9 @@ debug metrics. prometheus.exporter.azure "example" { subscriptions = SUBSCRIPTIONS resource_type = "Microsoft.Storage/storageAccounts" + regions = [ + "westeurope", + ] metric_namespace = "Microsoft.Storage/storageAccounts/blobServices" metrics = [ "Availability", @@ -120,8 +135,11 @@ prometheus.exporter.azure "example" { "SuccessServerLatency", "Transactions", ] + included_dimensions = [ + "ApiName", + "TransactionType", + ] timespan = "PT1H" - resource_graph_query_filter = "where location == 'westeurope'" } // Configure a prometheus.scrape component to send metrics to. diff --git a/docs/sources/static/configuration/integrations/azure-exporter-config.md b/docs/sources/static/configuration/integrations/azure-exporter-config.md index 981b0a523142..80f4eeda3dde 100644 --- a/docs/sources/static/configuration/integrations/azure-exporter-config.md +++ b/docs/sources/static/configuration/integrations/azure-exporter-config.md @@ -13,9 +13,16 @@ title: azure_exporter_config ## Overview The `azure_exporter_config` block configures the `azure_exporter` integration, an embedded version of [`azure-metrics-exporter`](https://github.com/webdevops/azure-metrics-exporter), used to -collect metrics from [Azure Monitor](https://azure.microsoft.com/en-us/products/monitor). The -exporter uses [Azure Resource Graph](https://azure.microsoft.com/en-us/get-started/azure-portal/resource-graph/#overview) -queries to identify resources for gathering metrics. +collect metrics from [Azure Monitor](https://azure.microsoft.com/en-us/products/monitor). + +The exporter offers two options for gathering metrics, +1. (Default) Use an [Azure Resource Graph](https://azure.microsoft.com/en-us/get-started/azure-portal/resource-graph/#overview) query to identify resources for gathering metrics + 1. This will make 1 API call per resource identified + 1. Subscriptions with a reasonable amount of resources are liable to hit the [12000 requests per hour rate limit](https://learn.microsoft.com/en-us/azure/azure-resource-manager/management/request-limits-and-throttling#subscription-and-tenant-limits) azure enforces +1. Set the regions to gather metrics from and get metrics for all resources across those regions + 1. This will make an API call per subscription reducing the number of API calls dramatically + 1. This approach does not work with all resource types and Azure's does not document which resource types do/do not work + 1. A resource type which is not support will produce errors which look like `Resource type: microsoft.containerservice/managedclusters not enabled for Cross Resource metrics` ## List of Supported Services and Metrics The exporter supports all metrics defined by Azure Monitor. The complete list of available metrics can be found in the [Azure Monitor documentation](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported). @@ -95,7 +102,14 @@ The account used by Grafana Agent needs: # Optional: The [kusto query](https://learn.microsoft.com/en-us/azure/data-explorer/kusto/query/) filter to apply when searching for resources # This value will be embedded in to a template query of the form `Resources | where type =~ "" | project id, tags` + # Cannot be used if `regions` is set. [resource_graph_query_filter: ] + + # Optional: The list of regions for gathering metrics. Enables gather metrics for all resources in the subscription. + # The list of available `regions` to your subscription can be found by running the azure CLI command `az account list-locations --query '[].name'`. + # Cannot be used if `resource_graph_query_filter` is set. + regions: + [ - ... ] # Optional: Aggregation to apply for the metrics produced. Valid values are minimum, maximum, average, total, and count # If no aggregation is specified the value for `Aggregation Type` on the `Metric` is used from https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported @@ -167,6 +181,7 @@ The account used by Grafana Agent needs: included_dimensions: - node - nodepool + - device ``` #### Blob Storage Metrics @@ -178,6 +193,8 @@ The account used by Grafana Agent needs: - resource_type: Microsoft.Storage/storageAccounts metric_namespace: Microsoft.Storage/storageAccounts/blobServices + regions: + - westeurope metrics: - Availability - BlobCapacity @@ -189,8 +206,10 @@ The account used by Grafana Agent needs: - SuccessE2ELatency - SuccessServerLatency - Transactions + included_dimensions: + - ApiName + - TransactionType timespan: PT1H - resource_graph_query_filter: where location == "westeurope" ``` ### Multiple Azure Services in a single config @@ -222,6 +241,8 @@ metrics: subscriptions: - 179c4f30-ebd8-489e-92bc-fb64588dadb3 resource_type: ["Microsoft.Storage/storageAccounts"] + regions: + - westeurope metric_namespace: ["Microsoft.Storage/storageAccounts/blobServices"] metrics: - Availability @@ -234,8 +255,10 @@ metrics: - SuccessE2ELatency - SuccessServerLatency - Transactions + included_dimensions: + - ApiName + - TransactionType timespan: ["PT1H"] - resource_graph_query_filter: ["where location == 'westeurope'"] - job_name: azure-kubernetes-node scrape_interval: 1m scrape_timeout: 50s @@ -263,6 +286,7 @@ metrics: included_dimensions: - node - nodepool + - device ``` In this example, all `azure_exporter`-specific configuration settings have been moved to the `scrape_config`. This method supports all available configuration options except `azure_cloud_environment`, which must be configured on the `azure_exporter`. For this method, if a field supports a singular value like `resource_graph_query_filter`, you diff --git a/pkg/integrations/azure_exporter/azure_exporter.go b/pkg/integrations/azure_exporter/azure_exporter.go index 864485732d7d..1ecd77d2b553 100644 --- a/pkg/integrations/azure_exporter/azure_exporter.go +++ b/pkg/integrations/azure_exporter/azure_exporter.go @@ -24,7 +24,7 @@ type Exporter struct { } func (e Exporter) MetricsHandler() (http.Handler, error) { - //Safe to re-use as it doesn't connect to anything directly + // Safe to re-use as it doesn't connect to anything directly client, err := armclient.NewArmClientWithCloudName(e.cfg.AzureCloudEnvironment, e.logger) if err != nil { return nil, fmt.Errorf("failed to create azure client, %v", err) @@ -35,7 +35,13 @@ func (e Exporter) MetricsHandler() (http.Handler, error) { ctx := context.Background() params := req.URL.Query() - mergedConfig := MergeConfigWithQueryParams(e.cfg, params) + mergedConfig, err := MergeConfigWithQueryParams(e.cfg, params) + if err != nil { + err = fmt.Errorf("failed to merge config with query parameters, %v", err) + e.logger.Error(err) + http.Error(resp, err.Error(), http.StatusBadRequest) + return + } if err := mergedConfig.Validate(); err != nil { err = fmt.Errorf("config to be used for scraping was invalid, %v", err) @@ -71,15 +77,23 @@ func (e Exporter) MetricsHandler() (http.Handler, error) { prober.SetPrometheusRegistry(reg) prober.SetAzureResourceTagManager(tagManager) - err = prober.ServiceDiscovery.FindResourceGraph(ctx, settings.Subscriptions, settings.ResourceType, settings.Filter) - if err != nil { - e.logger.Error(fmt.Errorf("service discovery failed, %v", err)) - http.Error(resp, "Failed to discovery azure resources", http.StatusInternalServerError) - return + // When this filter does not have a value then the request is for all resources in the subscription. + // "RunOnSubscriptionScope" uses a different API, https://github.com/Azure/azure-rest-api-specs/blob/main/specification/monitor/resource-manager/Microsoft.Insights/stable/2021-05-01/metrics_API.json#L40, + // which can get metric data for all resources in a single API call reducing overhead/likelihood of being rate limited. + // Limiting to specific resources requires 1 API call per resource to get metrics which can easily lead to rate limiting + if settings.Filter == "" { + prober.RunOnSubscriptionScope() + } else { + err = prober.ServiceDiscovery.FindResourceGraph(ctx, settings.Subscriptions, settings.ResourceType, settings.Filter) + if err != nil { + e.logger.Error(fmt.Errorf("service discovery failed, %v", err)) + http.Error(resp, "Failed to discovery azure resources", http.StatusInternalServerError) + return + } + + prober.Run() } - prober.Run() - promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(resp, req) }) return h, nil diff --git a/pkg/integrations/azure_exporter/config.go b/pkg/integrations/azure_exporter/config.go index 9f6615ccb2d4..17893dc435a0 100644 --- a/pkg/integrations/azure_exporter/config.go +++ b/pkg/integrations/azure_exporter/config.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/url" + "strconv" "strings" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" @@ -42,6 +43,7 @@ var DefaultConfig = Config{ type Config struct { Subscriptions []string `yaml:"subscriptions"` // Required ResourceGraphQueryFilter string `yaml:"resource_graph_query_filter"` // Optional + Regions []string `yaml:"regions"` // Optional // Valid values can be derived from https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported // Required: Root level names ex. Microsoft.DataShare/accounts @@ -125,6 +127,10 @@ func (c *Config) Validate() error { configErrors = append(configErrors, "metrics cannot be empty") } + if len(c.Regions) > 0 && c.ResourceGraphQueryFilter != "" { + configErrors = append(configErrors, "regions and resource_graph_query_filter cannot be used together. If you want to target specific resources add a region filter to the resource_graph_query_filter. Otherwise, remove your resource_graph_query_filter to get metrics without further filtering.") + } + validAggregations := []string{"minimum", "maximum", "average", "total", "count"} for _, aggregation := range c.MetricAggregations { @@ -167,6 +173,7 @@ func (c *Config) ToScrapeSettings() (*metrics.RequestMetricSettings, error) { MetricTemplate: c.MetricNameTemplate, HelpTemplate: c.MetricHelpTemplate, ValidateDimensions: c.ValidateDimensions, + Regions: c.Regions, // Interval controls data aggregation timeframe ie 1 minute or 5 minutes aggregations // Timespan controls query start and end time @@ -211,7 +218,7 @@ func (c *Config) ToScrapeSettings() (*metrics.RequestMetricSettings, error) { // MergeConfigWithQueryParams will map values from params which where the key // matches a yaml tag of the Config struct -func MergeConfigWithQueryParams(cfg Config, params url.Values) Config { +func MergeConfigWithQueryParams(cfg Config, params url.Values) (Config, error) { if subscriptions, exists := params["subscriptions"]; exists { cfg.Subscriptions = subscriptions } @@ -262,7 +269,20 @@ func MergeConfigWithQueryParams(cfg Config, params url.Values) Config { cfg.MetricHelpTemplate = helpTemplate } - return cfg + if regions, exists := params["regions"]; exists { + cfg.Regions = regions + } + + validateDimensions := params.Get("validate_dimensions") + if len(validateDimensions) != 0 { + v, err := strconv.ParseBool(validateDimensions) + if err != nil { + return Config{}, fmt.Errorf("invalid boolean value %s for validate_dimensions", validateDimensions) + } + cfg.ValidateDimensions = v + } + + return cfg, nil } // getHash calculates the MD5 hash of the yaml representation of the config diff --git a/pkg/integrations/azure_exporter/config_test.go b/pkg/integrations/azure_exporter/config_test.go index afae4b2746b2..4690184a59c2 100644 --- a/pkg/integrations/azure_exporter/config_test.go +++ b/pkg/integrations/azure_exporter/config_test.go @@ -38,7 +38,7 @@ func TestConfig_ToScrapeSettings(t *testing.T) { MetricTemplate: "name_template_me", HelpTemplate: "help_template_me", - //Should not be set + // Should not be set Name: "", MetricTop: nil, MetricFilter: "", @@ -177,6 +177,14 @@ func TestConfig_Validate(t *testing.T) { return config }, }, + { + name: "includes Regions and ResourceGraphQueryFilter", + toInvalidConfig: func(config azure_exporter.Config) azure_exporter.Config { + config.ResourceGraphQueryFilter = "filter the resources" + config.Regions = []string{"uswest", "useast"} + return config + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -193,7 +201,7 @@ func TestMergeConfigWithQueryParams_MapsAllExpectedFieldsByYamlNameFromConfig(t var mappableFields []reflect.StructField for i := 0; i < thing.NumField(); i++ { field := thing.Field(i) - //Not available to be mapped via query param + // Not available to be mapped via query param if field.Name == "AzureCloudEnvironment" { continue } @@ -216,6 +224,9 @@ func TestMergeConfigWithQueryParams_MapsAllExpectedFieldsByYamlNameFromConfig(t value := []string{"fake string 1", "fake string 2"} fieldValue = value urlParams[yamlFieldName] = value + case "bool": + urlParams[yamlFieldName] = []string{"false"} + fieldValue = false default: t.Fatalf("Attempting to map %s, discovered unexpected type %s", mappableField.Name, mappableField.Type.String()) } @@ -223,7 +234,8 @@ func TestMergeConfigWithQueryParams_MapsAllExpectedFieldsByYamlNameFromConfig(t expectedConfig := &azure_exporter.Config{} reflect.ValueOf(expectedConfig).Elem().FieldByName(mappableField.Name).Set(reflect.ValueOf(fieldValue)) - actualConfig := azure_exporter.MergeConfigWithQueryParams(azure_exporter.Config{}, urlParams) + actualConfig, err := azure_exporter.MergeConfigWithQueryParams(azure_exporter.Config{}, urlParams) + require.NoError(t, err) require.Equal(t, *expectedConfig, actualConfig) }) }