Merge pull request #4994 from consideRatio/pr/dashboards-dev

AWS cost attribution: distinguish total costs for attributable costs and the AWS account's total

grafana-dashboards/README.md

@@ -0,0 +1,21 @@
+# About these grafana dashboard files
+
+These are dashboard definitions as jsonnet templates. They are deployed using a
+Python script from https://github.com/jupyterhub/grafana-dashboard, which can be
+done via the deployer command:
+
+```bash
+deployer grafana deploy-dashboards $CLUSTER_NAME
+```
+
+Running this command has a pre-requisite that you have jsonnet installed,
+specifically the jsonnet binary built using golang called go-jsonnet.
+
+To just render the jsonnet templates, which is relevant during development, you
+can:
+
+1. Clone https://github.com/jupyterhub/grafana-dashboard somewhere
+2. Go to that folder, and then run something like:
+   ```bash
+   jsonnet -J vendor /some/path/2i2c-org/infrastructure/grafana-dashboards/cloud-cost-aws.jsonnet
+   ```

grafana-dashboards/cloud-cost-aws.jsonnet

@@ -19,10 +19,6 @@ local totalDailyCosts =
       common.queryTarget
       + {
           url: "http://aws-ce-grafana-backend.support.svc.cluster.local/total-costs?from=${__from:date}&to=${__to:date}",
-          columns: [
-            {selector: "cost", text: "Cost", type: "number"},
-            {selector: "date", text: "Date", type: "timestamp"},
-          ],
         }
   ]);
 
@@ -39,11 +35,6 @@ local totalDailyCostsPerHub =
       common.queryTarget
       + {
           url: "http://aws-ce-grafana-backend.support.svc.cluster.local/total-costs-per-hub?from=${__from:date}&to=${__to:date}",
-          columns: [
-            {selector: "date", text: "Date", type: "timestamp"},
-            {selector: "name", text: "Name", type: "string"},
-            {selector: "cost", text: "Cost", type: "number"}
-          ],
         }
   ]);
 
@@ -60,11 +51,6 @@ local totalDailyCostsPerComponent =
       common.queryTarget
       + {
           url: "http://aws-ce-grafana-backend.support.svc.cluster.local/total-costs-per-component?from=${__from:date}&to=${__to:date}",
-          columns: [
-            {selector: "date", text: "Date", type: "timestamp"},
-            {selector: "name", text: "Name", type: "string"},
-            {selector: "cost", text: "Cost", type: "number"}
-          ],
         }
   ]);
 
@@ -83,16 +69,16 @@ local totalDailyCostsPerComponentAndHub =
       common.queryTarget
       + {
           url: "http://aws-ce-grafana-backend.support.svc.cluster.local/total-costs-per-component?from=${__from:date}&to=${__to:date}&hub=${hub}",
-          columns: [
-            {selector: "date", text: "Date", type: "timestamp"},
-            {selector: "name", text: "Name", type: "string"},
-            {selector: "cost", text: "Cost", type: "number"}
-          ],
         }
   ]);
 
 
 // grafonnet ref: https://grafana.github.io/grafonnet/API/dashboard/index.html
+//
+// A dashboard description can be provided, but isn't used much it seems, due to
+// that we aren't providing one atm.
+// See https://community.grafana.com/t/dashboard-description-is-it-used-anywhere/53273.
+//
 dashboard.new("Cloud cost attribution")
 + dashboard.withUid("cloud-cost-aws")
 + dashboard.withTimezone("utc")

grafana-dashboards/common.libsonnet

@@ -42,6 +42,11 @@ local ts = grafonnet.panel.timeSeries;
       type: "yesoreyeram-infinity-datasource",
       uid: "${infinity_datasource}",
     },
+    columns: [
+      {selector: "date", text: "Date", type: "timestamp"},
+      {selector: "name", text: "Name", type: "string"},
+      {selector: "cost", text: "Cost", type: "number"}
+    ],
     parser: "backend",
     type: "json",
     source: "url",

helm-charts/aws-ce-grafana-backend/mounted-files/README.md

@@ -12,20 +12,20 @@ faster.
 
 ### Testing Python changes locally
 
-First authenticate yourself against the AWS openscapes account.
+First authenticate yourself against an AWS account.
 
 ```bash
 cd helm-charts/aws-ce-grafana-backend/mounted-files
-export AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME=openscapeshub
+export AWS_CE_GRAFANA_BACKEND__CLUSTER_NAME=<name of cluster according to eksctl config>
 python -m flask --app=webserver run --port=8080
 
-# visit http://localhost:8080/aws
+# visit http://localhost:8080/hub-names
 ```
 
 ### Testing Python changes in k8s
 
-This was initially developed in the openscapes cluster. It depends on a k8s
-ServiceAccount coupled to an IAM Role there as well.
+This requires a k8s ServiceAccount coupled to an IAM Role prepared in advance
+via terraform.
 
 The image shouldn't need to be rebuilt unless additional dependencies needs to
 be installed etc, so if you've only made code changes, you can do the following
@@ -35,7 +35,7 @@ During development, a procedure like below can be used to iterate faster than by
 using the deployer.
 
 ```bash
-deployer use-cluster-credentials openscapes
+deployer use-cluster-credentials $CLUSTER_NAME
 
 cd helm-charts/aws-ce-grafana-backend
 helm upgrade --install --create-namespace -n support --values my-test-config.yaml aws-ce-grafana-backend .
@@ -45,7 +45,7 @@ helm upgrade --install --create-namespace -n support --values my-test-config.yam
 # restarts.
 kubectl port-forward -n support service/aws-ce-grafana-backend 8080:http
 
-# visit http://localhost:8080/total-costs and other urls
+# visit http://localhost:8080/hub-names and other urls
 ```
 
 It assumes that you have a `my-test-config.yaml` file looking like this:

helm-charts/aws-ce-grafana-backend/mounted-files/query.py

@@ -94,20 +94,54 @@ def query_hub_names(from_date, to_date):
 @ttl_lru_cache(seconds_to_live=3600)
 def query_total_costs(from_date, to_date):
     """
-    A query with processing of the response tailored query to report hub
-    independent total costs.
+    A query with processing of the response tailored to report both the total
+    AWS account cost, and the total attributable cost.
+
+    Not all costs will be successfully attributed, such as the cost of accessing
+    the AWS Cost Explorer API - its not something that can be attributed based
+    on a tag.
+    """
+    total_account_costs = _query_total_costs(
+        from_date, to_date, add_attributable_costs_filter=False
+    )
+    total_attributable_costs = _query_total_costs(
+        from_date, to_date, add_attributable_costs_filter=True
+    )
+
+    processed_response = total_account_costs + total_attributable_costs
+
+    # the infinity plugin appears needs us to sort by date, otherwise it fails
+    # to distinguish time series by the name field for some reason
+    processed_response = sorted(processed_response, key=lambda x: x["date"])
+
+    return processed_response
+
+
+@ttl_lru_cache(seconds_to_live=3600)
+def _query_total_costs(from_date, to_date, add_attributable_costs_filter):
     """
+    A query with processing of the response tailored to report total costs.
+
+    It can either be the total account costs, or only the attributable costs.
+    """
+    if add_attributable_costs_filter:
+        name = "attributable"
+        filter = {
+            "And": [
+                FILTER_USAGE_COSTS,
+                FILTER_ATTRIBUTABLE_COSTS,
+            ]
+        }
+    else:
+        name = "account"
+        filter = FILTER_USAGE_COSTS
+
     response = query_aws_cost_explorer(
         metrics=[METRICS_UNBLENDED_COST],
         granularity=GRANULARITY_DAILY,
         from_date=from_date,
         to_date=to_date,
-        filter={
-            "And": [
-                FILTER_USAGE_COSTS,
-                FILTER_ATTRIBUTABLE_COSTS,
-            ]
-        },
+        filter=filter,
         group_by=[],
     )
 
@@ -144,6 +178,7 @@ def query_total_costs(from_date, to_date):
         {
             "date": e["TimePeriod"]["Start"],
             "cost": f'{float(e["Total"]["UnblendedCost"]["Amount"]):.2f}',
+            "name": name,
         }
         for e in response["ResultsByTime"]
     ]
@@ -153,9 +188,8 @@ def query_total_costs(from_date, to_date):
 @ttl_lru_cache(seconds_to_live=3600)
 def query_total_costs_per_hub(from_date, to_date):
     """
-    A query with processing of the response tailored query to report total costs
-    per hub, where costs not attributed to a specific hub is listed under
-    'shared'.
+    A query with processing of the response tailored to report total costs per
+    hub, where costs not attributed to a specific hub is listed under 'shared'.
     """
     response = query_aws_cost_explorer(
         metrics=[METRICS_UNBLENDED_COST],
@@ -238,8 +272,8 @@ def query_total_costs_per_hub(from_date, to_date):
 @ttl_lru_cache(seconds_to_live=3600)
 def query_total_costs_per_component(from_date, to_date, hub_name=None):
     """
-    A query with processing of the response tailored query to report total costs
-    per component - a grouping of services.
+    A query with processing of the response tailored to report total costs per
+    component - a grouping of services.
 
     If a hub_name is specified, component costs are filtered to only consider
     costs directly attributable to the hub name.