Add UOM mapping

component/class/defaults.yml

@@ -66,3 +66,6 @@ parameters:
       # How many days ago to get the metrics
       # The result is always 1 day of metrics
       days: 1
+
+    # Unit of measure map "cloud service value": "odoo16 value"
+    uom: {}

component/component/main.jsonnet

@@ -104,7 +104,7 @@ local serviceAccount(name, clusterRole) = {
   rb: rb,
 };
 
-local deployment(name, args, cm) =
+local deployment(name, args, config) =
   kube.Deployment(name) {
     metadata+: {
       labels+: labels,
@@ -122,7 +122,7 @@ local deployment(name, args, cm) =
               envFrom: [
                 {
                   configMapRef: {
-                    name: cm,
+                    name: config,
                   },
                 },
                 {
@@ -149,6 +149,7 @@ local config(name, extraConfig) = kube.ConfigMap(name) {
     CLUSTER_ID: std.toString(params.clusterId),
     APPUIO_MANAGED_SALES_ORDER: if appuioManaged then std.toString(params.appuioManaged.salesOrder) else '',
     PROM_URL: if !appuioManaged then std.toString(params.promUrl) else '',
+    UOM: std.toString(params.uom),
   },
 } + extraConfig;
 
@@ -184,7 +185,6 @@ local config(name, extraConfig) = kube.ConfigMap(name) {
    exoDbaasRoleBinding: sa.rb,
    exoDbaasConfigMap: cm,
    exoDbaasExporter: deployment(name, [ 'exoscale', 'dbaas' ], name + '-env'),
-
  } else {})
 +
 (if params.exoscale.enabled && params.exoscale.objectStorage.enabled then {

component/tests/fromenv.yml

@@ -37,3 +37,9 @@ parameters:
 
     cloudscale:
       enabled: true
+
+    uom:
+      GBDay: uom_uom_78_b847edc1
+      KReq: uom_uom_60_a83156c7
+      GB: uom_uom_71_6f28fc21
+      Instance-Hour: uom_uom_45_1e112771

component/tests/golden/cloudscale-metrics-collector-cloud/billing-collector-cloudservices/billing-collector-cloudservices/cloudscaleConfigMap.yaml

@@ -8,6 +8,7 @@ data:
   ODOO_OAUTH_TOKEN_URL: https://test.central.vshn.ch/api/v2/authentication/oauth2/token
   ODOO_URL: https://test.central.vshn.ch/api/v2/product_usage_report_POST
   PROM_URL: localhost:9090
+  UOM: '{ }'
 kind: ConfigMap
 metadata:
   name: cloudscale-env

component/tests/golden/cloudscale-metrics-collector-managed/billing-collector-cloudservices/billing-collector-cloudservices/cloudscaleConfigMap.yaml

@@ -8,6 +8,7 @@ data:
   ODOO_OAUTH_TOKEN_URL: https://test.central.vshn.ch/api/v2/authentication/oauth2/token
   ODOO_URL: https://test.central.vshn.ch/api/v2/product_usage_report_POST
   PROM_URL: ''
+  UOM: '{ }'
 kind: ConfigMap
 metadata:
   name: cloudscale-env

component/tests/golden/exoscale-metrics-collector-cloud/billing-collector-cloudservices/billing-collector-cloudservices/exoDbaasConfigMap.yaml

@@ -6,6 +6,7 @@ data:
   ODOO_OAUTH_TOKEN_URL: https://test.central.vshn.ch/api/v2/authentication/oauth2/token
   ODOO_URL: https://test.central.vshn.ch/api/v2/product_usage_report_POST
   PROM_URL: localhost:9090
+  UOM: '{ }'
 kind: ConfigMap
 metadata:
   name: exoscale-dbaas-env

component/tests/golden/exoscale-metrics-collector-cloud/billing-collector-cloudservices/billing-collector-cloudservices/exoObjectStorageConfigMap.yaml

@@ -7,6 +7,7 @@ data:
   ODOO_OAUTH_TOKEN_URL: https://test.central.vshn.ch/api/v2/authentication/oauth2/token
   ODOO_URL: https://test.central.vshn.ch/api/v2/product_usage_report_POST
   PROM_URL: localhost:9090
+  UOM: '{ }'
 kind: ConfigMap
 metadata:
   name: exoscale-objectstorage-env

component/tests/golden/exoscale-metrics-collector-managed/billing-collector-cloudservices/billing-collector-cloudservices/exoDbaasConfigMap.yaml

@@ -6,6 +6,7 @@ data:
   ODOO_OAUTH_TOKEN_URL: https://test.central.vshn.ch/api/v2/authentication/oauth2/token
   ODOO_URL: https://test.central.vshn.ch/api/v2/product_usage_report_POST
   PROM_URL: ''
+  UOM: '{ }'
 kind: ConfigMap
 metadata:
   name: exoscale-dbaas-env

component/tests/golden/exoscale-metrics-collector-managed/billing-collector-cloudservices/billing-collector-cloudservices/exoObjectStorageConfigMap.yaml

@@ -7,6 +7,7 @@ data:
   ODOO_OAUTH_TOKEN_URL: https://test.central.vshn.ch/api/v2/authentication/oauth2/token
   ODOO_URL: https://test.central.vshn.ch/api/v2/product_usage_report_POST
   PROM_URL: ''
+  UOM: '{ }'
 kind: ConfigMap
 metadata:
   name: exoscale-objectstorage-env

pkg/cloudscale/fixtures.go

@@ -1,5 +1,7 @@
 package cloudscale
 
+import "github.com/vshn/billing-collector-cloudservices/pkg/odoo"
+
 const (
 	// source format: 'query:zone:tenant:namespace' or 'query:zone:tenant:namespace:class'
 	// We do not have real (prometheus) queries here, just random hardcoded strings.
@@ -15,7 +17,7 @@ var (
 )
 
 var units = map[string]string{
-	productIdStorage:       "GBDay",
-	productIdTrafficOut:    "GB",
-	productIdQueryRequests: "KReq",
+	productIdStorage:       odoo.GBDay,
+	productIdTrafficOut:    odoo.GB,
+	productIdQueryRequests: odoo.KReq,
 }

pkg/cloudscale/objectstorage.go

@@ -27,20 +27,21 @@ type ObjectStorage struct {
 	promClient   apiv1.API
 	salesOrderId string
 	clusterId    string
+	uomMapping   map[string]string
 }
 
 const (
-	organizationLabel = "appuio.io/organization"
-	namespaceLabel    = "crossplane.io/claim-namespace"
+	namespaceLabel = "crossplane.io/claim-namespace"
 )
 
-func NewObjectStorage(client *cloudscale.Client, k8sClient client.Client, promClient apiv1.API, salesOrderId, clusterId string) (*ObjectStorage, error) {
+func NewObjectStorage(client *cloudscale.Client, k8sClient client.Client, promClient apiv1.API, salesOrderId, clusterId string, uomMapping map[string]string) (*ObjectStorage, error) {
 	return &ObjectStorage{
 		client:       client,
 		k8sClient:    k8sClient,
 		promClient:   promClient,
 		salesOrderId: salesOrderId,
 		clusterId:    clusterId,
+		uomMapping:   uomMapping,
 	}, nil
 }
 
@@ -90,7 +91,7 @@ func (o *ObjectStorage) GetMetrics(ctx context.Context, billingDate time.Time) (
 				continue
 			}
 		}
-		records, err := createOdooRecord(bucketMetricsData, bd, appuioManaged, o.salesOrderId, o.clusterId)
+		records, err := o.createOdooRecord(bucketMetricsData, bd, appuioManaged)
 		if err != nil {
 			logger.Error(err, "unable to create Odoo Record", "namespace", bd.Namespace)
 			continue
@@ -102,7 +103,7 @@ func (o *ObjectStorage) GetMetrics(ctx context.Context, billingDate time.Time) (
 	return allRecords, nil
 }
 
-func createOdooRecord(bucketMetricsData cloudscale.BucketMetricsData, b BucketDetail, appuioManaged bool, salesOrderId, clusterId string) ([]odoo.OdooMeteredBillingRecord, error) {
+func (o *ObjectStorage) createOdooRecord(bucketMetricsData cloudscale.BucketMetricsData, b BucketDetail, appuioManaged bool) ([]odoo.OdooMeteredBillingRecord, error) {
 	if len(bucketMetricsData.TimeSeries) != 1 {
 		return nil, fmt.Errorf("there must be exactly one metrics data point, found %d", len(bucketMetricsData.TimeSeries))
 	}
@@ -122,9 +123,9 @@ func createOdooRecord(bucketMetricsData cloudscale.BucketMetricsData, b BucketDe
 
 	itemGroup := ""
 	if appuioManaged {
-		itemGroup = fmt.Sprintf("APPUiO Managed - Zone: %s / Namespace: %s", clusterId, b.Namespace)
+		itemGroup = fmt.Sprintf("APPUiO Managed - Zone: %s / Namespace: %s", o.clusterId, b.Namespace)
 	} else {
-		itemGroup = fmt.Sprintf("APPUiO Cloud - Zone: %s / Namespace: %s", clusterId, b.Namespace)
+		itemGroup = fmt.Sprintf("APPUiO Cloud - Zone: %s / Namespace: %s", o.clusterId, b.Namespace)
 	}
 
 	instanceId := fmt.Sprintf("%s/%s", b.Zone, bucketMetricsData.Subject.BucketName)
@@ -135,8 +136,8 @@ func createOdooRecord(bucketMetricsData cloudscale.BucketMetricsData, b BucketDe
 			InstanceID:           instanceId,
 			ItemDescription:      "AppCat Cloudscale ObjectStorage",
 			ItemGroupDescription: itemGroup,
-			SalesOrderID:         salesOrderId,
-			UnitID:               units[productIdStorage],
+			SalesOrderID:         o.salesOrderId,
+			UnitID:               o.uomMapping[units[productIdStorage]],
 			ConsumedUnits:        storageBytesValue,
 			TimeRange: odoo.TimeRange{
 				From: bucketMetricsData.TimeSeries[0].Start,
@@ -148,8 +149,8 @@ func createOdooRecord(bucketMetricsData cloudscale.BucketMetricsData, b BucketDe
 			InstanceID:           instanceId,
 			ItemDescription:      "AppCat Cloudscale ObjectStorage",
 			ItemGroupDescription: itemGroup,
-			SalesOrderID:         salesOrderId,
-			UnitID:               units[productIdTrafficOut],
+			SalesOrderID:         o.salesOrderId,
+			UnitID:               o.uomMapping[units[productIdTrafficOut]],
 			ConsumedUnits:        trafficOutValue,
 			TimeRange: odoo.TimeRange{
 				From: bucketMetricsData.TimeSeries[0].Start,
@@ -161,8 +162,8 @@ func createOdooRecord(bucketMetricsData cloudscale.BucketMetricsData, b BucketDe
 			InstanceID:           instanceId,
 			ItemDescription:      "AppCat Cloudscale ObjectStorage",
 			ItemGroupDescription: itemGroup,
-			SalesOrderID:         salesOrderId,
-			UnitID:               units[productIdQueryRequests],
+			SalesOrderID:         o.salesOrderId,
+			UnitID:               o.uomMapping[units[productIdQueryRequests]],
 			ConsumedUnits:        queryRequestsValue,
 			TimeRange: odoo.TimeRange{
 				From: bucketMetricsData.TimeSeries[0].Start,
@@ -189,6 +190,13 @@ func fetchBuckets(ctx context.Context, k8sclient client.Client) (map[string]Buck
 	return bucketDetails, nil
 }
 
+func CheckUnitExistence(mapping map[string]string) error {
+	if mapping[odoo.GB] == "" || mapping[odoo.GBDay] == "" || mapping[odoo.KReq] == "" {
+		return fmt.Errorf("missing UOM mapping %s, %s or %s", odoo.GB, odoo.GBDay, odoo.KReq)
+	}
+	return nil
+}
+
 func convertUnit(unit string, value uint64) (float64, error) {
 	if unit == "GB" || unit == "GBDay" {
 		return float64(value) / 1000 / 1000 / 1000, nil

pkg/cmd/cloudscale.go

@@ -34,6 +34,7 @@ func CloudscaleCmds() *cli.Command {
 		salesOrderId      string
 		prometheusURL     string
 		clusterId         string
+		uom               string
 	)
 	return &cli.Command{
 		Name:  "cloudscale",
@@ -59,6 +60,8 @@ func CloudscaleCmds() *cli.Command {
 				EnvVars: []string{"CLUSTER_ID"}, Destination: &clusterId, Required: true, DefaultText: defaultTextForRequiredFlags},
 			&cli.StringFlag{Name: "prom-url", Usage: "Prometheus connection URL in the form of http://host:port, required for APPUiO Cloud",
 				EnvVars: []string{"PROM_URL"}, Destination: &prometheusURL, Value: "http://localhost:9090"},
+			&cli.StringFlag{Name: "uom", Usage: "Unit of measure mapping between cloud services and Odoo16 in json format",
+				EnvVars: []string{"UOM"}, Destination: &uom, Required: true, DefaultText: defaultTextForRequiredFlags},
 			&cli.IntFlag{Name: "collect-interval", Usage: "How often to collect the metrics from the Cloud Service in hours - 1-23",
 				EnvVars: []string{"COLLECT_INTERVAL"}, Destination: &collectInterval, Required: true, DefaultText: defaultTextForRequiredFlags},
 			&cli.IntFlag{Name: "billing-hour", Usage: "At what time to start collect the metrics (ex 6 would start running from 6)",
@@ -69,6 +72,16 @@ func CloudscaleCmds() *cli.Command {
 			logger := log.Logger(c.Context)
 			var wg sync.WaitGroup
 
+			logger.Info("Checking UOM mappings")
+			mapping, err := odoo.LoadUOM(uom)
+			if err != nil {
+				return err
+			}
+			err = cs.CheckUnitExistence(mapping)
+			if err != nil {
+				return err
+			}
+
 			logger.Info("Creating cloudscale client")
 			cloudscaleClient := cloudscale.NewClient(http.DefaultClient)
 			cloudscaleClient.AuthToken = apiToken
@@ -94,7 +107,7 @@ func CloudscaleCmds() *cli.Command {
 				return fmt.Errorf("load loaction: %w", err)
 			}
 
-			o, err := cs.NewObjectStorage(cloudscaleClient, k8sClient, promClient, salesOrderId, clusterId)
+			o, err := cs.NewObjectStorage(cloudscaleClient, k8sClient, promClient, salesOrderId, clusterId, mapping)
 			if err != nil {
 				return fmt.Errorf("object storage: %w", err)
 			}

pkg/cmd/exoscale.go

@@ -32,6 +32,7 @@ func ExoscaleCmds() *cli.Command {
 		salesOrderId      string
 		prometheusURL     string
 		clusterId         string
+		uom               string
 		collectInterval   int
 		billingHour       int
 	)
@@ -63,6 +64,8 @@ func ExoscaleCmds() *cli.Command {
 				EnvVars: []string{"CLUSTER_ID"}, Destination: &clusterId, Required: true, DefaultText: defaultTextForRequiredFlags},
 			&cli.StringFlag{Name: "prom-url", Usage: "Prometheus connection URL in the form of http://host:port, required for APPUiO Cloud",
 				EnvVars: []string{"PROM_URL"}, Destination: &prometheusURL, Value: "http://localhost:9090"},
+			&cli.StringFlag{Name: "uom", Usage: "Unit of measure mapping between cloud services and Odoo16 in json format",
+				EnvVars: []string{"UOM"}, Destination: &uom, Required: true, DefaultText: defaultTextForRequiredFlags},
 		},
 		Before: addCommandName,
 		Subcommands: []*cli.Command{
@@ -80,6 +83,16 @@ func ExoscaleCmds() *cli.Command {
 						return fmt.Errorf("exoscale client: %w", err)
 					}
 
+					logger.Info("Checking UOM mappings")
+					mapping, err := odoo.LoadUOM(uom)
+					if err != nil {
+						return err
+					}
+					err = exoscale.CheckObjectStorageUOMExistence(mapping)
+					if err != nil {
+						return err
+					}
+
 					logger.Info("Creating k8s client")
 					k8sClient, err := kubernetes.NewClient(kubeconfig)
 					if err != nil {
@@ -96,16 +109,16 @@ func ExoscaleCmds() *cli.Command {
 						}
 					}
 
-					o, err := exoscale.NewObjectStorage(exoscaleClient, k8sClient, promClient, salesOrderId, clusterId)
-					if err != nil {
-						return fmt.Errorf("objectbucket service: %w", err)
-					}
-
 					if collectInterval < 1 || collectInterval > 23 {
 						// Set to run once a day after billingHour in case the collectInterval is out of boundaries
 						collectInterval = 23
 					}
 
+					o, err := exoscale.NewObjectStorage(exoscaleClient, k8sClient, promClient, salesOrderId, clusterId, mapping)
+					if err != nil {
+						return fmt.Errorf("objectbucket service: %w", err)
+					}
+
 					wg.Add(1)
 					go func() {
 						for {
@@ -153,6 +166,16 @@ func ExoscaleCmds() *cli.Command {
 						return fmt.Errorf("exoscale client: %w", err)
 					}
 
+					logger.Info("Checking UOM mappings")
+					mapping, err := odoo.LoadUOM(uom)
+					if err != nil {
+						return err
+					}
+					err = exoscale.CheckDBaaSUOMExistence(mapping)
+					if err != nil {
+						return err
+					}
+
 					logger.Info("Creating k8s client")
 					k8sClient, err := kubernetes.NewClient(kubeconfig)
 					if err != nil {
@@ -169,21 +192,21 @@ func ExoscaleCmds() *cli.Command {
 						}
 					}
 
-					d, err := exoscale.NewDBaaS(exoscaleClient, k8sClient, promClient, salesOrderId, clusterId)
-					if err != nil {
-						return fmt.Errorf("dbaas service: %w", err)
-					}
-
 					if collectInterval < 1 || collectInterval > 24 {
 						// Set to run once a day after billingHour in case the collectInterval is out of boundaries
 						collectInterval = 1
 					}
 
+					d, err := exoscale.NewDBaaS(exoscaleClient, k8sClient, promClient, collectInterval, salesOrderId, clusterId, mapping)
+					if err != nil {
+						return fmt.Errorf("dbaas service: %w", err)
+					}
+
 					wg.Add(1)
 					go func() {
 						for {
 							logger.Info("Collecting DBaaS metrics")
-							metrics, err := d.GetMetrics(c.Context, collectInterval, salesOrderId)
+							metrics, err := d.GetMetrics(c.Context)
 							if err != nil {
 								logger.Error(err, "cannot execute dbaas collector")
 								os.Exit(1)

pkg/exofixtures/types.go

@@ -11,9 +11,8 @@ const (
 	Provider = "exoscale"
 
 	// SosType represents object storage storage type
-	SosType        ObjectType = "appcat_object-storage-storage"
-	QuerySos                  = string(SosType) + ":" + Provider
-	DefaultUnitSos            = "GBDay"
+	SosType  ObjectType = "appcat_object-storage-storage"
+	QuerySos            = string(SosType) + ":" + Provider
 )
 
 var ObjectStorage = InitConfig{