Skip to content

Commit

Permalink
Adjust component for the exporter
Browse files Browse the repository at this point in the history 
The component needed a few changes to accomondate for the new exporter
model.

* Added prometheus rule for aggregations
* Exoscale and Cloudscale can be run in the same instance, it will
  spawn two deployments though
* No more cronjobs, the pods will have their own intervalls
* There's a tenant override for deploying to APPUiO Managed
  • Loading branch information
@Kidswiss
Kidswiss authored and Gabriel Saratura committed Nov 22, 2023
1 parent f227fa4 commit e6014af
Show file tree
Hide file tree
Showing 37 changed files with 692 additions and 523 deletions.
5 changes: 5 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,8 @@
# Configuration
env
.env

__debug*

# Golden test that may contain live API keys for local testing
fromenv/
17 changes: 17 additions & 0 deletions component/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,20 @@ $(test_instances):
.PHONY: clean
clean: ## Clean the project
rm -rf .cache compiled dependencies vendor helmcharts jsonnetfile*.json || true

.PHONY: gen-golden-from-env
instance=fromenv
gen-golden-from-env: set-credentials gen-golden reset-credentials

.PHONY: set-credentials
set-credentials:
@test -n "$(EXOSCALE_API_KEY)"
@test -n "$(EXOSCALE_API_SECRET)"
@test -n "$(CLOUDSCALE_API_TOKEN)"
@yq e -i '.parameters.billing_collector_cloudservices.secrets.exoscale.credentials.stringData.EXOSCALE_API_KEY = "$(EXOSCALE_API_KEY)"' tests/fromenv.yml
@yq e -i '.parameters.billing_collector_cloudservices.secrets.exoscale.credentials.stringData.EXOSCALE_API_SECRET = "$(EXOSCALE_API_SECRET)"' tests/fromenv.yml
@yq e -i '.parameters.billing_collector_cloudservices.secrets.cloudscale.credentials.stringData.CLOUDSCALE_API_TOKEN = "$(CLOUDSCALE_API_TOKEN)"' tests/fromenv.yml

.PHONY: reset-credentials
reset-credentials:
git checkout tests/fromenv.yml
38 changes: 10 additions & 28 deletions component/class/defaults.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,51 +5,33 @@ parameters:

namespace: appuio-cloud-reporting

database: ${appuio_cloud_reporting:database}
database_secret: ${appuio_cloud_reporting:database_secret}
database_env: ${appuio_cloud_reporting:database_env}
extra_volumes: ${appuio_cloud_reporting:extra_volumes}

cloud_reporting_dbsecret_name: reporting-db

secrets:
exoscale:
credentials:
stringData:
EXOSCALE_API_KEY: "?{vaultkv:${cluster:tenant}/${cluster:name}/billing-collector-cloudservices/${_instance}/exoscale-key}"
EXOSCALE_API_SECRET: "?{vaultkv:${cluster:tenant}/${cluster:name}/billing-collector-cloudservices/${_instance}/exoscale-secret}"
KUBERNETES_SERVER_URL: "?{vaultkv:${cluster:tenant}/${cluster:name}/billing-collector-cloudservices/${_instance}/cluster-server}"
KUBERNETES_SERVER_TOKEN: "?{vaultkv:${cluster:tenant}/${cluster:name}/billing-collector-cloudservices/${_instance}/cluster-token}"
cloudscale:
credentials:
stringData:
CLOUDSCALE_API_TOKEN: "?{vaultkv:${cluster:tenant}/${cluster:name}/billing-collector-cloudservices/${_instance}/token}"
KUBERNETES_SERVER_URL: "?{vaultkv:${cluster:tenant}/${cluster:name}/billing-collector-cloudservices/${_instance}/cluster-server}"
KUBERNETES_SERVER_TOKEN: "?{vaultkv:${cluster:tenant}/${cluster:name}/billing-collector-cloudservices/${_instance}/cluster-token}"
images:
collector:
registry: 'ghcr.io'
repository: 'vshn/billing-collector-cloudservices'
tag: v1.0.3
tag: change_exporter

appuioManaged: true
tenantID: ${cluster:tenant}

exoscale:
enabled: false
objectStorage:
enabled: true
# Times in UTC! Don't run job around midnight as exoscale API may return incomplete data
# schedule for objectstorage cronjob
# default: Every day at minute 10 past hour 10, 16 and 20.
schedule: '10 10,16,20 * * *'

dbaas:
# enable DBaaS cronjob in addition to objectstorage cronjob.
enabled: false
# schedule for DBaaS cronjob every 15min
schedule: '*/15 * * * *'
# Exoscale metrics are generally fast
# and we are less likely to miss and dbaas instances
intervall: 60

cloudscale:
enabled: false
objectStorage:
enabled: true
# Times in UTC!
schedule: '10 4,10,16 * * *'
# cloudscale.ch queries are rather slow.
# Also the metrics are fetched for the previous day, the won't change often.
intervall: 600
243 changes: 163 additions & 80 deletions component/component/main.jsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,8 @@ local params = inv.parameters.billing_collector_cloudservices;
local kube = import 'lib/kube.libjsonnet';
local com = import 'lib/commodore.libjsonnet';
local collectorImage = '%(registry)s/%(repository)s:%(tag)s' % params.images.collector;
local alias = inv.parameters._instance;
local alias_suffix = '-' + alias;
local credentials_secret_name = 'credentials' + alias_suffix;
local component_name = 'billing-collector-cloudservices';

assert std.member(inv.applications, 'appuio-cloud-reporting') : 'Component appuio-cloud-reporting must be installed';

local labels = {
'app.kubernetes.io/name': component_name,
Expand All @@ -20,118 +16,205 @@ local labels = {

local secret(key) = [
if params.secrets[key][s] != null then
kube.Secret(s + alias_suffix) {
kube.Secret(s + '-' + key) {
metadata+: {
namespace: params.namespace,
},
} + com.makeMergeable(params.secrets[key][s])
for s in std.objectFields(params.secrets[key])
];

local dbEnv = [
{
name: name,
valueFrom: {
secretKeyRef: {
name: params.cloud_reporting_dbsecret_name,
key: name,
},
local healthProbe = {
httpGet: {
path: '/metrics',
port: 9123,
},
periodSeconds: 30,
};

local exoClusterRole = kube.ClusterRole('appcat:cloudcollector:exoscale') + {
rules: [
{
apiGroups: [ '*' ],
resources: [ 'namespaces' ],
verbs: [ 'get', 'list' ],
},
{
apiGroups: [ 'exoscale.crossplane.io' ],
resources: [
'buckets',
'postgresqls',
'mysqls',
'redis',
'opensearches',
'kafkas',
],
verbs: [
'get',
'list',
'watch',
],
},
],
};

local cloudscaleClusterRole = kube.ClusterRole('appcat:cloudcollector:cloudscale') + {
rules: [
{
apiGroups: [ '' ],
resources: [ 'namespaces' ],
verbs: [ 'get', 'list' ],
},
{
apiGroups: [ 'cloudscale.crossplane.io' ],
resources: [
'buckets',
],
verbs: [
'get',
'list',
'watch',
],
},
],
};

local serviceAccount(name, clusterRole) = {
local sa = kube.ServiceAccount(name) + {
metadata+: {
namespace: params.namespace,
},
}
for name in std.objectFields(params.database_secret)
] + [
{
name: name,
[if std.type(params.database_env[name]) == 'string' then 'value' else 'valueFrom']: params.database_env[name],
}
for name in std.objectFields(params.database_env)
] + [
assert params.database.url != null : 'database.url must be set.';
{
name: 'DB_PARAMS',
value: params.database.parameters,
},
{
name: 'ACR_DB_URL',
value: params.database.url,
local rb = kube.ClusterRoleBinding(name) {
roleRef_: clusterRole,
subjects_: [ sa ],
},
];
sa: sa,
rb: rb,
};

local cronjob(name, args, schedule) = {
kind: 'CronJob',
apiVersion: 'batch/v1',
metadata: {
name: name,
namespace: params.namespace,
labels+: labels,
},
spec: {
concurrencyPolicy: 'Forbid',
failedJobsHistoryLimit: 5,
jobTemplate: {
spec: {
template: {
spec: {
restartPolicy: 'OnFailure',
containers: [
{
name: 'billing-collector-cloudservices-backfill',
image: collectorImage,
args: args,
envFrom: [
{
secretRef: {
name: credentials_secret_name,
},
local deployment(name, args) =
kube.Deployment(name) {
metadata+: {
labels+: labels,
namespace: params.namespace,
},
spec+: {
template+: {
spec+: {
serviceAccount: name,
containers_:: {
exporter: kube.Container('exporter') {
imagePullPolicy: 'IfNotPresent',
image: collectorImage,
args: args,
envFrom: [
{
secretRef: {
name: 'credentials-' + name,
},
],
env: dbEnv,
resources: {},
[if std.length(params.extra_volumes) > 0 then 'volumeMounts']: [
{ name: name } + params.extra_volumes[name].mount_spec
for name in std.objectFields(params.extra_volumes)
],
},
],
ports_:: {
exporter: {
containerPort: 9123,
},
},
],
[if std.length(params.extra_volumes) > 0 then 'volumes']: [
{ name: name } + params.extra_volumes[name].volume_spec
for name in std.objectFields(params.extra_volumes)
],
readinessProbe: healthProbe,
livenessProbe: healthProbe,
},
},
},
},
},
schedule: schedule,
successfulJobsHistoryLimit: 3,
};

local podMonitor(name) = kube._Object('monitoring.coreos.com/v1', 'PodMonitor', 'hello') + {
metadata: {
name: name + '-podmonitor',
namespace: params.namespace,
},
spec: {
podMetricsEndpoints: [
{
port: 'exporter',
},
],
selector: {
matchLabels: {
name: name,
},
},
},
};

local promRule = kube._Object('monitoring.coreos.com/v1', 'PrometheusRule', 'appcat-cloud-billing') {
metadata+: {
namespace: params.namespace,
},
spec: {
groups: [
{
name: 'appcat:billing:cloudservices',
rules: [
{
expr: 'max_over_time(appcat:raw:billing{type="dbaas"}[1h])',
record: 'appcat:billing',
},
{
expr: 'appcat:raw:billing{type!="dbaas"}',
record: 'appcat:billing',
},
],
},
],
},
};

assert params.exoscale.enabled != params.cloudscale.enabled : 'only one of the components can be enabled: cloudscale or exoscale. not both and not neither.';
local orgOverride = (
if params.appuioManaged
then
[ '--organizationOverride', params.tenantID ]
else
[]
);

(if params.exoscale.enabled || params.cloudscale.enabled then {
promRule: promRule,
} else {}) +
(if params.exoscale.enabled then {
local secrets = params.secrets.exoscale,
local intervall = std.toString(params.exoscale.intervall),
local name = 'exoscale',
local sa = serviceAccount(name, exoClusterRole),
assert secrets != null : 'secrets must be set.',
assert secrets.credentials != null : 'secrets.credentials must be set.',
assert secrets.credentials.stringData != null : 'secrets.credentials.stringData must be set.',
assert secrets.credentials.stringData.EXOSCALE_API_KEY != null : 'secrets.credentials.stringData.EXOSCALE_API_KEY must be set.',
assert secrets.credentials.stringData.EXOSCALE_API_SECRET != null : 'secrets.credentials.stringData.EXOSCALE_API_SECRET must be set.',
assert secrets.credentials.stringData.KUBERNETES_SERVER_URL != null : 'secrets.credentials.stringData.KUBERNETES_SERVER_URL must be set.',
assert secrets.credentials.stringData.KUBERNETES_SERVER_TOKEN != null : 'secrets.credentials.stringData.KUBERNETES_SERVER_TOKEN must be set.',

secrets: std.filter(function(it) it != null, secret('exoscale')),
objectStorageCronjob: cronjob(alias + '-objectstorage', [ 'exoscale', 'objectstorage' ], params.exoscale.objectStorage.schedule),
[if params.exoscale.dbaas.enabled then 'dbaasCronjob']: cronjob(alias + '-dbaas', [ 'exoscale', 'dbaas' ], params.exoscale.dbaas.schedule),
exoSecrets: std.filter(function(it) it != null, secret(name)),
exoPodMonitor: podMonitor(name),
exoClusterRole: exoClusterRole,
exoServiceAccount: sa.sa,
exoRoleBinding: sa.rb,
exoscaleExporter: deployment(name, orgOverride + [ '--collectInterval', intervall, name ]),
} else {})
+
(if params.cloudscale.enabled then {
local secrets = params.secrets.cloudscale,
local intervall = std.toString(params.cloudscale.intervall),
local name = 'cloudscale',
local sa = serviceAccount(name, cloudscaleClusterRole),
assert secrets != null : 'secrets must be set.',
assert secrets.credentials != null : 'secrets.credentials must be set.',
assert secrets.credentials.stringData != null : 'secrets.credentials.stringData must be set.',
assert secrets.credentials.stringData.CLOUDSCALE_API_TOKEN != null : 'secrets.credentials.stringData.CLOUDSCALE_API_TOKEN must be set.',
assert secrets.credentials.stringData.KUBERNETES_SERVER_URL != null : 'secrets.credentials.stringData.KUBERNETES_SERVER_URL must be set.',
assert secrets.credentials.stringData.KUBERNETES_SERVER_TOKEN != null : 'secrets.credentials.stringData.KUBERNETES_SERVER_TOKEN must be set.',

secrets: std.filter(function(it) it != null, secret('cloudscale')),
[if params.cloudscale.objectStorage.enabled then 'objectStorageCronjob']: cronjob(alias + '-objectstorage', [ 'cloudscale', 'objectstorage' ], params.cloudscale.objectStorage.schedule),
cloudscaleSecrets: std.filter(function(it) it != null, secret(name)),
cloudscalePodMonitor: podMonitor(name),
cloudscaleClusterRole: cloudscaleClusterRole,
cloudscaleServiceAccount: sa.sa,
cloudscaleRolebinding: sa.rb,
cloudscaleExporter: deployment(name, orgOverride + [ '--collectInterval', intervall, name ]),
} else {})
Loading

0 comments on commit e6014af

Please sign in to comment.