From b6d20181c62ca6dd1464e7bd3cfa4fbacf72223e Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Wed, 18 Dec 2024 14:48:47 -0800 Subject: [PATCH 1/2] feat: add temporary helm repos Signed-off-by: Tarun Gupta Akirala --- common/helm-repositories/deleteme.yaml | 11 +++++++++++ common/helm-repositories/kubecost.yaml | 10 ++++++++++ common/helm-repositories/kustomization.yaml | 2 ++ 3 files changed, 23 insertions(+) create mode 100644 common/helm-repositories/deleteme.yaml create mode 100644 common/helm-repositories/kubecost.yaml diff --git a/common/helm-repositories/deleteme.yaml b/common/helm-repositories/deleteme.yaml new file mode 100644 index 000000000..829bdabd6 --- /dev/null +++ b/common/helm-repositories/deleteme.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: takirala + namespace: kommander-flux +spec: + interval: 10m + timeout: 1m + url: "${helmMirrorURL:=https://takirala.github.io/charts/stable}" +--- diff --git a/common/helm-repositories/kubecost.yaml b/common/helm-repositories/kubecost.yaml new file mode 100644 index 000000000..6d3c9ad37 --- /dev/null +++ b/common/helm-repositories/kubecost.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: kubecost + namespace: kommander-flux +spec: + interval: 10m + timeout: 1m + url: "${helmMirrorURL:=https://kubecost.github.io/cost-analyzer/}" diff --git a/common/helm-repositories/kustomization.yaml b/common/helm-repositories/kustomization.yaml index 6eaf6759e..eb4dab744 100644 --- a/common/helm-repositories/kustomization.yaml +++ b/common/helm-repositories/kustomization.yaml @@ -5,6 +5,7 @@ resources: - bitnami.yaml - cert-manager.yaml - dashboard.yaml + - deleteme.yaml - fluent.yaml - gatekeeper.yaml - grafana.yaml @@ -13,6 +14,7 @@ resources: - kommander-ui.yaml - kommander.yaml - kube-logging.yaml + - kubecost.yaml - kubefed.yaml - kubetunnel.yaml - mesosphere-repos.yaml From 6cb89a4c723a0081e9f08b26ac6c54e76434c6c1 Mon Sep 17 00:00:00 2001 From: Tarun Gupta Akirala Date: Wed, 18 Dec 2024 14:49:34 -0800 Subject: [PATCH 2/2] feat: upgrade to kubecost v2 Signed-off-by: Tarun Gupta Akirala --- .../0.37.8/defaults/cm.yaml | 153 ----------- .../0.37.8/release/release.yaml | 156 ----------- .../2.5.0/cosi-storage.yaml | 23 ++ .../2.5.0/cosi-storage/kustomization.yaml | 5 + .../move-to-rook-ceph-cluster-driver.yaml | 45 ++++ ...new-chart-in-mesosphere_charts_stable.yaml | 22 ++ .../2.5.0/defaults/cm.yaml | 243 ++++++++++++++++++ .../defaults/kustomization.yaml | 0 .../2.5.0/kustomization.yaml | 8 + .../2.5.0/move-to-konvoy.yaml | 20 ++ .../2.5.0/move-to-konvoy}/kustomization.yaml | 2 +- .../move-to-konvoy-cosi-hr.yaml | 26 ++ .../{0.37.8 => 2.5.0}/post-install-jobs.yaml | 3 +- .../post-install-jobs/post-install-jobs.yaml | 0 .../2.5.0/prerequisites.yaml | 24 ++ .../2.5.0/prerequisites/prerequisites.yaml | 115 +++++++++ .../{0.37.8 => 2.5.0}/release.yaml | 5 +- .../2.5.0/release/release.yaml | 102 ++++++++ services/kubecost/0.37.9/defaults/cm.yaml | 130 ---------- services/kubecost/0.37.9/kubecost.yaml | 84 ------ services/kubecost/2.5.0/defaults/cm.yaml | 53 ++++ .../defaults/kustomization.yaml | 0 .../2.5.0}/kustomization.yaml | 2 +- services/kubecost/2.5.0/prerequisites.yaml | 24 ++ .../2.5.0/prerequisites/prerequisites.yaml | 53 ++++ services/kubecost/2.5.0/release.yaml | 23 ++ services/kubecost/2.5.0/release/release.yaml | 31 +++ 27 files changed, 825 insertions(+), 527 deletions(-) delete mode 100644 services/centralized-kubecost/0.37.8/defaults/cm.yaml delete mode 100644 services/centralized-kubecost/0.37.8/release/release.yaml create mode 100644 services/centralized-kubecost/2.5.0/cosi-storage.yaml create mode 100644 services/centralized-kubecost/2.5.0/cosi-storage/kustomization.yaml create mode 100644 services/centralized-kubecost/2.5.0/cosi-storage/move-to-rook-ceph-cluster-driver.yaml create mode 100644 services/centralized-kubecost/2.5.0/cosi-storage/todo-create-a-new-chart-in-mesosphere_charts_stable.yaml create mode 100644 services/centralized-kubecost/2.5.0/defaults/cm.yaml rename services/centralized-kubecost/{0.37.8 => 2.5.0}/defaults/kustomization.yaml (100%) create mode 100644 services/centralized-kubecost/2.5.0/kustomization.yaml create mode 100644 services/centralized-kubecost/2.5.0/move-to-konvoy.yaml rename services/{kubecost/0.37.9 => centralized-kubecost/2.5.0/move-to-konvoy}/kustomization.yaml (71%) create mode 100644 services/centralized-kubecost/2.5.0/move-to-konvoy/move-to-konvoy-cosi-hr.yaml rename services/centralized-kubecost/{0.37.8 => 2.5.0}/post-install-jobs.yaml (82%) rename services/centralized-kubecost/{0.37.8 => 2.5.0}/post-install-jobs/post-install-jobs.yaml (100%) create mode 100644 services/centralized-kubecost/2.5.0/prerequisites.yaml create mode 100644 services/centralized-kubecost/2.5.0/prerequisites/prerequisites.yaml rename services/centralized-kubecost/{0.37.8 => 2.5.0}/release.yaml (73%) create mode 100644 services/centralized-kubecost/2.5.0/release/release.yaml delete mode 100644 services/kubecost/0.37.9/defaults/cm.yaml delete mode 100644 services/kubecost/0.37.9/kubecost.yaml create mode 100644 services/kubecost/2.5.0/defaults/cm.yaml rename services/kubecost/{0.37.9 => 2.5.0}/defaults/kustomization.yaml (100%) rename services/{centralized-kubecost/0.37.8 => kubecost/2.5.0}/kustomization.yaml (77%) create mode 100644 services/kubecost/2.5.0/prerequisites.yaml create mode 100644 services/kubecost/2.5.0/prerequisites/prerequisites.yaml create mode 100644 services/kubecost/2.5.0/release.yaml create mode 100644 services/kubecost/2.5.0/release/release.yaml diff --git a/services/centralized-kubecost/0.37.8/defaults/cm.yaml b/services/centralized-kubecost/0.37.8/defaults/cm.yaml deleted file mode 100644 index e1ba33317..000000000 --- a/services/centralized-kubecost/0.37.8/defaults/cm.yaml +++ /dev/null @@ -1,153 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: centralized-kubecost-0.37.8-d2iq-defaults - namespace: ${releaseNamespace} -data: - values.yaml: | - --- - hooks: - clusterID: - kubectlImage: "${kubetoolsImageRepository:=bitnami/kubectl}:${kubetoolsImageTag:=1.30.5}" - priorityClassName: dkp-high-priority - - cost-analyzer: - fullnameOverride: "kommander-kubecost-cost-analyzer" - kubecostFrontend: - fullImageName: ghcr.io/mesosphere/dkp-container-images/gcr.io/kubecost1/frontend:prod-1.108.1-d2iq.0 - priority: - enabled: true - name: dkp-high-priority - diagnostics: - enabled: false - global: - prometheus: - fqdn: http://kubecost-prometheus-server.kommander.svc.cluster.local - enabled: false - - thanos: - enabled: true - queryService: http://kommander-kubecost-thanos-query-http.kubecost.svc.cluster.local:10902 - # The wait time before Kommander begins querying cost data for all attached clusters - queryOffset: 5m - query: - deploymentAnnotations: - secret.reloader.stakater.com/reload: kommander-kubecost-thanos-client-tls - - grafana: - enabled: false - # Use kommander monitoring Grafana instance - domainName: centralized-grafana.${releaseNamespace}.svc.cluster.local - - # For Thanos Installs, Allow Higher Concurrency from Cost-Model - # Still may require tweaking for some installs, but the thanos-query-frontend - # will greatly assist in reduction memory bloat in query. - kubecostModel: - maxQueryConcurrency: 5 - # This configuration is applied to thanos only. Expresses the resolution to - # use for longer query ranges. Options: raw, 5m, 1h - Default: raw - maxSourceResolution: 5m - - ingress: - enabled: true - annotations: - kubernetes.io/ingress.class: kommander-traefik - ingress.kubernetes.io/auth-response-headers: X-Forwarded-User - traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.middlewares: "${releaseNamespace}-stripprefixes@kubernetescrd,${releaseNamespace}-forwardauth@kubernetescrd" - paths: - - "/dkp/kommander/kubecost/frontend/" - hosts: - - "" - tls: [] - - kubecostDeployment: - labels: - vendor.kubecost.io/partner: d2iq - - podSecurityPolicy: - enabled: false - - grafana: - # These values are set so that kubecost grafana dashboards are installed. - # Grafana itself is not installed. - sidecar: - image: - repository: docker.io/kiwigrid/k8s-sidecar - tag: 1.28.0 - dashboards: - enabled: true - label: grafana_dashboard_kommander - datasources: - enabled: true - defaultDatasourceEnabled: false - label: grafana_datasource_kommander - - prometheus: - fullnameOverride: "kommander-kubecost-prometheus" - server: - fullnameOverride: "kommander-kubecost-prometheus-server" - priorityClassName: dkp-high-priority - alertmanager: - fullnameOverride: "kommander-kubecost-prometheus-alertmanager" - priorityClassName: dkp-high-priority - kube-state-metrics: - fullnameOverride: "kommander-kubecost-prometheus-kube-state-metrics" - priorityClassName: dkp-high-priority - - thanos: - image: - repository: quay.io/thanos/thanos - tag: v0.37.1 - fullnameOverride: "kommander-kubecost-thanos" - nameOverride: "kubecost-thanos" - priorityClassName: dkp-high-priority - query: - enabled: true - timeout: 3m - maxConcurrent: 10 - # Name of HTTP request header used for dynamic prefixing of UI links and redirects. - webPrefixHeader: "X-Forwarded-Prefix" - resources: - limits: - cpu: 2000m - memory: 16Gi - requests: - cpu: 1000m - memory: 4Gi - http: - service: - labels: - servicemonitor.kommander.mesosphere.io/path: "metrics" - ingress: - enabled: true - annotations: - kubernetes.io/ingress.class: kommander-traefik - traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.middlewares: "${releaseNamespace}-stripprefixes@kubernetescrd,${releaseNamespace}-forwardauth@kubernetescrd" - path: "/dkp/kommander/kubecost/query" - hosts: - - "" - tls: [] - # Enable DNS discovery for stores - storeDNSDiscovery: false - # Enable DNS discovery for sidecars (this is for the chart built-in sidecar service) - sidecarDNSDiscovery: false - # Names of configmaps that contain addresses of store API servers, used for file service discovery. - serviceDiscoveryFileConfigMaps: - - kubecost-thanos-query-stores - # Refresh interval to re-read file SD files. It is used as a resync fallback. - serviceDiscoveryInterval: 5m - extraArgs: - - "--log.format=json" - - "--grpc-client-tls-secure" - - "--grpc-client-tls-cert=/etc/certs/tls.crt" - - "--grpc-client-tls-key=/etc/certs/tls.key" - - "--grpc-client-tls-ca=/etc/certs/ca.crt" - - "--grpc-client-server-name=server.thanos.kubecost.localhost.localdomain" - certSecretName: kommander-kubecost-thanos-client-tls - - kubecostProductConfigs: - grafanaURL: "/dkp/kommander/monitoring/grafana" - # used for display in Kubecost UI - clusterName: "Kommander Host" diff --git a/services/centralized-kubecost/0.37.8/release/release.yaml b/services/centralized-kubecost/0.37.8/release/release.yaml deleted file mode 100644 index 88df95b7f..000000000 --- a/services/centralized-kubecost/0.37.8/release/release.yaml +++ /dev/null @@ -1,156 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2beta2 -kind: HelmRelease -metadata: - name: centralized-kubecost - namespace: ${releaseNamespace} -spec: - chart: - spec: - chart: kubecost - sourceRef: - kind: HelmRepository - name: mesosphere.github.io-charts-stable - namespace: kommander-flux - version: 0.37.4 - interval: 15s - install: - crds: CreateReplace - remediation: - retries: 30 - createNamespace: true - upgrade: - crds: CreateReplace - remediation: - retries: 30 - releaseName: centralized-kubecost - valuesFrom: - - kind: ConfigMap - name: centralized-kubecost-0.37.8-d2iq-defaults - targetNamespace: kubecost ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubecost-thanos-configmap-edit - namespace: kubecost ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: kubecost-thanos-configmap-edit - namespace: kubecost -rules: - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "update", "patch", "delete"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: kubecost-thanos-configmap-edit - namespace: kubecost -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kubecost-thanos-configmap-edit -subjects: - - kind: ServiceAccount - name: kubecost-thanos-configmap-edit - namespace: kubecost ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: create-kubecost-thanos-query-stores-configmap - namespace: kubecost -spec: - template: - metadata: - name: create-kubecost-thanos-query-stores-configmap - spec: - serviceAccountName: kubecost-thanos-configmap-edit - restartPolicy: OnFailure - priorityClassName: dkp-high-priority - containers: - - name: kubectl - image: "${kubetoolsImageRepository:=bitnami/kubectl}:${kubetoolsImageTag:=1.30.5}" - command: - - sh - - "-c" - - |- - /bin/bash <<'EOF' - set -o nounset - set -o errexit - set -o pipefail - - echo "checking if kubecost-thanos-query-stores configmap exists" - - RES=$(set -o errexit; kubectl get configmap --ignore-not-found kubecost-thanos-query-stores) - if [[ $RES == "" ]]; then - echo "kubecost-thanos-query-stores configmap does not exist - creating" - printf '%s\n' "apiVersion: v1" "kind: ConfigMap" "metadata:" " name: kubecost-thanos-query-stores" "data:" " stores.yaml: |-" " - targets: []" > /tmp/kubecost-thanos-query-stores.yaml - kubectl apply -f /tmp/kubecost-thanos-query-stores.yaml - exit 0 - fi - - echo "kubecost-thanos-query-stores configmap already exists - no need to create" - EOF ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: kommander-kubecost-thanos-client-cert - namespace: kubecost -spec: - commonName: client.thanos.kubecost.localhost.localdomain - dnsNames: - - client.thanos.kubecost.localhost.localdomain - duration: 87600h - subject: - organizations: - - Nutanix - secretName: kommander-kubecost-thanos-client-tls - issuerRef: - name: kommander-ca - kind: ClusterIssuer ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: dkp-centralized-kubecost-view -rules: - - nonResourceURLs: - - /dkp/kommander/kubecost - - /dkp/kommander/kubecost/* - verbs: - - get - - head ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: dkp-centralized-kubecost-edit -rules: - - nonResourceURLs: - - /dkp/kommander/kubecost - - /dkp/kommander/kubecost/* - verbs: - - get - - head - - post - - put ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: dkp-centralized-kubecost-admin -rules: - - nonResourceURLs: - - /dkp/kommander/kubecost - - /dkp/kommander/kubecost/* - verbs: - - get - - head - - post - - put - - delete diff --git a/services/centralized-kubecost/2.5.0/cosi-storage.yaml b/services/centralized-kubecost/2.5.0/cosi-storage.yaml new file mode 100644 index 000000000..380691256 --- /dev/null +++ b/services/centralized-kubecost/2.5.0/cosi-storage.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: kubecost-cosi-storage + namespace: ${releaseNamespace} +spec: + force: true + prune: true + wait: true + interval: 6h + retryInterval: 1m + path: ./services/centralized-kubecost/2.5.0/cosi-storage + sourceRef: + kind: GitRepository + name: management + namespace: kommander-flux + timeout: 1m + dependsOn: + - name: todo-move-to-konvoy + namespace: ${releaseNamespace} + postBuild: + substitute: + releaseNamespace: ${releaseNamespace} diff --git a/services/centralized-kubecost/2.5.0/cosi-storage/kustomization.yaml b/services/centralized-kubecost/2.5.0/cosi-storage/kustomization.yaml new file mode 100644 index 000000000..97c19c90a --- /dev/null +++ b/services/centralized-kubecost/2.5.0/cosi-storage/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- move-to-rook-ceph-cluster-driver.yaml +- todo-create-a-new-chart-in-mesosphere_charts_stable.yaml diff --git a/services/centralized-kubecost/2.5.0/cosi-storage/move-to-rook-ceph-cluster-driver.yaml b/services/centralized-kubecost/2.5.0/cosi-storage/move-to-rook-ceph-cluster-driver.yaml new file mode 100644 index 000000000..9b8ec583b --- /dev/null +++ b/services/centralized-kubecost/2.5.0/cosi-storage/move-to-rook-ceph-cluster-driver.yaml @@ -0,0 +1,45 @@ +apiVersion: ceph.rook.io/v1 +kind: CephCOSIDriver +metadata: + name: ceph-cosi-driver + namespace: kommander +spec: + deploymentStrategy: "Auto" +--- +# The Ceph-COSI driver needs a privileged user for each CephObjectStore +# in order to provision buckets and users +apiVersion: ceph.rook.io/v1 +kind: CephObjectStoreUser +metadata: + name: cosi-admin + namespace: kommander +spec: + displayName: "cosi user" + store: dkp-object-store # name of the CephObjectStore + capabilities: + bucket: "*" + user: "*" +--- +# Following are "ADMIN" operations +--- +kind: BucketClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: test-bc + # Cluster scoped resource +driverName: rook-ceph.ceph.objectstorage.k8s.io +deletionPolicy: Delete +parameters: + objectStoreUserSecretName: rook-ceph-object-user-dkp-object-store-cosi-admin + objectStoreUserSecretNamespace: kommander +--- +kind: BucketAccessClass +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: test-bac + # Cluster scoped resource +driverName: rook-ceph.ceph.objectstorage.k8s.io +authenticationType: KEY +parameters: + objectStoreUserSecretName: rook-ceph-object-user-dkp-object-store-cosi-admin + objectStoreUserSecretNamespace: kommander diff --git a/services/centralized-kubecost/2.5.0/cosi-storage/todo-create-a-new-chart-in-mesosphere_charts_stable.yaml b/services/centralized-kubecost/2.5.0/cosi-storage/todo-create-a-new-chart-in-mesosphere_charts_stable.yaml new file mode 100644 index 000000000..88ff3b89a --- /dev/null +++ b/services/centralized-kubecost/2.5.0/cosi-storage/todo-create-a-new-chart-in-mesosphere_charts_stable.yaml @@ -0,0 +1,22 @@ +--- +kind: BucketClaim +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: test-bclaim + namespace: kubecost +spec: + bucketClassName: test-bc + protocols: + - s3 +--- +kind: BucketAccess +apiVersion: objectstorage.k8s.io/v1alpha1 +metadata: + name: test-ba + namespace: kubecost +spec: + bucketAccessClassName: test-bac + bucketClaimName: test-bclaim + protocol: s3 + credentialsSecretName: federated-store +--- diff --git a/services/centralized-kubecost/2.5.0/defaults/cm.yaml b/services/centralized-kubecost/2.5.0/defaults/cm.yaml new file mode 100644 index 000000000..487c3d87f --- /dev/null +++ b/services/centralized-kubecost/2.5.0/defaults/cm.yaml @@ -0,0 +1,243 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: centralized-kubecost-2.5.0-d2iq-defaults + namespace: ${releaseNamespace} +data: + values.yaml: | + --- + global: + prometheus: + enabled: false + fqdn: http://kubecost-prometheus-server.kommander.svc.cluster.local + + savedReports: + # Refer https://docs.kubecost.com/using-kubecost/navigating-the-kubecost-ui/reports to create reports + enabled: false + + grafana: + enabled: false + # Use kommander monitoring Grafana instance + domainName: centralized-grafana.${releaseNamespace}.svc.cluster.local + + notifications: + alertmanager: + # If true, allow kubecost to write to alertmanager + enabled: true + + kubecostModel: + federatedStorageConfigSecret: "federated-store" # Secret should have a key named "federated-store.yaml" with the federated storage credentials + + kubecostAggregator: + # deployMethod determines how Aggregator is deployed. Current options are + # "singlepod" (within cost-analyzer Pod) "statefulset" (separate + # StatefulSet), and "disabled". + deployMethod: statefulset + cloudCost: + # The cloudCost component of Aggregator depends on + # kubecostAggregator.deployMethod: + # kA.dM = "singlepod" -> cloudCost is run as container inside cost-analyzer + # kA.dM = "statefulset" -> cloudCost is run as single-replica Deployment + enabled: false # TODO: document how to enable here + # Log level for the aggregator container. Options are "trace", "debug", "info", "warn", "error", "fatal", "panic" + logLevel: info + resources: + requests: + cpu: 1000m + memory: 1Gi + jaeger: + # Enable this to use jaeger for tracing, useful for debugging + enabled: false + image: jaegertracing/all-in-one + imageVersion: 1.64.0 # Pin the image here to avoid pulling in latest as that would affect CVE scans + + federatedETL: + # Centralized kubecost should not write data to s3 storage. It's read-only. + federatedCluster: false + + kubecostFrontend: + enabled: true + fullImageName: gcr.io/kubecost1/frontend:prod-2.5.0 + deployMethod: singlepod # Other possible value is `haMode` that is supported only with enterprise license. + + priority: + enabled: true + name: dkp-high-priority + + forecasting: + # Enable this to use kubecost's cost forecosting model + # TODO(takirala): do we enable this and create yet another pod or disable this but ship the image for airgap bundle? + enabled: false + + # Define persistence volume for cost-analyzer, more information at https://github.com/kubecost/docs/blob/master/storage.md + persistentVolume: + # Upgrades from original default 0.2Gi may break if automatic disk resize is not supported + # https://github.com/kubecost/cost-analyzer-helm-chart/issues/507 + size: 32Gi + # Note that setting this to false means configurations will be wiped out on pod restart. + enabled: true + # storageClass: "-" + + ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: kommander-traefik + ingress.kubernetes.io/auth-response-headers: X-Forwarded-User + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.middlewares: "${releaseNamespace}-stripprefixes@kubernetescrd,${releaseNamespace}-forwardauth@kubernetescrd" + paths: + - "/dkp/kommander/kubecost/frontend/" + hosts: + - "" + tls: [] + + podSecurityPolicy: + enabled: false + + prometheus: + kube-state-metrics: + fullnameOverride: "kommander-kubecost-prometheus-kube-state-metrics" + priorityClassName: dkp-high-priority + extraScrapeConfigs: | + - job_name: kubecost + honor_labels: true + scrape_interval: 1m + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + dns_sd_configs: + - names: + - {{ .Release.Name }}-cost-analyzer + type: 'A' + port: 9003 + - job_name: kubecost-networking + kubernetes_sd_configs: + - role: pod + relabel_configs: + # Scrape only the the targets matching the following metadata + - source_labels: [__meta_kubernetes_pod_label_app] + action: keep + regex: {{ .Release.Name }}-network-costs + + server: + fullnameOverride: "kommander-kubecost-prometheus-server" + image: + repository: quay.io/prometheus/prometheus + tag: v2.55.0 + # If clusterIDConfigmap is defined, instead use user-generated configmap with key CLUSTER_ID + # to use as unique cluster ID in kubecost cost-analyzer deployment. + # This overrides the cluster_id set in prometheus.server.global.external_labels. + # NOTE: This does not affect the external_labels set in prometheus config. + clusterIDConfigmap: kubecost-cluster-info-configmap + extraFlags: + - web.enable-admin-api + - web.enable-lifecycle + - storage.tsdb.wal-compression + resources: + limits: + cpu: 1000m + memory: 2500Mi + requests: + cpu: 300m + memory: 1500Mi + global: + scrape_interval: 1m + scrape_timeout: 10s + evaluation_interval: 1m + external_labels: + cluster_id: $CLUSTER_ID + persistentVolume: + size: 32Gi + enabled: true + extraArgs: + log.level: info + log.format: json + storage.tsdb.min-block-duration: 2h + storage.tsdb.max-block-duration: 2h + query.max-concurrency: 1 + query.max-samples: 100000000 + enableAdminApi: true + service: + gRPC: + enabled: true + priorityClassName: dkp-high-priority + configmapReload: + prometheus: + enabled: true + #image: + #repository: ghcr.io/jimmidyson/configmap-reload + #tag: v0.14.0 + alertmanager: + enabled: true + #image: + #repository: ghcr.io/jimmidyson/configmap-reload + #tag: v0.14.0 + alertmanager: + fullnameOverride: "kommander-kubecost-prometheus-alertmanager" + priorityClassName: dkp-high-priority + enabled: true + image: + repository: quay.io/prometheus/alertmanager + tag: v0.27.0 + resources: + limits: + cpu: 50m + memory: 100Mi + requests: + cpu: 10m + memory: 50Mi + persistentVolume: + enabled: true + pushgateway: + enabled: false + persistentVolume: + enabled: false + serverFiles: + alerts: + groups: + - name: Kubecost + rules: + - alert: kubecostDown + expr: up{job="kubecost"} == 0 + annotations: + message: 'Kubecost metrics endpoint is not being scraped successfully.' + for: 10m + labels: + severity: warning + - alert: kubecostMetricsUnavailable + expr: sum(sum_over_time(node_cpu_hourly_cost[5m])) == 0 + annotations: + message: 'Kubecost metrics are not available in Prometheus.' + for: 10m + labels: + severity: warning + - alert: kubecostRecordingRulesNotEvaluated + expr: avg_over_time(kubecost_cluster_memory_working_set_bytes[5m]) == 0 + annotations: + message: 'Kubecost recording rules are not being successfully evaluated.' + for: 10m + labels: + severity: warning + + grafana: + sidecar: + image: + repository: docker.io/kiwigrid/k8s-sidecar + tag: 1.28.1 + dashboards: + enabled: true + label: grafana_dashboard_kommander + datasources: + enabled: true + defaultDatasourceEnabled: false + label: grafana_datasource_kommander + + kubecostProductConfigs: + grafanaURL: "/dkp/kommander/monitoring/grafana" + # used for display in Kubecost UI + clusterName: "" + clusterProfile: production + cloudIntegrationSecret: # TODO: figure out how to add cloud integration secret + productKey: + enabled: false + #key: YOUR_KEY diff --git a/services/centralized-kubecost/0.37.8/defaults/kustomization.yaml b/services/centralized-kubecost/2.5.0/defaults/kustomization.yaml similarity index 100% rename from services/centralized-kubecost/0.37.8/defaults/kustomization.yaml rename to services/centralized-kubecost/2.5.0/defaults/kustomization.yaml diff --git a/services/centralized-kubecost/2.5.0/kustomization.yaml b/services/centralized-kubecost/2.5.0/kustomization.yaml new file mode 100644 index 000000000..c1809a075 --- /dev/null +++ b/services/centralized-kubecost/2.5.0/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - move-to-konvoy.yaml + - cosi-storage.yaml + - prerequisites.yaml + - release.yaml + - post-install-jobs.yaml diff --git a/services/centralized-kubecost/2.5.0/move-to-konvoy.yaml b/services/centralized-kubecost/2.5.0/move-to-konvoy.yaml new file mode 100644 index 000000000..51d48aae2 --- /dev/null +++ b/services/centralized-kubecost/2.5.0/move-to-konvoy.yaml @@ -0,0 +1,20 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: todo-move-to-konvoy + namespace: ${releaseNamespace} +spec: + force: true + prune: true + wait: true + interval: 6h + retryInterval: 1m + path: ./services/centralized-kubecost/2.5.0/move-to-konvoy + sourceRef: + kind: GitRepository + name: management + namespace: kommander-flux + timeout: 1m + postBuild: + substitute: + releaseNamespace: ${releaseNamespace} diff --git a/services/kubecost/0.37.9/kustomization.yaml b/services/centralized-kubecost/2.5.0/move-to-konvoy/kustomization.yaml similarity index 71% rename from services/kubecost/0.37.9/kustomization.yaml rename to services/centralized-kubecost/2.5.0/move-to-konvoy/kustomization.yaml index f867bff56..25813daec 100644 --- a/services/kubecost/0.37.9/kustomization.yaml +++ b/services/centralized-kubecost/2.5.0/move-to-konvoy/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - kubecost.yaml +- move-to-konvoy-cosi-hr.yaml diff --git a/services/centralized-kubecost/2.5.0/move-to-konvoy/move-to-konvoy-cosi-hr.yaml b/services/centralized-kubecost/2.5.0/move-to-konvoy/move-to-konvoy-cosi-hr.yaml new file mode 100644 index 000000000..d8bc3a8f1 --- /dev/null +++ b/services/centralized-kubecost/2.5.0/move-to-konvoy/move-to-konvoy-cosi-hr.yaml @@ -0,0 +1,26 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: cosi-move-to-konvoy-todo + namespace: ${releaseNamespace} +spec: + chart: + spec: + chart: cosi + sourceRef: + kind: HelmRepository + name: takirala + namespace: kommander-flux + version: 0.0.1 + interval: 15s + install: + crds: CreateReplace + remediation: + retries: 30 + createNamespace: true + upgrade: + crds: CreateReplace + remediation: + retries: 30 + releaseName: does-not-matter + targetNamespace: container-object-storage-system diff --git a/services/centralized-kubecost/0.37.8/post-install-jobs.yaml b/services/centralized-kubecost/2.5.0/post-install-jobs.yaml similarity index 82% rename from services/centralized-kubecost/0.37.8/post-install-jobs.yaml rename to services/centralized-kubecost/2.5.0/post-install-jobs.yaml index 3a33acf9c..38a6c7b68 100644 --- a/services/centralized-kubecost/0.37.8/post-install-jobs.yaml +++ b/services/centralized-kubecost/2.5.0/post-install-jobs.yaml @@ -9,9 +9,10 @@ spec: wait: true interval: 6h retryInterval: 1m - path: ./services/centralized-kubecost/0.37.8/post-install-jobs + path: ./services/centralized-kubecost/2.5.0/post-install-jobs dependsOn: - name: centralized-kubecost-release + namespace: ${releaseNamespace} sourceRef: kind: GitRepository name: management diff --git a/services/centralized-kubecost/0.37.8/post-install-jobs/post-install-jobs.yaml b/services/centralized-kubecost/2.5.0/post-install-jobs/post-install-jobs.yaml similarity index 100% rename from services/centralized-kubecost/0.37.8/post-install-jobs/post-install-jobs.yaml rename to services/centralized-kubecost/2.5.0/post-install-jobs/post-install-jobs.yaml diff --git a/services/centralized-kubecost/2.5.0/prerequisites.yaml b/services/centralized-kubecost/2.5.0/prerequisites.yaml new file mode 100644 index 000000000..58e18268a --- /dev/null +++ b/services/centralized-kubecost/2.5.0/prerequisites.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: centralized-kubecost-prerequisites + namespace: ${releaseNamespace} +spec: + force: true + prune: true + wait: true + interval: 6h + retryInterval: 1m + path: ./services/centralized-kubecost/2.5.0/prerequisites + dependsOn: + - name: kubecost-cosi-storage + namespace: ${releaseNamespace} + sourceRef: + kind: GitRepository + name: management + namespace: kommander-flux + timeout: 1m + postBuild: + substituteFrom: + - kind: ConfigMap + name: substitution-vars diff --git a/services/centralized-kubecost/2.5.0/prerequisites/prerequisites.yaml b/services/centralized-kubecost/2.5.0/prerequisites/prerequisites.yaml new file mode 100644 index 000000000..aa989026f --- /dev/null +++ b/services/centralized-kubecost/2.5.0/prerequisites/prerequisites.yaml @@ -0,0 +1,115 @@ +# Copy grafana-datasource cm after it has been created in the release. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: centralized-kubecost-prerequisites + namespace: kubecost +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: centralized-kubecost-prerequisites +rules: + - apiGroups: [""] + resources: ["configmaps", "namespaces"] + verbs: ["get", "list", "create"] + - apiGroups: [ "" ] + resources: [ "secrets" ] + verbs: [ "get", "list", "patch" ] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: centralized-kubecost-prerequisites +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: centralized-kubecost-prerequisites +subjects: + - kind: ServiceAccount + name: centralized-kubecost-prerequisites + namespace: kubecost +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: centralized-kubecost-prerequisites + namespace: kubecost +spec: + template: + metadata: + name: centralized-kubecost-prerequisites + spec: + serviceAccountName: centralized-kubecost-prerequisites + restartPolicy: OnFailure + priorityClassName: dkp-high-priority + containers: + - name: create-kubecost-cluster-info-configmap + image: "${kubetoolsImageRepository:=bitnami/kubectl}:${kubetoolsImageTag:=1.30.5}" + command: + - sh + - -c + - | + /bin/bash <<'EOF' + set -o nounset + set -o errexit + set -o pipefail + # Skip if the configmap already exists. + if kubectl get configmap -n kubecost kubecost-cluster-info-configmap; then + echo "Configmap kubecost-cluster-info-configmap already exists. Skipping the step." + exit 0 + fi + kubectl create configmap kubecost-cluster-info-configmap -n kubecost -oyaml --dry-run --save-config --from-literal=CLUSTER_ID=$(kubectl get namespace kube-system -o jsonpath="{.metadata.uid}") | kubectl apply -f - + echo "Done replacing \$CLUSTER_ID with $CLUSTERID" + EOF + - name: transform-cosi-secret-to-kubecost-secret + image: "${kubetoolsImageRepository:=bitnami/kubectl}:${kubetoolsImageTag:=1.30.5}" + command: + - sh + - -c + - | + set -o nounset + set -o errexit + set +x + + # Wait until federated-store secret is found. + while ! kubectl get secret -n kubecost federated-store; do + echo "federated-store secret not found. Waiting for it to be created." + sleep 5 + done + + echo "federated-store secret found. Fetching bucketInfo..." + bucketInfo=$(kubectl get secret -n kubecost federated-store -o go-template='{{ .data.BucketInfo | base64decode }}') + tmpfile=$(mktemp /tmp/federated-store.XXXXXX) + + echo "Fetched bucketInfo from federated-store secret. Processing it..." + echo "$bucketInfo" | yq eval ' + { + "type": "S3", + "config": { + "bucket": .spec.bucketName, + "endpoint": .spec.secretS3.endpoint | sub(":\\d+$", "") | sub("^http://", "") | sub("^https://", ""), # Remove port and protocol (if any). + "region": .spec.secretS3.region, + "access_key": .spec.secretS3.accessKeyID, + "secret_key": .spec.secretS3.accessSecretKey, + "insecure": .spec.secretS3.endpoint | test("^http://"), # Use insecure if endpoint is http (e.g.: cluster internal endpoint). + "signature_version2": false, # Use signature version 4. + "put_user_metadata": { + "X-Amz-Acl": "bucket-owner-full-control" + }, + "http_config": { + "idle_conn_timeout": "90s", + "response_header_timeout": "2m", + "insecure_skip_verify": false + }, + "trace": { + "enable": false # Enable to debug errors (if any) + }, + "part_size": 10240 # TODO(takirala): Deduce this value logically. + } + }' > "$tmpfile" + echo "Transformed bucketInfo to federated-store.yaml. Updating federated-store secret..." + kubectl create secret generic federated-store -n kubecost --from-file=federated-store.yaml="$tmpfile" --dry-run=client -o yaml | kubectl apply -f - + kubectl label secret federated-store -n kubecost app.kubernetes.io/processed-by-kommander-centralized-kubecost=true --overwrite + rm "$tmpfile" + # TODO(takirala): Test and support azure, nutanix cosi secrets. diff --git a/services/centralized-kubecost/0.37.8/release.yaml b/services/centralized-kubecost/2.5.0/release.yaml similarity index 73% rename from services/centralized-kubecost/0.37.8/release.yaml rename to services/centralized-kubecost/2.5.0/release.yaml index d745445c6..74c007209 100644 --- a/services/centralized-kubecost/0.37.8/release.yaml +++ b/services/centralized-kubecost/2.5.0/release.yaml @@ -9,7 +9,7 @@ spec: wait: true interval: 6h retryInterval: 1m - path: ./services/centralized-kubecost/0.37.8/release + path: ./services/centralized-kubecost/2.5.0/release sourceRef: kind: GitRepository name: management @@ -19,3 +19,6 @@ spec: substituteFrom: - kind: ConfigMap name: substitution-vars + dependsOn: + - name: centralized-kubecost-prerequisites + namespace: ${releaseNamespace} diff --git a/services/centralized-kubecost/2.5.0/release/release.yaml b/services/centralized-kubecost/2.5.0/release/release.yaml new file mode 100644 index 000000000..8b8a18dcb --- /dev/null +++ b/services/centralized-kubecost/2.5.0/release/release.yaml @@ -0,0 +1,102 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: centralized-kubecost + namespace: ${releaseNamespace} +spec: + chart: + spec: + chart: cost-analyzer + sourceRef: + kind: HelmRepository + name: kubecost + namespace: kommander-flux + version: 2.5.0 + interval: 15s + install: + crds: CreateReplace + remediation: + retries: 30 + createNamespace: true + upgrade: + crds: CreateReplace + remediation: + retries: 30 + driftDetection: + mode: enabled + releaseName: centralized-kubecost + valuesFrom: + - kind: ConfigMap + name: centralized-kubecost-2.5.0-d2iq-defaults + targetNamespace: kubecost +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubecost-thanos-configmap-edit + namespace: kubecost +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubecost-thanos-configmap-edit + namespace: kubecost +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubecost-thanos-configmap-edit + namespace: kubecost +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubecost-thanos-configmap-edit +subjects: + - kind: ServiceAccount + name: kubecost-thanos-configmap-edit + namespace: kubecost +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dkp-centralized-kubecost-view +rules: + - nonResourceURLs: + - /dkp/kommander/kubecost + - /dkp/kommander/kubecost/* + verbs: + - get + - head +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dkp-centralized-kubecost-edit +rules: + - nonResourceURLs: + - /dkp/kommander/kubecost + - /dkp/kommander/kubecost/* + verbs: + - get + - head + - post + - put +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dkp-centralized-kubecost-admin +rules: + - nonResourceURLs: + - /dkp/kommander/kubecost + - /dkp/kommander/kubecost/* + verbs: + - get + - head + - post + - put + - delete diff --git a/services/kubecost/0.37.9/defaults/cm.yaml b/services/kubecost/0.37.9/defaults/cm.yaml deleted file mode 100644 index dbe45c556..000000000 --- a/services/kubecost/0.37.9/defaults/cm.yaml +++ /dev/null @@ -1,130 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: kubecost-0.37.9-d2iq-defaults - namespace: ${releaseNamespace} -data: - values.yaml: | - --- - hooks: - clusterID: - kubectlImage: "${kubetoolsImageRepository:=bitnami/kubectl}:${kubetoolsImageTag:=1.30.5}" - priorityClassName: dkp-high-priority - - cost-analyzer: - kubecostFrontend: - fullImageName: ghcr.io/mesosphere/dkp-container-images/gcr.io/kubecost1/frontend:prod-1.108.1-d2iq.0 - priority: - enabled: true - name: dkp-high-priority - diagnostics: - enabled: false - global: - prometheus: - enabled: true - grafana: - enabled: true - - ingress: - enabled: true - annotations: - kubernetes.io/ingress.class: kommander-traefik - ingress.kubernetes.io/auth-response-headers: X-Forwarded-User - traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.middlewares: "${workspaceNamespace}-stripprefixes@kubernetescrd,${workspaceNamespace}-forwardauth@kubernetescrd" - paths: - - "/dkp/kubecost/frontend/" - hosts: - - "" - tls: [] - - podSecurityPolicy: - enabled: false - - prometheus: - kubeStateMetrics: - enabled: false - kube-state-metrics: - disabled: true - server: - priorityClassName: dkp-high-priority - image: - tag: v2.55.0 - sidecarContainers: - - name: thanos-sidecar - image: quay.io/thanos/thanos:v0.37.1 - args: - - sidecar - - --log.level=debug - - --tsdb.path=/data/ - - --prometheus.url=http://127.0.0.1:9090 - - --reloader.config-file=/etc/config/prometheus.yml - # Start of time range limit to serve. Thanos sidecar will serve only metrics, which happened - # later than this value. Option can be a constant time in RFC3339 format or time duration - # relative to current time, such as -1d or 2h45m. Valid duration units are ms, s, m, h, d, w, y. - - --min-time=-3h - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - ports: - - name: sidecar-http - containerPort: 10902 - - name: grpc - containerPort: 10901 - - name: cluster - containerPort: 10900 - volumeMounts: - - name: config-volume - mountPath: /etc/config - - name: storage-volume - mountPath: /data - subPath: "" - alertmanager: - priorityClassName: dkp-high-priority - image: - repository: quay.io/prometheus/alertmanager - tag: v0.27.0 - grafana: - priorityClassName: dkp-high-priority - image: - repository: ghcr.io/mesosphere/dkp-container-images/docker.io/grafana/grafana - tag: 10.3.3-d2iq.0 - ingress: - enabled: true - annotations: - kubernetes.io/ingress.class: kommander-traefik - ingress.kubernetes.io/auth-response-headers: X-Forwarded-User - traefik.ingress.kubernetes.io/router.tls: "true" - traefik.ingress.kubernetes.io/router.middlewares: "${workspaceNamespace}-stripprefixes@kubernetescrd,${workspaceNamespace}-forwardauth@kubernetescrd" - hosts: [""] - path: "/dkp/kubecost/grafana" - grafana.ini: - log: - level: warn - server: - protocol: http - enable_gzip: true - root_url: "%(protocol)s://%(domain)s:%(http_port)s/dkp/kubecost/grafana" - serve_from_sub_path: false # Set to false on Grafana v10+ - auth.proxy: - enabled: true - header_name: X-Forwarded-User - auto-sign-up: true - auth.basic: - enabled: false - users: - auto_assign_org_role: Admin - analytics: - reporting_enabled: false - check_for_updates: false - sidecar: - image: - repository: docker.io/kiwigrid/k8s-sidecar - tag: 1.28.0 - - kubecostProductConfigs: - grafanaURL: "/dkp/kubecost/grafana" - # used for display in Kubecost UI - clusterName: "Kommander Managed Cluster" diff --git a/services/kubecost/0.37.9/kubecost.yaml b/services/kubecost/0.37.9/kubecost.yaml deleted file mode 100644 index 8385fb221..000000000 --- a/services/kubecost/0.37.9/kubecost.yaml +++ /dev/null @@ -1,84 +0,0 @@ -apiVersion: helm.toolkit.fluxcd.io/v2beta2 -kind: HelmRelease -metadata: - name: kubecost - namespace: ${releaseNamespace} -spec: - chart: - spec: - chart: kubecost - sourceRef: - kind: HelmRepository - name: mesosphere.github.io-charts-stable - namespace: kommander-flux - version: 0.37.4 - interval: 15s - install: - crds: CreateReplace - remediation: - retries: 30 - createNamespace: true - upgrade: - crds: CreateReplace - remediation: - retries: 30 - releaseName: kubecost - valuesFrom: - - kind: ConfigMap - name: kubecost-0.37.9-d2iq-defaults - targetNamespace: ${releaseNamespace} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: kubecost-app-dashboard-info - namespace: ${releaseNamespace} - labels: - "kommander.d2iq.io/application": "kubecost" -data: - name: "Kubecost" - dashboardLink: "/dkp/kubecost/frontend/overview.html" - docsLink: "http://docs.kubecost.com/" - # From: https://github.com/mesosphere/charts/blob/master/stable/kubecost/Chart.yaml#L2 - version: "1.104.0" ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: dkp-kubecost-view -rules: - - nonResourceURLs: - - /dkp/kubecost/frontend - - /dkp/kubecost/frontend/* - verbs: - - get - - head ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: dkp-kubecost-edit -rules: - - nonResourceURLs: - - /dkp/kubecost/frontend - - /dkp/kubecost/frontend/* - verbs: - - get - - head - - post - - put ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: dkp-kubecost-admin -rules: - - nonResourceURLs: - - /dkp/kubecost/frontend - - /dkp/kubecost/frontend/* - verbs: - - get - - head - - post - - put - - delete diff --git a/services/kubecost/2.5.0/defaults/cm.yaml b/services/kubecost/2.5.0/defaults/cm.yaml new file mode 100644 index 000000000..9df0d96f0 --- /dev/null +++ b/services/kubecost/2.5.0/defaults/cm.yaml @@ -0,0 +1,53 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubecost-2.5.0-d2iq-defaults + namespace: ${releaseNamespace} +data: + values.yaml: | + --- + global: + prometheus: + enabled: true + + grafana: + enabled: true #TODO: Disable Grafana + proxy: false + forecasting: + # Enable this to use kubecost's cost forecosting model + enabled: false + upgrade: + toV2: false # TODO: Handle upgrades. + + federatedETL: + federatedCluster: true + agentOnly: true # disables UI + + ingress: + enabled: false + + kubecostModel: + federatedStorageConfigSecret: "federated-store" # Secret should have a key named "federated-store.yaml" with the federated storage credentials + + kubecostAggregator: + deployMethod: disabled + + priority: + enabled: true + name: dkp-high-priority + + prometheus: + server: + retention: 14d + clusterIDConfigmap: kubecost-cluster-info-configmap + global: + scrape_interval: 1m + scrape_timeout: 10s + evaluation_interval: 1m + external_labels: + cluster_id: $CLUSTER_ID + + kubecostProductConfigs: + # used for display in Kubecost UI + clusterName: "" + clusterProfile: production diff --git a/services/kubecost/0.37.9/defaults/kustomization.yaml b/services/kubecost/2.5.0/defaults/kustomization.yaml similarity index 100% rename from services/kubecost/0.37.9/defaults/kustomization.yaml rename to services/kubecost/2.5.0/defaults/kustomization.yaml diff --git a/services/centralized-kubecost/0.37.8/kustomization.yaml b/services/kubecost/2.5.0/kustomization.yaml similarity index 77% rename from services/centralized-kubecost/0.37.8/kustomization.yaml rename to services/kubecost/2.5.0/kustomization.yaml index 1b10bcdf3..840f4608e 100644 --- a/services/centralized-kubecost/0.37.8/kustomization.yaml +++ b/services/kubecost/2.5.0/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - release.yaml - - post-install-jobs.yaml + - prerequisites.yaml diff --git a/services/kubecost/2.5.0/prerequisites.yaml b/services/kubecost/2.5.0/prerequisites.yaml new file mode 100644 index 000000000..a96387924 --- /dev/null +++ b/services/kubecost/2.5.0/prerequisites.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: kubecost-prerequisites + namespace: ${releaseNamespace} +spec: + force: true + prune: true + wait: true + interval: 6h + retryInterval: 1m + path: ./services/kubecost/2.5.0/prerequisites + sourceRef: + kind: GitRepository + name: management + namespace: kommander-flux + timeout: 1m + postBuild: + # The var values which are specified in-line with substitute take precedence over the ones in substituteFrom. + substitute: + releaseNamespace: ${releaseNamespace} + substituteFrom: + - kind: ConfigMap + name: substitution-vars diff --git a/services/kubecost/2.5.0/prerequisites/prerequisites.yaml b/services/kubecost/2.5.0/prerequisites/prerequisites.yaml new file mode 100644 index 000000000..95876336e --- /dev/null +++ b/services/kubecost/2.5.0/prerequisites/prerequisites.yaml @@ -0,0 +1,53 @@ +# Copy grafana-datasource cm after it has been created in the release. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubecost-prerequisites + namespace: ${releaseNamespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubecost-prerequisites +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "patch"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubecost-prerequisites +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubecost-prerequisites +subjects: + - kind: ServiceAccount + name: kubecost-prerequisites + namespace: ${releaseNamespace} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-kubecost-cluster-info-configmap + namespace: ${releaseNamespace} +spec: + template: + metadata: + name: create-kubecost-cluster-info-configmap + spec: + serviceAccountName: kubecost-prerequisites + restartPolicy: OnFailure + priorityClassName: dkp-high-priority + containers: + - name: kubectl + image: "${kubetoolsImageRepository:=bitnami/kubectl}:${kubetoolsImageTag:=1.30.5}" + command: + - sh + - -c + - | + kubectl create configmap kubecost-cluster-info-configmap -n ${releaseNamespace} -oyaml --dry-run=client --save-config --from-literal=CLUSTER_ID=$(kubectl get namespace kube-system -o jsonpath="{.metadata.uid}") | kubectl apply -f - diff --git a/services/kubecost/2.5.0/release.yaml b/services/kubecost/2.5.0/release.yaml new file mode 100644 index 000000000..3743e76ae --- /dev/null +++ b/services/kubecost/2.5.0/release.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: kubecost-release + namespace: ${releaseNamespace} +spec: + force: true + prune: true + wait: true + interval: 6h + retryInterval: 1m + dependsOn: + - name: kubecost-prerequisites + namespace: ${releaseNamespace} + path: ./services/kubecost/2.5.0/release + sourceRef: + kind: GitRepository + name: management + namespace: kommander-flux + timeout: 1m + postBuild: + substitute: + releaseNamespace: ${releaseNamespace} diff --git a/services/kubecost/2.5.0/release/release.yaml b/services/kubecost/2.5.0/release/release.yaml new file mode 100644 index 000000000..13d1902c3 --- /dev/null +++ b/services/kubecost/2.5.0/release/release.yaml @@ -0,0 +1,31 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: kubecost + namespace: ${releaseNamespace} +spec: + chart: + spec: + chart: cost-analyzer + sourceRef: + kind: HelmRepository + name: kubecost + namespace: kommander-flux + version: 2.5.0 + interval: 15s + install: + crds: CreateReplace + remediation: + retries: 30 + createNamespace: true + upgrade: + crds: CreateReplace + remediation: + retries: 30 + driftDetection: + mode: enabled + releaseName: kubecost + valuesFrom: + - kind: ConfigMap + name: kubecost-2.5.0-d2iq-defaults + targetNamespace: ${releaseNamespace}