From 6e3e83c207942fc99de53ea208e157f28d92d7e8 Mon Sep 17 00:00:00 2001 From: Dmitry Kartsev Date: Thu, 7 Mar 2024 12:12:19 +0200 Subject: [PATCH] wip --- charts/cnvrg-mlops-idc/Chart.yaml | 6 + .../crds/mlops.cnvrg.io_cnvrgapps.yaml | 979 ++++++++++++++++++ charts/cnvrg-mlops-idc/templates/NOTES.txt | 13 + charts/cnvrg-mlops-idc/templates/cap.yml | 377 +++++++ charts/cnvrg-mlops-idc/templates/hooks.yml | 32 + charts/cnvrg-mlops-idc/templates/operator.yml | 88 ++ charts/cnvrg-mlops-idc/values.yaml | 374 +++++++ hack/coredns-deploy.yaml | 120 +++ hack/permission-fix.yaml | 38 + hack/split-dns.yaml | 38 + 10 files changed, 2065 insertions(+) create mode 100644 charts/cnvrg-mlops-idc/Chart.yaml create mode 100644 charts/cnvrg-mlops-idc/crds/mlops.cnvrg.io_cnvrgapps.yaml create mode 100644 charts/cnvrg-mlops-idc/templates/NOTES.txt create mode 100644 charts/cnvrg-mlops-idc/templates/cap.yml create mode 100644 charts/cnvrg-mlops-idc/templates/hooks.yml create mode 100644 charts/cnvrg-mlops-idc/templates/operator.yml create mode 100644 charts/cnvrg-mlops-idc/values.yaml create mode 100644 hack/coredns-deploy.yaml create mode 100644 hack/permission-fix.yaml create mode 100644 hack/split-dns.yaml diff --git a/charts/cnvrg-mlops-idc/Chart.yaml b/charts/cnvrg-mlops-idc/Chart.yaml new file mode 100644 index 00000000..d6f0f9cc --- /dev/null +++ b/charts/cnvrg-mlops-idc/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: cnvrg-mlops-platform +description: A cnvrg.io operator chart for K8s +type: application +version: 5.0.0 +appVersion: 1.2.3 diff --git a/charts/cnvrg-mlops-idc/crds/mlops.cnvrg.io_cnvrgapps.yaml b/charts/cnvrg-mlops-idc/crds/mlops.cnvrg.io_cnvrgapps.yaml new file mode 100644 index 00000000..fd142fd7 --- /dev/null +++ b/charts/cnvrg-mlops-idc/crds/mlops.cnvrg.io_cnvrgapps.yaml @@ -0,0 +1,979 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: cnvrgapps.mlops.cnvrg.io +spec: + group: mlops.cnvrg.io + names: + kind: CnvrgApp + listKind: CnvrgAppList + plural: cnvrgapps + shortNames: + - cap + singular: cnvrgapp + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.controlPlane.image + name: Version + type: string + - jsonPath: .status.status + name: Status + type: string + - jsonPath: .status.message + name: Message + type: string + name: v1 + schema: + openAPIV3Schema: + description: CnvrgApp represent the cnvrg.io AI/MLOps control plane stack, + which includes frontend and backend services & persistent workloads (DBs). + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + annotations: + additionalProperties: + type: string + type: object + clusterDomain: + type: string + clusterInternalDomain: + type: string + controlPlane: + properties: + baseConfig: + properties: + agentCustomTag: + type: string + cnvrgJobRbacStrict: + type: boolean + cnvrgJobUid: + type: string + cnvrgPrivilegedJob: + type: boolean + featureFlags: + additionalProperties: + type: string + type: object + intercom: + type: string + jobsStorageClass: + type: string + metagpuEnabled: + type: boolean + runJobsOnSelfCluster: + type: string + sentryUrl: + type: string + type: object + cnvrgClusterProvisionerOperator: + properties: + awsCredsRef: + type: string + enabled: + type: boolean + image: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + type: object + cnvrgRouter: + properties: + enabled: + type: boolean + image: + type: string + nodePort: + type: integer + svcName: + type: string + type: object + cnvrgScheduler: + properties: + enabled: + type: boolean + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + type: object + hyper: + properties: + cpuLimit: + type: string + enabled: + type: boolean + image: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + memoryLimit: + type: string + nodePort: + type: integer + port: + type: integer + readinessPeriodSeconds: + type: integer + readinessTimeoutSeconds: + type: integer + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + svcName: + type: string + token: + type: string + type: object + image: + type: string + ldap: + properties: + account: + type: string + adminPassword: + type: string + adminUser: + type: string + base: + type: string + enabled: + type: boolean + host: + type: string + port: + type: string + ssl: + type: string + type: object + mpi: + properties: + enabled: + type: boolean + extraArgs: + additionalProperties: + type: string + type: object + image: + type: string + kubectlDeliveryImage: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + registry: + properties: + name: + type: string + password: + type: string + url: + type: string + user: + type: string + type: object + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + type: object + nomex: + properties: + enabled: + type: boolean + image: + type: string + type: object + objectStorage: + properties: + accessKey: + type: string + azureAccountName: + type: string + azureContainer: + type: string + bucket: + type: string + endpoint: + type: string + gcpProject: + type: string + gcpSecretRef: + type: string + region: + type: string + secretKey: + type: string + type: + enum: + - minio + - aws + - azure + - gcp + type: string + type: object + searchkiq: + properties: + enabled: + type: boolean + hpa: + properties: + enabled: + type: boolean + maxReplicas: + type: integer + utilization: + type: integer + type: object + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + type: object + sidekiq: + properties: + enabled: + type: boolean + hpa: + properties: + enabled: + type: boolean + maxReplicas: + type: integer + utilization: + type: integer + type: object + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + split: + type: boolean + type: object + smtp: + properties: + domain: + type: string + opensslVerifyMode: + type: string + password: + type: string + port: + type: integer + sender: + type: string + server: + type: string + username: + type: string + type: object + systemkiq: + properties: + enabled: + type: boolean + hpa: + properties: + enabled: + type: boolean + maxReplicas: + type: integer + utilization: + type: integer + type: object + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + type: object + webapp: + properties: + enabled: + type: boolean + failureThreshold: + type: integer + hpa: + properties: + enabled: + type: boolean + maxReplicas: + type: integer + utilization: + type: integer + type: object + initialDelaySeconds: + type: integer + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + nodePort: + type: integer + passengerMaxPoolSize: + type: integer + port: + type: integer + readinessPeriodSeconds: + type: integer + readinessTimeoutSeconds: + type: integer + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + svcName: + type: string + type: object + type: object + dbs: + properties: + es: + properties: + cleanupPolicy: + properties: + all: + type: string + app: + type: string + endpoints: + type: string + jobs: + type: string + type: object + credsRef: + type: string + elastalert: + properties: + authProxyImage: + type: string + credsRef: + type: string + enabled: + type: boolean + image: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + nodePort: + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + port: + type: integer + pvcName: + type: string + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + storageClass: + type: string + storageSize: + type: string + svcName: + type: string + type: object + enabled: + type: boolean + image: + type: string + javaOpts: + type: string + kibana: + properties: + credsRef: + type: string + enabled: + type: boolean + image: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + nodePort: + type: integer + port: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + serviceAccount: + type: string + svcName: + type: string + type: object + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + nodePort: + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + patchEsNodes: + type: boolean + port: + type: integer + pvcName: + type: string + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + serviceAccount: + type: string + storageClass: + type: string + storageSize: + type: string + svcName: + type: string + type: object + minio: + properties: + enabled: + type: boolean + image: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + nodePort: + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + port: + type: integer + pvcName: + type: string + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + serviceAccount: + type: string + storageClass: + type: string + storageSize: + type: string + svcName: + type: string + type: object + pg: + properties: + credsRef: + type: string + effectiveCacheSize: + type: string + enabled: + type: boolean + hugePages: + properties: + enabled: + type: boolean + memory: + type: string + size: + type: string + type: object + image: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + maxConnections: + type: integer + nodeSelector: + additionalProperties: + type: string + type: object + port: + type: integer + pvcName: + type: string + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + serviceAccount: + type: string + sharedBuffers: + type: string + storageClass: + type: string + storageSize: + type: string + svcName: + type: string + type: object + prom: + properties: + credsRef: + type: string + enabled: + type: boolean + extraScrapeConfigs: + items: + properties: + labelSelector: + type: string + namespace: + type: string + role: + type: string + type: object + type: array + grafana: + properties: + credsRef: + type: string + enabled: + type: boolean + image: + type: string + nodePort: + type: integer + port: + type: integer + svcName: + type: string + type: object + image: + type: string + port: + type: integer + storageClass: + type: string + storageSize: + type: string + svcName: + type: string + type: object + redis: + properties: + credsRef: + type: string + enabled: + type: boolean + image: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + port: + type: integer + pvcName: + type: string + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + serviceAccount: + type: string + storageClass: + type: string + storageSize: + type: string + svcName: + type: string + type: object + type: object + imageHub: + type: string + labels: + additionalProperties: + type: string + type: object + networking: + properties: + https: + properties: + cert: + type: string + certSecret: + type: string + enabled: + type: boolean + key: + type: string + type: object + ingress: + properties: + istioGwEnabled: + type: boolean + istioGwName: + type: string + istioIngressSelectorKey: + type: string + istioIngressSelectorValue: + type: string + ocpSecureRoutes: + type: boolean + perTryTimeout: + type: string + retriesAttempts: + type: integer + timeout: + type: string + type: + type: string + type: object + proxy: + properties: + configRef: + type: string + enabled: + type: boolean + httpProxy: + items: + type: string + type: array + httpsProxy: + items: + type: string + type: array + noProxy: + items: + type: string + type: array + type: object + type: object + priorityClass: + properties: + appClassRef: + type: string + jobClassRef: + type: string + type: object + registry: + properties: + name: + type: string + password: + type: string + url: + type: string + user: + type: string + type: object + sso: + properties: + central: + properties: + adminUser: + type: string + centralUiImage: + type: string + clientId: + type: string + clientSecret: + type: string + cnvrgProxyImage: + type: string + cookieDomain: + type: string + emailDomain: + items: + type: string + type: array + enabled: + type: boolean + groupsAuth: + type: boolean + insecureOidcAllowUnverifiedEmail: + type: boolean + jwksURL: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + oauthProxyImage: + type: string + oidcIssuerUrl: + type: string + provider: + type: string + publicUrl: + type: string + readiness: + type: boolean + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + scope: + type: string + serviceUrl: + type: string + sslInsecureSkipVerify: + type: boolean + svcName: + type: string + whitelistDomain: + type: string + type: object + enabled: + type: boolean + jwks: + properties: + cacheImage: + type: string + enabled: + type: boolean + image: + type: string + nodeSelector: + additionalProperties: + type: string + type: object + replicas: + type: integer + svcName: + type: string + type: object + pki: + properties: + enabled: + type: boolean + privateKeySecret: + type: string + publicKeySecret: + type: string + rootCaSecret: + type: string + type: object + proxy: + properties: + address: + type: string + enabled: + type: boolean + image: + type: string + limits: + properties: + cpu: + type: string + memory: + type: string + type: object + nodeSelector: + additionalProperties: + type: string + type: object + readiness: + type: boolean + replicas: + type: integer + requests: + properties: + cpu: + type: string + memory: + type: string + type: object + svcName: + type: string + type: object + version: + type: string + type: object + tenancy: + properties: + enabled: + type: boolean + key: + type: string + value: + type: string + type: object + type: object + status: + properties: + message: + type: string + progress: + type: integer + stackReadiness: + additionalProperties: + type: boolean + type: object + status: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/cnvrg-mlops-idc/templates/NOTES.txt b/charts/cnvrg-mlops-idc/templates/NOTES.txt new file mode 100644 index 00000000..aa20e8a6 --- /dev/null +++ b/charts/cnvrg-mlops-idc/templates/NOTES.txt @@ -0,0 +1,13 @@ +🚀 Thank you for installing cnvrg.io! + +Wait until all the pods in the {{ .Release.Namespace }} namespace are running and healthy. +Once all pods are running, get the app url by running: +kubectl get cap cnvrg-app -n{{ .Release.Namespace }} -ojson | grep clusterDomain | awk '{print "app."$2}' | tr -d '"|,' +If something goes wrong (how come? 🙀), check app init container logs: +kubectl logs -n{{ .Release.Namespace }} --tail 10 -lapp=app -c ingresscheck + +Join our community at https://community.cnvrg.io +Talk to our team via email at hi@cnvrg.io + + + diff --git a/charts/cnvrg-mlops-idc/templates/cap.yml b/charts/cnvrg-mlops-idc/templates/cap.yml new file mode 100644 index 00000000..8d39f7b2 --- /dev/null +++ b/charts/cnvrg-mlops-idc/templates/cap.yml @@ -0,0 +1,377 @@ +apiVersion: mlops.cnvrg.io/v1 +kind: CnvrgApp +metadata: + name: cnvrg-app + namespace: {{ .Release.Namespace}} +spec: + clusterDomain: {{.Values.clusterDomain}} + clusterInternalDomain: {{.Values.clusterInternalDomain}} + imageHub: {{.Values.imageHub}} + controlPlane: + image: {{.Values.controlPlane.image}} + webapp: + replicas: {{.Values.controlPlane.webapp.replicas}} + enabled: {{.Values.controlPlane.webapp.enabled}} + port: {{.Values.controlPlane.webapp.port}} + requests: + cpu: "{{.Values.controlPlane.webapp.requests.cpu}}" + memory: {{.Values.controlPlane.webapp.requests.memory}} + limits: + cpu: "{{.Values.controlPlane.webapp.limits.cpu}}" + memory: {{.Values.controlPlane.webapp.limits.memory}} + svcName: {{.Values.controlPlane.webapp.svcName}} + nodePort: {{.Values.controlPlane.webapp.nodePort}} + passengerMaxPoolSize: {{.Values.controlPlane.webapp.passengerMaxPoolSize}} + initialDelaySeconds: {{.Values.controlPlane.webapp.initialDelaySeconds}} + readinessPeriodSeconds: {{.Values.controlPlane.webapp.readinessPeriodSeconds}} + readinessTimeoutSeconds: {{.Values.controlPlane.webapp.readinessTimeoutSeconds}} + failureThreshold: {{.Values.controlPlane.webapp.failureThreshold}} + hpa: + enabled: {{.Values.controlPlane.webapp.hpa.enabled}} + utilization: {{.Values.controlPlane.webapp.hpa.utilization}} + maxReplicas: {{.Values.controlPlane.webapp.hpa.maxReplicas}} + sidekiq: + enabled: {{.Values.controlPlane.sidekiq.enabled}} + split: {{.Values.controlPlane.sidekiq.split}} + requests: + cpu: "{{.Values.controlPlane.sidekiq.requests.cpu}}" + memory: {{.Values.controlPlane.sidekiq.requests.memory}} + limits: + cpu: "{{.Values.controlPlane.sidekiq.limits.cpu}}" + memory: {{.Values.controlPlane.sidekiq.limits.memory}} + replicas: {{.Values.controlPlane.sidekiq.replicas}} + hpa: + enabled: {{.Values.controlPlane.sidekiq.hpa.enabled}} + utilization: {{.Values.controlPlane.sidekiq.hpa.utilization}} + maxReplicas: {{.Values.controlPlane.sidekiq.hpa.maxReplicas}} + searchkiq: + enabled: {{.Values.controlPlane.searchkiq.enabled}} + requests: + cpu: "{{.Values.controlPlane.searchkiq.requests.cpu}}" + memory: {{.Values.controlPlane.searchkiq.requests.memory}} + limits: + cpu: "{{.Values.controlPlane.searchkiq.limits.cpu}}" + memory: "{{.Values.controlPlane.searchkiq.limits.memory}}" + replicas: {{.Values.controlPlane.searchkiq.replicas}} + hpa: + enabled: {{.Values.controlPlane.searchkiq.hpa.enabled}} + utilization: {{.Values.controlPlane.searchkiq.hpa.utilization}} + maxReplicas: {{.Values.controlPlane.searchkiq.hpa.maxReplicas}} + systemkiq: + enabled: {{.Values.controlPlane.systemkiq.enabled}} + requests: + cpu: "{{.Values.controlPlane.systemkiq.requests.cpu}}" + memory: {{.Values.controlPlane.systemkiq.requests.memory}} + limits: + cpu: "{{.Values.controlPlane.systemkiq.limits.cpu}}" + memory: {{.Values.controlPlane.systemkiq.limits.memory}} + replicas: {{.Values.controlPlane.systemkiq.replicas}} + hpa: + enabled: {{.Values.controlPlane.systemkiq.hpa.enabled}} + utilization: {{.Values.controlPlane.systemkiq.hpa.utilization}} + maxReplicas: {{.Values.controlPlane.systemkiq.hpa.maxReplicas}} + hyper: + enabled: {{.Values.controlPlane.hyper.enabled}} + image: {{.Values.controlPlane.hyper.image}} + port: {{.Values.controlPlane.hyper.port}} + replicas: {{.Values.controlPlane.hyper.replicas}} + nodePort: {{.Values.controlPlane.hyper.nodePort}} + svcName: {{.Values.controlPlane.hyper.svcName}} + token: {{.Values.controlPlane.hyper.token}} + requests: + cpu: {{.Values.controlPlane.hyper.requests.cpu}} + memory: {{.Values.controlPlane.hyper.requests.memory}} + limits: + cpu: "{{.Values.controlPlane.hyper.limits.cpu}}" + memory: {{.Values.controlPlane.hyper.limits.memory}} + cpuLimit: {{.Values.controlPlane.hyper.cpuLimit}} + memoryLimit: {{.Values.controlPlane.hyper.memoryLimit}} + readinessPeriodSeconds: {{.Values.controlPlane.hyper.readinessPeriodSeconds}} + readinessTimeoutSeconds: {{.Values.controlPlane.hyper.readinessTimeoutSeconds}} + cnvrgScheduler: + enabled: {{.Values.controlPlane.cnvrgScheduler.enabled}} + requests: + cpu: {{.Values.controlPlane.cnvrgScheduler.requests.cpu}} + memory: {{.Values.controlPlane.cnvrgScheduler.requests.memory}} + limits: + cpu: "{{.Values.controlPlane.cnvrgScheduler.limits.cpu}}" + memory: {{.Values.controlPlane.cnvrgScheduler.limits.memory}} + replicas: {{.Values.controlPlane.cnvrgScheduler.replicas}} + cnvrgRouter: + enabled: {{.Values.controlPlane.cnvrgRouter.enabled}} + image: {{.Values.controlPlane.cnvrgRouter.image}} + svcName: {{.Values.controlPlane.cnvrgRouter.svcName}} + nodePort: {{.Values.controlPlane.cnvrgRouter.nodePort}} + baseConfig: + jobsStorageClass: {{.Values.controlPlane.baseConfig.jobsStorageClass}} + featureFlags: + {{- range $fk, $fv := .Values.controlPlane.baseConfig.featureFlags }} + {{$fk}}: "{{$fv}}" + {{- end }} + sentryUrl: {{.Values.controlPlane.baseConfig.sentryUrl}} + runJobsOnSelfCluster: {{.Values.controlPlane.baseConfig.runJobsOnSelfCluster}} + agentCustomTag: {{.Values.controlPlane.baseConfig.agentCustomTag}} + intercom: "{{.Values.controlPlane.baseConfig.intercom}}" + cnvrgJobUid: "{{.Values.controlPlane.baseConfig.cnvrgJobUid}}" + cnvrgJobRbacStrict: {{.Values.controlPlane.baseConfig.cnvrgJobRbacStrict}} + cnvrgPrivilegedJob: {{.Values.controlPlane.baseConfig.cnvrgPrivilegedJob}} + metagpuEnabled: {{.Values.controlPlane.baseConfig.metagpuEnabled}} + ldap: + enabled: {{.Values.controlPlane.ldap.enabled}} + host: {{.Values.controlPlane.ldap.host}} + port: {{.Values.controlPlane.ldap.port}} + account: {{.Values.controlPlane.ldap.account}} + base: {{.Values.controlPlane.ldap.base}} + adminUser: {{.Values.controlPlane.ldap.adminUser}} + adminPassword: {{.Values.controlPlane.ldap.adminPassword}} + ssl: {{.Values.controlPlane.ldap.ssl}} + smtp: + server: {{.Values.controlPlane.smtp.server}} + port: {{.Values.controlPlane.smtp.port}} + username: {{.Values.controlPlane.smtp.username}} + password: {{.Values.controlPlane.smtp.password}} + domain: {{.Values.controlPlane.smtp.domain}} + opensslVerifyMode: {{.Values.controlPlane.smtp.opensslVerifyMode}} + sender: {{.Values.controlPlane.smtp.sender}} + objectStorage: + type: {{.Values.controlPlane.objectStorage.type}} + bucket: {{.Values.controlPlane.objectStorage.bucket}} + region: {{.Values.controlPlane.objectStorage.region}} + accessKey: {{.Values.controlPlane.objectStorage.accessKey}} + secretKey: {{.Values.controlPlane.objectStorage.secretKey}} + endpoint: {{.Values.controlPlane.objectStorage.endpoint}} + azureAccountName: {{.Values.controlPlane.objectStorage.azureAccountName}} + azureContainer: {{.Values.controlPlane.objectStorage.azureContainer}} + gcpProject: {{.Values.controlPlane.objectStorage.gcpProject}} + gcpSecretRef: {{.Values.controlPlane.objectStorage.gcpSecretRef}} + mpi: + enabled: {{.Values.controlPlane.mpi.enabled}} + image: {{.Values.controlPlane.mpi.image}} + kubectlDeliveryImage: {{.Values.controlPlane.mpi.kubectlDeliveryImage}} + extraArgs: null + registry: + name: {{.Values.controlPlane.mpi.registry.name}} + url: {{.Values.controlPlane.mpi.registry.url}} + user: {{.Values.controlPlane.mpi.registry.user}} + password: {{.Values.controlPlane.mpi.registry.password}} + requests: + cpu: {{.Values.controlPlane.mpi.requests.cpu}} + memory: {{.Values.controlPlane.mpi.requests.memory}} + limits: + cpu: {{.Values.controlPlane.mpi.limits.cpu}} + memory: {{.Values.controlPlane.mpi.limits.memory}} + nomex: + enabled: {{.Values.controlPlane.nomex.enabled}} + image: {{.Values.controlPlane.nomex.image}} + registry: + name: {{.Values.registry.name}} + url: {{.Values.registry.url}} + user: {{.Values.registry.user}} + password: {{.Values.registry.password}} + dbs: + pg: + enabled: {{.Values.dbs.pg.enabled}} + serviceAccount: {{.Values.dbs.pg.serviceAccount}} + image: {{.Values.dbs.pg.image}} + port: {{.Values.dbs.pg.port}} + storageSize: {{.Values.dbs.pg.storageSize}} + svcName: {{.Values.dbs.pg.svcName}} + storageClass: {{.Values.dbs.pg.storageClass}} + requests: + cpu: "{{.Values.dbs.pg.requests.cpu}}" + memory: {{.Values.dbs.pg.requests.memory}} + limits: + cpu: "{{.Values.dbs.pg.limits.cpu}}" + memory: {{.Values.dbs.pg.limits.memory}} + maxConnections: {{.Values.dbs.pg.maxConnections}} + sharedBuffers: {{.Values.dbs.pg.sharedBuffers}} + effectiveCacheSize: {{.Values.dbs.pg.effectiveCacheSize}} + hugePages: + enabled: {{.Values.dbs.pg.hugePages.enabled}} + size: {{.Values.dbs.pg.hugePages.size}} + memory: {{.Values.dbs.pg.hugePages.memory}} + nodeSelector: null + credsRef: {{.Values.dbs.pg.credsRef}} + pvcName: {{.Values.dbs.pg.pvcName}} + redis: + enabled: {{.Values.dbs.redis.enabled}} + serviceAccount: {{.Values.dbs.redis.serviceAccount}} + image: {{.Values.dbs.redis.image}} + svcName: {{.Values.dbs.redis.svcName}} + port: {{.Values.dbs.redis.port}} + storageSize: {{.Values.dbs.redis.storageSize}} + storageClass: {{.Values.dbs.redis.storageClass}} + requests: + cpu: "{{.Values.dbs.redis.requests.cpu}}" + memory: {{.Values.dbs.redis.requests.memory}} + limits: + cpu: "{{.Values.dbs.redis.limits.cpu}}" + memory: {{.Values.dbs.redis.limits.memory}} + nodeSelector: null + credsRef: {{.Values.dbs.redis.credsRef}} + pvcName: {{.Values.dbs.redis.pvcName}} + minio: + enabled: {{.Values.dbs.minio.enabled}} + serviceAccount: {{.Values.dbs.minio.serviceAccount}} + replicas: {{.Values.dbs.minio.replicas}} + image: {{.Values.dbs.minio.image}} + port: {{.Values.dbs.minio.port}} + storageSize: {{.Values.dbs.minio.storageSize}} + svcName: {{.Values.dbs.minio.svcName}} + nodePort: {{.Values.dbs.minio.nodePort}} + storageClass: {{.Values.dbs.minio.storageClass}} + requests: + cpu: "{{.Values.dbs.minio.requests.cpu}}" + memory: {{.Values.dbs.minio.requests.memory}} + limits: + cpu: "{{.Values.dbs.minio.limits.cpu}}" + memory: {{.Values.dbs.minio.limits.memory}} + nodeSelector: null + pvcName: {{.Values.dbs.minio.pvcName}} + es: + enabled: {{.Values.dbs.es.enabled}} + serviceAccount: {{.Values.dbs.es.serviceAccount}} + image: {{.Values.dbs.es.image}} + port: {{.Values.dbs.es.port}} + storageSize: {{.Values.dbs.es.storageSize}} + svcName: {{.Values.dbs.es.svcName}} + nodePort: {{.Values.dbs.es.nodePort}} + storageClass: {{.Values.dbs.es.storageClass}} + requests: + cpu: "{{.Values.dbs.es.requests.cpu}}" + memory: {{.Values.dbs.es.requests.memory}} + limits: + cpu: "{{.Values.dbs.es.limits.cpu}}" + memory: {{.Values.dbs.es.limits.memory}} + javaOpts: {{.Values.dbs.es.javaOpts}} + patchEsNodes: {{.Values.dbs.es.patchEsNodes}} + nodeSelector: null + credsRef: {{.Values.dbs.es.credsRef}} + pvcName: {{.Values.dbs.es.pvcName}} + cleanupPolicy: + all: {{.Values.dbs.es.cleanupPolicy.all}} + app: {{.Values.dbs.es.cleanupPolicy.app}} + jobs: {{.Values.dbs.es.cleanupPolicy.jobs}} + endpoints: {{.Values.dbs.es.cleanupPolicy.endpoints}} + kibana: + enabled: {{.Values.dbs.es.kibana.enabled}} + serviceAccount: {{.Values.dbs.es.kibana.serviceAccount}} + svcName: {{.Values.dbs.es.kibana.svcName}} + port: {{.Values.dbs.es.kibana.port}} + image: {{.Values.dbs.es.kibana.image}} + nodePort: {{.Values.dbs.es.kibana.nodePort}} + requests: + cpu: {{.Values.dbs.es.kibana.requests.cpu}} + memory: {{.Values.dbs.es.kibana.requests.memory}} + limits: + cpu: {{.Values.dbs.es.kibana.limits.cpu}} + memory: {{.Values.dbs.es.kibana.limits.memory}} + credsRef: {{.Values.dbs.es.kibana.credsRef}} + elastalert: + enabled: {{.Values.dbs.es.elastalert.enabled}} + image: {{.Values.dbs.es.elastalert.image}} + authProxyImage: {{.Values.dbs.es.elastalert.authProxyImage}} + credsRef: {{.Values.dbs.es.elastalert.credsRef}} + port: {{.Values.dbs.es.elastalert.port}} + nodePort: {{.Values.dbs.es.elastalert.nodePort}} + storageSize: {{.Values.dbs.es.elastalert.storageSize}} + svcName: {{.Values.dbs.es.elastalert.svcName}} + storageClass: {{.Values.dbs.es.elastalert.storageClass}} + requests: + cpu: {{.Values.dbs.es.elastalert.requests.cpu}} + memory: {{.Values.dbs.es.elastalert.requests.memory}} + limits: + cpu: {{.Values.dbs.es.elastalert.limits.cpu}} + memory: {{.Values.dbs.es.elastalert.limits.memory}} + nodeSelector: null + pvcName: {{.Values.dbs.es.elastalert.pvcName}} + prom: + enabled: {{.Values.dbs.prom.enabled}} + credsRef: {{.Values.dbs.prom.credsRef}} + extraScrapeConfigs: null + image: {{.Values.dbs.prom.image}} + storageClass: {{.Values.dbs.prom.storageClass}} + storageSize: {{.Values.dbs.prom.storageSize}} + grafana: + enabled: {{.Values.dbs.prom.grafana.enabled}} + image: {{.Values.dbs.prom.grafana.image}} + svcName: {{.Values.dbs.prom.grafana.svcName}} + port: {{.Values.dbs.prom.grafana.port}} + nodePort: {{.Values.dbs.prom.grafana.nodePort}} + credsRef: {{.Values.dbs.prom.grafana.credsRef}} + networking: + ingress: + type: {{.Values.networking.ingress.type}} + timeout: {{.Values.networking.ingress.timeout}} + retriesAttempts: {{.Values.networking.ingress.retriesAttempts}} + perTryTimeout: {{.Values.networking.ingress.perTryTimeout}} + istioGwEnabled: {{.Values.networking.ingress.istioGwEnabled}} + istioGwName: {{.Values.networking.ingress.istioGwName}} + istioIngressSelectorKey: {{.Values.networking.ingress.istioIngressSelectorKey}} + istioIngressSelectorValue: {{.Values.networking.ingress.istioIngressSelectorValue}} + ocpSecureRoutes: {{.Values.networking.ingress.ocpSecureRoutes}} + https: + enabled: {{.Values.networking.https.enabled}} + certSecret: {{.Values.networking.https.certSecret}} + cert: {{.Values.networking.https.cert}} + key: {{.Values.networking.https.key}} + proxy: + enabled: {{.Values.networking.proxy.enabled}} + configRef: {{.Values.networking.proxy.configRef}} + httpProxy: {{ toJson .Values.networking.proxy.httpProxy }} + httpsProxy: {{ toJson .Values.networking.proxy.httpsProxy}} + noProxy: {{ toJson .Values.networking.proxy.noProxy}} + sso: + enabled: {{.Values.sso.enabled}} + version: {{.Values.sso.version}} + pki: + enabled: {{.Values.sso.pki.enabled}} + rootCaSecret: {{.Values.sso.pki.rootCaSecret}} + privateKeySecret: {{.Values.sso.pki.privateKeySecret}} + publicKeySecret: {{.Values.sso.pki.publicKeySecret}} + jwks: + enabled: {{.Values.sso.jwks.enabled}} + image: {{.Values.sso.jwks.image}} + cacheImage: {{.Values.sso.jwks.cacheImage}} + central: + enabled: {{.Values.sso.central.enabled}} + publicUrl: {{.Values.sso.central.publicUrl}} + oauthProxyImage: {{.Values.sso.central.oauthProxyImage}} + centralUiImage: {{.Values.sso.central.centralUiImage}} + adminUser: {{.Values.sso.central.adminUser}} + provider: {{.Values.sso.central.provider}} + emailDomain: {{ toJson .Values.sso.central.emailDomain }} + clientId: {{.Values.sso.central.clientId}} + clientSecret: {{.Values.sso.central.clientSecret}} + oidcIssuerUrl: {{.Values.sso.central.oidcIssuerUrl}} + serviceUrl: {{.Values.sso.central.serviceUrl}} + scope: {{.Values.sso.central.scope}} + insecureOidcAllowUnverifiedEmail: {{.Values.sso.central.insecureOidcAllowUnverifiedEmail}} + whitelistDomain: {{.Values.sso.central.whitelistDomain}} + cookieDomain: {{.Values.sso.central.cookieDomain}} + groupsAuth: {{.Values.sso.central.groupsAuth}} + readiness: {{.Values.sso.central.readiness}} + requests: + cpu: "{{.Values.sso.central.requests.cpu}}" + memory: "{{.Values.sso.central.requests.memory}}" + limits: + cpu: "{{.Values.sso.central.limits.cpu}}" + memory: "{{.Values.sso.central.limits.memory}}" + proxy: + enabled: {{.Values.sso.proxy.enabled}} + image: {{.Values.sso.proxy.image}} + address: {{.Values.sso.proxy.address}} + readiness: {{.Values.sso.proxy.readiness}} + requests: + cpu: "{{.Values.sso.proxy.requests.cpu}}" + memory: "{{.Values.sso.proxy.requests.memory}}" + limits: + cpu: "{{.Values.sso.proxy.limits.cpu}}" + memory: "{{.Values.sso.proxy.limits.memory}}" + tenancy: + enabled: {{.Values.tenancy.enabled}} + key: {{.Values.tenancy.key}} + value: {{.Values.tenancy.value}} + priorityClass: + appClassRef: {{.Values.priorityClass.appClassRef}} + jobClassRef: {{.Values.priorityClass.jobClassRef}} diff --git a/charts/cnvrg-mlops-idc/templates/hooks.yml b/charts/cnvrg-mlops-idc/templates/hooks.yml new file mode 100644 index 00000000..e593c24c --- /dev/null +++ b/charts/cnvrg-mlops-idc/templates/hooks.yml @@ -0,0 +1,32 @@ +####################################### Uninstall hooks ####################################### +apiVersion: batch/v1 +kind: Job +metadata: + name: pre-cnvrg-cleanup + namespace: {{ .Release.Namespace}} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-weight": "2" + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + template: + spec: + serviceAccountName: cnvrg-operator + imagePullSecrets: + - name: operator-pull-secret + containers: + - name: delete-cnvrgapp + image: "{{ .Values.imageHub }}/cnvrg-tools:v0.3" + args: + - /bin/bash + - -c + - | + set -x + echo "running cleanup" + kubectl delete cnvrgapp cnvrg-app -n {{ .Release.Namespace }} --ignore-not-found + while (( $(kubectl get cnvrgapp cnvrg-app -n {{ .Release.Namespace }} --ignore-not-found | grep cnvrg-app | wc -l ) != 0 )); do + echo "waiting for cnvrgapp will be deleted. . . "; + sleep 1 + done + restartPolicy: Never + backoffLimit: 4 diff --git a/charts/cnvrg-mlops-idc/templates/operator.yml b/charts/cnvrg-mlops-idc/templates/operator.yml new file mode 100644 index 00000000..dc9f5fbf --- /dev/null +++ b/charts/cnvrg-mlops-idc/templates/operator.yml @@ -0,0 +1,88 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cnvrg-operator + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cnvrg-operator-role + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - "*" + resources: + - "*" + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cnvrg-operator-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cnvrg-operator-role +subjects: + - kind: ServiceAccount + name: cnvrg-operator +--- +apiVersion: v1 +kind: Secret +type: kubernetes.io/dockerconfigjson +metadata: + name: operator-pull-secret + namespace: {{ .Release.Namespace }} +data: + .dockerconfigjson: {{ printf "{\"auths\":{\"%s\":{\"username\":\"%s\",\"password\":\"%s\",\"auth\":\"%s\"}}}" .Values.registry.url .Values.registry.user .Values.registry.password (printf "%s:%s" .Values.registry.user .Values.registry.password | b64enc) | b64enc }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: cnvrg-operator + name: cnvrg-operator + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + control-plane: cnvrg-operator + template: + metadata: + labels: + control-plane: cnvrg-operator + spec: + imagePullSecrets: + - name: operator-pull-secret + containers: + - command: + - /opt/app-root/cnvrg-operator + - start + - --max-concurrent-reconciles + - "3" + - --namespace + - {{.Release.Namespace}} + image: "{{.Values.imageHub}}/cnvrg-operator:v5.0.1.snp" + imagePullPolicy: Always + name: cnvrg-operator + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 500m + memory: 200Mi + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + capabilities: + drop: + - "ALL" + serviceAccountName: cnvrg-operator + terminationGracePeriodSeconds: 10 \ No newline at end of file diff --git a/charts/cnvrg-mlops-idc/values.yaml b/charts/cnvrg-mlops-idc/values.yaml new file mode 100644 index 00000000..7f2a02a1 --- /dev/null +++ b/charts/cnvrg-mlops-idc/values.yaml @@ -0,0 +1,374 @@ +clusterDomain: '' +clusterInternalDomain: cluster.local +imageHub: docker.io/cnvrg +controlPlane: + image: core:3.6.99 + webapp: + replicas: 1 + enabled: true + port: 8080 + requests: + cpu: 500m + memory: 4Gi + limits: + cpu: '4' + memory: 8Gi + svcName: app + nodePort: 30080 + passengerMaxPoolSize: 50 + initialDelaySeconds: 10 + readinessPeriodSeconds: 25 + readinessTimeoutSeconds: 20 + failureThreshold: 5 + hpa: + enabled: true + utilization: 85 + maxReplicas: 5 + sidekiq: + enabled: true + split: true + requests: + cpu: 200m + memory: 3750Mi + limits: + cpu: '2' + memory: 8Gi + replicas: 2 + hpa: + enabled: true + utilization: 85 + maxReplicas: 5 + searchkiq: + enabled: true + requests: + cpu: 200m + memory: 1Gi + limits: + cpu: '2' + memory: 8Gi + replicas: 1 + hpa: + enabled: true + utilization: 85 + maxReplicas: 5 + systemkiq: + enabled: true + requests: + cpu: 300m + memory: 2Gi + limits: + cpu: '2' + memory: 8Gi + replicas: 1 + hpa: + enabled: true + utilization: 85 + maxReplicas: 5 + hyper: + enabled: true + image: hyper-server:latest + port: 5050 + replicas: 1 + nodePort: 30050 + svcName: hyper + token: token + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: '2' + memory: 4Gi + cpuLimit: '' + memoryLimit: '' + readinessPeriodSeconds: 100 + readinessTimeoutSeconds: 60 + cnvrgScheduler: + enabled: false + requests: + cpu: 200m + memory: 1000Mi + limits: + cpu: '2' + memory: 4Gi + replicas: 1 + cnvrgRouter: + enabled: false + image: nginx:1.21.0 + svcName: cnvrg-router + nodePort: 30081 + baseConfig: + jobsStorageClass: '' + featureFlags: + CNVRG_ENABLE_MOUNT_FOLDERS: false + CNVRG_MOUNT_HOST_FOLDERS: false + CNVRG_PROMETHEUS_METRICS: true + sentryUrl: '' + runJobsOnSelfCluster: '' + agentCustomTag: agnostic-logs + intercom: 'true' + cnvrgJobUid: '0' + cnvrgJobRbacStrict: false + cnvrgPrivilegedJob: true + metagpuEnabled: false + ldap: + enabled: false + host: '' + port: '' + account: userPrincipalName + base: '' + adminUser: '' + adminPassword: '' + ssl: '' + smtp: + server: '' + port: 587 + username: '' + password: '' + domain: '' + opensslVerifyMode: '' + sender: info@cnvrg.io + objectStorage: + type: minio + bucket: cnvrg-storage + region: eastus + accessKey: '' + secretKey: '' + endpoint: '' + azureAccountName: '' + azureContainer: '' + gcpProject: '' + gcpSecretRef: gcp-storage-secret + mpi: + enabled: false + image: mpioperator/mpi-operator:v0.2.3 + kubectlDeliveryImage: mpioperator/kubectl-delivery:v0.2.3 + extraArgs: + registry: + name: mpi-private-registry + url: docker.io + user: '' + password: '' + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1000m + memory: 1Gi + nomex: + enabled: false + image: nomex:v1.0.0 +registry: + name: cnvrg-app-registry + url: docker.io + user: '' + password: '' +dbs: + pg: + enabled: true + serviceAccount: pg + image: postgresql-12-centos7:latest + port: 5432 + storageSize: 10Gi + svcName: postgres + storageClass: '' + requests: + cpu: '1' + memory: 4Gi + limits: + cpu: '12' + memory: 32Gi + maxConnections: 500 + sharedBuffers: 1024MB + effectiveCacheSize: 2048MB + hugePages: + enabled: false + size: 2Mi + memory: '' + nodeSelector: + credsRef: pg-creds + pvcName: pg-storage + redis: + enabled: true + serviceAccount: redis + image: cnvrg-redis:v3.0.5.c2 + svcName: redis + port: 6379 + storageSize: 3Gi + storageClass: '' + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 1000m + memory: 2Gi + nodeSelector: + credsRef: redis-creds + pvcName: redis-storage + minio: + enabled: true + serviceAccount: minio + replicas: 1 + image: minio:RELEASE.2021-05-22T02-34-39Z + port: 9000 + storageSize: 10Gi + svcName: minio + nodePort: 30090 + storageClass: '' + requests: + cpu: 200m + memory: 2Gi + limits: + cpu: '8' + memory: 20Gi + nodeSelector: + pvcName: minio-storage + es: + enabled: true + serviceAccount: es + image: cnvrg-es:7.17.5 + port: 9200 + storageSize: 10Gi + svcName: elasticsearch + nodePort: 32200 + storageClass: '' + requests: + cpu: 500m + memory: 4Gi + limits: + cpu: '4' + memory: 8Gi + javaOpts: '' + patchEsNodes: false + nodeSelector: + credsRef: es-creds + pvcName: es-storage + cleanupPolicy: + all: 3d + app: 30d + jobs: 14d + endpoints: 1825d + kibana: + enabled: true + serviceAccount: kibana + svcName: kibana + port: 8080 + image: kibana-oss:7.8.1 + nodePort: 30601 + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 1000m + memory: 2Gi + credsRef: kibana-creds + elastalert: + enabled: true + image: elastalert:3.0.0-beta.1 + authProxyImage: nginx:1.20 + credsRef: elastalert-creds + port: 8080 + nodePort: 32030 + storageSize: 3Gi + svcName: elastalert + storageClass: '' + requests: + cpu: 100m + memory: 200Mi + limits: + cpu: 400m + memory: 800Mi + nodeSelector: + pvcName: elastalert-storage + prom: + enabled: true + credsRef: prom-creds + extraScrapeConfigs: + image: prometheus:v2.37.1 + storageClass: "" + storageSize: 5Gi + grafana: + enabled: true + image: grafana-oss:9.1.7 + svcName: grafana + port: 8080 + nodePort: 30012 + credsRef: grafana-creds +networking: + ingress: + type: istio + timeout: 18000s + retriesAttempts: 5 + perTryTimeout: 3600s + istioGwEnabled: true + istioGwName: '' + istioIngressSelectorKey: 'istio' + istioIngressSelectorValue: 'ingressgateway' + ocpSecureRoutes: false + https: + enabled: false + certSecret: '' + cert: '' + key: '' + proxy: + enabled: false + configRef: cp-proxy + httpProxy: [] + httpsProxy: [] + noProxy: [] +sso: + enabled: false + version: v3 + pki: + enabled: false + rootCaSecret: sso-idp-root-ca + privateKeySecret: sso-idp-private-key + publicKeySecret: sso-idp-pki-public-key + jwks: + enabled: false + name: cnvrg-jwks + image: jwks:ns-watch-scope + cacheImage: redis:7.0.5 + central: + enabled: false + publicUrl: '' + oauthProxyImage: oauth2-proxy:v7.4.ssov3.p6 + centralUiImage: centralsso:latest + adminUser: '' + provider: '' + emailDomain: + - "*" + clientId: '' + clientSecret: '' + oidcIssuerUrl: '' + serviceUrl: '' + scope: openid email profile + insecureOidcAllowUnverifiedEmail: true + whitelistDomain: '' + cookieDomain: '' + groupsAuth: false + readiness: true + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2 + memory: 4Gi + proxy: + enabled: false + image: cnvrg-proxy:ns-watch-scope + address: "" + readiness: true + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 2 + memory: 4Gi + +tenancy: + enabled: false + key: purpose + value: cnvrg-control-plane +priorityClass: + appClassRef: "" + jobClassRef: "" \ No newline at end of file diff --git a/hack/coredns-deploy.yaml b/hack/coredns-deploy.yaml new file mode 100644 index 00000000..6d260cb4 --- /dev/null +++ b/hack/coredns-deploy.yaml @@ -0,0 +1,120 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system +spec: + progressDeadlineSeconds: 600 + replicas: 3 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kube-dns + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + k8s-app: kube-dns + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: + - kube-dns + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - -conf + - /etc/coredns/Corefile + image: registry.k8s.io/coredns/coredns:v1.11.1 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /ready + port: 8181 + scheme: HTTP + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + resources: + limits: + memory: 500Mi + requests: + cpu: 100m + memory: 70Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /tmp/custom-zone + name: custom-zone + readOnly: true + - mountPath: /etc/coredns + name: config-volume + readOnly: true + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + restartPolicy: Always + schedulerName: default-scheduler + securityContext: + seccompProfile: + type: RuntimeDefault + serviceAccountName: coredns + terminationGracePeriodSeconds: 30 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + volumes: + - configMap: + defaultMode: 420 + name: custom-zone + name: custom-zone + - configMap: + defaultMode: 420 + items: + - key: Corefile + path: Corefile + name: coredns + name: config-volume diff --git a/hack/permission-fix.yaml b/hack/permission-fix.yaml new file mode 100644 index 00000000..8d06056f --- /dev/null +++ b/hack/permission-fix.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: minikube-pv-hack +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: minikube-pv-hack + namespace: minikube-pv-hack +spec: + selector: + matchLabels: + name: minikube-pv-hack + template: + metadata: + labels: + name: minikube-pv-hack + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: minikube-pv-hack + image: registry.access.redhat.com/ubi8:latest + command: + - bash + - -c + - | + while : ; do + chmod 777 /target + sleep 1 + done + volumeMounts: + - name: host-vol + mountPath: /target + volumes: + - name: host-vol + hostPath: + path: /mnt/hostpath \ No newline at end of file diff --git a/hack/split-dns.yaml b/hack/split-dns.yaml new file mode 100644 index 00000000..17c529da --- /dev/null +++ b/hack/split-dns.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: custom-zone + namespace: kube-system +data: + custom-zone: | + $ORIGIN cnvrg-on-idc.azops.cnvrg.io. + cnvrg-on-idc.azops.cnvrg.io. IN SOA ns.cnvrg-on-idc.azops.cnvrg.io. username.cnvrg-on-idc.azops.cnvrg.io. ( 2007120710 1d 2h 4w 1h ) + * IN A 100.82.189.186 + * IN A 100.82.189.93 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + Corefile: | + cnvrg-on-idc.azops.cnvrg.io:53 { + file /tmp/custom-zone/custom-zone + } + .:53 { + errors + health + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + ttl 30 + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } \ No newline at end of file