diff --git a/.github/gitleaks.yaml b/.github/workflows/gitleaks.yaml similarity index 100% rename from .github/gitleaks.yaml rename to .github/workflows/gitleaks.yaml diff --git a/.github/kubeconform.yaml b/.github/workflows/kubeconform.yaml similarity index 100% rename from .github/kubeconform.yaml rename to .github/workflows/kubeconform.yaml diff --git a/kubernetes/arc1/apps/kube-system/nvidia-device-plugin/app/helmrelease.yaml b/kubernetes/arc1/apps/kube-system/nvidia-device-plugin/app/helmrelease.yaml index 799055f..e0001e7 100644 --- a/kubernetes/arc1/apps/kube-system/nvidia-device-plugin/app/helmrelease.yaml +++ b/kubernetes/arc1/apps/kube-system/nvidia-device-plugin/app/helmrelease.yaml @@ -63,4 +63,4 @@ spec: resources: - name: nvidia.com/gpu replicas: 7 - default: "single" + default: "default" diff --git a/kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml b/kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml index b198781..4b7f857 100644 --- a/kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml +++ b/kubernetes/arc1/apps/machine-learning/colabfold/app/helmrelease.yaml @@ -26,7 +26,7 @@ spec: retries: 3 values: controllers: - ollama: + colabfold: type: deployment annotations: reloader.stakater.com/auto: "true" @@ -56,11 +56,11 @@ spec: requests: cpu: 200m memory: 4Gi - gpu.intel.com/i915: "4" + nvidia.com/gpu: 2 limits: cpu: 32000m memory: 64Gi - gpu.intel.com/i915: "4" + nvidia.com/gpu: 4 service: app: controller: *app @@ -88,6 +88,7 @@ spec: - secretName: colabfold-tls hosts: [*host] persistence: + # TODO: Replace with existing PVC data: storageClass: local-nvme accessMode: ReadWriteMany diff --git a/kubernetes/arc1/apps/machine-learning/kustomization.yaml b/kubernetes/arc1/apps/machine-learning/kustomization.yaml index 20d4825..f7f14e5 100644 --- a/kubernetes/arc1/apps/machine-learning/kustomization.yaml +++ b/kubernetes/arc1/apps/machine-learning/kustomization.yaml @@ -4,5 +4,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml + - ./qlora/ks.yaml #- ./ollama/ks.yaml #- ./jupyterhub/ks.yaml diff --git a/kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml b/kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml index 52381ca..455f7e1 100644 --- a/kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml +++ b/kubernetes/arc1/apps/machine-learning/mmseqs2/app/helmrelease.yaml @@ -5,7 +5,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: - name: &app ollama + name: &app mmseqs2 spec: interval: 30m chart: @@ -26,7 +26,7 @@ spec: retries: 3 values: controllers: - ollama: + mmseqs2: type: deployment annotations: reloader.stakater.com/auto: "true" @@ -41,11 +41,11 @@ spec: requests: cpu: 200m memory: 4Gi - # gpu.intel.com/i915: "1" + nvidia.com/gpu: 2 limits: - cpu: 8000m - memory: 8Gi - # gpu.intel.com/i915: "1" + cpu: 32000m + memory: 64Gi + nvidia.com/gpu: 4 service: app: controller: *app @@ -74,6 +74,7 @@ spec: - secretName: mmseqs2-tls hosts: [*host] persistence: + # TODO: Replace with existing PVC data: storageClass: local-nvme accessMode: ReadWriteMany diff --git a/kubernetes/arc1/apps/machine-learning/qlora/app/helmrelease.yaml b/kubernetes/arc1/apps/machine-learning/qlora/app/helmrelease.yaml new file mode 100644 index 0000000..53b40f7 --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/qlora/app/helmrelease.yaml @@ -0,0 +1,102 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +# TODO: Finish this +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: &app qlora +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 3.5.1 + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + strategy: rollback + retries: 3 + values: + controllers: + qlora: + type: deployment + annotations: + reloader.stakater.com/auto: "true" + pod: + runtimeClassName: nvidia + terminationGracePeriodSeconds: 1 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + containers: + app: + image: + repository: ghcr.io/rarecompute/qlora-docker + tag: main@sha256:e56350596e17af5198bfc848b0de3a5a11cb98d97e1e02dbb322467269342541 + env: + TZ: ${TIMEZONE} + # GITHUB_REPO: https://github.com/RareCompute/example-models + NVIDIA_VISIBLE_DEVICES: all + NVIDIA_DRIVER_CAPABILITIES: all + securityContext: + capabilities.drop: ["ALL"] + resources: + requests: + cpu: 200m + memory: 8Gi + limits: + cpu: 16 + memory: 32Gi + nvidia.com/gpu: 4 + # probes: + # liveness: + # enabled: true + # readiness: + # enabled: true + # startup: + # enabled: false + # spec: + # failureThreshold: 30 + # periodSeconds: 5 + service: + app: + controller: *app + annotations: + teleport.dev/name: *app + labels: + teleport: enabled + ports: + http: + port: &port 80 + persistence: + app: + storageClass: local-nvme + accessMode: ReadWriteOnce + size: 2Gi + globalMounts: + - path: /app + workspace: + storageClass: local-nvme + # TODO: OpenEBS only support ReadWriteOnce + accessMode: ReadWriteOnce + size: 2048Gi + retain: true + globalMounts: + - path: /workspace + tmp: + type: emptyDir + globalMounts: + - path: /tmp diff --git a/kubernetes/arc1/apps/machine-learning/qlora/app/kustomization.yaml b/kubernetes/arc1/apps/machine-learning/qlora/app/kustomization.yaml new file mode 100644 index 0000000..17cbc72 --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/qlora/app/kustomization.yaml @@ -0,0 +1,6 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/arc1/apps/machine-learning/qlora/ks.yaml b/kubernetes/arc1/apps/machine-learning/qlora/ks.yaml new file mode 100644 index 0000000..b89a998 --- /dev/null +++ b/kubernetes/arc1/apps/machine-learning/qlora/ks.yaml @@ -0,0 +1,24 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app qlora + namespace: flux-system +spec: + targetNamespace: machine-learning + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/arc1/apps/machine-learning/qlora/app + prune: true + sourceRef: + kind: GitRepository + name: k8s-gitops + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m + postBuild: + substitute: + APP: *app diff --git a/kubernetes/arc1/apps/network-system/external-dns/app/helmrelease.yaml b/kubernetes/arc1/apps/network-system/external-dns/app/helmrelease.yaml new file mode 100644 index 0000000..194f0b6 --- /dev/null +++ b/kubernetes/arc1/apps/network-system/external-dns/app/helmrelease.yaml @@ -0,0 +1,48 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: &app external-dns +spec: + interval: 30m + chart: + spec: + chart: external-dns + version: 1.15.0 + sourceRef: + kind: HelmRepository + name: external-dns + namespace: flux-system + install: + crds: CreateReplace + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + crds: CreateReplace + remediation: + strategy: rollback + retries: 3 + values: + fullnameOverride: *app + provider: cloudflare + env: + - name: CF_API_TOKEN + valueFrom: + secretKeyRef: + name: external-dns-secret + key: api-token + extraArgs: + - --ingress-class=external + - --cloudflare-proxied + - --crd-source-apiversion=externaldns.k8s.io/v1alpha1 + - --crd-source-kind=DNSEndpoint + policy: sync + sources: ["crd", "ingress"] + txtPrefix: k8s. + txtOwnerId: default + domainFilters: ["${SECRET_EXTERNAL_DOMAIN}"] + serviceMonitor: + enabled: true + podAnnotations: + secret.reloader.stakater.com/reload: external-dns-secret diff --git a/kubernetes/arc1/apps/network-system/external-dns/app/kustomization.yaml b/kubernetes/arc1/apps/network-system/external-dns/app/kustomization.yaml new file mode 100644 index 0000000..95bf474 --- /dev/null +++ b/kubernetes/arc1/apps/network-system/external-dns/app/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./secret.sops.yaml + - ./helmrelease.yaml diff --git a/kubernetes/arc1/apps/network-system/external-dns/app/secret.sops.yaml b/kubernetes/arc1/apps/network-system/external-dns/app/secret.sops.yaml new file mode 100644 index 0000000..2cb9d05 --- /dev/null +++ b/kubernetes/arc1/apps/network-system/external-dns/app/secret.sops.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Secret +metadata: + name: external-dns-secret +stringData: + api-token: ENC[AES256_GCM,data:JTphCAIBIt4+al2z37lUck1Zq7hlW2W5V9/KASVQae5jwkxI03OVmQ==,iv:ZxU2FjPgVtVpP8/xmxOun3jjoxeJ3gp2vIF3ml+A04c=,tag:GjT20j6rAp0sDhaq2WAkfw==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1ey3reuxyffqynll464r4q3tlhq5v73nxesyktr44lfez8jzxm94s0644n7 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBrNVdPRFFxekJvQlprM0ps + QXBNenFZR2k3bUt3dFloRkNUbFFDWldYUlc0CnpOMVg3cytnblBIOUptdHRsTGFa + Qk4vay9lWjAyR2gwTmUxQ0U4QVVZc1kKLS0tIHRUWGYxYSttaTMrZGFUeloxNDh5 + T2pIZXdHNEt5azJPU2lpcVpydjlIMVUKQm2uTdBw+T1RMWS1XTZ1TACEujS7nr8e + VWTpDUY+LRU62sJx30evTAUitn+qTxF2jhMudLC7YZBwrQD+pPTISA== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-11-19T02:05:15Z" + mac: ENC[AES256_GCM,data:xTYv3RcaPMbV9gwhrtlYXVQ++y5ozSlzorHnloOc+40kI7dc5ue9cmQufthIhsoL6ebcbDz6/cN1/rjMcBrlPxBop0nGkvyyjL/PybEEwrHwa82/fEuGkdJcJKZF76FYdOSXeJY41m7QOJlh31xOIkRmZUAZg+2N94VCeyPrJ5U=,iv:wLBRew0er+yA2EWqnnipPdUa297Ph2O4WBchuizuOVM=,tag:8Ni9BfQvcoSQ1PwJrlOlBw==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.9.1 diff --git a/kubernetes/arc1/apps/network-system/external-dns/ks.yaml b/kubernetes/arc1/apps/network-system/external-dns/ks.yaml new file mode 100644 index 0000000..5b0fcfc --- /dev/null +++ b/kubernetes/arc1/apps/network-system/external-dns/ks.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app external-dns + namespace: flux-system +spec: + targetNamespace: network-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/arc1/apps/network-system/external-dns/app + prune: true + sourceRef: + kind: GitRepository + name: k8s-gitops + wait: true + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/arc1/apps/network-system/kustomization.yaml b/kubernetes/arc1/apps/network-system/kustomization.yaml index 6ecd406..c9a87bd 100644 --- a/kubernetes/arc1/apps/network-system/kustomization.yaml +++ b/kubernetes/arc1/apps/network-system/kustomization.yaml @@ -5,4 +5,5 @@ kind: Kustomization resources: - ./namespace.yaml - ./echo-server/ks.yaml + - ./external-dns/ks.yaml - ./k8tz/ks.yaml diff --git a/kubernetes/arc1/apps/observability/healthchecks/app/helmrelease.yaml b/kubernetes/arc1/apps/observability/healthchecks/app/helmrelease.yaml new file mode 100644 index 0000000..132f2bf --- /dev/null +++ b/kubernetes/arc1/apps/observability/healthchecks/app/helmrelease.yaml @@ -0,0 +1,186 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: healthchecks +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 3.5.1 + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + values: + controllers: + healthchecks: + strategy: RollingUpdate + annotations: + reloader.stakater.com/auto: "true" + initContainers: + init-db: + image: + repository: ghcr.io/onedr0p/postgres-init + tag: 16 + env: + INIT_POSTGRES_HOST: &dbHost postgres16-rw.database.svc.cluster.local + INIT_POSTGRES_DBNAME: &dbName healthchecks + INIT_POSTGRES_USER: + valueFrom: + secretKeyRef: + name: healthchecks-secret + key: DB_USER + INIT_POSTGRES_PASS: + valueFrom: + secretKeyRef: + name: healthchecks-secret + key: DB_PASSWORD + INIT_POSTGRES_SUPER_PASS: + valueFrom: + secretKeyRef: + name: cloudnative-pg-secret + key: password + init-user: + dependsOn: [init-db] + image: + repository: docker.io/healthchecks/healthchecks + tag: v3.7 + command: [python3] + args: + - manage.py + - shell + - -v + - '3' + - -c + # https://github.com/linuxserver/docker-healthchecks/blob/9aedb6911bd4dd49f637145b04ad2aeb4339e78b/root/etc/s6-overlay/s6-rc.d/init-healthchecks-config/run#L52-L66 + - |- + """ + from django.contrib.auth.models import User; + from hc.accounts.views import _make_user; + + email = '$SUPERUSER_EMAIL'; + password = '$SUPERUSER_PASSWORD'; + + if User.objects.filter(email=email).count()==0: + user = _make_user(email); + user.set_password(password); + user.is_staff = True; + user.is_superuser = True; + user.save(); + print('Superuser created.'); + else: + print('Superuser creation skipped. Already exists.'); + """ + env: + SUPERUSER_EMAIL: + valueFrom: + secretKeyRef: + name: healthchecks-secret + key: SUPERUSER_EMAIL + SUPERUSER_PASSWORD: + valueFrom: + secretKeyRef: + name: healthchecks-secret + key: SUPERUSER_PASSWORD + containers: + app: + image: + repository: docker.io/healthchecks/healthchecks + tag: v3.7 + # https://healthchecks.io/docs/self_hosted_configuration/ + env: + DEBUG: "False" + REGISTRATION_OPEN: "False" + SITE_ROOT: "https://healthchecks.${SECRET_EXTERNAL_DOMAIN}" + SITE_NAME: Healthchecks + SITE_LOGO_URL: /static/img/logo.svg + DEFAULT_FROM_EMAIL: "Healthchecks <${SECRET_SMTP_FROM}>" + EMAIL_HOST: maddy.default.svc.cluster.local + EMAIL_PORT: 25 + EMAIL_USE_TLS: "False" + EMAIL_USE_VERIFICATION: "False" + INTEGRATIONS_ALLOW_PRIVATE_IPS: "True" + DB: postgres + DB_HOST: *dbHost + DB_NAME: *dbName + DB_PORT: 5432 + ADMINS: + valueFrom: + secretKeyRef: + name: healthchecks-secret + key: SUPERUSER_EMAIL + PUSHOVER_EMERGENCY_RETRY_DELAY: 300 # 5 minutes + PUSHOVER_EMERGENCY_EXPIRATION: 86400 # 24 hours + envFrom: + - secretRef: + name: healthchecks-secret + probes: + startup: + enabled: true + spec: + failureThreshold: 30 + periodSeconds: 5 + liveness: + enabled: true + readiness: + enabled: true + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + resources: + limits: + memory: 512Mi + defaultPodOptions: + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + seccompProfile: { type: RuntimeDefault } + service: + app: + controller: healthchecks + ports: + http: + port: 80 + targetPort: 8000 + serviceMonitor: + healthchecks: + enabled: true + serviceName: healthchecks + endpoints: + - port: http + scheme: http + path: ${service_monitor_path} + ingress: + app: + className: external + annotations: + external-dns.alpha.kubernetes.io/target: "external.${SECRET_EXTERNAL_DOMAIN}" + hosts: + - host: "healthchecks.${SECRET_EXTERNAL_DOMAIN}" + paths: + - path: / + service: + identifier: app + port: http + persistence: + logo: + type: configMap + name: healthchecks-config + globalMounts: + # gets turned into `/static/img/logo.svg` + - path: /opt/healthchecks/static-collected/img/logo.svg + subPath: logo.svg + readOnly: true diff --git a/kubernetes/arc1/apps/observability/healthchecks/app/kustomization.yaml b/kubernetes/arc1/apps/observability/healthchecks/app/kustomization.yaml new file mode 100644 index 0000000..8415448 --- /dev/null +++ b/kubernetes/arc1/apps/observability/healthchecks/app/kustomization.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./secret.sops.yaml + - ../../../database/cloudnative-pg/app/secret.sops.yaml + - ./helmrelease.yaml + - ../../../../templates/gatus/external +configMapGenerator: + - name: healthchecks-config + files: + - logo.svg=./resources/logo.svg +generatorOptions: + disableNameSuffixHash: true diff --git a/kubernetes/arc1/apps/observability/healthchecks/app/resources/logo.svg b/kubernetes/arc1/apps/observability/healthchecks/app/resources/logo.svg new file mode 100644 index 0000000..5901c63 --- /dev/null +++ b/kubernetes/arc1/apps/observability/healthchecks/app/resources/logo.svg @@ -0,0 +1 @@ + diff --git a/kubernetes/arc1/apps/observability/healthchecks/app/secret.sops.yaml b/kubernetes/arc1/apps/observability/healthchecks/app/secret.sops.yaml new file mode 100644 index 0000000..424cb69 --- /dev/null +++ b/kubernetes/arc1/apps/observability/healthchecks/app/secret.sops.yaml @@ -0,0 +1,34 @@ +# yamllint disable +kind: Secret +apiVersion: v1 +type: Opaque +metadata: + name: healthchecks-secret +stringData: + SUPERUSER_EMAIL: ENC[AES256_GCM,data:pHG0WJarDrP7PD3vY6ZY5ujmd2c=,iv:h5s62B9yOiicezuprNzEEiZvElyINrDLKtXdOzxqCik=,tag:mhmYwclXZPrCxShtZziRZQ==,type:str] + SUPERUSER_PASSWORD: ENC[AES256_GCM,data:9ehrErSnrM/KqopibELYYXTZnt6uXeQXVDDjc59hMA==,iv:ApdnfLZqxcJBG5lsCaW6abKKgW6IsOAmv/Xy3ANpcas=,tag:neoIB4NTu9ndoMC2RfIb1Q==,type:str] + SECRET_KEY: ENC[AES256_GCM,data:Re/lil2sXZ5u6qXS2kc0Gx/ihRynVXngV64JRxS1FLDPnwyXaFiGnz1sU0eDbtOZAcJxJC6fqHIRr9U9Fsz9k4KgPqtfuJjIjRlA4Y48/gaEelpO2bDyHStnViNeFnJRJLGBYGTKsGFT0JDoFr8yJP9x/OC5dajXL674jfVGZgY=,iv:oBwDCc1dCGWO+GhlcvQRO+ilLQ9hycXf3yaSvS8HCBU=,tag:YyazrhoavMFJ9l5SKpCyAw==,type:str] + DB_USER: ENC[AES256_GCM,data:OgAHtzsuLy6NM4eY,iv:leUjOb65qm3uxBNqEQORpPOS8qlUCIVBOhO7oARMOtI=,tag:oS1BDwqT+fBueY5FM0a4OQ==,type:str] + DB_PASSWORD: ENC[AES256_GCM,data:AFt+9nTZIO2/Ko9tJPySh+MIxc+iUy/t6GPws7AOmuyoT+DPJdI64phZbjwNlXhOgkl2j0/aqthj3olzBPZJJA==,iv:T9IBM5RMumT7ZedpTOjRhQZzWwEwj5ohamOvrt0hTd0=,tag:Umb4BTz5Pxi7xV7Cp+xKWQ==,type:str] + PUSHOVER_API_TOKEN: ENC[AES256_GCM,data:hU2AVJ+cJsxUcu6yT5b9vHoZjDpBQzzmMD7QqDKh,iv:/RPzhz1XjnqheSoWz6alPShM0h2qpJdDhKOseOAiKD8=,tag:M1yXjvpYTLx/e8J73hbwtw==,type:str] + PUSHOVER_SUBSCRIPTION_URL: ENC[AES256_GCM,data:onpbih248NE1BoX9scwXdIrN4p/cmkio1VVvtfexhPhJbg6OfomOw2oidW6tiGasklSS/RWLb1oBSL4=,iv:pDbLY0R/OMtlB6IKCgHjDZh5cljnGIu7t4I+1prB7hE=,tag:p+8bUsAMFSyfWwME4oFFpQ==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age148wprsnqjq8jughvywnzmvs8gffhrkendpr7g60q8u4rdsj4jvuqk7ltrs + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBERWp2UXdDZ29naERzWGZw + OS9SM1RSMVh5cGw2Q3pMelYxcG9RdGNudTAwCkFneXFlTXdpZitic2daOW1Ob2hq + QXl3UnBIYmtyNCtxQjV1dVFaRjFGL0kKLS0tIEhOZTVmU0xJM3FEUDVjQ0d2Wml2 + UmFGTlVDd3JRN1UyRG1HNURlL1RKUHcK0NrD/pNpAIOo/iEDhxcf1GxE1YlaK/lj + JiKBbzi3Zf1PbCdogcPUtWj4U4E9OZXo7BgG9bfW172bgGporM/ehQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-05-24T18:15:38Z" + mac: ENC[AES256_GCM,data:nBfQkBmR3CX3v+XLN2jawtn9LKESGag7eiw/FM8bo9WsO1Kf1yFYsFL97L8umrtOuLbqQZ5ndiq33GQgvUIw3Z7QA7ni0/5pZ16G/2YxJwpR0nV/ahigUoCdHoQ2M0IptYgCvjxBKdtKxxIF/ds4DE4WFXecOjdFmalF/JswZnk=,iv:qrZgasdquQFb7EjxuvIAolDu1LzrUuJnOrar34ISYoU=,tag:GmQc2KIDWQAoQwad4hoJuw==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.8.1 diff --git a/kubernetes/arc1/apps/observability/healthchecks/ks.yaml b/kubernetes/arc1/apps/observability/healthchecks/ks.yaml new file mode 100644 index 0000000..d4e8401 --- /dev/null +++ b/kubernetes/arc1/apps/observability/healthchecks/ks.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app healthchecks + namespace: flux-system +spec: + targetNamespace: observability + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: cloudnative-pg-cluster + path: ./kubernetes/main/apps/observability/healthchecks/app + prune: true + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m + postBuild: + substitute: + APP: *app + GATUS_PATH: /accounts/login/ diff --git a/kubernetes/arc1/apps/security/kustomization.yaml b/kubernetes/arc1/apps/security/kustomization.yaml index e430c5f..b75ad0e 100644 --- a/kubernetes/arc1/apps/security/kustomization.yaml +++ b/kubernetes/arc1/apps/security/kustomization.yaml @@ -4,4 +4,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - #- ./teleport/ks.yaml + - ./teleport/ks.yaml diff --git a/kubernetes/arc1/apps/security/teleport/app/crt.yaml b/kubernetes/arc1/apps/security/teleport/app/crt.yaml new file mode 100644 index 0000000..a3be5b5 --- /dev/null +++ b/kubernetes/arc1/apps/security/teleport/app/crt.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: teleport-tls + namespace: security +spec: + secretName: teleport-tls + issuerRef: + name: letsencrypt-production + kind: ClusterIssuer + dnsNames: + - "teleport.${SECRET_EXTERNAL_DOMAIN}" + - "*.teleport.${SECRET_EXTERNAL_DOMAIN}" diff --git a/kubernetes/arc1/apps/security/teleport/app/helmrelease.yaml b/kubernetes/arc1/apps/security/teleport/app/helmrelease.yaml index 6b8a278..2d95b6a 100644 --- a/kubernetes/arc1/apps/security/teleport/app/helmrelease.yaml +++ b/kubernetes/arc1/apps/security/teleport/app/helmrelease.yaml @@ -8,7 +8,7 @@ spec: chart: spec: chart: teleport-cluster - version: 16.4.6 + version: 17.0.1 sourceRef: kind: HelmRepository name: teleport @@ -19,12 +19,11 @@ spec: upgrade: cleanupOnFail: true remediation: - strategy: rollback retries: 3 values: clusterName: teleport.${SECRET_EXTERNAL_DOMAIN} chartMode: standalone - kubeClusterName: ARC1 + kubeClusterName: arc1 validateConfigOnDeploy: true enterprise: false auth: @@ -54,22 +53,32 @@ spec: enabled: true suppressAutomaticWildcards: false spec: - ingressClassName: traefik + ingressClassName: traefik-external annotations: ingress: + external-dns.alpha.kubernetes.io/hostname: "teleport.${SECRET_EXTERNAL_DOMAIN}" cert-manager.io/cluster-issuer: "letsencrypt-production" gethomepage.dev/enabled: "true" gethomepage.dev/group: Services - gethomepage.dev/name: *app - gethomepage.dev/icon: teleport.png + gethomepage.dev/name: Teleport + gethomepage.dev/description: Teleport dashboard + gethomepage.dev/icon: teleport + traefik.ingress.kubernetes.io/router.entrypoints: "websecure" + service: + traefik.ingress.kubernetes.io/service.serversscheme: https + # highAvailability: + # certManager: + # enabled: true + # issuerName: "letsencrypt-production" + # issuerKind: "ClusterIssuer" tls: - existingSecretName: "teleport-cluster-tls" + existingSecretName: teleport-tls authentication: type: local proxyListenerMode: multiplex persistence: enabled: true - storageClassName: local-nvme + existingClaimName: teleport serviceAccount: create: true rbac: diff --git a/kubernetes/arc1/apps/security/teleport/app/kustomization.yaml b/kubernetes/arc1/apps/security/teleport/app/kustomization.yaml index 17cbc72..66c8272 100644 --- a/kubernetes/arc1/apps/security/teleport/app/kustomization.yaml +++ b/kubernetes/arc1/apps/security/teleport/app/kustomization.yaml @@ -3,4 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - ./pvc.yaml + - ./crt.yaml - ./helmrelease.yaml diff --git a/kubernetes/arc1/apps/security/teleport/app/pvc.yaml b/kubernetes/arc1/apps/security/teleport/app/pvc.yaml new file mode 100644 index 0000000..6ce438b --- /dev/null +++ b/kubernetes/arc1/apps/security/teleport/app/pvc.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: teleport + namespace: security +spec: + storageClassName: local-nvme + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 8Gi diff --git a/kubernetes/arc1/apps/security/teleport/app/resources/token.yaml b/kubernetes/arc1/apps/security/teleport/app/resources/token.yaml new file mode 100644 index 0000000..05d4595 --- /dev/null +++ b/kubernetes/arc1/apps/security/teleport/app/resources/token.yaml @@ -0,0 +1,18 @@ +kind: token +version: v2 +metadata: + name: kubernetes-token + # set a long expiry time, the default for tokens is only 30 minutes + expires: "2050-01-01T00:00:00Z" +spec: + # Use the minimal set of system roles required. + roles: [kube, app, discovery, node, windowsdesktop] + + # set the join method allowed for this token + join_method: kubernetes + + kubernetes: + type: in_cluster + allow: + # Service account names follow the format "namespace:serviceaccountname". + - service_account: "security:teleport-kube-agent" diff --git a/kubernetes/arc1/apps/traefik-ingress/traefik/app/helmrelease.yaml b/kubernetes/arc1/apps/traefik-ingress/traefik/app/helmrelease.yaml index 2200b17..77bffbb 100644 --- a/kubernetes/arc1/apps/traefik-ingress/traefik/app/helmrelease.yaml +++ b/kubernetes/arc1/apps/traefik-ingress/traefik/app/helmrelease.yaml @@ -3,7 +3,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: - name: traefik + name: &app traefik spec: interval: 30m chart: @@ -31,19 +31,69 @@ spec: service: annotations: io.cilium/lb-ipam-ips: ${LB_TRAEFIK} - # spec: - #externalTrafficPolicy: Local + env: + - name: TZ + value: "${TIMEZONE}" ingressClass: enabled: true isDefaultClass: true + ingressRoute: + dashboard: + enabled: false + globalArguments: + - "--serversTransport.insecureSkipVerify=true" + - "--global.sendanonymoususage=false" + additionalArguments: + - "--entrypoints.web.transport.respondingTimeouts.readTimeout=0" + - "--entrypoints.websecure.transport.respondingTimeouts.readTimeout=0" + ports: + traefik: + expose: + default: false + web: + redirectTo: + port: websecure + websecure: + tls: + enabled: true + options: default + forwardedHeaders: + trustedIPs: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + proxyProtocol: + trustedIPs: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + http3: + enabled: true + metrics: + expose: + default: false metrics: serviceMonitor: enabled: true namespaceSelector: any: true + pilot: + enabled: false + providers: + kubernetesCRD: + enabled: true + ingressClass: traefik + allowCrossNamespace: true + allowExternalNameServices: true + kubernetesIngress: + enabled: true + ingressClass: traefik + allowExternalNameServices: true + publishedService: + enabled: true resources: requests: - memory: 128Mi cpu: 100m + memory: 512Mi limits: memory: 1536Mi diff --git a/kubernetes/arc1/apps/traefik-ingress/traefik/external/app/helmrelease.yaml b/kubernetes/arc1/apps/traefik-ingress/traefik/external/app/helmrelease.yaml new file mode 100644 index 0000000..b63c3c2 --- /dev/null +++ b/kubernetes/arc1/apps/traefik-ingress/traefik/external/app/helmrelease.yaml @@ -0,0 +1,154 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: &app traefik-external +spec: + interval: 30m + chart: + spec: + chart: traefik + version: 33.0.0 + sourceRef: + kind: HelmRepository + name: traefik + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + dependsOn: + - name: cert-manager + namespace: cert-manager + values: + deployment: + enabled: true + replicas: 1 + service: + enabled: true + type: LoadBalancer + annotations: + external-dns.alpha.kubernetes.io/hostname: "external.${SECRET_EXTERNAL_DOMAIN}" + lbipam.cilium.io/ips: "${LB_TRAEFIK_EXTERNAL}" + spec: + externalTrafficPolicy: Cluster + env: + - name: TZ + value: "${TIMEZONE}" + logs: + general: + level: INFO + access: + enabled: true + ingressClass: + enabled: true + isDefaultClass: false + name: traefik-external + ingressRoute: + dashboard: + enabled: false + globalArguments: + - "--serversTransport.insecureSkipVerify=true" + - "--global.sendanonymoususage=false" + additionalArguments: + - "--entrypoints.web.transport.respondingTimeouts.readTimeout=0" + - "--entrypoints.websecure.transport.respondingTimeouts.readTimeout=0" + ports: + traefik: + expose: + default: false + web: + redirectTo: + port: websecure + websecure: + tls: + enabled: true + options: default + forwardedHeaders: + trustedIPs: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 103.21.244.0/22 + - 103.22.200.0/22 + - 103.31.4.0/22 + - 104.16.0.0/13 + - 104.24.0.0/14 + - 108.162.192.0/18 + - 131.0.72.0/22 + - 141.101.64.0/18 + - 162.158.0.0/15 + - 172.64.0.0/13 + - 173.245.48.0/20 + - 188.114.96.0/20 + - 190.93.240.0/20 + - 197.234.240.0/22 + - 198.41.128.0/17 + - 2400:cb00::/32 + - 2606:4700::/32 + - 2803:f800::/32 + - 2405:b500::/32 + - 2405:8100::/32 + - 2a06:98c0::/29 + - 2c0f:f248::/32 + proxyProtocol: + trustedIPs: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 103.21.244.0/22 + - 103.22.200.0/22 + - 103.31.4.0/22 + - 104.16.0.0/13 + - 104.24.0.0/14 + - 108.162.192.0/18 + - 131.0.72.0/22 + - 141.101.64.0/18 + - 162.158.0.0/15 + - 172.64.0.0/13 + - 173.245.48.0/20 + - 188.114.96.0/20 + - 190.93.240.0/20 + - 197.234.240.0/22 + - 198.41.128.0/17 + - 2400:cb00::/32 + - 2606:4700::/32 + - 2803:f800::/32 + - 2405:b500::/32 + - 2405:8100::/32 + - 2a06:98c0::/29 + - 2c0f:f248::/32 + http3: + enabled: true + metrics: + expose: + default: false + metrics: + prometheus: + entryPoint: metrics + service: + enabled: true + pilot: + enabled: false + providers: + kubernetesCRD: + enabled: true + ingressClass: traefik-external + allowCrossNamespace: true + allowExternalNameServices: true + kubernetesIngress: + enabled: true + ingressClass: traefik-external + allowExternalNameServices: true + publishedService: + enabled: true + resources: + requests: + cpu: 100m + memory: 768Mi + limits: + memory: 768Mi diff --git a/kubernetes/arc1/apps/traefik-ingress/traefik/external/app/kustomization.yaml b/kubernetes/arc1/apps/traefik-ingress/traefik/external/app/kustomization.yaml new file mode 100644 index 0000000..1889392 --- /dev/null +++ b/kubernetes/arc1/apps/traefik-ingress/traefik/external/app/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - helmrelease.yaml diff --git a/kubernetes/arc1/apps/traefik-ingress/traefik/ks.yaml b/kubernetes/arc1/apps/traefik-ingress/traefik/ks.yaml index 56c8975..e1e6999 100644 --- a/kubernetes/arc1/apps/traefik-ingress/traefik/ks.yaml +++ b/kubernetes/arc1/apps/traefik-ingress/traefik/ks.yaml @@ -18,3 +18,23 @@ spec: interval: 30m retryInterval: 1m timeout: 5m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app traefik-external + namespace: flux-system +spec: + targetNamespace: traefik-ingress + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/arc1/apps/traefik-ingress/traefik/external/app + prune: true + sourceRef: + kind: GitRepository + name: k8s-gitops + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m diff --git a/kubernetes/arc1/apps/volsync-system/kustomization.yaml b/kubernetes/arc1/apps/volsync-system/kustomization.yaml new file mode 100644 index 0000000..775f204 --- /dev/null +++ b/kubernetes/arc1/apps/volsync-system/kustomization.yaml @@ -0,0 +1,8 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./namespace.yaml + - ./snapshot-controller/ks.yaml + - ./volsync/ks.yaml diff --git a/kubernetes/arc1/apps/volsync-system/namespace.yaml b/kubernetes/arc1/apps/volsync-system/namespace.yaml new file mode 100644 index 0000000..05df292 --- /dev/null +++ b/kubernetes/arc1/apps/volsync-system/namespace.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: volsync-system + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + volsync.backube/privileged-movers: "true" diff --git a/kubernetes/arc1/apps/volsync-system/snapshot-controller/app/helmrelease.yaml b/kubernetes/arc1/apps/volsync-system/snapshot-controller/app/helmrelease.yaml new file mode 100644 index 0000000..6ec363d --- /dev/null +++ b/kubernetes/arc1/apps/volsync-system/snapshot-controller/app/helmrelease.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: snapshot-controller +spec: + interval: 30m + chart: + spec: + chart: snapshot-controller + version: 3.0.6 + sourceRef: + kind: HelmRepository + name: piraeus + namespace: flux-system + maxHistory: 2 + install: + crds: CreateReplace + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + crds: CreateReplace + remediation: + retries: 3 + uninstall: + keepHistory: false + values: + controller: + serviceMonitor: + create: true + webhook: + enabled: false diff --git a/kubernetes/arc1/apps/volsync-system/snapshot-controller/app/kustomization.yaml b/kubernetes/arc1/apps/volsync-system/snapshot-controller/app/kustomization.yaml new file mode 100644 index 0000000..17cbc72 --- /dev/null +++ b/kubernetes/arc1/apps/volsync-system/snapshot-controller/app/kustomization.yaml @@ -0,0 +1,6 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/arc1/apps/volsync-system/snapshot-controller/ks.yaml b/kubernetes/arc1/apps/volsync-system/snapshot-controller/ks.yaml new file mode 100644 index 0000000..065c8ef --- /dev/null +++ b/kubernetes/arc1/apps/volsync-system/snapshot-controller/ks.yaml @@ -0,0 +1,24 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app snapshot-controller + namespace: flux-system +spec: + targetNamespace: volsync-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/arc1/apps/volsync-system/snapshot-controller/app + prune: true + sourceRef: + kind: GitRepository + name: k8s-gitops + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m + postBuild: + substitute: + APP: *app diff --git a/kubernetes/arc1/apps/volsync-system/volsync/app/helmrelease.yaml b/kubernetes/arc1/apps/volsync-system/volsync/app/helmrelease.yaml new file mode 100644 index 0000000..002b6d0 --- /dev/null +++ b/kubernetes/arc1/apps/volsync-system/volsync/app/helmrelease.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: &app volsync +spec: + interval: 30m + chart: + spec: + chart: volsync + version: 0.11.0 + sourceRef: + kind: HelmRepository + name: backube + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: snapshot-controller + namespace: volsync-system + values: + manageCRDs: true + metrics: + disableAuth: true diff --git a/kubernetes/arc1/apps/volsync-system/volsync/ks.yaml b/kubernetes/arc1/apps/volsync-system/volsync/ks.yaml new file mode 100644 index 0000000..0c0d5a1 --- /dev/null +++ b/kubernetes/arc1/apps/volsync-system/volsync/ks.yaml @@ -0,0 +1,24 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app volsync + namespace: flux-system +spec: + targetNamespace: volsync-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/arc1/apps/volsync-system/volsync/app + prune: true + sourceRef: + kind: GitRepository + name: k8s-gitops + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m + postBuild: + substitute: + APP: *app diff --git a/kubernetes/arc1/flux/repositories/helm/backube.yaml b/kubernetes/arc1/flux/repositories/helm/backube.yaml new file mode 100644 index 0000000..4ba0742 --- /dev/null +++ b/kubernetes/arc1/flux/repositories/helm/backube.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1.json +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: backube + namespace: flux-system +spec: + interval: 2h + url: https://backube.github.io/helm-charts/ diff --git a/kubernetes/arc1/flux/repositories/helm/external-dns.yaml b/kubernetes/arc1/flux/repositories/helm/external-dns.yaml new file mode 100644 index 0000000..f38c48a --- /dev/null +++ b/kubernetes/arc1/flux/repositories/helm/external-dns.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1.json +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: external-dns + namespace: flux-system +spec: + interval: 1h + url: https://kubernetes-sigs.github.io/external-dns diff --git a/kubernetes/arc1/flux/repositories/helm/kustomization.yaml b/kubernetes/arc1/flux/repositories/helm/kustomization.yaml index fb4f48d..d0cc8a2 100644 --- a/kubernetes/arc1/flux/repositories/helm/kustomization.yaml +++ b/kubernetes/arc1/flux/repositories/helm/kustomization.yaml @@ -3,11 +3,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - ./backube.yaml - ./bitnami.yaml - ./bjw-s.yaml - ./cilium.yaml - ./cloudnative-pg.yaml - ./coredns.yaml + - ./external-dns.yaml - ./grafana.yaml - ./jetstack.yaml - ./k8tz.yaml @@ -15,6 +17,7 @@ resources: - ./node-feature-discovery.yaml - ./nvidia-device-plugin.yaml - ./openebs.yaml + - ./piraeus.yaml - ./postfinance.yaml - ./prometheus-community.yaml - ./spegel.yaml diff --git a/kubernetes/arc1/flux/repositories/helm/piraeus.yaml b/kubernetes/arc1/flux/repositories/helm/piraeus.yaml new file mode 100644 index 0000000..84f361a --- /dev/null +++ b/kubernetes/arc1/flux/repositories/helm/piraeus.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: piraeus + namespace: flux-system +spec: + interval: 2h + url: https://piraeus.io/helm-charts/ diff --git a/kubernetes/arc1/flux/vars/cluster-settings.yaml b/kubernetes/arc1/flux/vars/cluster-settings.yaml index d06f04e..a24660e 100644 --- a/kubernetes/arc1/flux/vars/cluster-settings.yaml +++ b/kubernetes/arc1/flux/vars/cluster-settings.yaml @@ -19,4 +19,5 @@ data: NFS_PATH: /mnt/r720xd-nvme/r720xd-nfs LB_TRAEFIK: 10.2.12.100 - LB_POSTGRES: 10.2.12.101 + LB_TRAEFIK_EXTERNAL: 10.2.12.101 + LB_POSTGRES: 10.2.12.102