diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index f9687d068..969206ece 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -26,11 +26,18 @@ jobs: with: token: "${{ steps.app-token.outputs.token }}" - - name: Setup OpenTofu - uses: opentofu/setup-opentofu@ae80d4ecaab946d8f5ff18397fbf6d0686c6d46a # v1.0.3 + - name: Install nix + uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27 + with: + nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + access-tokens = github.com=${{ steps.app-token.outputs.token }} + + - name: Enable develop shell + uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0 - - name: Setup TFLint - uses: terraform-linters/setup-tflint@19a52fbac37dacb22a09518e4ef6ee234f2d4987 # v4.0.0 + - name: Run linters + run: pre-commit run --all-files - name: Setup Homebrew uses: Homebrew/actions/setup-homebrew@master @@ -39,28 +46,6 @@ jobs: shell: bash run: brew install helm kubeconform yq - - name: Init TFLint - run: tflint --init - env: - GITHUB_TOKEN: "${{ steps.app-token.outputs.token }}" - - - name: Run ShellCheck - uses: ludeeus/action-shellcheck@00cae500b08a931fb5698e11e79bfbd38e612a38 # 2.0.0 - - - name: Run tofu formatter - run: tofu fmt -check -diff -recursive tofu/ - - - name: Run tofu lint - run: tflint -f compact - - - name: Run yamllint - uses: karancode/yamllint-github-action@fdef6bc189425ecc84cc4543b2674566c0827053 # v2.1.1 - with: - yamllint_strict: true - yamllint_comment: true - env: - GITHUB_ACCESS_TOKEN: "${{ steps.app-token.outputs.token }}" - - name: Run task lint:egress-comment run: ./.taskfiles/Lint/egress-comment-job.sh diff --git a/.github/workflows/lychee.yaml b/.github/workflows/lychee.yaml index a80e642f4..0e0a18b68 100644 --- a/.github/workflows/lychee.yaml +++ b/.github/workflows/lychee.yaml @@ -68,7 +68,7 @@ jobs: --json number \ | jq --raw-output '.[0].number' \ ) - echo "issue-number=${issue_number}" >> $GITHUB_OUTPUT + echo "issue-number=${issue_number}" >> "$GITHUB_OUTPUT" echo "${issue_number}" - name: Create or Update Issue diff --git a/.gitignore b/.gitignore index 1d9fa6284..57d638291 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,4 @@ talenv* # nix result +.pre-commit-config.yaml diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 000000000..8214650f5 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,5 @@ +--- +MD013: + # Number of characters + line_length: 120 + tables: false diff --git a/.taskfiles/Lint/yaml-json-schema-job.sh b/.taskfiles/Lint/yaml-json-schema-job.sh index 66b497293..aac5c987a 100755 --- a/.taskfiles/Lint/yaml-json-schema-job.sh +++ b/.taskfiles/Lint/yaml-json-schema-job.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +TMP_DIR="$(mktemp -d)" # search for manifests without JSON schema links yaml_files="$(sh -c "find . -name '*.y*ml' -not -name \"$(yq '.ignoreNames|join("\" -not -name \"")' "${SCRIPT_DIR}/../../.yaml-json-schema")\" -not -path ./\"$(yq '.ignorePaths|join("\" -not -path ./\"")' "${SCRIPT_DIR}/../../.yaml-json-schema")\"")" @@ -15,17 +16,17 @@ for file in $yaml_files; do fi fi # shellcheck disable=SC2016 - if ! yq -s '"/tmp/test_split_" + $index' "${file}" 2> /dev/null; then - cp "${file}" /tmp/test_split_0.yml + if ! yq -s '"${TMP_DIR}/test_split_" + $index' "${file}" 2> /dev/null; then + cp "${file}" "${TMP_DIR}/test_split_0.yml" fi - if grep -Hoc '# yaml-language-serve' /tmp/test_split_* | grep -q ':0$'; then + if grep -Hoc '# yaml-language-serve' "${TMP_DIR}/test_split_"* | grep -q ':0$'; then if [ $error == 0 ]; then echo "Found YAML files without valid JSON schema manifest links:" fi error=1 echo "${file}" else - for split in /tmp/test_split_*; do + for split in "${TMP_DIR}/test_split_"*; do if [ -z "${IGNORE_SCHEMA_FETCH}" ]; then schemaUrl="$(grep '# yaml-language-serve' "${split}" | head -n 1 | awk -F= '{print $2}')" if [ -z "$schemaUrl" ]; then @@ -46,7 +47,7 @@ for file in $yaml_files; do fi done fi - rm -rf /tmp/test_split_* + rm -rf "${TMP_DIR}/test_split_"* done if [ $error == 1 ]; then diff --git a/.yaml-json-schema b/.yaml-json-schema index 342d1961d..ba91947c9 100644 --- a/.yaml-json-schema +++ b/.yaml-json-schema @@ -3,6 +3,8 @@ ignorePaths: - talos/*/tal*.yaml ignoreNames: + - .markdownlint.yaml + - .pre-commit-config.yaml - protection.tmpl.yaml - "*.sops.y*ml" - "values.*y*ml" diff --git a/README.md b/README.md index eecdf230a..6622ca769 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ +
- + kubepepe
Art by @SkeletalGadget @@ -31,12 +32,16 @@ _... automated via [ArgoCD](https://argoproj.github.io/cd/), [Renovate](https:// [![Power-Usage](https://img.shields.io/endpoint?url=https%3A%2F%2Fkromgo.rzegocki.dev%2Fquery%3Fformat%3Dendpoint%26metric%3Dcluster_power_usage&style=flat-square&label=Power)](https://github.com/kashalls/kromgo/)
+ --- ## 📖 Overview -This is a repository for my home infrastructure and Kubernetes cluster. I try to adhere to Infrastructure as Code (IaC) and GitOps practices using tools like [OpenTofu](https://opentofu.org/), [Kubernetes](https://kubernetes.io), [ArgoCD](https://argoproj.github.io/cd/), [Renovate](https://github.com/renovatebot/renovate) and [GitHub Actions](https://github.com/features/actions). +This is a repository for my home infrastructure and Kubernetes cluster. +I try to adhere to Infrastructure as Code (IaC) and GitOps practices using tools like [OpenTofu](https://opentofu.org/), +[Kubernetes](https://kubernetes.io), [ArgoCD](https://argoproj.github.io/cd/), [Renovate](https://github.com/renovatebot/renovate) +and [GitHub Actions](https://github.com/features/actions). --- @@ -44,15 +49,20 @@ This is a repository for my home infrastructure and Kubernetes cluster. I try to ### Installation -This semi hyper-converged cluster runs [Talos Linux](https://talos.dev), an immutable and ephemeral Linux distribution built for [Kubernetes](https://kubernetes.io), deployed on bare-metal [Intel NUCs](https://www.intel.com/content/www/us/en/products/details/nuc.html). [Rook](https://rook.io) then provides my workloads with persistent block, and file storage; while a seperate server provides file storage for my media. +This semi hyper-converged cluster runs [Talos Linux](https://talos.dev), an immutable and ephemeral Linux distribution +built for [Kubernetes](https://kubernetes.io), deployed on bare-metal [Intel NUCs](https://www.intel.com/content/www/us/en/products/details/nuc.html). +[Rook](https://rook.io) then provides my workloads with persistent block, and file storage; +while a seperate server provides file storage for my media. ### Core Components - [actions-runner-controller](https://github.com/actions/actions-runner-controller): Self-hosted Github runners. - [cilium](https://cilium.io): Internal Kubernetes networking plugin. - [cert-manager](https://cert-manager.io): Creates SSL certificates for services in my Kubernetes cluster. -- [external-dns](https://github.com/kubernetes-sigs/external-dns): Automatically manages DNS records from my cluster in a cloud DNS provider. -- [ingress-nginx](https://github.com/kubernetes/ingress-nginx): Ingress controller for Kubernetes using NGINX as a reverse proxy and load balancer. +- [external-dns](https://github.com/kubernetes-sigs/external-dns): Automatically manages DNS records from my cluster + in a cloud DNS provider. +- [ingress-nginx](https://github.com/kubernetes/ingress-nginx): Ingress controller for Kubernetes using NGINX as + a reverse proxy and load balancer. - [rook](https://rook.io): Distributed block storage for peristent storage. - [spegel](https://github.com/XenitAB/spegel): Stateless cluster local OCI registry mirror. - [vault](https://www.vaultproject.io/): Safe and encrypted storage for all Kubernetes secrets. @@ -60,11 +70,15 @@ This semi hyper-converged cluster runs [Talos Linux](https://talos.dev), an immu ### GitOps -[ArgoCD](https://argoproj.github.io/cd/) watches the clusters in my [kubernetes](./kubernetes/) folder (see Directories below) and makes the changes to my clusters based on the state of my Git repository. +[ArgoCD](https://argoproj.github.io/cd/) watches the clusters in my [kubernetes](./kubernetes/) folder +(see Directories below), and makes the changes to my clusters based on the state of my Git repository. -The way ArgoCD works for me here is it will recursively search the `kubernetes/clusters/${cluster}` folder and deploys all `application.yaml` manifests. I follow "app of apps" pattern, so cluster apps can include other apps, which can be shared between clusters, and which live under `kubernetes/apps` directory. +The way ArgoCD works for me here is it will recursively search the `kubernetes/clusters/${cluster}` folder, +and deploys all `application.yaml` manifests. I follow "app of apps" pattern, so cluster apps can include other apps, +which can be shared between clusters, and which live under `kubernetes/apps` directory. -[Renovate](https://github.com/renovatebot/renovate) watches my **entire** repository looking for dependency updates, when they are found a PR is automatically created. When some PRs are merged Flux applies the changes to my cluster. +[Renovate](https://github.com/renovatebot/renovate) watches my **entire** repository looking for dependency updates. +When they are found a PR is automatically created. When some PRs are merged ArgoCD applies the changes to my cluster. ### Directories @@ -84,7 +98,9 @@ This Git repository contains the following directories under [Kubernetes](./kube ## ☁️ Cloud Dependencies -While most of my infrastructure and workloads are self-hosted I do rely upon the cloud for certain key parts of my setup. This saves me from having to worry about two things. (1) Dealing with chicken/egg scenarios and (2) services I critically need whether my cluster is online or not. +While most of my infrastructure and workloads are self-hosted I do rely upon the cloud for certain key parts of my setup. +This saves me from having to worry about two things. (1) Dealing with chicken/egg scenarios and (2) services I critically +need whether my cluster is online or not. | Service | Use | Cost | |-------------------------------------------|----------------------------------------------------------------|----------------| @@ -113,6 +129,7 @@ While most of my infrastructure and workloads are self-hosted I do rely upon the ## ⭐ Stargazers +
@@ -124,12 +141,15 @@ While most of my infrastructure and workloads are self-hosted I do rely upon the
+ --- ## 🤝 Gratitude and Thanks -Thanks to all the people who donate their time to the [Home Operations](https://discord.gg/home-operations) Discord community. Be sure to check out [kubesearch.dev](https://kubesearch.dev/) for ideas on how to deploy applications or get ideas on what you may deploy. +Thanks to all the people who donate their time to the [Home Operations](https://discord.gg/home-operations) Discord community. +Be sure to check out [kubesearch.dev](https://kubesearch.dev/) for ideas on how to deploy applications +or get ideas on what you may deploy. --- diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 87acac26f..4a086a416 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -1,3 +1,5 @@ +# INSTALL + ## Prepare ### Zap disks for CEPH diff --git a/flake.lock b/flake.lock index 98d36a8b9..12b4203c3 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" @@ -36,6 +52,27 @@ "type": "github" } }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1722421184, @@ -64,7 +101,39 @@ "url": "https://github.com/NixOS/nixpkgs/archive/5daf0514482af3f97abaefc78a6606365c9108e2.tar.gz" } }, + "nixpkgs-stable": { + "locked": { + "lastModified": 1720386169, + "narHash": "sha256-NGKVY4PjzwAa4upkGtAMz1npHGoRzWotlSnVlqI40mo=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "194846768975b7ad2c4988bdb82572c00222c0d7", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { + "locked": { + "lastModified": 1719082008, + "narHash": "sha256-jHJSUH619zBQ6WdC21fFAlDxHErKVDJ5fpN0Hgx4sjs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9693852a2070b398ee123a329e68f0dab5526681", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { "locked": { "lastModified": 1720418205, "narHash": "sha256-cPJoFPXU44GlhWg4pUk9oUPqurPlCFZ11ZQPk21GTPU=", @@ -80,10 +149,32 @@ "type": "github" } }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": "flake-compat", + "gitignore": "gitignore", + "nixpkgs": "nixpkgs_2", + "nixpkgs-stable": "nixpkgs-stable" + }, + "locked": { + "lastModified": 1723202784, + "narHash": "sha256-qbhjc/NEGaDbyy0ucycubq4N3//gDFFH3DOmp1D3u1Q=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "c7012d0c18567c889b948781bc74a501e92275d1", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, "root": { "inputs": { "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", + "pre-commit-hooks": "pre-commit-hooks", "talhelper": "talhelper" } }, @@ -105,7 +196,7 @@ "talhelper": { "inputs": { "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_3" }, "locked": { "lastModified": 1722560739, diff --git a/flake.nix b/flake.nix index c957eb857..4f71ad45d 100644 --- a/flake.nix +++ b/flake.nix @@ -2,40 +2,84 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; flake-utils.url = "github:numtide/flake-utils"; + pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; talhelper.url = "github:budimanjojo/talhelper"; }; - outputs = { self, nixpkgs, talhelper, flake-utils }: - flake-utils.lib.eachDefaultSystem - (system: - let - pkgs = import nixpkgs { - inherit system; + outputs = + { + self, + nixpkgs, + talhelper, + flake-utils, + pre-commit-hooks, + }: + flake-utils.lib.eachDefaultSystem ( + system: + let + pkgs = import nixpkgs { inherit system; }; + in + { + + checks.pre-commit-check = pre-commit-hooks.lib.${system}.run { + src = ./.; + hooks = { + actionlint.enable = true; + check-json = { + enable = true; + excludes = [ "kubernetes/apps/external/gokapi/files/config.json" ]; + }; + markdownlint.enable = true; + shellcheck.enable = true; + terraform-format.enable = true; + tflint.enable = true; + yamllint.enable = true; + + check-case-conflicts.enable = true; + check-shebang-scripts-are-executable.enable = true; + mixed-line-endings.enable = true; + + egress-comment = { + enable = true; + entry = "./.taskfiles/Lint/egress-comment-job.sh"; + }; + yaml-json-schema = { + enable = true; + entry = "./.taskfiles/Lint/yaml-json-schema-job.sh"; + }; + + deadnix.enable = true; + flake-checker.enable = true; + statix.enable = true; + nixfmt = { + enable = true; + package = nixpkgs.legacyPackages.${system}.nixfmt-rfc-style; + excludes = [ ".direnv" ]; + }; }; - in - { - devShells.default = pkgs.mkShell { - buildInputs = [ - pkgs.age - pkgs.go-task - pkgs.kubernetes-helm - pkgs.jq - pkgs.lefthook - pkgs.opentofu - pkgs.sops - talhelper.packages.${system}.default - pkgs.yq-go + }; + + devShells.default = pkgs.mkShell { + buildInputs = self.checks.${system}.pre-commit-check.enabledPackages ++ [ + pkgs.age + pkgs.go-task + pkgs.kubernetes-helm + pkgs.jq + pkgs.lefthook + pkgs.opentofu + pkgs.sops + talhelper.packages.${system}.default + pkgs.yq-go - # linters - pkgs.kubeconform - pkgs.shellcheck - pkgs.tflint - pkgs.yamllint - ]; + # extra linters + pkgs.kubeconform + ]; - shellHook = '' - sh -c 'cd opentofu && ${pkgs.opentofu}/bin/tofu init -backend-config=<(grep '^#' terraform.tfvars | sed "s@^# *@@g") -upgrade' + shellHook = + self.checks.${system}.pre-commit-check.shellHook + + '' + [ ! -f opentofu/terraform.tfvars ] || sh -c 'cd opentofu && ${pkgs.opentofu}/bin/tofu init -backend-config=<(grep '^#' terraform.tfvars | sed "s@^# *@@g") -upgrade' ''; - }; - } - ); + }; + } + ); } diff --git a/kubernetes/apps/networking/nfs-subdir-external-provisioner/templates/deployments.yaml b/kubernetes/apps/networking/nfs-subdir-external-provisioner/templates/deployments.yaml index e76ccef77..59d4be385 100644 --- a/kubernetes/apps/networking/nfs-subdir-external-provisioner/templates/deployments.yaml +++ b/kubernetes/apps/networking/nfs-subdir-external-provisioner/templates/deployments.yaml @@ -1,6 +1,6 @@ # yamllint disable rule:line-length --- -# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/backups-json-schema/master/master-standalone/deployment.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/master-standalone/deployment.json apiVersion: apps/v1 kind: Deployment metadata: diff --git a/kubernetes/apps/networking/nfs-subdir-external-provisioner/templates/storage_classes.yaml b/kubernetes/apps/networking/nfs-subdir-external-provisioner/templates/storage_classes.yaml index e911cc6b1..6ebeb1d72 100644 --- a/kubernetes/apps/networking/nfs-subdir-external-provisioner/templates/storage_classes.yaml +++ b/kubernetes/apps/networking/nfs-subdir-external-provisioner/templates/storage_classes.yaml @@ -1,6 +1,6 @@ # yamllint disable rule:line-length --- -# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/backups-json-schema/master/master-standalone/storageclass.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/yannh/kubernetes-json-schema/master/master-standalone/storageclass.json apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/kubernetes/apps/workers/synology/files/syncer.sh b/kubernetes/apps/workers/synology/files/syncer.sh old mode 100644 new mode 100755 diff --git a/opentofu/mail/variables.tf b/opentofu/mail/variables.tf index 7f3a750a5..8c42df655 100644 --- a/opentofu/mail/variables.tf +++ b/opentofu/mail/variables.tf @@ -17,11 +17,11 @@ variable "domain_aliases" { variable "extra_identities" { description = "List of extra identities allowed to send and receive" - type = map(object({ - name = string, - password = optional(string), + type = map(object({ + name = string, + password = optional(string), may_receive = optional(bool), - may_send = optional(bool) + may_send = optional(bool) })) } diff --git a/opentofu/variables.tf b/opentofu/variables.tf index 062b70b60..f03797f42 100644 --- a/opentofu/variables.tf +++ b/opentofu/variables.tf @@ -52,11 +52,11 @@ variable "mail_domain_aliases" { variable "mail_extra_identities" { description = "List of extra identities allowed to send and receive" - type = map(object({ - name = string, - password = optional(string), + type = map(object({ + name = string, + password = optional(string), may_receive = optional(bool), - may_send = optional(bool) + may_send = optional(bool) })) }