diff --git a/docs/adrs/007-add-kine-support.md b/docs/adrs/007-add-kine-support.md new file mode 100644 index 0000000000..f374b49000 --- /dev/null +++ b/docs/adrs/007-add-kine-support.md @@ -0,0 +1,49 @@ +# Add kine support to RKE2 + +## Established + +## Revisit by + +## Status + +Accepted + +## Context + +This ADR is a introduction of kine support for RKE2. However, for this support to be implemented, it was necessary to add kine with TLS in K3s. +Which was done in this [PR](https://github.com/k3s-io/k3s/pull/9572), It was needed since rke2 cannot connect to kine without tls via the api server. + +When rke2 is started with the `--datastore-endpoint` flag, it will disable the etcd pod and set the `cluster-init` flag to be `false`, to avoid the etcd part of k3s to be started. +Kine will use the etcd client certificate to authenticate the connection to the kine server that will be a `unixs` socket type. + +### Pros + +- With the integration of kine, it is now possible to use the `--datastore-endpoint` flag among others related to kine. This allows for a more versatile configuration of the datastore, +providing users with the flexibility to choose their preferred storage backend. + +### Cons + +- Kine can only be utilized with TLS due to the requirements of the API server. + +## Other changes needed in k3s to better support kine in rke2 + +When testing rke2 with kine, there was some changes to avoid panics (specially when we are talking about `etcd`) and to make it work with tls. The changes are that when the user +uses `--database-endpoint` and other flags related to `etcd only` nodes, we have to ignore this flags or simply end the process with a error message. + +We decided to set a error message and end the process, since it is not clear to the user that the flags are being ignored. + +### Pros of Ignoring the flags + +- It is possible to avoid panics and rke2 will run as expected. + +### Cons of Ignoring the flags + +- It will be not very clear to the user that the flags are being ignored. + +### Pros of Ending the process with a error message + +- Rke2 will run as expected with transparency to the user. + +### Cons of Ending the process with a error message + +- The user will have to change the flags to make rke2 run. diff --git a/go.mod b/go.mod index 309aee7b6a..c361ace829 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ replace ( github.com/docker/docker => github.com/docker/docker v24.0.0-rc.2.0.20230801142700-69c9adb7d386+incompatible github.com/docker/libnetwork => github.com/docker/libnetwork v0.8.0-dev.2.0.20190624125649-f0e46a78ea34 github.com/go-logr/logr => github.com/go-logr/logr v1.2.0 - github.com/golang/protobuf => github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf => github.com/golang/protobuf v1.5.4 github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.1.1 github.com/juju/errors => github.com/k3s-io/nocode v0.0.0-20200630202308-cb097102c09f github.com/kubernetes-sigs/cri-tools => github.com/k3s-io/cri-tools v1.26.0-rc.0-k3s1 @@ -102,7 +102,7 @@ require ( github.com/google/go-containerregistry v0.19.0 github.com/iamacarpet/go-win64api v0.0.0-20210311141720-fe38760bed28 github.com/k3s-io/helm-controller v0.15.9 - github.com/k3s-io/k3s v1.27.12-0.20240313063150-06ecbe807148 //release-1.27 + github.com/k3s-io/k3s v1.27.13-0.20240402144016-9176d7f68aa2 //release-1.27 github.com/libp2p/go-netroute v0.2.1 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/onsi/ginkgo/v2 v2.11.0 @@ -117,16 +117,16 @@ require ( github.com/tigera/operator v1.28.1 github.com/urfave/cli v1.22.14 golang.org/x/sync v0.6.0 - golang.org/x/sys v0.16.0 + golang.org/x/sys v0.18.0 google.golang.org/grpc v1.60.1 k8s.io/api v0.28.2 k8s.io/apimachinery v0.28.2 - k8s.io/apiserver v0.27.11 + k8s.io/apiserver v0.27.12 k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible - k8s.io/component-base v0.27.11 + k8s.io/component-base v0.27.12 k8s.io/cri-api v0.29.0-alpha.0 k8s.io/klog/v2 v2.100.1 - k8s.io/kubernetes v1.27.11 + k8s.io/kubernetes v1.27.12 k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/yaml v1.4.0 ) @@ -188,7 +188,6 @@ require ( github.com/coreos/go-iptables v0.7.0 // indirect github.com/coreos/go-oidc v2.1.0+incompatible // indirect github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect @@ -239,7 +238,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/mock v1.6.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect github.com/google/cadvisor v0.47.2 // indirect @@ -280,7 +279,8 @@ require ( github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.4.2 // indirect + github.com/jackc/pgx/v5 v5.5.4 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -289,9 +289,9 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/k3s-io/kine v0.11.4 // indirect + github.com/k3s-io/kine v0.11.7 // indirect github.com/karrick/godirwalk v1.17.0 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/leodido/go-urn v1.2.4 // indirect @@ -356,10 +356,10 @@ require ( github.com/mwitkow/go-http-dialer v0.0.0-20161116154839-378f744fb2b8 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee // indirect - github.com/nats-io/jwt/v2 v2.5.3 // indirect - github.com/nats-io/nats-server/v2 v2.10.5 // indirect - github.com/nats-io/nats.go v1.31.0 // indirect - github.com/nats-io/nkeys v0.4.6 // indirect + github.com/nats-io/jwt/v2 v2.5.5 // indirect + github.com/nats-io/nats-server/v2 v2.10.12 // indirect + github.com/nats-io/nats.go v1.34.0 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc5 // indirect @@ -440,10 +440,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.18.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -455,7 +455,7 @@ require ( google.golang.org/api v0.152.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/gcfg.v1 v1.2.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect @@ -466,9 +466,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect inet.af/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252 // indirect k8s.io/apiextensions-apiserver v0.25.4 // indirect - k8s.io/cloud-provider v0.27.11 // indirect + k8s.io/cloud-provider v0.27.12 // indirect k8s.io/cluster-bootstrap v0.0.0 // indirect - k8s.io/component-helpers v0.27.11 // indirect + k8s.io/component-helpers v0.27.12 // indirect k8s.io/controller-manager v0.25.4 // indirect k8s.io/csi-translation-lib v0.0.0 // indirect k8s.io/dynamic-resource-allocation v0.0.0 // indirect @@ -482,7 +482,7 @@ require ( k8s.io/kubelet v0.0.0 // indirect k8s.io/legacy-cloud-providers v0.0.0 // indirect k8s.io/metrics v0.0.0 // indirect - k8s.io/mount-utils v0.27.11 // indirect + k8s.io/mount-utils v0.27.12 // indirect k8s.io/pod-security-admission v0.0.0 // indirect lukechampine.com/blake3 v1.2.1 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect diff --git a/go.sum b/go.sum index 3e3427bbcb..f045b8a5bc 100644 --- a/go.sum +++ b/go.sum @@ -379,8 +379,6 @@ github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmf github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -671,8 +669,8 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -933,8 +931,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.4.2 h1:u1gmGDwbdRUZiwisBm/Ky2M14uQyUP65bG8+20nnyrg= -github.com/jackc/pgx/v5 v5.4.2/go.mod h1:q6iHT8uDNXWiFNOlRqJzBTaSH3+2xCXkokxHZC5qWFY= +github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= +github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= @@ -1002,10 +1002,10 @@ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1 h1:B3039IkTPnwQEt4tIMjC6yd6b1Q3Z9ZZ github.com/k3s-io/etcd/server/v3 v3.5.9-k3s1/go.mod h1:GgI1fQClQCFIzuVjlvdbMxNbnISt90gdfYyqiAIt65g= github.com/k3s-io/helm-controller v0.15.9 h1:eBZq0KkZCDyWh4og+tyI43Nt9T5TNjc7QCFhAt1aR64= github.com/k3s-io/helm-controller v0.15.9/go.mod h1:AYitg40howLjKloL/zdjDDOPL1jg/K5R4af0tQcyPR8= -github.com/k3s-io/k3s v1.27.12-0.20240313063150-06ecbe807148 h1:3CBgHIzTDFbHZXrUE06O9Bwje/pshazlSEnkjmk0F6g= -github.com/k3s-io/k3s v1.27.12-0.20240313063150-06ecbe807148/go.mod h1:N27+ujeEVoz+aRXIliRvDqQDH6u0p22RCykBt1e76Y0= -github.com/k3s-io/kine v0.11.4 h1:ZIXQT4vPPKNL9DwLF4dQ11tWtpJ1C/7OKNIpFmTkImo= -github.com/k3s-io/kine v0.11.4/go.mod h1:NmwOWsWgB3aScq5+LEYytAaceqkG7lmCLLjjrWug8v4= +github.com/k3s-io/k3s v1.27.13-0.20240402144016-9176d7f68aa2 h1:pzJBe6Ci5XJ7aWRU1SUwmpybU5Hgu8TmPuCsJbhmnYE= +github.com/k3s-io/k3s v1.27.13-0.20240402144016-9176d7f68aa2/go.mod h1:h8Y2GYm4R0CoNfIrQZVvK75JtATKZ5hvZDUWgJ6CUOY= +github.com/k3s-io/kine v0.11.7 h1:+I4TrxozQv4cdmD8RULI35r4o5G+A7gOD3F75lfjDP0= +github.com/k3s-io/kine v0.11.7/go.mod h1:4C/zNVwl3FU1EubA2ju1Hq36JIjp8gAZaM+Hfnuvqt4= github.com/k3s-io/klog v1.0.0-k3s2/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= github.com/k3s-io/klog/v2 v2.90.1-k3s1 h1:QOJ/1xi4ERgpPGmSI3n1f989XjII+OvPdgdiMOKqV9s= github.com/k3s-io/klog/v2 v2.90.1-k3s1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= @@ -1084,8 +1084,8 @@ github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -1347,17 +1347,17 @@ github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6Yf github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee h1:+l6i7zS8N1LOokm7dzShezI9STRGrzp0O49Pw8Jetdk= github.com/nats-io/jsm.go v0.0.31-0.20220317133147-fe318f464eee/go.mod h1:EKSYvbvWAoh0hIfuZ+ieWm8u0VOTRTeDfuQvNPKRqEg= github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= -github.com/nats-io/jwt/v2 v2.5.3 h1:/9SWvzc6hTfamcgXJ3uYRpgj+QuY2aLNqRiqrKcrpEo= -github.com/nats-io/jwt/v2 v2.5.3/go.mod h1:iysuPemFcc7p4IoYots3IuELSI4EDe9Y0bQMe+I3Bf4= +github.com/nats-io/jwt/v2 v2.5.5 h1:ROfXb50elFq5c9+1ztaUbdlrArNFl2+fQWP6B8HGEq4= +github.com/nats-io/jwt/v2 v2.5.5/go.mod h1:ZdWS1nZa6WMZfFwwgpEaqBV8EPGVgOTDHN/wTbz0Y5A= github.com/nats-io/nats-server/v2 v2.7.5-0.20220309212130-5c0d1999ff72/go.mod h1:1vZ2Nijh8tcyNe8BDVyTviCd9NYzRbubQYiEHsvOQWc= -github.com/nats-io/nats-server/v2 v2.10.5 h1:hhWt6m9ja/mNnm6ixc85jCthDaiUFPaeJI79K/MD980= -github.com/nats-io/nats-server/v2 v2.10.5/go.mod h1:xUMTU4kS//SDkJCSvFwN9SyJ9nUuLhSkzB/Qz0dvjjg= +github.com/nats-io/nats-server/v2 v2.10.12 h1:G6u+RDrHkw4bkwn7I911O5jqys7jJVRY6MwgndyUsnE= +github.com/nats-io/nats-server/v2 v2.10.12/go.mod h1:H1n6zXtYLFCgXcf/SF8QNTSIFuS8tyZQMN9NguUHdEs= github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= -github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= -github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= +github.com/nats-io/nats.go v1.34.0 h1:fnxnPCNiwIG5w08rlMcEKTUw4AV/nKyGCOJE8TdhSPk= +github.com/nats-io/nats.go v1.34.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= -github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= -github.com/nats-io/nkeys v0.4.6/go.mod h1:4DxZNzenSVd1cYQoAa8948QY3QDjrHfcfVADymtkpts= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= @@ -2170,8 +2170,8 @@ google.golang.org/protobuf v1.29.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index 439a6dc855..cc1c4b58cc 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -73,10 +73,11 @@ var ( "kube-controller-manager-arg": copyFlag, "kube-cloud-controller-manager-arg": copyFlag, "kube-cloud-controller-arg": dropFlag, // deprecated version of kube-cloud-controller-manager-arg - "datastore-endpoint": dropFlag, - "datastore-cafile": dropFlag, - "datastore-certfile": dropFlag, - "datastore-keyfile": dropFlag, + "datastore-endpoint": copyFlag, + "datastore-cafile": copyFlag, + "datastore-certfile": copyFlag, + "datastore-keyfile": copyFlag, + "kine-tls": dropFlag, "default-local-storage-path": dropFlag, "disable": { Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(DisableItems, ", ") + ")", diff --git a/pkg/podexecutor/staticpod.go b/pkg/podexecutor/staticpod.go index 3ddc50dbfc..4c65cde532 100644 --- a/pkg/podexecutor/staticpod.go +++ b/pkg/podexecutor/staticpod.go @@ -109,19 +109,20 @@ type StaticPodConfig struct { ControlPlaneProbeConfs ControlPlaneEnv ControlPlaneMounts - ManifestsDir string - ImagesDir string - Resolver *images.Resolver - CloudProvider *CloudProviderConfig - DataDir string - AuditPolicyFile string - PSAConfigFile string - KubeletPath string - RuntimeEndpoint string - KubeProxyChan chan struct{} - CISMode bool - DisableETCD bool - IsServer bool + ManifestsDir string + ImagesDir string + Resolver *images.Resolver + CloudProvider *CloudProviderConfig + DataDir string + AuditPolicyFile string + PSAConfigFile string + KubeletPath string + RuntimeEndpoint string + KubeProxyChan chan struct{} + CISMode bool + DisableETCD bool + ExternalDatabase bool + IsServer bool stopKubelet context.CancelFunc } @@ -264,6 +265,11 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{} case "--advertise-port", "--basic-auth-file": // This is an option k3s adds that does not exist upstream args = append(args[:i], args[i+1:]...) + case "--etcd-servers": + if s.ExternalDatabase { + args = append(args[:i], args[i+1:]...) + args = append([]string{"--etcd-servers=" + "unixs://" + filepath.Join(s.DataDir, "server", "kine.sock")}, args...) + } case "--audit-log-path": auditLogFile = value case "--kubelet-preferred-address-types": @@ -309,47 +315,56 @@ func (s *StaticPodConfig) APIServer(_ context.Context, etcdReady <-chan struct{} if !s.DisableETCD { files = append(files, etcdNameFile(s.DataDir)) } + + sockets := []string{} + if s.ExternalDatabase { + sockets = append(sockets, kineSock(s.DataDir)) + } + dirs := onlyExisting(ssldirs) if auditLogFile != "" && auditLogFile != "-" { dirs = append(dirs, filepath.Dir(auditLogFile)) excludeFiles = append(excludeFiles, auditLogFile) } + apiServerArgs := staticpod.Args{ + Command: "kube-apiserver", + Args: args, + Image: image, + Dirs: dirs, + CISMode: s.CISMode, + CPURequest: s.ControlPlaneResources.KubeAPIServerCPURequest, + CPULimit: s.ControlPlaneResources.KubeAPIServerCPULimit, + MemoryRequest: s.ControlPlaneResources.KubeAPIServerMemoryRequest, + MemoryLimit: s.ControlPlaneResources.KubeAPIServerMemoryLimit, + ExtraEnv: s.ControlPlaneEnv.KubeAPIServer, + ExtraMounts: s.ControlPlaneMounts.KubeAPIServer, + ProbeConfs: s.ControlPlaneProbeConfs.KubeAPIServer, + Sockets: sockets, + Files: files, + ExcludeFiles: excludeFiles, + HealthExec: []string{ + "kubectl", + "get", + "--server=https://localhost:6443/", + "--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt", + "--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key", + "--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt", + "--raw=/livez", + }, + ReadyExec: []string{ + "kubectl", + "get", + "--server=https://localhost:6443/", + "--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt", + "--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key", + "--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt", + "--raw=/readyz", + }, + } + return after(etcdReady, func() error { - return staticpod.Run(s.ManifestsDir, staticpod.Args{ - Command: "kube-apiserver", - Args: args, - Image: image, - Dirs: dirs, - CISMode: s.CISMode, - CPURequest: s.ControlPlaneResources.KubeAPIServerCPURequest, - CPULimit: s.ControlPlaneResources.KubeAPIServerCPULimit, - MemoryRequest: s.ControlPlaneResources.KubeAPIServerMemoryRequest, - MemoryLimit: s.ControlPlaneResources.KubeAPIServerMemoryLimit, - ExtraEnv: s.ControlPlaneEnv.KubeAPIServer, - ExtraMounts: s.ControlPlaneMounts.KubeAPIServer, - ProbeConfs: s.ControlPlaneProbeConfs.KubeAPIServer, - Files: files, - ExcludeFiles: excludeFiles, - HealthExec: []string{ - "kubectl", - "get", - "--server=https://localhost:6443/", - "--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt", - "--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key", - "--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt", - "--raw=/livez", - }, - ReadyExec: []string{ - "kubectl", - "get", - "--server=https://localhost:6443/", - "--client-certificate=" + s.DataDir + "/server/tls/client-kube-apiserver.crt", - "--client-key=" + s.DataDir + "/server/tls/client-kube-apiserver.key", - "--certificate-authority=" + s.DataDir + "/server/tls/server-ca.crt", - "--raw=/readyz", - }, - }) + return staticpod.Run(s.ManifestsDir, apiServerArgs) }) } @@ -368,6 +383,12 @@ func (s *StaticPodConfig) Scheduler(_ context.Context, apiReady <-chan struct{}, if !s.DisableETCD { files = append(files, etcdNameFile(s.DataDir)) } + + sockets := []string{} + if s.ExternalDatabase { + sockets = append(sockets, kineSock(s.DataDir)) + } + args = append(permitPortSharingFlag, args...) return after(apiReady, func() error { return staticpod.Run(s.ManifestsDir, staticpod.Args{ @@ -385,6 +406,7 @@ func (s *StaticPodConfig) Scheduler(_ context.Context, apiReady <-chan struct{}, ExtraMounts: s.ControlPlaneMounts.KubeScheduler, ProbeConfs: s.ControlPlaneProbeConfs.KubeScheduler, Files: files, + Sockets: sockets, }) }) } @@ -433,6 +455,12 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s if !s.DisableETCD { files = append(files, etcdNameFile(s.DataDir)) } + + sockets := []string{} + if s.ExternalDatabase { + sockets = append(sockets, kineSock(s.DataDir)) + } + return after(apiReady, func() error { extraArgs := []string{ "--flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins", @@ -455,6 +483,7 @@ func (s *StaticPodConfig) ControllerManager(_ context.Context, apiReady <-chan s ExtraMounts: s.ControlPlaneMounts.KubeControllerManager, ProbeConfs: s.ControlPlaneProbeConfs.KubeControllerManager, Files: files, + Sockets: sockets, }) }) } @@ -686,6 +715,10 @@ func chownr(path string, uid, gid int) error { }) } +func kineSock(dataDir string) string { + return filepath.Join(dataDir, "server", "kine.sock") +} + func etcdNameFile(dataDir string) string { return filepath.Join(dataDir, "server", "db", "etcd", "name") } diff --git a/pkg/rke2/rke2.go b/pkg/rke2/rke2.go index 35ac9e514d..08e8303b1b 100644 --- a/pkg/rke2/rke2.go +++ b/pkg/rke2/rke2.go @@ -154,17 +154,18 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error { } // check for missing db name file on a server running etcd, indicating we're rejoining after cluster reset on a different node - if _, err := os.Stat(etcdNameFile(dataDir)); err != nil && os.IsNotExist(err) && isServer && !clx.Bool("disable-etcd") { + if _, err := os.Stat(etcdNameFile(dataDir)); err != nil && os.IsNotExist(err) && isServer && !clx.Bool("disable-etcd") && !clx.IsSet("datastore-endpoint") { clusterReset = true } disabledItems := map[string]bool{ "cloud-controller-manager": !isServer || forceRestart || clx.Bool("disable-cloud-controller"), - "etcd": !isServer || forceRestart || clx.Bool("disable-etcd"), + "etcd": !isServer || forceRestart || clx.Bool("disable-etcd") || clx.IsSet("datastore-endpoint"), "kube-apiserver": !isServer || forceRestart || clx.Bool("disable-apiserver"), "kube-controller-manager": !isServer || forceRestart || clx.Bool("disable-controller-manager"), "kube-scheduler": !isServer || forceRestart || clx.Bool("disable-scheduler"), } + // adding force restart file when cluster reset restore path is passed if clusterResetRestorePath != "" { forceRestartFile := ForceRestartFile(dataDir) @@ -175,6 +176,7 @@ func setup(clx *cli.Context, cfg Config, isServer bool) error { return err } } + return removeDisabledPods(dataDir, containerRuntimeEndpoint, disabledItems, clusterReset) } diff --git a/pkg/rke2/rke2_linux.go b/pkg/rke2/rke2_linux.go index 1b7b59c41c..f8da3b7da4 100644 --- a/pkg/rke2/rke2_linux.go +++ b/pkg/rke2/rke2_linux.go @@ -57,11 +57,20 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta return nil, err } + // Verify if the user want to use kine as the datastore + // and then remove the etcd from the static pod + ExternalDatabase := false + if cmds.ServerConfig.DatastoreEndpoint != "" { + cmds.ServerConfig.ClusterInit = false + cmds.ServerConfig.KineTLS = true + ExternalDatabase = true + } else { + managed.RegisterDriver(&etcd.ETCD{}) + } + agentManifestsDir := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath) agentImagesDir := filepath.Join(dataDir, "agent", "images") - managed.RegisterDriver(&etcd.ETCD{}) - if clx.IsSet("cloud-provider-config") || clx.IsSet("cloud-provider-name") { if clx.IsSet("node-external-ip") { return nil, errors.New("can't set node-external-ip while using cloud provider") @@ -115,6 +124,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta if err != nil { return nil, err } + // Adding PSAs podSecurityConfigFile := clx.String("pod-security-admission-config-file") if podSecurityConfigFile == "" { @@ -142,6 +152,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta RuntimeEndpoint: containerRuntimeEndpoint, KubeProxyChan: make(chan struct{}), DisableETCD: clx.Bool("disable-etcd"), + ExternalDatabase: ExternalDatabase, IsServer: isServer, ControlPlaneResources: *controlPlaneResources, ControlPlaneProbeConfs: *controlPlaneProbeConfs, @@ -155,7 +166,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources, // resources is a map of the component (kube-apiserver, kube-controller-manager, etc.) to a map[string]*string, // where the key of the downstream map is the `cpu-request`, `cpu-limit`, `memory-request`, or `memory-limit` and // the value corresponds to a pointer to the component resources array - var resources = map[string]map[string]*string{ + resources := map[string]map[string]*string{ KubeAPIServer: { CPURequest: &controlPlaneResources.KubeAPIServerCPURequest, CPULimit: &controlPlaneResources.KubeAPIServerCPULimit, @@ -195,7 +206,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources, } // defaultResources contains a map of default resources for each component, used if not explicitly configured. - var defaultResources = map[string]map[string]string{ + defaultResources := map[string]map[string]string{ KubeAPIServer: { CPURequest: "250m", MemoryRequest: "1024Mi", @@ -222,7 +233,7 @@ func parseControlPlaneResources(cfg Config) (*podexecutor.ControlPlaneResources, }, } - var parsedRequestsLimits = make(map[string]string) + parsedRequestsLimits := make(map[string]string) for _, requests := range cfg.ControlPlaneResourceRequests { for _, rawRequest := range strings.Split(requests, ",") { @@ -262,7 +273,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf var controlPlaneProbes podexecutor.ControlPlaneProbeConfs // probes is a map of the component (kube-apiserver, kube-controller-manager, etc.) probe type, and setting, where // the value corresponds to a pointer to the component probes array. - var probes = map[string]map[string]map[string]*int32{ + probes := map[string]map[string]map[string]*int32{ KubeAPIServer: { Liveness: { InitialDelaySeconds: &controlPlaneProbes.KubeAPIServer.Liveness.InitialDelaySeconds, @@ -386,7 +397,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf } // defaultProbeConf contains a map of default probe settings for each type, used if not explicitly configured. - var defaultProbeConf = map[string]map[string]int32{ + defaultProbeConf := map[string]map[string]int32{ // https://github.com/kubernetes/kubernetes/blob/v1.24.0/cmd/kubeadm/app/util/staticpod/utils.go#L246 Liveness: { InitialDelaySeconds: 10, @@ -410,7 +421,7 @@ func parseControlPlaneProbeConfs(cfg Config) (*podexecutor.ControlPlaneProbeConf }, } - var parsedProbeConf = make(map[string]int32) + parsedProbeConf := make(map[string]int32) for _, conf := range cfg.ControlPlaneProbeConf { for _, rawConf := range strings.Split(conf, ",") { diff --git a/pkg/staticpod/staticpod.go b/pkg/staticpod/staticpod.go index e016a72b08..a41a696b95 100644 --- a/pkg/staticpod/staticpod.go +++ b/pkg/staticpod/staticpod.go @@ -28,8 +28,13 @@ import ( "sigs.k8s.io/yaml" ) +type typeVolume string + const ( - extraMountPrefix = "extra-mount" + extraMountPrefix = "extra-mount" + socket typeVolume = "socket" + dir typeVolume = "dir" + file typeVolume = "file" ) type ProbeConf struct { @@ -51,6 +56,7 @@ type Args struct { Image name.Reference Dirs []string Files []string + Sockets []string CISMode bool // CIS requires that the manifest be saved with 600 permissions ExcludeFiles []string HealthExec []string @@ -278,8 +284,9 @@ func pod(args Args) (*v1.Pod, error) { } } - addVolumes(p, args.Dirs, true) - addVolumes(p, args.Files, false) + addVolumes(p, args.Sockets, socket) + addVolumes(p, args.Dirs, dir) + addVolumes(p, args.Files, file) addExtraMounts(p, args.ExtraMounts) addExtraEnv(p, args.ExtraEnv) @@ -287,14 +294,22 @@ func pod(args Args) (*v1.Pod, error) { return p, nil } -func addVolumes(p *v1.Pod, src []string, dir bool) { +func addVolumes(p *v1.Pod, src []string, volume typeVolume) { var ( - prefix = "dir" - sourceType = v1.HostPathDirectoryOrCreate - readOnly = false + prefix string + sourceType v1.HostPathType + readOnly bool ) - if !dir { - prefix = "file" + + prefix = string(volume) + switch volume { + case dir: + sourceType = v1.HostPathDirectoryOrCreate + readOnly = false + case socket: + sourceType = v1.HostPathSocket + readOnly = false + default: sourceType = v1.HostPathFile readOnly = true }