diff --git a/Dockerfile b/Dockerfile index febab62238..eb144e406e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,14 @@ FROM golang:1.14.2 AS build # Shell used for debugging FROM build AS shell RUN apt-get update && \ - apt-get install -y iptables socat vim + apt-get install -y iptables socat vim bash-completion less psmisc libseccomp-dev sudo +RUN cp -f /etc/skel/.bashrc /etc/skel/.profile /root/ && \ + echo 'alias abort="echo -e '\''q\ny\n'\'' | ./bin/dlv connect :2345"' >> /root/.bashrc +ENV PATH=/var/lib/rancher/rke2/bin:$PATH +ENV KUBECONFIG=/etc/rancher/k3s/k3s.yaml VOLUME /var/lib/rancher/rke2 +# This makes it so we can run and debug k3s too +VOLUME /var/lib/rancher/k3s # Dapper/Drone/CI environment FROM build AS dapper @@ -18,18 +24,17 @@ ENV DAPPER_RUN_ARGS "-v rke2-pkg:/go/pkg -v rke2-cache:/root/.cache/go-build" WORKDIR /source # End Dapper stuff -# rke-runtime Dockerfile. This image includes any host level programs that we might need. All binaries +# rke-runtime image +# This image includes any host level programs that we might need. All binaries # must be placed in bin/ of the file image and subdirectories of bin/ will be flattened during installation. # This means bin/foo/bar will become bin/bar when rke2 installs this to the host - FROM k8s.gcr.io/hyperkube:v1.18.2 AS k8s FROM rancher/k3s:v1.18.2-rc2-k3s1 AS k3s - FROM ubuntu:18.04 AS containerd ENV CONTAINERD_VERION=1.3.4 -ENV CONTAINERD_URL=https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERION}/containerd-${CONTAINERD_VERION}.linux-amd64.tar.gz ENV CONTAINERD_HASH=61e65c9589e5abfded1daa353e6dfb4b8c2436199bbc5507fc45809a3bb80c1d +ENV CONTAINERD_URL=https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERION}/containerd-${CONTAINERD_VERION}.linux-amd64.tar.gz RUN apt-get update && \ apt-get install -y curl diff --git a/Makefile b/Makefile index 45a7c0f918..cfc74d1aad 100644 --- a/Makefile +++ b/Makefile @@ -46,11 +46,10 @@ RELEASE=${PROG}-$(VERSION).${GOOS}-${GOARCH} ifdef BUILDTAGS GO_BUILDTAGS = ${BUILDTAGS} endif -# Build tags seccomp, apparmor and selinux are needed by CRI plugin. -GO_BUILDTAGS ?= todo +GO_BUILDTAGS ?= no_embedded_executor GO_BUILDTAGS += ${DEBUG_TAGS} GO_TAGS=$(if $(GO_BUILDTAGS),-tags "$(GO_BUILDTAGS)",) -GO_LDFLAGS=-ldflags '-X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PACKAGE) $(EXTRA_LDFLAGS)' +GO_LDFLAGS=-ldflags '-X $(PKG)/version.Program=$(PROG) -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PACKAGE) $(EXTRA_LDFLAGS)' default: in-docker-build ## Build using docker environment (default target) @@ -87,17 +86,21 @@ validate: ## Run go fmt/vet validate-ci: validate bin/golangci-lint ## Run more validation for CI ./bin/golangci-lint run +COMMAND ?= "server" run: build-debug - ./bin/${PROG} server + ./bin/${PROG} ${COMMAND} ${ARGS} bin/dlv: go build -o bin/dlv github.com/go-delve/delve/cmd/dlv remote-debug: build-debug bin/dlv ## Run with remote debugging listening on :2345 - ./bin/dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./bin/${PROG} server + CATTLE_DEV_MODE=true ./bin/dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec -- ./bin/${PROG} ${COMMAND} ${ARGS} + +remote-debug-exit: bin/dlv ## Kill dlv started with make remote-debug + echo exit | ./bin/dlv connect :2345 --init /dev/stdin .dev-shell-build: - docker build -t ${PROG}-dev --target shell -f Dockerfile --target dapper . + docker build -t ${PROG}-dev --target shell . clean-cache: ## Clean up docker base caches used for development docker rm -fv ${PROG}-dev-shell @@ -107,11 +110,18 @@ clean: ## Clean up workspace rm -rf bin dist dev-shell: .dev-shell-build ## Launch a development shell to run test builds - docker run --rm --name ${PROG}-dev-shell -ti -v $${HOME}:$${HOME} -v ${PROG} -w $$(pwd) --privileged --net=host -v ${PROG}-pkg:/go/pkg -v ${PROG}-cache:/root/.cache/go-build ${PROG}-dev bash + docker run --rm --name ${PROG}-dev-shell --hostname ${PROG}-server -ti -e WORKSPACE=$$(pwd) -p 127.0.0.1:2345:2345 -v $${HOME}:$${HOME} -v ${PROG} -w $$(pwd) --privileged -v ${PROG}-pkg:/go/pkg -v ${PROG}-cache:/root/.cache/go-build ${PROG}-dev bash dev-shell-enter: ## Enter the development shell on another terminal docker exec -it ${PROG}-dev-shell bash +PEER ?= 1 +dev-peer: .dev-shell-build ## Launch a server peer to run test builds + docker run --rm --link ${PROG}-dev-shell:${PROG}-server --name ${PROG}-peer${PEER} --hostname ${PROG}-peer${PEER} -p 127.0.0.1:234${PEER}:2345 -ti -e WORKSPACE=$$(pwd) -v $${HOME}:$${HOME} -v ${PROG} -w $$(pwd) --privileged -v ${PROG}-pkg:/go/pkg -v ${PROG}-cache:/root/.cache/go-build ${PROG}-dev bash + +dev-peer-enter: ## Enter the peer shell on another terminal + docker exec -it ${PROG}-peer${PEER} bash + artifacts: build mkdir -p dist/artifacts cp bin/${PROG} dist/artifacts/${RELEASE} diff --git a/go.mod b/go.mod index 8e5f855af0..99e0560ffc 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,13 @@ module github.com/rancher/rke2 go 1.14 replace ( - github.com/Azure/go-autorest => github.com/Azure/go-autorest v14.0.0+incompatible - github.com/Microsoft/hcsshim => github.com/Microsoft/hcsshim v0.8.7-0.20190926181021-82c7525d98c8 github.com/containerd/btrfs => github.com/containerd/btrfs v0.0.0-20181101203652-af5082808c83 github.com/containerd/cgroups => github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601 github.com/containerd/console => github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50 github.com/containerd/containerd => github.com/rancher/containerd v1.3.3-k3s2 github.com/containerd/continuity => github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02 - github.com/containerd/cri => github.com/rancher/cri v1.3.0-k3s.5 + github.com/containerd/cri => github.com/rancher/cri v1.3.0-k3s.6 github.com/containerd/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c github.com/containerd/go-runc => github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda github.com/containerd/typeurl => github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd @@ -34,10 +32,7 @@ replace ( github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 github.com/prometheus/common => github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a - github.com/rancher/helm-controller => github.com/ibuildthecloud/helm-controller-1 v0.1.2-0.20200427180043-a4b3f2c320ef - - github.com/rancher/k3s => github.com/ibuildthecloud/k3s-dev v0.1.0-rc6.0.20200427184827-21712abb0a8a - + github.com/rancher/k3s => github.com/ibuildthecloud/k3s-dev v0.1.0-rc6.0.20200507231729-5cf25b189e0e k8s.io/api => github.com/rancher/kubernetes/staging/src/k8s.io/api v1.18.2-k3s.1 k8s.io/apiextensions-apiserver => github.com/rancher/kubernetes/staging/src/k8s.io/apiextensions-apiserver v1.18.2-k3s.1 k8s.io/apimachinery => github.com/rancher/kubernetes/staging/src/k8s.io/apimachinery v1.18.2-k3s.1 @@ -68,11 +63,15 @@ replace ( require ( github.com/google/go-containerregistry v0.0.0-20200424115305-087a4bdef7c4 + github.com/pkg/errors v0.9.1 github.com/rancher/k3s v0.0.0 - github.com/rancher/wrangler v0.6.2-0.20200427172034-da9b142ae061 + github.com/rancher/wrangler v0.6.1 github.com/sirupsen/logrus v1.4.2 github.com/urfave/cli v1.22.2 + google.golang.org/grpc v1.26.0 k8s.io/api v0.18.0 k8s.io/apimachinery v0.18.0 k8s.io/apiserver v0.0.0 + k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible + sigs.k8s.io/yaml v1.2.0 ) diff --git a/go.sum b/go.sum index b2f02992cb..e6b7cfbb20 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,7 @@ github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1 github.com/checkpoint-restore/go-criu v0.0.0-20181120144056-17b0214f6c48 h1:AQMF0Xixllgf29MKlx/TGEhRk7bEDX5kxz8Ui8lOvEs= github.com/checkpoint-restore/go-criu v0.0.0-20181120144056-17b0214f6c48/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= +github.com/cilium/ebpf v0.0.0-20191025125908-95b36a581eed h1:/UgmF+cZTm9kp4uJ122y/9cVhczNJCgAgAeH2FfzPeg= github.com/cilium/ebpf v0.0.0-20191025125908-95b36a581eed/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= @@ -219,8 +220,6 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4 h1:GY1+t5Dr9OKADM64SYnQjw/w99HMYvQ0A8/JoUkxVmc= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -234,8 +233,6 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0 github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M= github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA= github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -430,13 +427,9 @@ github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mo github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ibuildthecloud/helm-controller-1 v0.1.2-0.20200427180043-a4b3f2c320ef h1:1RnWARqAp7RzM1TknUAsFvvPgkrV1I4l3emNfGfJteg= -github.com/ibuildthecloud/helm-controller-1 v0.1.2-0.20200427180043-a4b3f2c320ef/go.mod h1:4tTIyZASaQXltQ0Noc9fq+X2wsU3E6BQo0D8NGlaWec= -github.com/ibuildthecloud/k3s-dev v0.1.0-rc6.0.20200427184827-21712abb0a8a h1:4tfBRflKvwis9lxBl8dMdkkTkTY2jPWlLqi+XmTGJ+Q= -github.com/ibuildthecloud/k3s-dev v0.1.0-rc6.0.20200427184827-21712abb0a8a/go.mod h1:FsXo6qy/t6ih2xWkmXaAYhLqusvRp2Gzto9AtH38zww= +github.com/ibuildthecloud/k3s-dev v0.1.0-rc6.0.20200507231729-5cf25b189e0e h1:p9up77xlMMPyweLX+89hCJ1eSXyrbO7uusC6XMb5q/U= +github.com/ibuildthecloud/k3s-dev v0.1.0-rc6.0.20200507231729-5cf25b189e0e/go.mod h1:/5+T1vYOwutUBKv8eJ6HNIfSk1HdCQ6dlFIg9klx8UU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -460,13 +453,10 @@ github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d h1:hJXjZMxj0SWlMoQkzeZDLi2cmeiWKa7y1B8Rg+qaoEc= github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 h1:UUHMLvzt/31azWTN/ifGWef4WUqvXk0iRqdhdy/2uzI= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2 h1:Pp8RxiF4rSoXP9SED26WCfNB28/dwTDpPXS8XMJR8rc= github.com/juju/testing v0.0.0-20190723135506-ce30eb24acd2/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= @@ -507,8 +497,6 @@ github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk= github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao= github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58= -github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455 h1:gQQV7It0kjZxMLJkS/+5Mc6w0zM6pKGzl3OS0h2RHrY= -github.com/lxc/lxd v0.0.0-20191108214106-60ea15630455/go.mod h1:2BaZflfwsv8a3uy3/Vw+de4Avn4DSrAiqaHJjCIXMV4= github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -593,18 +581,16 @@ github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= -github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc10 h1:AbmCEuSZXVflng0/cboQkpdEOeBsPMjz6tmq4Pv8MZw= github.com/opencontainers/runc v1.0.0-rc10/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU4LguQVtc= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v0.0.0-20180911193056-5684b8af48c1 h1:AcjCvgprf9I23wEYTHuyuHcuuQAp4hK5l+u1FUXgVaM= github.com/opencontainers/runtime-spec v0.0.0-20180911193056-5684b8af48c1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= @@ -639,12 +625,14 @@ github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8 h1:83l9gPhYtgxODl github.com/rakelkar/gonetsh v0.0.0-20190719023240-501daadcadf8/go.mod h1:4XHkfaUj+URzGO9sohoAgt2V9Y8nIW7fugpu0E6gShk= github.com/rancher/containerd v1.3.3-k3s2 h1:RZr+TqFt7+YsrSYkyytlhW4HmneWeFNM7IymNOoGW6A= github.com/rancher/containerd v1.3.3-k3s2/go.mod h1:ZMfzmqce2Z+QSEqdHMfeJs1TZ/UeJ1aDrazjpQT4ehM= -github.com/rancher/cri v1.3.0-k3s.5/go.mod h1:Ht5T1dIKzm+4NExmb7wDVG6qR+j0xeXIjjhCv1d9geY= +github.com/rancher/cri v1.3.0-k3s.6/go.mod h1:Ht5T1dIKzm+4NExmb7wDVG6qR+j0xeXIjjhCv1d9geY= github.com/rancher/cri-tools v1.18.0-k3s1/go.mod h1:Ij/GWNRcEDP6zVN6eQpvN/s0nhuJVtPQFy7RAdl+Wu8= github.com/rancher/dynamiclistener v0.2.0 h1:KucYwJXVVGhZ/NndfMCeQoCafT/VN7kvqSGgmlX8Lxk= github.com/rancher/dynamiclistener v0.2.0/go.mod h1:fs/dxyNcB3YT6W9fVz4bDGfhmSQS17QQup6BIcGF++s= github.com/rancher/flannel v0.11.0-k3s.2 h1:0GVr5ORAIvcri1LYTE8eMQ+NrRbuPeIniPaW51IzLco= github.com/rancher/flannel v0.11.0-k3s.2/go.mod h1:Hn4ZV+eq0LhLZP63xZnxdGwXEoRSxs5sxELxu27M3UA= +github.com/rancher/helm-controller v0.6.0 h1:nFptBZFWpHga65M6bP04BZGLlzeMgezAXdsOwaB8t+4= +github.com/rancher/helm-controller v0.6.0/go.mod h1:ZylsxIMGNADRPRNW+NiBWhrwwks9vnKLQiCHYWb6Bi0= github.com/rancher/kine v0.4.0 h1:1IhWy3TzjExG8xnj46eyUEWdzqNAD1WrgL4eEBKm6Uc= github.com/rancher/kine v0.4.0/go.mod h1:IImtCJ68AIkE+VY/kUI0NkyJL5q5WzO8QvMsSXqbrpA= github.com/rancher/kubernetes v1.18.2-k3s.1 h1:LhWNObWF7dL/+T57LkYpuRKtsCBpt0P5G6dRVFG+Ncs= @@ -688,18 +676,17 @@ github.com/rancher/kubernetes/staging/src/k8s.io/legacy-cloud-providers v1.18.2- github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.18.2-k3s.1 h1:ha8xCCbv3iPsXg+TjB+ZHHxxRyuiWWB9bgTDkgHmLCk= github.com/rancher/kubernetes/staging/src/k8s.io/metrics v1.18.2-k3s.1/go.mod h1:xZM9EdJpWjqIWPvLiCP7vYKUEMwIgc0S8nc/MlLVK3Y= github.com/rancher/kubernetes/staging/src/k8s.io/sample-apiserver v1.18.2-k3s.1/go.mod h1:gpiIUEAyQvSEXKbsH2taOEzkrHXvoZwHuArWgR+DpG8= -github.com/rancher/lasso v0.0.0-20200427171700-e0509f89f319 h1:w9ex9f8KIiD1G84cy2RBYpYSCPcuHIXAMnTlIVI9mNw= -github.com/rancher/lasso v0.0.0-20200427171700-e0509f89f319/go.mod h1:6Dw19z1lDIpL887eelVjyqH/mna1hfR61ddCFOG78lw= github.com/rancher/moq v0.0.0-20190404221404-ee5226d43009/go.mod h1:wpITyDPTi/Na5h73XkbuEf2AP9fbgrIGqqxVzFhYD6U= github.com/rancher/remotedialer v0.2.0 h1:xD7t3K6JYwTdAsxmGtTHQMkEkFgKouQ1foLxVW424Dc= github.com/rancher/remotedialer v0.2.0/go.mod h1:tkU8ZvrR5lRgaKWaX71nAy6daeqvPFx/lJEnbW7tXSI= github.com/rancher/wrangler v0.1.4/go.mod h1:EYP7cqpg42YqElaCm+U9ieSrGQKAXxUH5xsr+XGpWyE= github.com/rancher/wrangler v0.4.0/go.mod h1:1cR91WLhZgkZ+U4fV9nVuXqKurWbgXcIReU4wnQvTN8= -github.com/rancher/wrangler v0.6.2-0.20200427172034-da9b142ae061 h1:oash8kayb/AlGop5KPn/cjUZv2m7oKMU3mwTNqd126c= -github.com/rancher/wrangler v0.6.2-0.20200427172034-da9b142ae061/go.mod h1:n5Du/gGD7WoiqnEo0SHnPirDIp1V9Zu+6guc8lXS2dk= +github.com/rancher/wrangler v0.6.0/go.mod h1:L4HtjPeX8iqLgsxfJgz+JjKMcX2q3qbRXSeTlC/CSd4= +github.com/rancher/wrangler v0.6.1 h1:7tyLk/FV2zCQkYg5SEtT4lSlsHNwa5yMOa797/VJhiQ= +github.com/rancher/wrangler v0.6.1/go.mod h1:L4HtjPeX8iqLgsxfJgz+JjKMcX2q3qbRXSeTlC/CSd4= github.com/rancher/wrangler-api v0.2.0/go.mod h1:zTPdNLZO07KvRaVOx6XQbKBSV55Fnn4s7nqmrMPJqd8= -github.com/rancher/wrangler-api v0.6.1-0.20200427172631-a7c2f09b783e h1:UJpGtw6IKs0dHPTF+6Wd12lskeCZZAejl8/ie/fc1+0= -github.com/rancher/wrangler-api v0.6.1-0.20200427172631-a7c2f09b783e/go.mod h1:2lcWR98q8HU3U4mVETnXc8quNG0uXxrt8vKd6cAa/30= +github.com/rancher/wrangler-api v0.6.0 h1:d/b0AkgZ+x41EYLIcUJiogtU3Y11Mqss2zr9VEKycRk= +github.com/rancher/wrangler-api v0.6.0/go.mod h1:RbuDkPNHhxcXuwAbLVvEAhH+UPAh+MIkpEd2fcGc0MM= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY= github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= @@ -819,6 +806,8 @@ go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 h1:C7kWARE8r64ppRadl40yfNo6pag+G6ocvGU2xZ6yNes= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2 h1:jxcFYjlkl8xaERsgLo+RNquI0epW6zuy/ZRQs6jnrFA= @@ -958,7 +947,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1024,9 +1012,8 @@ google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -1043,8 +1030,6 @@ gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3M gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+VuiKTDG/v1U2stad0QzddfJC7kY= -gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= gopkg.in/square/go-jose.v2 v2.2.2 h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go new file mode 100644 index 0000000000..4736849964 --- /dev/null +++ b/pkg/auth/auth.go @@ -0,0 +1,47 @@ +package auth + +import ( + "strings" + + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/group" + "k8s.io/apiserver/pkg/authentication/request/union" + "k8s.io/apiserver/pkg/authentication/request/x509" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile" + "k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth" +) + +func FromArgs(args []string) (authenticator.Request, error) { + var authenticators []authenticator.Request + basicFile := getArg("--basic-auth-file=", args) + if basicFile != "" { + basicAuthenticator, err := passwordfile.NewCSV(basicFile) + if err != nil { + return nil, err + } + authenticators = append(authenticators, basicauth.New(basicAuthenticator)) + } + + clientCA := getArg("--client-ca-file", args) + if clientCA != "" { + ca, err := dynamiccertificates.NewDynamicCAContentFromFile("client-ca", clientCA) + if err != nil { + return nil, err + } + authenticators = append(authenticators, x509.NewDynamic(ca.VerifyOptions, x509.CommonNameUserConversion)) + } + + auth := union.New(authenticators...) + return group.NewAuthenticatedGroupAdder(auth), nil +} + +func getArg(key string, args []string) string { + for _, arg := range args { + if !strings.HasPrefix(arg, key) { + continue + } + return strings.SplitN(arg, "=", 2)[1] + } + return "" +} diff --git a/pkg/bootstrap/stage.go b/pkg/bootstrap/stage.go index ee1ffe9775..e9dac87462 100644 --- a/pkg/bootstrap/stage.go +++ b/pkg/bootstrap/stage.go @@ -27,6 +27,10 @@ func dataDirFor(dataDir, dataName string) string { return filepath.Join(dataDir, "data", dataName, "bin") } +func symlinkDataDir(dataDir string) string { + return filepath.Join(dataDir, "bin") +} + func dirExists(dir string) bool { if s, err := os.Stat(dir); err == nil && s.IsDir() { return true @@ -82,7 +86,16 @@ func Stage(dataDir string, images images.Images) (string, error) { return "", err } - return dir, os.Rename(tempDir, dir) + err = os.Rename(tempDir, dir) + if err != nil { + return "", err + } + + // ignore errors + _ = os.RemoveAll(symlinkDataDir(dataDir)) + _ = os.Symlink(dir, symlinkDataDir(dataDir)) + + return dir, err } func extract(image, targetDir string, reader io.Reader) error { diff --git a/pkg/cli/cmds/agent.go b/pkg/cli/cmds/agent.go index 9d8df9572f..e58d48f9e2 100644 --- a/pkg/cli/cmds/agent.go +++ b/pkg/cli/cmds/agent.go @@ -1,13 +1,13 @@ package cmds import ( - "github.com/rancher/k3s/pkg/cli/agent" "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/rke2/pkg/rke2" "github.com/urfave/cli" ) var ( - k3sAgentBase = mustCmdFromK3S(cmds.NewAgentCommand(agent.Run), map[string]*K3SFlagOption{ + k3sAgentBase = mustCmdFromK3S(cmds.NewAgentCommand(AgentRun), map[string]*K3SFlagOption{ "v": Hide, "vmodule": Hide, "log": Hide, @@ -26,7 +26,7 @@ var ( "docker": Drop, "container-runtime-endpoint": Drop, "pause-image": Drop, - "private-registry": Drop, + "private-registry": nil, "node-ip": nil, "node-external-ip": Drop, "resolv-conf": nil, @@ -42,5 +42,11 @@ var ( ) func NewAgentCommand() cli.Command { - return k3sAgentBase + cmd := k3sAgentBase + cmd.Flags = append(cmd.Flags, commonFlag...) + return cmd +} + +func AgentRun(app *cli.Context) error { + return rke2.Agent(app, config) } diff --git a/pkg/cli/cmds/k3sopts.go b/pkg/cli/cmds/k3sopts.go index ad3fe34865..c41e8d1983 100644 --- a/pkg/cli/cmds/k3sopts.go +++ b/pkg/cli/cmds/k3sopts.go @@ -62,10 +62,6 @@ func commandFromK3S(cmd cli.Command, flagOpts map[string]*K3SFlagOption) (cli.Co if opt.Default != "" { strFlag.Value = opt.Default } - if strFlag.EnvVar != "" { - strFlag.EnvVar = strings.Replace(strFlag.EnvVar, "K3S_", "RKE2_", 1) - } - strFlag.Usage = strings.Replace(strFlag.Usage, "k3s", "RKE2", -1) if opt.Hide { strFlag.Hidden = true } @@ -74,10 +70,6 @@ func commandFromK3S(cmd cli.Command, flagOpts map[string]*K3SFlagOption) (cli.Co if opt.Usage != "" { intFlag.Usage = opt.Usage } - if intFlag.EnvVar != "" { - intFlag.EnvVar = strings.Replace(intFlag.EnvVar, "K3S_", "RKE2_", 1) - } - intFlag.Usage = strings.Replace(intFlag.Usage, "k3s", "RKE2", -1) if opt.Hide { intFlag.Hidden = true } @@ -86,10 +78,6 @@ func commandFromK3S(cmd cli.Command, flagOpts map[string]*K3SFlagOption) (cli.Co if opt.Usage != "" { boolFlag.Usage = opt.Usage } - if boolFlag.EnvVar != "" { - boolFlag.EnvVar = strings.Replace(boolFlag.EnvVar, "K3S_", "RKE2_", 1) - } - boolFlag.Usage = strings.Replace(boolFlag.Usage, "k3s", "RKE2", -1) if opt.Hide { boolFlag.Hidden = true } diff --git a/pkg/cli/cmds/root.go b/pkg/cli/cmds/root.go index caf2bbc1b5..c35a538f5e 100644 --- a/pkg/cli/cmds/root.go +++ b/pkg/cli/cmds/root.go @@ -11,8 +11,17 @@ import ( ) var ( - debug bool - appName = filepath.Base(os.Args[0]) + debug bool + appName = filepath.Base(os.Args[0]) + commonFlag = []cli.Flag{ + cli.StringFlag{ + Name: "repo", + Usage: "(image) Image repository to use for RKE2 images", + EnvVar: "RKE2_REPO", + Value: "rancher", + Destination: &config.Repo, + }, + } ) func init() { diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index bd8edea787..1142124a70 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -2,12 +2,13 @@ package cmds import ( "github.com/rancher/k3s/pkg/cli/cmds" - "github.com/rancher/rke2/pkg/server" + "github.com/rancher/rke2/pkg/rke2" "github.com/urfave/cli" ) var ( - config server.Config + config rke2.Config + k3sServerBase = mustCmdFromK3S(cmds.NewServerCommand(ServerRun), map[string]*K3SFlagOption{ "v": Hide, "vmodule": Hide, @@ -53,6 +54,7 @@ var ( "disable-scheduler": Drop, "disable-cloud-controller": Drop, "disable-network-policy": Drop, + "disable-kube-proxy": Drop, "node-name": nil, "with-node-id": Drop, "node-label": nil, @@ -60,7 +62,7 @@ var ( "docker": Drop, "container-runtime-endpoint": nil, "pause-image": Drop, - "private-registry": Drop, + "private-registry": nil, "node-ip": nil, "node-external-ip": Drop, "resolv-conf": nil, @@ -81,22 +83,10 @@ var ( func NewServerCommand() cli.Command { cmd := k3sServerBase - cmd.Flags = append(cmd.Flags, cli.StringFlag{ - Name: "version", - Usage: "(image) Label used for the image download", - EnvVar: "RKE2_VERSION", - Value: "latest", - Destination: &config.Version, - }, cli.StringFlag{ - Name: "repo", - Usage: "(image) Image repository to use for RKE2 images", - EnvVar: "RKE2_REPO", - Value: "rancher", - Destination: &config.Repo, - }) + cmd.Flags = append(cmd.Flags, commonFlag...) return cmd } func ServerRun(app *cli.Context) error { - return server.Run(app, config) + return rke2.Server(app, config) } diff --git a/pkg/cli/defaults/defaults.go b/pkg/cli/defaults/defaults.go index 2f7db4000c..6ef35204a3 100644 --- a/pkg/cli/defaults/defaults.go +++ b/pkg/cli/defaults/defaults.go @@ -1,12 +1,25 @@ package defaults import ( + "io/ioutil" + "os" + "path/filepath" + + "google.golang.org/grpc/grpclog" + + "github.com/pkg/errors" + "github.com/rancher/k3s/pkg/cli/cmds" "github.com/rancher/rke2/pkg/images" ) -func Set(images images.Images) { - cmds.ServerConfig.DatastoreEndpoint = "http://localhost:2379" +func Set(images images.Images, dataDir string) error { + logsDir := filepath.Join(dataDir, "agent", "logs") + if err := os.MkdirAll(logsDir, 0755); err != nil { + return errors.Wrapf(err, "failed to create directory %s", logsDir) + } + + cmds.ServerConfig.DatastoreEndpoint = "etcd" cmds.ServerConfig.DisableCCM = true cmds.ServerConfig.DisableNPC = true cmds.ServerConfig.FlannelBackend = "none" @@ -14,7 +27,20 @@ func Set(images images.Images) { cmds.ServerConfig.SupervisorPort = 9345 cmds.ServerConfig.HTTPSPort = 6443 cmds.ServerConfig.APIServerPort = 6443 + cmds.ServerConfig.APIServerBindAddress = "0.0.0.0" cmds.ServerConfig.DisableKubeProxy = true cmds.AgentConfig.PauseImage = images.Pause cmds.AgentConfig.NoFlannel = true + cmds.AgentConfig.ExtraKubeletArgs = append(cmds.AgentConfig.ExtraKubeletArgs, + "stderrthreshold=FATAL", + "log-file-max-size=50", + "alsologtostderr=false", + "logtostderr=false", "log-file="+filepath.Join(logsDir, "kubelet.log")) + + if !cmds.Debug { + l := grpclog.NewLoggerV2(ioutil.Discard, ioutil.Discard, os.Stderr) + grpclog.SetLoggerV2(l) + } + + return nil } diff --git a/pkg/images/images.go b/pkg/images/images.go index a903a85462..e0dd11040b 100644 --- a/pkg/images/images.go +++ b/pkg/images/images.go @@ -1,28 +1,34 @@ package images +import ( + "os" + + "github.com/rancher/k3s/pkg/version" +) + +var ( + // These should not be used, these are settings to help with development + apiServer = os.Getenv("RKE2_KUBE_APISERVER_IMAGE") + controllerManager = os.Getenv("RKE2_KUBE_CONTROLLER_MANAGER_IMAGE") + scheduler = os.Getenv("RKE2_KUBE_SCHEDULER_IMAGE") + pause = os.Getenv("RKE2_PAUSE_IMAGE") + runtime = os.Getenv("RKE2_RUNTIME_IMAGE") + etcd = os.Getenv("RKE2_ETCD_IMAGE") + + KubernetesVersion = "v1.18.2" + PauseVersion = "3.2" + EtcdVersion = "3.4.3-0" +) + type Images struct { KubeAPIServer string `json:"kube-apiserver"` KubeControllManager string `json:"kube-controller-manager"` KubeScheduler string `json:"kube-scheduler"` Pause string `json:"pause"` Runtime string `json:"runtime"` + ETCD string `json:"etcd"` } -var ( - // These should not be used, these are settings to help with development - //apiServer = os.Getenv("RKE2_KUBE_APISERVER_IMAGE") - //controllerManager = os.Getenv("RKE2_KUBE_CONTROLLER_MANAGER_IMAGE") - //scheduler = os.Getenv("RKE2_KUBE_SCHEDULER_IMAGE") - //pause = os.Getenv("RKE2_PAUSE_IMAGE") - //runtime = os.Getenv("RKE2_RUNTIME_IMAGE") - - apiServer = "k8s.gcr.io/kube-apiserver:v1.18.2" - controllerManager = "k8s.gcr.io/kube-controller-manager:v1.18.2" - scheduler = "k8s.gcr.io/kube-scheduler:v1.18.2" - pause = "k8s.gcr.io/pause:3.2" - runtime = "ibuildthecloud/rke2-runtime:latest" -) - func override(str, override string) string { if override != "" { return override @@ -30,12 +36,13 @@ func override(str, override string) string { return str } -func New(repo, version string) Images { +func New(repo string) Images { return Images{ - KubeAPIServer: override(repo+"/rke2-kube-apiserver:"+version, apiServer), - KubeControllManager: override(repo+"/rke2-kube-controller-maanger:"+version, controllerManager), - KubeScheduler: override(repo+"/rke2-kube-scheduler:"+version, scheduler), - Pause: override(repo+"/rke2-pause:"+version, pause), - Runtime: override(repo+"/rke2-runtime:"+version, runtime), + KubeAPIServer: override(override("k8s.gcr.io", repo)+"/kube-apiserver:"+KubernetesVersion, apiServer), + KubeControllManager: override(override("k8s.gcr.io", repo)+"/kube-controller-maanger:"+KubernetesVersion, controllerManager), + KubeScheduler: override(override("k8s.gcr.io", repo)+"/kube-scheduler:"+KubernetesVersion, scheduler), + Pause: override(override("k8s.gcr.io", repo)+"/pause:"+PauseVersion, pause), + Runtime: override(override("rancher", repo)+"/rke2-runtime:"+version.Version, runtime), + ETCD: override(override("k8s.gcr.io", repo)+"/etcd:"+EtcdVersion, etcd), } } diff --git a/pkg/images/pull.go b/pkg/images/pull.go new file mode 100644 index 0000000000..db5dccb034 --- /dev/null +++ b/pkg/images/pull.go @@ -0,0 +1,24 @@ +package images + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +func Pull(dir, name, image string) error { + if dir == "" { + return nil + } + + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + + dest := filepath.Join(dir, name+".txt") + if err := ioutil.WriteFile(dest, []byte(image+"\n"), 0644); err != nil { + return err + } + + return nil +} diff --git a/pkg/podexecutor/command.go b/pkg/podexecutor/command.go new file mode 100644 index 0000000000..3693f9c36a --- /dev/null +++ b/pkg/podexecutor/command.go @@ -0,0 +1,15 @@ +// +build !windows + +package podexecutor + +import ( + "os/exec" + "syscall" +) + +func addDeathSig(cmd *exec.Cmd) { + // not supported in this OS + cmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGKILL, + } +} diff --git a/pkg/podexecutor/command_windows.go b/pkg/podexecutor/command_windows.go new file mode 100644 index 0000000000..f5a93c7cf8 --- /dev/null +++ b/pkg/podexecutor/command_windows.go @@ -0,0 +1,7 @@ +package podexecutor + +import "os/exec" + +func addDeathSig(_ *exec.Cmd) { + // not supported in this OS +} diff --git a/pkg/podexecutor/staticpod.go b/pkg/podexecutor/staticpod.go index b7efaaa8db..9c7906085f 100644 --- a/pkg/podexecutor/staticpod.go +++ b/pkg/podexecutor/staticpod.go @@ -2,23 +2,24 @@ package podexecutor import ( "context" - "fmt" + "encoding/json" "io/ioutil" "net/http" "os" "os/exec" "path/filepath" "strings" + "time" - "k8s.io/apiserver/plugin/pkg/authenticator/password/passwordfile" - "k8s.io/apiserver/plugin/pkg/authenticator/request/basicauth" + "sigs.k8s.io/yaml" - "github.com/rancher/rke2/pkg/images" - "github.com/rancher/wrangler/pkg/yaml" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/rancher/k3s/pkg/daemons/executor" + "github.com/rancher/rke2/pkg/auth" + "github.com/rancher/rke2/pkg/images" + "github.com/rancher/rke2/pkg/staticpod" + "github.com/sirupsen/logrus" "k8s.io/apiserver/pkg/authentication/authenticator" ) @@ -32,28 +33,23 @@ var ( ) type StaticPod struct { - Manifests string - Images images.Images -} - -type PodArgs struct { - Command string - Args []string - Image string - Dirs []string - Files []string - HealthPort int32 - CPUMillis int64 + Manifests string + PullImages string + Images images.Images } func (s *StaticPod) Kubelet(args []string) error { go func() { for { - fmt.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! START", args) cmd := exec.Command("kubelet", args...) cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Run() + //cmd.Stderr = os.Stderr + addDeathSig(cmd) + + err := cmd.Run() + logrus.Errorf("Kubelet exited: %v", err) + + time.Sleep(5 * time.Second) } }() @@ -64,203 +60,130 @@ func (s *StaticPod) KubeProxy(args []string) error { panic("kube-proxy unsupported") } -func (s *StaticPod) APIServer(ctx context.Context, args []string) (authenticator.Request, http.Handler, bool, error) { +func (s *StaticPod) APIServer(ctx context.Context, etcdReady <-chan struct{}, args []string) (authenticator.Request, http.Handler, error) { + if err := images.Pull(s.PullImages, "kube-apiserver", s.Images.KubeAPIServer); err != nil { + return nil, nil, err + } + for i, arg := range args { + // This is an option k3s adds that does not exist upstream if strings.HasPrefix(arg, "--advertise-port=") { args = append(args[:i], args[i+1:]...) break } } - err := RunPod(s.Manifests, PodArgs{ - Command: "kube-apiserver", - Args: args, - Image: s.Images.KubeAPIServer, - Dirs: ssldirs, - HealthPort: 6443, - CPUMillis: 250, + after(etcdReady, func() error { + return staticpod.Run(s.Manifests, staticpod.Args{ + Command: "kube-apiserver", + Args: args, + Image: s.Images.KubeAPIServer, + Dirs: ssldirs, + CPUMillis: 250, + }) }) - if err != nil { - return nil, nil, false, err - } - auth, err := auth(args) - return auth, http.NotFoundHandler(), false, err -} -func auth(args []string) (authenticator.Request, error) { - for _, arg := range args { - if !strings.HasPrefix(arg, "--basic-auth-file=") { - continue - } - file := strings.SplitN(arg, "=", 2)[1] - basicAuthenticator, err := passwordfile.NewCSV(file) - if err != nil { - return nil, err - } + auth, err := auth.FromArgs(args) + return auth, http.NotFoundHandler(), err +} - return basicauth.New(basicAuthenticator), nil +func (s *StaticPod) Scheduler(apiReady <-chan struct{}, args []string) error { + if err := images.Pull(s.PullImages, "kube-scheduler", s.Images.KubeScheduler); err != nil { + return err } + return after(apiReady, func() error { + return staticpod.Run(s.Manifests, staticpod.Args{ + Command: "kube-scheduler", + Args: args, + Image: s.Images.KubeScheduler, + HealthPort: 10251, + HealthProto: "HTTP", + CPUMillis: 100, + }) + }) +} - return nil, nil +func after(after <-chan struct{}, f func() error) error { + go func() { + <-after + if err := f(); err != nil { + logrus.Fatal(err) + } + }() + return nil } -func (s *StaticPod) Scheduler(args []string) error { - if true { - return nil +func (s *StaticPod) ControllerManager(apiReady <-chan struct{}, args []string) error { + if err := images.Pull(s.PullImages, "kube-controller-manager", s.Images.KubeControllManager); err != nil { + return err } - return RunPod(s.Manifests, PodArgs{ - Command: "kube-scheduler", - Args: args, - Image: s.Images.KubeScheduler, - HealthPort: 10259, - CPUMillis: 100, + return after(apiReady, func() error { + return staticpod.Run(s.Manifests, staticpod.Args{ + Command: "kube-controller-manager", + Args: append(args, + "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"), + Image: s.Images.KubeControllManager, + HealthPort: 10252, + HealthProto: "HTTP", + CPUMillis: 200, + }) }) } -func (s *StaticPod) ControllerManager(args []string) error { - if true { - return nil +func (s *StaticPod) CurrentETCDOptions() (opts executor.InitialOptions, err error) { + bytes, err := ioutil.ReadFile(filepath.Join(s.Manifests, "etcd.yaml")) + if os.IsNotExist(err) { + return } - return RunPod(s.Manifests, PodArgs{ - Command: "kube-controller-manager", - Args: append(args, - "/usr/libexec/kubernetes/kubelet-plugins/volume/exec"), - Image: s.Images.KubeControllManager, - HealthPort: 10257, - CPUMillis: 200, - }) -} -func RunPod(dir string, args PodArgs) error { - seen := map[string]bool{} - for _, arg := range args.Args { - parts := strings.SplitN(arg, "=", 2) - if len(parts) == 2 && strings.HasPrefix(parts[1], "/") { - if stat, err := os.Stat(parts[1]); err == nil && !stat.IsDir() { - if seen[parts[1]] { - continue - } - seen[parts[1]] = true - args.Files = append(args.Files, parts[1]) - } - } + pod := &v1.Pod{} + if err := yaml.Unmarshal(bytes, pod); err != nil { + return opts, err } - pod := pod(args) - bytes, err := yaml.Export(pod) - if err != nil { - return err + v, ok := pod.Annotations["etcd.k3s.io/initial"] + if ok { + return opts, json.NewDecoder(strings.NewReader(v)).Decode(&opts) } - return writeFile(dir, args.Command, bytes) + return } -func writeFile(dir, name string, content []byte) error { - if err := os.MkdirAll(dir, 0700); err != nil { +func (s *StaticPod) ETCD(args executor.ETCDConfig) error { + if err := images.Pull(s.PullImages, "etcd", s.Images.ETCD); err != nil { return err } - dest := filepath.Join(dir, name+".yaml") - tmp := filepath.Join(dir, name+".tmp") - if err := ioutil.WriteFile(tmp, content, 0777); err != nil { + initial, err := json.Marshal(args.InitialOptions) + if err != nil { return err } - return os.Rename(tmp, dest) -} - -func pod(args PodArgs) *v1.Pod { - p := &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: args.Command, - Namespace: "kube-system", - Labels: map[string]string{ - "component": args.Command, - "tier": "control-plane", - }, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Command: append([]string{args.Command}, args.Args...), - Image: args.Image, - ImagePullPolicy: v1.PullIfNotPresent, - LivenessProbe: &v1.Probe{ - Handler: v1.Handler{ - HTTPGet: &v1.HTTPGetAction{ - Path: "/healthz", - Port: intstr.IntOrString{ - IntVal: args.HealthPort, - }, - Host: "127.0.0.1", - Scheme: "HTTPS", - }, - }, - InitialDelaySeconds: 15, - TimeoutSeconds: 15, - FailureThreshold: 8, - }, - Name: args.Command, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: *resource.NewMilliQuantity(args.CPUMillis, resource.DecimalSI), - }, - }, - }, - }, - HostNetwork: true, - PriorityClassName: "system-cluster-critical", - }, - } - - for _, env := range os.Environ() { - parts := strings.SplitN(env, "=", 2) - if !strings.Contains(strings.ToLower(parts[0]), "proxy") { - continue - } - if len(parts) == 1 { - p.Spec.Containers[0].Env = append(p.Spec.Containers[0].Env, v1.EnvVar{ - Name: parts[0], - }) - } else { - p.Spec.Containers[0].Env = append(p.Spec.Containers[0].Env, v1.EnvVar{ - Name: parts[0], - Value: parts[1], - }) - } - } - addVolumes(p, args.Dirs, true) - addVolumes(p, args.Files, false) - - return p -} - -func addVolumes(p *v1.Pod, src []string, dir bool) { - var ( - prefix = "dir" - sourceType = v1.HostPathDirectoryOrCreate - ) - if !dir { - prefix = "file" - sourceType = v1.HostPathFile - } - - for i, src := range src { - name := fmt.Sprintf("%s%d", prefix, i) - p.Spec.Volumes = append(p.Spec.Volumes, v1.Volume{ - Name: name, - VolumeSource: v1.VolumeSource{ - HostPath: &v1.HostPathVolumeSource{ - Path: src, - Type: &sourceType, - }, - }, - }) - p.Spec.Containers[0].VolumeMounts = append(p.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ - Name: name, - ReadOnly: true, - MountPath: src, - }) + confFile, err := args.ToConfigFile() + if err != nil { + return err } + return staticpod.Run(s.Manifests, staticpod.Args{ + Annotations: map[string]string{ + "etcd.k3s.io/initial": string(initial), + }, + Command: "etcd", + Args: []string{ + "--config-file=" + confFile, + }, + Image: s.Images.ETCD, + Dirs: []string{args.DataDir}, + Files: []string{ + args.ServerTrust.CertFile, + args.ServerTrust.KeyFile, + args.ServerTrust.TrustedCAFile, + args.PeerTrust.CertFile, + args.PeerTrust.KeyFile, + args.PeerTrust.TrustedCAFile, + }, + HealthPort: 2381, + HealthPath: "/health", + HealthProto: "HTTP", + }) } diff --git a/pkg/rke2/server.go b/pkg/rke2/server.go new file mode 100644 index 0000000000..09e28d09b4 --- /dev/null +++ b/pkg/rke2/server.go @@ -0,0 +1,80 @@ +package rke2 + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/rancher/k3s/pkg/agent/config" + + "github.com/rancher/k3s/pkg/cli/agent" + "github.com/rancher/k3s/pkg/cli/cmds" + "github.com/rancher/k3s/pkg/cli/server" + "github.com/rancher/k3s/pkg/cluster/managed" + "github.com/rancher/k3s/pkg/daemons/executor" + "github.com/rancher/k3s/pkg/etcd" + "github.com/rancher/rke2/pkg/bootstrap" + "github.com/rancher/rke2/pkg/cli/defaults" + "github.com/rancher/rke2/pkg/images" + "github.com/rancher/rke2/pkg/podexecutor" + "github.com/urfave/cli" +) + +type Config struct { + Repo string +} + +func Server(ctx *cli.Context, cfg Config) error { + if err := setup(ctx, cfg); err != nil { + return err + } + if err := ctx.Set("disable", cmds.DisableItems); err != nil { + return err + } + return server.Run(ctx) +} + +func Agent(ctx *cli.Context, cfg Config) error { + if err := setup(ctx, cfg); err != nil { + return err + } + return agent.Run(ctx) +} + +func setup(ctx *cli.Context, cfg Config) error { + if err := cmds.InitLogging(); err != nil { + return err + } + + dataDir := cmds.ServerConfig.DataDir + if dataDir == "" { + dataDir = cmds.AgentConfig.DataDir + } + + images := images.New(cfg.Repo) + if err := defaults.Set(images, dataDir); err != nil { + return err + } + + execPath, err := bootstrap.Stage(dataDir, images) + if err != nil { + return err + } + + if err := os.Setenv("PATH", fmt.Sprintf("%s:%s", execPath, os.Getenv("PATH"))); err != nil { + return err + } + + manifests := filepath.Join(dataDir, "agent", config.DefaultPodManifestPath) + pullImages := filepath.Join(dataDir, "agent", "images") + + managed.RegisterDriver(&etcd.ETCD{}) + + executor.Set(&podexecutor.StaticPod{ + Images: images, + PullImages: pullImages, + Manifests: manifests, + }) + + return nil +} diff --git a/pkg/server/server.go b/pkg/server/server.go deleted file mode 100644 index 4a17f1fcfd..0000000000 --- a/pkg/server/server.go +++ /dev/null @@ -1,51 +0,0 @@ -package server - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/rancher/rke2/pkg/bootstrap" - - "github.com/rancher/k3s/pkg/cli/cmds" - "github.com/rancher/k3s/pkg/cli/server" - "github.com/rancher/k3s/pkg/daemons/executor" - "github.com/rancher/rke2/pkg/cli/defaults" - "github.com/rancher/rke2/pkg/images" - "github.com/rancher/rke2/pkg/podexecutor" - "github.com/urfave/cli" -) - -type Config struct { - Version string - Repo string -} - -func Run(ctx *cli.Context, cfg Config) error { - if err := cmds.InitLogging(); err != nil { - return err - } - - if err := ctx.Set("disable", cmds.DisableItems); err != nil { - return err - } - - images := images.New(cfg.Repo, cfg.Version) - defaults.Set(images) - - execPath, err := bootstrap.Stage(cmds.ServerConfig.DataDir, images) - if err != nil { - return err - } - - if err := os.Setenv("PATH", fmt.Sprintf("%s:%s", execPath, os.Getenv("PATH"))); err != nil { - return err - } - - executor.Set(&podexecutor.StaticPod{ - Images: images, - Manifests: filepath.Join(cmds.ServerConfig.DataDir, "agent", "manifests"), - }) - - return server.Run(ctx) -} diff --git a/pkg/staticpod/staticpod.go b/pkg/staticpod/staticpod.go new file mode 100644 index 0000000000..797422aa3c --- /dev/null +++ b/pkg/staticpod/staticpod.go @@ -0,0 +1,270 @@ +package staticpod + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/rancher/wrangler/pkg/yaml" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/clientcmd" +) + +type Args struct { + Command string + Args []string + Image string + Dirs []string + Files []string + HealthPort int32 + HealthProto string + HealthPath string + CPUMillis int64 + Annotations map[string]string +} + +func Run(dir string, args Args) error { + files, err := readFiles(args.Args) + if err != nil { + return err + } + + args.Files = append(args.Files, files...) + pod, err := pod(args) + if err != nil { + return err + } + + bytes, err := yaml.Export(pod) + if err != nil { + return err + } + + return writeFile(dir, args.Command, bytes) +} + +func writeFile(dir, name string, content []byte) error { + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + + dest := filepath.Join(dir, name+".yaml") + + existing, err := ioutil.ReadFile(dest) + if err == nil && bytes.Equal(existing, content) { + return nil + } + + tmp := filepath.Join(dir, name+".tmp") + if err := ioutil.WriteFile(tmp, content, 0777); err != nil { + return err + } + return os.Rename(tmp, dest) +} + +func hashFiles(files []string) (string, error) { + h := sha256.New() + for _, file := range files { + f, err := os.Open(file) + if err != nil { + return "", err + } + _, err = io.Copy(h, f) + f.Close() + if err != nil { + return "", err + } + } + + return hex.EncodeToString(h.Sum(nil)), nil +} + +func pod(args Args) (*v1.Pod, error) { + filehash, err := hashFiles(args.Files) + if err != nil { + return nil, err + } + + p := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: args.Command, + Namespace: "kube-system", + Labels: map[string]string{ + "component": args.Command, + "tier": "control-plane", + }, + Annotations: args.Annotations, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Command: append([]string{args.Command}, args.Args...), + Image: args.Image, + ImagePullPolicy: v1.PullIfNotPresent, + Env: []v1.EnvVar{ + { + Name: "FILE_HASH", + Value: filehash, + }, + }, + Name: args.Command, + }, + }, + HostNetwork: true, + PriorityClassName: "system-cluster-critical", + }, + } + + if args.CPUMillis > 0 { + p.Spec.Containers[0].Resources = v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewMilliQuantity(args.CPUMillis, resource.DecimalSI), + }, + } + } + + if args.HealthPort != 0 { + scheme := args.HealthProto + if scheme == "" { + scheme = "HTTPS" + } + path := args.HealthPath + if path == "" { + path = "/healthz" + } + p.Spec.Containers[0].LivenessProbe = &v1.Probe{ + Handler: v1.Handler{ + HTTPGet: &v1.HTTPGetAction{ + Path: path, + Port: intstr.IntOrString{ + IntVal: args.HealthPort, + }, + Host: "127.0.0.1", + Scheme: v1.URIScheme(scheme), + }, + }, + InitialDelaySeconds: 15, + TimeoutSeconds: 15, + FailureThreshold: 8, + } + } + + for _, env := range os.Environ() { + parts := strings.SplitN(env, "=", 2) + if !strings.Contains(strings.ToLower(parts[0]), "proxy") { + continue + } + if len(parts) == 1 { + p.Spec.Containers[0].Env = append(p.Spec.Containers[0].Env, v1.EnvVar{ + Name: parts[0], + }) + } else { + p.Spec.Containers[0].Env = append(p.Spec.Containers[0].Env, v1.EnvVar{ + Name: parts[0], + Value: parts[1], + }) + } + } + + addVolumes(p, args.Dirs, true) + addVolumes(p, args.Files, false) + + return p, nil +} + +func addVolumes(p *v1.Pod, src []string, dir bool) { + var ( + prefix = "dir" + sourceType = v1.HostPathDirectoryOrCreate + readOnly = false + ) + if !dir { + prefix = "file" + sourceType = v1.HostPathFile + readOnly = true + } + + for i, src := range src { + name := fmt.Sprintf("%s%d", prefix, i) + p.Spec.Volumes = append(p.Spec.Volumes, v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: src, + Type: &sourceType, + }, + }, + }) + p.Spec.Containers[0].VolumeMounts = append(p.Spec.Containers[0].VolumeMounts, v1.VolumeMount{ + Name: name, + ReadOnly: readOnly, + MountPath: src, + }) + } +} + +func readFiles(args []string) ([]string, error) { + files := map[string]bool{} + + for _, arg := range args { + parts := strings.SplitN(arg, "=", 2) + if len(parts) == 2 && strings.HasPrefix(parts[1], "/") { + if stat, err := os.Stat(parts[1]); err == nil && !stat.IsDir() { + files[parts[1]] = true + + if parts[0] == "--kubeconfig" { + certs, err := kubeconfigFiles(parts[1]) + if err != nil { + return nil, err + } + for _, cert := range certs { + files[cert] = true + } + } + } + } + } + + var result []string + for k := range files { + result = append(result, k) + } + sort.Strings(result) + return result, nil +} + +func kubeconfigFiles(kubeconfig string) ([]string, error) { + var result []string + + kc, err := clientcmd.LoadFromFile(kubeconfig) + if err != nil { + return nil, err + } + + for _, cluster := range kc.Clusters { + if cluster.CertificateAuthority != "" { + result = append(result, cluster.CertificateAuthority) + } + } + + for _, auth := range kc.AuthInfos { + if auth.ClientKey != "" { + result = append(result, auth.ClientKey) + } + if auth.ClientCertificate != "" { + result = append(result, auth.ClientCertificate) + } + } + + return result, nil +}