From c98d68f9198b739f65ea813f1b3869b49aa7e149 Mon Sep 17 00:00:00 2001 From: ChrisLiu Date: Wed, 21 Sep 2022 14:27:27 +0800 Subject: [PATCH] Kruise-Game init Signed-off-by: ChrisLiu --- .dockerignore | 4 + .gitignore | 10 +- Dockerfile | 26 + Makefile | 133 +++ PROJECT | 25 + README.md | 49 +- apis/v1alpha1/doc.go | 19 + apis/v1alpha1/gameserver_types.go | 156 +++ apis/v1alpha1/gameserverset_types.go | 170 +++ apis/v1alpha1/groupversion_info.go | 43 + apis/v1alpha1/zz_generated.deepcopy.go | 562 +++++++++ .../crd/bases/game.kruise.io_gameservers.yaml | 838 ++++++++++++++ .../bases/game.kruise.io_gameserversets.yaml | 665 +++++++++++ config/crd/kustomization.yaml | 23 + config/crd/kustomizeconfig.yaml | 19 + config/default/kustomization.yaml | 74 ++ config/default/manager_auth_proxy_patch.yaml | 39 + config/default/manager_webhook_patch.yaml | 23 + config/default/webhookcainjection_patch.yaml | 8 + config/manager/controller_manager_config.yaml | 21 + config/manager/kustomization.yaml | 16 + config/manager/manager.yaml | 70 ++ config/prometheus/kustomization.yaml | 2 + config/prometheus/monitor.yaml | 20 + .../rbac/auth_proxy_client_clusterrole.yaml | 9 + config/rbac/auth_proxy_role.yaml | 17 + config/rbac/auth_proxy_role_binding.yaml | 12 + config/rbac/auth_proxy_service.yaml | 15 + config/rbac/gameserver_editor_role.yaml | 24 + config/rbac/gameserver_viewer_role.yaml | 20 + config/rbac/gameserverset_editor_role.yaml | 24 + config/rbac/gameserverset_viewer_role.yaml | 20 + config/rbac/kustomization.yaml | 18 + config/rbac/leader_election_role.yaml | 37 + config/rbac/leader_election_role_binding.yaml | 12 + config/rbac/role.yaml | 131 +++ config/rbac/role_binding.yaml | 12 + config/rbac/service_account.yaml | 5 + .../game.kruise.io_v1alpha1_gameserver.yaml | 0 ...game.kruise.io_v1alpha1_gameserverset.yaml | 0 config/webhook/kustomization.yaml | 2 + config/webhook/service.yaml | 12 + docs/getting_started/installation.md | 26 + docs/getting_started/introduction.md | 30 + docs/images/logo.jpg | Bin 0 -> 82885 bytes docs/tutorials/basic_usage.md | 222 ++++ go.mod | 89 ++ go.sum | 1017 +++++++++++++++++ hack/boilerplate.go.txt | 15 + hack/gencerts.sh | 170 +++ hack/tools.go | 25 + main.go | 150 +++ pkg/client/clientset/versioned/clientset.go | 120 ++ pkg/client/clientset/versioned/doc.go | 19 + .../versioned/fake/clientset_generated.go | 84 ++ pkg/client/clientset/versioned/fake/doc.go | 19 + .../clientset/versioned/fake/register.go | 55 + pkg/client/clientset/versioned/scheme/doc.go | 19 + .../clientset/versioned/scheme/register.go | 55 + .../typed/apis/v1alpha1/apis_client.go | 111 ++ .../versioned/typed/apis/v1alpha1/doc.go | 19 + .../versioned/typed/apis/v1alpha1/fake/doc.go | 19 + .../apis/v1alpha1/fake/fake_apis_client.go | 43 + .../apis/v1alpha1/fake/fake_gameserver.go | 141 +++ .../apis/v1alpha1/fake/fake_gameserverset.go | 141 +++ .../typed/apis/v1alpha1/gameserver.go | 194 ++++ .../typed/apis/v1alpha1/gameserverset.go | 194 ++++ .../apis/v1alpha1/generated_expansion.go | 22 + pkg/client/generic_client.go | 64 ++ .../externalversions/apis/interface.go | 45 + .../apis/v1alpha1/gameserver.go | 89 ++ .../apis/v1alpha1/gameserverset.go | 89 ++ .../apis/v1alpha1/interface.go | 51 + .../informers/externalversions/factory.go | 179 +++ .../informers/externalversions/generic.go | 63 + .../internalinterfaces/factory_interfaces.go | 39 + .../apis/v1alpha1/expansion_generated.go | 34 + .../listers/apis/v1alpha1/gameserver.go | 98 ++ .../listers/apis/v1alpha1/gameserverset.go | 98 ++ pkg/client/registry.go | 56 + pkg/client/versioned/clientset.go | 18 + pkg/client/versioned/doc.go | 19 + .../versioned/fake/clientset_generated.go | 18 + pkg/client/versioned/fake/doc.go | 19 + pkg/client/versioned/fake/register.go | 18 + pkg/client/versioned/scheme/doc.go | 19 + pkg/client/versioned/scheme/register.go | 18 + pkg/controllers/controller.go | 45 + .../gameserver/gameserver_controller.go | 251 ++++ .../gameserver/gameserver_manager.go | 214 ++++ .../gameserverset/gameserverset_controller.go | 286 +++++ .../gameserverset/gameserverset_manager.go | 246 ++++ pkg/util/client/delegating_client.go | 167 +++ pkg/util/client/no_deepcopy_lister.go | 198 ++++ pkg/util/discovery/discovery.go | 97 ++ pkg/util/gameserver.go | 164 +++ pkg/util/gameserver_test.go | 110 ++ pkg/util/hash.go | 44 + pkg/util/hash_test.go | 62 + pkg/util/slice.go | 141 +++ pkg/util/slice_test.go | 257 +++++ pkg/webhook/mutating_pod.go | 53 + pkg/webhook/util/generator/certgenerator.go | 41 + .../util/generator/fake/certgenerator.go | 54 + pkg/webhook/util/generator/selfsigned.go | 192 ++++ pkg/webhook/util/generator/util.go | 62 + .../util/writer/atomic/atomic_writer.go | 452 ++++++++ pkg/webhook/util/writer/certwriter.go | 107 ++ pkg/webhook/util/writer/error.go | 44 + pkg/webhook/util/writer/fs.go | 229 ++++ pkg/webhook/util/writer/secret.go | 179 +++ pkg/webhook/validating_gss.go | 61 + pkg/webhook/webhook.go | 262 +++++ scripts/generate_client.sh | 8 + test/e2e/client/client.go | 137 +++ test/e2e/e2e_test.go | 35 + test/e2e/framework/framework.go | 216 ++++ test/e2e/testcase/testcase.go | 78 ++ 118 files changed, 11902 insertions(+), 8 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 PROJECT create mode 100644 apis/v1alpha1/doc.go create mode 100644 apis/v1alpha1/gameserver_types.go create mode 100644 apis/v1alpha1/gameserverset_types.go create mode 100644 apis/v1alpha1/groupversion_info.go create mode 100644 apis/v1alpha1/zz_generated.deepcopy.go create mode 100644 config/crd/bases/game.kruise.io_gameservers.yaml create mode 100644 config/crd/bases/game.kruise.io_gameserversets.yaml create mode 100644 config/crd/kustomization.yaml create mode 100644 config/crd/kustomizeconfig.yaml create mode 100644 config/default/kustomization.yaml create mode 100644 config/default/manager_auth_proxy_patch.yaml create mode 100644 config/default/manager_webhook_patch.yaml create mode 100644 config/default/webhookcainjection_patch.yaml create mode 100644 config/manager/controller_manager_config.yaml create mode 100644 config/manager/kustomization.yaml create mode 100644 config/manager/manager.yaml create mode 100644 config/prometheus/kustomization.yaml create mode 100644 config/prometheus/monitor.yaml create mode 100644 config/rbac/auth_proxy_client_clusterrole.yaml create mode 100644 config/rbac/auth_proxy_role.yaml create mode 100644 config/rbac/auth_proxy_role_binding.yaml create mode 100644 config/rbac/auth_proxy_service.yaml create mode 100644 config/rbac/gameserver_editor_role.yaml create mode 100644 config/rbac/gameserver_viewer_role.yaml create mode 100644 config/rbac/gameserverset_editor_role.yaml create mode 100644 config/rbac/gameserverset_viewer_role.yaml create mode 100644 config/rbac/kustomization.yaml create mode 100644 config/rbac/leader_election_role.yaml create mode 100644 config/rbac/leader_election_role_binding.yaml create mode 100644 config/rbac/role.yaml create mode 100644 config/rbac/role_binding.yaml create mode 100644 config/rbac/service_account.yaml create mode 100644 config/samples/game.kruise.io_v1alpha1_gameserver.yaml create mode 100644 config/samples/game.kruise.io_v1alpha1_gameserverset.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/service.yaml create mode 100644 docs/getting_started/installation.md create mode 100644 docs/getting_started/introduction.md create mode 100644 docs/images/logo.jpg create mode 100644 docs/tutorials/basic_usage.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 hack/boilerplate.go.txt create mode 100644 hack/gencerts.sh create mode 100644 hack/tools.go create mode 100644 main.go create mode 100644 pkg/client/clientset/versioned/clientset.go create mode 100644 pkg/client/clientset/versioned/doc.go create mode 100644 pkg/client/clientset/versioned/fake/clientset_generated.go create mode 100644 pkg/client/clientset/versioned/fake/doc.go create mode 100644 pkg/client/clientset/versioned/fake/register.go create mode 100644 pkg/client/clientset/versioned/scheme/doc.go create mode 100644 pkg/client/clientset/versioned/scheme/register.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/apis_client.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/doc.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/doc.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserver.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserverset.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserver.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserverset.go create mode 100644 pkg/client/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go create mode 100644 pkg/client/generic_client.go create mode 100644 pkg/client/informers/externalversions/apis/interface.go create mode 100644 pkg/client/informers/externalversions/apis/v1alpha1/gameserver.go create mode 100644 pkg/client/informers/externalversions/apis/v1alpha1/gameserverset.go create mode 100644 pkg/client/informers/externalversions/apis/v1alpha1/interface.go create mode 100644 pkg/client/informers/externalversions/factory.go create mode 100644 pkg/client/informers/externalversions/generic.go create mode 100644 pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go create mode 100644 pkg/client/listers/apis/v1alpha1/expansion_generated.go create mode 100644 pkg/client/listers/apis/v1alpha1/gameserver.go create mode 100644 pkg/client/listers/apis/v1alpha1/gameserverset.go create mode 100644 pkg/client/registry.go create mode 100644 pkg/client/versioned/clientset.go create mode 100644 pkg/client/versioned/doc.go create mode 100644 pkg/client/versioned/fake/clientset_generated.go create mode 100644 pkg/client/versioned/fake/doc.go create mode 100644 pkg/client/versioned/fake/register.go create mode 100644 pkg/client/versioned/scheme/doc.go create mode 100644 pkg/client/versioned/scheme/register.go create mode 100644 pkg/controllers/controller.go create mode 100644 pkg/controllers/gameserver/gameserver_controller.go create mode 100644 pkg/controllers/gameserver/gameserver_manager.go create mode 100644 pkg/controllers/gameserverset/gameserverset_controller.go create mode 100644 pkg/controllers/gameserverset/gameserverset_manager.go create mode 100644 pkg/util/client/delegating_client.go create mode 100644 pkg/util/client/no_deepcopy_lister.go create mode 100644 pkg/util/discovery/discovery.go create mode 100644 pkg/util/gameserver.go create mode 100644 pkg/util/gameserver_test.go create mode 100644 pkg/util/hash.go create mode 100644 pkg/util/hash_test.go create mode 100644 pkg/util/slice.go create mode 100644 pkg/util/slice_test.go create mode 100644 pkg/webhook/mutating_pod.go create mode 100644 pkg/webhook/util/generator/certgenerator.go create mode 100644 pkg/webhook/util/generator/fake/certgenerator.go create mode 100644 pkg/webhook/util/generator/selfsigned.go create mode 100644 pkg/webhook/util/generator/util.go create mode 100644 pkg/webhook/util/writer/atomic/atomic_writer.go create mode 100644 pkg/webhook/util/writer/certwriter.go create mode 100644 pkg/webhook/util/writer/error.go create mode 100644 pkg/webhook/util/writer/fs.go create mode 100644 pkg/webhook/util/writer/secret.go create mode 100644 pkg/webhook/validating_gss.go create mode 100644 pkg/webhook/webhook.go create mode 100755 scripts/generate_client.sh create mode 100644 test/e2e/client/client.go create mode 100644 test/e2e/e2e_test.go create mode 100644 test/e2e/framework/framework.go create mode 100644 test/e2e/testcase/testcase.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..0f046820 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ +testbin/ diff --git a/.gitignore b/.gitignore index cfe9bae1..c0a7a54c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,15 @@ + # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib -bin/ -testbin/ -.temp +bin +testbin/* # Test binary, build with `go test -c` *.test -test/e2e/generated/bindata.go # Output of the go coverage tool, specifically when used with LiteIDE *.out @@ -24,6 +23,3 @@ test/e2e/generated/bindata.go *.swp *.swo *~ -.vscode - -.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b2ce32b3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# Build the manager binary +FROM golang:1.18 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY apis/ apis/ +COPY pkg/ pkg/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go + +# Use distroless as minimal base images to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM alpine:3.14 +WORKDIR / +COPY --from=builder /workspace/manager . + +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..19496598 --- /dev/null +++ b/Makefile @@ -0,0 +1,133 @@ + +# Image URL to use all building/pushing images targets +IMG ?= kruise-game-manager:test +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.24.1 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out + +##@ Build + +.PHONY: build +build: generate fmt vet ## Build manager binary. + go build -o bin/manager main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./main.go + +.PHONY: docker-build +docker-build: ## Build docker images with the manager. + docker build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker images with the manager. + docker push ${IMG} + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +.PHONY: undeploy +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +##@ Build Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest + +## Tool Versions +KUSTOMIZE_VERSION ?= v4.5.5 +CONTROLLER_TOOLS_VERSION ?= v0.9.0 + +KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest diff --git a/PROJECT b/PROJECT new file mode 100644 index 00000000..2bf8d697 --- /dev/null +++ b/PROJECT @@ -0,0 +1,25 @@ +domain: my.domain +layout: +- go.kubebuilder.io/v3 +projectName: kruise-game +repo: github.com/openkruise/kruise-game +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: kruise.io + group: game + kind: GameServerSet + path: github.com/openkruise/kruise-game/apis/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: kruise.io + group: game + kind: GameServer + path: github.com/openkruise/kruise-game/apis/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/README.md b/README.md index 2dce58b5..edeed89d 100644 --- a/README.md +++ b/README.md @@ -1 +1,48 @@ -# kruise-game \ No newline at end of file +# kruise-game + +## Introduction +`Kruise-Game` is an open source project based on OpenKruise, to solve the problem of game server landing in Kubernetes. + +OpenKruiseGame logo + + +## Why is Kruise-Game? +Game servers are stateful services, and there are differences in the operation and maintenance of each game server, which also increases with time. In Kubernetes, general workloads manages a batch of game servers according to pod templates, which cannot take into account the differences in game server status. Batch management and directional management are in conflict in k8s. **Kruise-Game** was born to resolve that. Kruise-Game contains two CRDs, GameServer and GameServerSet: + +- `GameServer` is responsible for the management of game server status. Users can customize the game server status to reflect the differences between game servers; +- `GameServerSet` is responsible for batch management of game servers. Users can customize update/reduction strategies according to the status of game servers. + +Features: +- Game server status management + - Mark game servers status without effecting to its lifecycle +- Flexible scaling/deletion mechanism + - Support scaling down by user-defined status & priority + - Support specifying game server to delete directly +- Flexible update mechanism + - Support hot update (in-place update) + - Support updating game server by user-defined priority + - Can control the range of the game servers to be updated + - Can control the pace of the entire update process +- Custom service quality + - Support probing game servers‘ containers and mark game servers status automatically + +## Quick Start + +- [Installation](./docs/getting_started/installation.md) +- [Basic Usage](./docs/tutorials/basic_usage.md) + +## License + +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/apis/v1alpha1/doc.go b/apis/v1alpha1/doc.go new file mode 100644 index 00000000..4f1b3af6 --- /dev/null +++ b/apis/v1alpha1/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:openapi-gen=true +// +groupName=game.kruise.io +package v1alpha1 diff --git a/apis/v1alpha1/gameserver_types.go b/apis/v1alpha1/gameserver_types.go new file mode 100644 index 00000000..108ed317 --- /dev/null +++ b/apis/v1alpha1/gameserver_types.go @@ -0,0 +1,156 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + GameServerStateKey = "game.kruise.io/gs-state" + GameServerOpsStateKey = "game.kruise.io/gs-opsState" + GameServerUpdatePriorityKey = "game.kruise.io/gs-update-priority" + GameServerDeletePriorityKey = "game.kruise.io/gs-delete-priority" +) + +// GameServerSpec defines the desired state of GameServer +type GameServerSpec struct { + OpsState OpsState `json:"opsState,omitempty"` + UpdatePriority *intstr.IntOrString `json:"updatePriority,omitempty"` + DeletionPriority *intstr.IntOrString `json:"deletionPriority,omitempty"` + NetworkDisabled bool `json:"networkDisabled,omitempty"` +} + +type GameServerState string + +const ( + Unknown GameServerState = "Unknown" + Creating GameServerState = "Creating" + Ready GameServerState = "Ready" + NotReady GameServerState = "NotReady" + Crash GameServerState = "Crash" + Updating GameServerState = "Updating" + Deleting GameServerState = "Deleting" +) + +type OpsState string + +const ( + Maintaining OpsState = "Maintaining" + WaitToDelete OpsState = "WaitToBeDeleted" + None OpsState = "None" +) + +type ServiceQuality struct { + corev1.Probe `json:",inline"` + Name string `json:"name"` + ContainerName string `json:"containerName,omitempty"` + ServiceQualityAction []ServiceQualityAction `json:"serviceQualityAction,omitempty"` +} + +type ServiceQualityCondition struct { + Name string `json:"name"` + Status string `json:"status,omitempty"` + LastProbeTime metav1.Time `json:"lastProbeTime,omitempty"` + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + LastActionTransitionTime metav1.Time `json:"lastActionTransitionTime,omitempty"` +} + +type ServiceQualityAction struct { + Permanent bool `json:"permanent,omitempty"` // default: true + State bool `json:"state"` + GameServerSpec `json:",inline"` +} + +// GameServerStatus defines the observed state of GameServer +type GameServerStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + DesiredState GameServerState `json:"desiredState,omitempty"` + CurrentState GameServerState `json:"currentState,omitempty"` + NetworkStatus NetworkStatus `json:"networkStatus,omitempty"` + PodStatus corev1.PodStatus `json:"podStatus,omitempty"` + ServiceQualitiesCondition []ServiceQualityCondition `json:"serviceQualitiesConditions,omitempty"` + // Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update. + UpdatePriority *intstr.IntOrString `json:"updatePriority,omitempty"` + DeletionPriority *intstr.IntOrString `json:"deletionPriority,omitempty"` + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` +} + +type NetworkStatus struct { + NetworkType string `json:"networkType,omitempty"` + InternalAddresses []NetworkAddress `json:"internalAddresses,omitempty"` + ExternalAddresses []NetworkAddress `json:"externalAddresses,omitempty"` + DesiredNetworkState NetworkState `json:"desiredNetworkState,omitempty"` + CurrentNetworkState NetworkState `json:"currentNetworkState,omitempty"` + CreateTime metav1.Time `json:"createTime,omitempty"` + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` +} + +type NetworkState string + +type NetworkAddress struct { + IP string `json:"ip"` + // TODO add IPv6 + Ports []NetworkPort `json:"ports,omitempty"` + PortRange NetworkPortRange `json:"portRange,omitempty"` + EndPoint string `json:"endPoint,omitempty"` +} + +type NetworkPort struct { + Name string `json:"name"` + Protocol corev1.Protocol `json:"protocol,omitempty"` + Port *intstr.IntOrString `json:"port,omitempty"` +} + +type NetworkPortRange struct { + Protocol corev1.Protocol `json:"protocol,omitempty"` + PortRange string `json:"portRange,omitempty"` +} + +//+genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="STATE",type="string",JSONPath=".status.currentState",description="The current state of GameServer" +//+kubebuilder:printcolumn:name="OPSSTATE",type="string",JSONPath=".spec.opsState",description="The operations state of GameServer" +//+kubebuilder:printcolumn:name="DP",type="string",JSONPath=".status.deletionPriority",description="The current deletionPriority of GameServer" +//+kubebuilder:printcolumn:name="UP",type="string",JSONPath=".status.updatePriority",description="The current updatePriority of GameServer" +//+kubebuilder:resource:shortName=gs + +// GameServer is the Schema for the gameservers API +type GameServer struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GameServerSpec `json:"spec,omitempty"` + Status GameServerStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// GameServerList contains a list of GameServer +type GameServerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GameServer `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GameServer{}, &GameServerList{}) +} diff --git a/apis/v1alpha1/gameserverset_types.go b/apis/v1alpha1/gameserverset_types.go new file mode 100644 index 00000000..f32a30ea --- /dev/null +++ b/apis/v1alpha1/gameserverset_types.go @@ -0,0 +1,170 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + appspub "github.com/openkruise/kruise-api/apps/pub" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +const ( + GameServerOwnerGssKey = "game.kruise.io/owner-gss" + GameServerSetReserveIdsKey = "game.kruise.io/reserve-ids" + GameServerSetNotExistIdsKey = "game.kruise.io/not-exist-ids" + AstsHashKey = "game.kruise.io/asts-hash" +) + +// GameServerSetSpec defines the desired state of GameServerSet +type GameServerSetSpec struct { + // replicas is the desired number of replicas of the given Template. + // These are replicas in the sense that they are instantiations of the + // same Template, but individual replicas also have a consistent identity. + //+kubebuilder:validation:Required + //+kubebuilder:validation:Minimum=0 + Replicas *int32 `json:"replicas"` + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + GameServerTemplate GameServerTemplate `json:"gameServerTemplate,omitempty"` + ReserveGameServerIds []int `json:"reserveGameServerIds,omitempty"` + ServiceQualities []ServiceQuality `json:"serviceQualities,omitempty"` + UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"` + ScaleStrategy ScaleStrategy `json:"scaleStrategy,omitempty"` + Network *Network `json:"network,omitempty"` +} + +type GameServerTemplate struct { + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + corev1.PodTemplateSpec `json:",inline"` + VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` +} + +type Network struct { + NetworkType string `json:"networkType,omitempty"` + NetworkConf []NetworkConfParams `json:"networkConf,omitempty"` +} + +type NetworkConfParams KVParams + +type KVParams struct { + Name string `json:"name,omitempty"` + Value string `json:"value,omitempty"` +} + +type UpdateStrategy struct { + // Type indicates the type of the StatefulSetUpdateStrategy. + // Default is RollingUpdate. + // +optional + Type apps.StatefulSetUpdateStrategyType `json:"type,omitempty"` + // RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType. + // +optional + RollingUpdate *RollingUpdateStatefulSetStrategy `json:"rollingUpdate,omitempty"` +} + +type RollingUpdateStatefulSetStrategy struct { + // Partition indicates the ordinal at which the StatefulSet should be partitioned by default. + // But if unorderedUpdate has been set: + // - Partition indicates the number of pods with non-updated revisions when rolling update. + // - It means controller will update $(replicas - partition) number of pod. + // Default value is 0. + // +optional + Partition *int32 `json:"partition,omitempty"` + // The maximum number of pods that can be unavailable during the update. + // Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + // Absolute number is calculated from percentage by rounding down. + // Also, maxUnavailable can just be allowed to work with Parallel podManagementPolicy. + // Defaults to 1. + // +optional + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + // PodUpdatePolicy indicates how pods should be updated + // Default value is "ReCreate" + // +optional + PodUpdatePolicy kruiseV1beta1.PodUpdateStrategyType `json:"podUpdatePolicy,omitempty"` + // Paused indicates that the StatefulSet is paused. + // Default value is false + // +optional + Paused bool `json:"paused,omitempty"` + // UnorderedUpdate contains strategies for non-ordered update. + // If it is not nil, pods will be updated with non-ordered sequence. + // Noted that UnorderedUpdate can only be allowed to work with Parallel podManagementPolicy + // +optional + // UnorderedUpdate *kruiseV1beta1.UnorderedUpdateStrategy `json:"unorderedUpdate,omitempty"` + // InPlaceUpdateStrategy contains strategies for in-place update. + // +optional + InPlaceUpdateStrategy *appspub.InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"` + // MinReadySeconds indicates how long will the pod be considered ready after it's updated. + // MinReadySeconds works with both OrderedReady and Parallel podManagementPolicy. + // It affects the pod scale up speed when the podManagementPolicy is set to be OrderedReady. + // Combined with MaxUnavailable, it affects the pod update speed regardless of podManagementPolicy. + // Default value is 0, max is 300. + // +optional + MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` +} + +type ScaleStrategy struct { + kruiseV1beta1.StatefulSetScaleStrategy `json:",inline"` +} + +// GameServerSetStatus defines the observed state of GameServerSet +type GameServerSetStatus struct { + // replicas from advancedStatefulSet + Replicas int32 `json:"replicas"` + ReadyReplicas int32 `json:"readyReplicas"` + AvailableReplicas int32 `json:"availableReplicas"` + CurrentReplicas int32 `json:"currentReplicas"` + UpdatedReplicas int32 `json:"updatedReplicas"` + UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"` + MaintainingReplicas *int32 `json:"maintainingReplicas,omitempty"` + WaitToBeDeletedReplicas *int32 `json:"waitToBeDeletedReplicas,omitempty"` + // LabelSelector is label selectors for query over pods that should match the replica count used by HPA. + LabelSelector string `json:"labelSelector,omitempty"` +} + +//+genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector +//+kubebuilder:resource:shortName=gss + +// GameServerSet is the Schema for the gameserversets API +type GameServerSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GameServerSetSpec `json:"spec,omitempty"` + Status GameServerSetStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// GameServerSetList contains a list of GameServerSet +type GameServerSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GameServerSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GameServerSet{}, &GameServerSetList{}) +} diff --git a/apis/v1alpha1/groupversion_info.go b/apis/v1alpha1/groupversion_info.go new file mode 100644 index 00000000..0726d726 --- /dev/null +++ b/apis/v1alpha1/groupversion_info.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the game.kruise.io v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=game.kruise.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "game.kruise.io", Version: "v1alpha1"} + + SchemeGroupVersion = GroupVersion + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource is required by pkg/client/listers/... +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..dfa22766 --- /dev/null +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,562 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "github.com/openkruise/kruise-api/apps/pub" + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServer) DeepCopyInto(out *GameServer) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServer. +func (in *GameServer) DeepCopy() *GameServer { + if in == nil { + return nil + } + out := new(GameServer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GameServer) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerList) DeepCopyInto(out *GameServerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GameServer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerList. +func (in *GameServerList) DeepCopy() *GameServerList { + if in == nil { + return nil + } + out := new(GameServerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GameServerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSet) DeepCopyInto(out *GameServerSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSet. +func (in *GameServerSet) DeepCopy() *GameServerSet { + if in == nil { + return nil + } + out := new(GameServerSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GameServerSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSetList) DeepCopyInto(out *GameServerSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GameServerSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSetList. +func (in *GameServerSetList) DeepCopy() *GameServerSetList { + if in == nil { + return nil + } + out := new(GameServerSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GameServerSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSetSpec) DeepCopyInto(out *GameServerSetSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.GameServerTemplate.DeepCopyInto(&out.GameServerTemplate) + if in.ReserveGameServerIds != nil { + in, out := &in.ReserveGameServerIds, &out.ReserveGameServerIds + *out = make([]int, len(*in)) + copy(*out, *in) + } + if in.ServiceQualities != nil { + in, out := &in.ServiceQualities, &out.ServiceQualities + *out = make([]ServiceQuality, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) + in.ScaleStrategy.DeepCopyInto(&out.ScaleStrategy) + if in.Network != nil { + in, out := &in.Network, &out.Network + *out = new(Network) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSetSpec. +func (in *GameServerSetSpec) DeepCopy() *GameServerSetSpec { + if in == nil { + return nil + } + out := new(GameServerSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSetStatus) DeepCopyInto(out *GameServerSetStatus) { + *out = *in + if in.MaintainingReplicas != nil { + in, out := &in.MaintainingReplicas, &out.MaintainingReplicas + *out = new(int32) + **out = **in + } + if in.WaitToBeDeletedReplicas != nil { + in, out := &in.WaitToBeDeletedReplicas, &out.WaitToBeDeletedReplicas + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSetStatus. +func (in *GameServerSetStatus) DeepCopy() *GameServerSetStatus { + if in == nil { + return nil + } + out := new(GameServerSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSpec) DeepCopyInto(out *GameServerSpec) { + *out = *in + if in.UpdatePriority != nil { + in, out := &in.UpdatePriority, &out.UpdatePriority + *out = new(intstr.IntOrString) + **out = **in + } + if in.DeletionPriority != nil { + in, out := &in.DeletionPriority, &out.DeletionPriority + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSpec. +func (in *GameServerSpec) DeepCopy() *GameServerSpec { + if in == nil { + return nil + } + out := new(GameServerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerStatus) DeepCopyInto(out *GameServerStatus) { + *out = *in + in.NetworkStatus.DeepCopyInto(&out.NetworkStatus) + in.PodStatus.DeepCopyInto(&out.PodStatus) + if in.ServiceQualitiesCondition != nil { + in, out := &in.ServiceQualitiesCondition, &out.ServiceQualitiesCondition + *out = make([]ServiceQualityCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.UpdatePriority != nil { + in, out := &in.UpdatePriority, &out.UpdatePriority + *out = new(intstr.IntOrString) + **out = **in + } + if in.DeletionPriority != nil { + in, out := &in.DeletionPriority, &out.DeletionPriority + *out = new(intstr.IntOrString) + **out = **in + } + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerStatus. +func (in *GameServerStatus) DeepCopy() *GameServerStatus { + if in == nil { + return nil + } + out := new(GameServerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerTemplate) DeepCopyInto(out *GameServerTemplate) { + *out = *in + in.PodTemplateSpec.DeepCopyInto(&out.PodTemplateSpec) + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]v1.PersistentVolumeClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerTemplate. +func (in *GameServerTemplate) DeepCopy() *GameServerTemplate { + if in == nil { + return nil + } + out := new(GameServerTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KVParams) DeepCopyInto(out *KVParams) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KVParams. +func (in *KVParams) DeepCopy() *KVParams { + if in == nil { + return nil + } + out := new(KVParams) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Network) DeepCopyInto(out *Network) { + *out = *in + if in.NetworkConf != nil { + in, out := &in.NetworkConf, &out.NetworkConf + *out = make([]NetworkConfParams, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. +func (in *Network) DeepCopy() *Network { + if in == nil { + return nil + } + out := new(Network) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkAddress) DeepCopyInto(out *NetworkAddress) { + *out = *in + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]NetworkPort, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.PortRange = in.PortRange +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkAddress. +func (in *NetworkAddress) DeepCopy() *NetworkAddress { + if in == nil { + return nil + } + out := new(NetworkAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkConfParams) DeepCopyInto(out *NetworkConfParams) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkConfParams. +func (in *NetworkConfParams) DeepCopy() *NetworkConfParams { + if in == nil { + return nil + } + out := new(NetworkConfParams) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkPort) DeepCopyInto(out *NetworkPort) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkPort. +func (in *NetworkPort) DeepCopy() *NetworkPort { + if in == nil { + return nil + } + out := new(NetworkPort) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkPortRange) DeepCopyInto(out *NetworkPortRange) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkPortRange. +func (in *NetworkPortRange) DeepCopy() *NetworkPortRange { + if in == nil { + return nil + } + out := new(NetworkPortRange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkStatus) DeepCopyInto(out *NetworkStatus) { + *out = *in + if in.InternalAddresses != nil { + in, out := &in.InternalAddresses, &out.InternalAddresses + *out = make([]NetworkAddress, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ExternalAddresses != nil { + in, out := &in.ExternalAddresses, &out.ExternalAddresses + *out = make([]NetworkAddress, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.CreateTime.DeepCopyInto(&out.CreateTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkStatus. +func (in *NetworkStatus) DeepCopy() *NetworkStatus { + if in == nil { + return nil + } + out := new(NetworkStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpdateStatefulSetStrategy) DeepCopyInto(out *RollingUpdateStatefulSetStrategy) { + *out = *in + if in.Partition != nil { + in, out := &in.Partition, &out.Partition + *out = new(int32) + **out = **in + } + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(intstr.IntOrString) + **out = **in + } + if in.InPlaceUpdateStrategy != nil { + in, out := &in.InPlaceUpdateStrategy, &out.InPlaceUpdateStrategy + *out = new(pub.InPlaceUpdateStrategy) + **out = **in + } + if in.MinReadySeconds != nil { + in, out := &in.MinReadySeconds, &out.MinReadySeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateStatefulSetStrategy. +func (in *RollingUpdateStatefulSetStrategy) DeepCopy() *RollingUpdateStatefulSetStrategy { + if in == nil { + return nil + } + out := new(RollingUpdateStatefulSetStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScaleStrategy) DeepCopyInto(out *ScaleStrategy) { + *out = *in + in.StatefulSetScaleStrategy.DeepCopyInto(&out.StatefulSetScaleStrategy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleStrategy. +func (in *ScaleStrategy) DeepCopy() *ScaleStrategy { + if in == nil { + return nil + } + out := new(ScaleStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceQuality) DeepCopyInto(out *ServiceQuality) { + *out = *in + in.Probe.DeepCopyInto(&out.Probe) + if in.ServiceQualityAction != nil { + in, out := &in.ServiceQualityAction, &out.ServiceQualityAction + *out = make([]ServiceQualityAction, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceQuality. +func (in *ServiceQuality) DeepCopy() *ServiceQuality { + if in == nil { + return nil + } + out := new(ServiceQuality) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceQualityAction) DeepCopyInto(out *ServiceQualityAction) { + *out = *in + in.GameServerSpec.DeepCopyInto(&out.GameServerSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceQualityAction. +func (in *ServiceQualityAction) DeepCopy() *ServiceQualityAction { + if in == nil { + return nil + } + out := new(ServiceQualityAction) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceQualityCondition) DeepCopyInto(out *ServiceQualityCondition) { + *out = *in + in.LastProbeTime.DeepCopyInto(&out.LastProbeTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + in.LastActionTransitionTime.DeepCopyInto(&out.LastActionTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceQualityCondition. +func (in *ServiceQualityCondition) DeepCopy() *ServiceQualityCondition { + if in == nil { + return nil + } + out := new(ServiceQualityCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateStrategy) DeepCopyInto(out *UpdateStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdateStatefulSetStrategy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStrategy. +func (in *UpdateStrategy) DeepCopy() *UpdateStrategy { + if in == nil { + return nil + } + out := new(UpdateStrategy) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/game.kruise.io_gameservers.yaml b/config/crd/bases/game.kruise.io_gameservers.yaml new file mode 100644 index 00000000..abd95b3d --- /dev/null +++ b/config/crd/bases/game.kruise.io_gameservers.yaml @@ -0,0 +1,838 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: gameservers.game.kruise.io +spec: + group: game.kruise.io + names: + kind: GameServer + listKind: GameServerList + plural: gameservers + shortNames: + - gs + singular: gameserver + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of GameServer + jsonPath: .status.currentState + name: STATE + type: string + - description: The operations state of GameServer + jsonPath: .spec.opsState + name: OPSSTATE + type: string + - description: The current deletionPriority of GameServer + jsonPath: .status.deletionPriority + name: DP + type: string + - description: The current updatePriority of GameServer + jsonPath: .status.updatePriority + name: UP + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: GameServer is the Schema for the gameservers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GameServerSpec defines the desired state of GameServer + properties: + deletionPriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + networkDisabled: + type: boolean + opsState: + type: string + updatePriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + status: + description: GameServerStatus defines the observed state of GameServer + properties: + currentState: + type: string + deletionPriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + desiredState: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: string + lastTransitionTime: + format: date-time + type: string + networkStatus: + properties: + createTime: + format: date-time + type: string + currentNetworkState: + type: string + desiredNetworkState: + type: string + externalAddresses: + items: + properties: + endPoint: + type: string + ip: + type: string + portRange: + properties: + portRange: + type: string + protocol: + default: TCP + type: string + type: object + ports: + description: TODO add IPv6 + items: + properties: + name: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + default: TCP + type: string + required: + - name + type: object + type: array + required: + - ip + type: object + type: array + internalAddresses: + items: + properties: + endPoint: + type: string + ip: + type: string + portRange: + properties: + portRange: + type: string + protocol: + default: TCP + type: string + type: object + ports: + description: TODO add IPv6 + items: + properties: + name: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + default: TCP + type: string + required: + - name + type: object + type: array + required: + - ip + type: object + type: array + lastTransitionTime: + format: date-time + type: string + networkType: + type: string + type: object + podStatus: + description: PodStatus represents information about the status of + a pod. Status may trail the actual state of a system, especially + if the node that hosts the pod cannot contact the control plane. + properties: + conditions: + description: 'Current service state of pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + items: + description: PodCondition contains details for the current condition + of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can + be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + containerStatuses: + description: 'The list has one entry per container in the manifest. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-and-container-status' + items: + description: ContainerStatus contains details for the current + status of this container. + properties: + containerID: + description: Container's ID in the format '://'. + type: string + image: + description: 'The image the container is running. More info: + https://kubernetes.io/docs/concepts/containers/images.' + type: string + imageID: + description: ImageID of the container's image. + type: string + lastState: + description: Details about the container's last termination + condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + name: + description: This must be a DNS_LABEL. Each container in + a pod must have a unique name. Cannot be updated. + type: string + ready: + description: Specifies whether the container has passed + its readiness probe. + type: boolean + restartCount: + description: The number of times the container has been + restarted. + format: int32 + type: integer + started: + description: Specifies whether the container has passed + its startup probe. Initialized as false, becomes true + after startupProbe is considered successful. Resets to + false when the container is restarted, or if kubelet loses + state temporarily. Is always true when no startupProbe + is defined. + type: boolean + state: + description: Details about the container's current condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + required: + - image + - imageID + - name + - ready + - restartCount + type: object + type: array + ephemeralContainerStatuses: + description: Status for any ephemeral containers that have run + in this pod. This field is beta-level and available on clusters + that haven't disabled the EphemeralContainers feature gate. + items: + description: ContainerStatus contains details for the current + status of this container. + properties: + containerID: + description: Container's ID in the format '://'. + type: string + image: + description: 'The image the container is running. More info: + https://kubernetes.io/docs/concepts/containers/images.' + type: string + imageID: + description: ImageID of the container's image. + type: string + lastState: + description: Details about the container's last termination + condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + name: + description: This must be a DNS_LABEL. Each container in + a pod must have a unique name. Cannot be updated. + type: string + ready: + description: Specifies whether the container has passed + its readiness probe. + type: boolean + restartCount: + description: The number of times the container has been + restarted. + format: int32 + type: integer + started: + description: Specifies whether the container has passed + its startup probe. Initialized as false, becomes true + after startupProbe is considered successful. Resets to + false when the container is restarted, or if kubelet loses + state temporarily. Is always true when no startupProbe + is defined. + type: boolean + state: + description: Details about the container's current condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + required: + - image + - imageID + - name + - ready + - restartCount + type: object + type: array + hostIP: + description: IP address of the host to which the pod is assigned. + Empty if not yet scheduled. + type: string + initContainerStatuses: + description: 'The list has one entry per init container in the + manifest. The most recent successful init container will have + ready = true, the most recently started container will have + startTime set. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-and-container-status' + items: + description: ContainerStatus contains details for the current + status of this container. + properties: + containerID: + description: Container's ID in the format '://'. + type: string + image: + description: 'The image the container is running. More info: + https://kubernetes.io/docs/concepts/containers/images.' + type: string + imageID: + description: ImageID of the container's image. + type: string + lastState: + description: Details about the container's last termination + condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + name: + description: This must be a DNS_LABEL. Each container in + a pod must have a unique name. Cannot be updated. + type: string + ready: + description: Specifies whether the container has passed + its readiness probe. + type: boolean + restartCount: + description: The number of times the container has been + restarted. + format: int32 + type: integer + started: + description: Specifies whether the container has passed + its startup probe. Initialized as false, becomes true + after startupProbe is considered successful. Resets to + false when the container is restarted, or if kubelet loses + state temporarily. Is always true when no startupProbe + is defined. + type: boolean + state: + description: Details about the container's current condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + required: + - image + - imageID + - name + - ready + - restartCount + type: object + type: array + message: + description: A human readable message indicating details about + why the pod is in this condition. + type: string + nominatedNodeName: + description: nominatedNodeName is set only when this pod preempts + other pods on the node, but it cannot be scheduled right away + as preemption victims receive their graceful termination periods. + This field does not guarantee that the pod will be scheduled + on this node. Scheduler may decide to place the pod elsewhere + if other nodes become available sooner. Scheduler may also decide + to give the resources on this node to a higher priority pod + that is created after preemption. As a result, this field may + be different than PodSpec.nodeName when the pod is scheduled. + type: string + phase: + description: "The phase of a Pod is a simple, high-level summary + of where the Pod is in its lifecycle. The conditions array, + the reason and message fields, and the individual container + status arrays contain more detail about the pod's status. There + are five possible phase values: \n Pending: The pod has been + accepted by the Kubernetes system, but one or more of the container + images has not been created. This includes time before being + scheduled as well as time spent downloading images over the + network, which could take a while. Running: The pod has been + bound to a node, and all of the containers have been created. + At least one container is still running, or is in the process + of starting or restarting. Succeeded: All containers in the + pod have terminated in success, and will not be restarted. Failed: + All containers in the pod have terminated, and at least one + container has terminated in failure. The container either exited + with non-zero status or was terminated by the system. Unknown: + For some reason the state of the pod could not be obtained, + typically due to an error in communicating with the host of + the pod. \n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-phase" + type: string + podIP: + description: IP address allocated to the pod. Routable at least + within the cluster. Empty if not yet allocated. + type: string + podIPs: + description: podIPs holds the IP addresses allocated to the pod. + If this field is specified, the 0th entry must match the podIP + field. Pods may be allocated at most 1 value for each of IPv4 + and IPv6. This list is empty if no IPs have been allocated yet. + items: + description: 'IP address information for entries in the (plural) + PodIPs field. Each entry includes: IP: An IP address allocated + to the pod. Routable at least within the cluster.' + properties: + ip: + description: ip is an IP address (IPv4 or IPv6) assigned + to the pod + type: string + type: object + type: array + qosClass: + description: 'The Quality of Service (QOS) classification assigned + to the pod based on resource requirements See PodQOSClass type + for available QOS classes More info: https://git.k8s.io/community/contributors/design-proposals/node/resource-qos.md' + type: string + reason: + description: A brief CamelCase message indicating details about + why the pod is in this state. e.g. 'Evicted' + type: string + startTime: + description: RFC 3339 date and time at which the object was acknowledged + by the Kubelet. This is before the Kubelet pulled the container + image(s) for the pod. + format: date-time + type: string + type: object + serviceQualitiesConditions: + items: + properties: + lastActionTransitionTime: + format: date-time + type: string + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + name: + type: string + status: + type: string + required: + - name + type: object + type: array + updatePriority: + anyOf: + - type: integer + - type: string + description: Lifecycle defines the lifecycle hooks for Pods pre-delete, + in-place update. + x-kubernetes-int-or-string: true + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/game.kruise.io_gameserversets.yaml b/config/crd/bases/game.kruise.io_gameserversets.yaml new file mode 100644 index 00000000..ea9002c1 --- /dev/null +++ b/config/crd/bases/game.kruise.io_gameserversets.yaml @@ -0,0 +1,665 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: gameserversets.game.kruise.io +spec: + group: game.kruise.io + names: + kind: GameServerSet + listKind: GameServerSetList + plural: gameserversets + shortNames: + - gss + singular: gameserverset + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: GameServerSet is the Schema for the gameserversets API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GameServerSetSpec defines the desired state of GameServerSet + properties: + gameServerTemplate: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file' + properties: + volumeClaimTemplates: + items: + description: PersistentVolumeClaim is a user's request for and + claim to a persistent volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'spec defines the desired characteristics of + a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on + the contents of the specified data source. If the + AnyVolumeDataSource feature gate is enabled, this + field will always have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any local object from + a non-empty API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the DataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource and + DataSourceRef) will be set to the same value automatically + if one of them is empty and the other is non-empty. + There are two important differences between DataSource + and DataSourceRef: * While DataSource only allows + two specific types of objects, DataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed values + (dropping them), DataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the + status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + status: + description: 'status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'accessModes contains the actual access + modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: allocatedResources is the storage resource + within AllocatedResources tracks the capacity allocated + to a PVC. It may be larger than the actual capacity + when a volume expansion operation is requested. For + storage quota, the larger value from allocatedResources + and PVC.spec.resources is used. If allocatedResources + is not set, PVC.spec.resources alone is used for quota + calculation. If a volume expansion capacity request + is lowered, allocatedResources is only lowered if + there are no expansion operations in progress and + if the actual volume capacity is equal or lower than + the requested capacity. This is an alpha field and + requires enabling RecoverVolumeExpansionFailure feature. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: capacity represents the actual resources + of the underlying volume. + type: object + conditions: + description: conditions is the current Condition of + persistent volume claim. If underlying persistent + volume is being resized then the Condition will be + set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails + details about state of pvc + properties: + lastProbeTime: + description: lastProbeTime is the time we probed + the condition. + format: date-time + type: string + lastTransitionTime: + description: lastTransitionTime is the time the + condition transitioned from one status to another. + format: date-time + type: string + message: + description: message is the human-readable message + indicating details about last transition. + type: string + reason: + description: reason is a unique, this should be + a short, machine understandable string that + gives the reason for condition's last transition. + If it reports "ResizeStarted" that means the + underlying persistent volume is being resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType + is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: phase represents the current phase of PersistentVolumeClaim. + type: string + resizeStatus: + description: resizeStatus stores status of resize operation. + ResizeStatus is not set by default but when expansion + is complete resizeStatus is set to empty string by + resize controller or kubelet. This is an alpha field + and requires enabling RecoverVolumeExpansionFailure + feature. + type: string + type: object + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + network: + properties: + networkConf: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + networkType: + type: string + type: object + replicas: + description: replicas is the desired number of replicas of the given + Template. These are replicas in the sense that they are instantiations + of the same Template, but individual replicas also have a consistent + identity. + format: int32 + minimum: 0 + type: integer + reserveGameServerIds: + items: + type: integer + type: array + scaleStrategy: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can be unavailable + during scaling. Value can be an absolute number (ex: 5) or a + percentage of desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding down. It can just be allowed to + work with Parallel podManagementPolicy.' + x-kubernetes-int-or-string: true + type: object + serviceQualities: + items: + properties: + containerName: + type: string + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be + considered failed after having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is a beta field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place + in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior is defined + by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod + IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults + to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + name: + type: string + periodSeconds: + description: How often (in seconds) to perform the probe. Default + to 10 seconds. Minimum value is 1. + format: int32 + type: integer + serviceQualityAction: + items: + properties: + deletionPriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + networkDisabled: + type: boolean + opsState: + type: string + permanent: + type: boolean + state: + type: boolean + updatePriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - state + type: object + type: array + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate + gracefully upon probe failure. The grace period is the duration + in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer than the + expected cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. Value + must be non-negative integer. The value zero indicates stop + immediately via the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + required: + - name + type: object + type: array + updateStrategy: + properties: + rollingUpdate: + description: RollingUpdate is used to communicate parameters when + Type is RollingUpdateStatefulSetStrategyType. + properties: + inPlaceUpdateStrategy: + description: UnorderedUpdate contains strategies for non-ordered + update. If it is not nil, pods will be updated with non-ordered + sequence. Noted that UnorderedUpdate can only be allowed + to work with Parallel podManagementPolicy UnorderedUpdate + *kruiseV1beta1.UnorderedUpdateStrategy `json:"unorderedUpdate,omitempty"` + InPlaceUpdateStrategy contains strategies for in-place update. + properties: + gracePeriodSeconds: + description: GracePeriodSeconds is the timespan between + set Pod status to not-ready and update images in Pod + spec when in-place update a Pod. + format: int32 + type: integer + type: object + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can be unavailable + during the update. Value can be an absolute number (ex: + 5) or a percentage of desired pods (ex: 10%). Absolute number + is calculated from percentage by rounding down. Also, maxUnavailable + can just be allowed to work with Parallel podManagementPolicy. + Defaults to 1.' + x-kubernetes-int-or-string: true + minReadySeconds: + description: MinReadySeconds indicates how long will the pod + be considered ready after it's updated. MinReadySeconds + works with both OrderedReady and Parallel podManagementPolicy. + It affects the pod scale up speed when the podManagementPolicy + is set to be OrderedReady. Combined with MaxUnavailable, + it affects the pod update speed regardless of podManagementPolicy. + Default value is 0, max is 300. + format: int32 + type: integer + partition: + description: 'Partition indicates the ordinal at which the + StatefulSet should be partitioned by default. But if unorderedUpdate + has been set: - Partition indicates the number of pods with + non-updated revisions when rolling update. - It means controller + will update $(replicas - partition) number of pod. Default + value is 0.' + format: int32 + type: integer + paused: + description: Paused indicates that the StatefulSet is paused. + Default value is false + type: boolean + podUpdatePolicy: + description: PodUpdatePolicy indicates how pods should be + updated Default value is "ReCreate" + type: string + type: object + type: + description: Type indicates the type of the StatefulSetUpdateStrategy. + Default is RollingUpdate. + type: string + type: object + required: + - replicas + type: object + status: + description: GameServerSetStatus defines the observed state of GameServerSet + properties: + availableReplicas: + format: int32 + type: integer + currentReplicas: + format: int32 + type: integer + labelSelector: + description: LabelSelector is label selectors for query over pods + that should match the replica count used by HPA. + type: string + maintainingReplicas: + format: int32 + type: integer + readyReplicas: + format: int32 + type: integer + replicas: + description: replicas from advancedStatefulSet + format: int32 + type: integer + updatedReadyReplicas: + format: int32 + type: integer + updatedReplicas: + format: int32 + type: integer + waitToBeDeletedReplicas: + format: int32 + type: integer + required: + - availableReplicas + - currentReplicas + - readyReplicas + - replicas + - updatedReplicas + type: object + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.labelSelector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 00000000..33f766b1 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,23 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/game.kruise.io_gameserversets.yaml +- bases/game.kruise.io_gameservers.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_gameserversets.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_gameserversets.yaml +#- patches/cainjection_in_gameservers.yaml +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 00000000..ec5c150a --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 00000000..a4607e42 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,74 @@ +# Adds namespace to all resources. +namespace: kruise-game-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kruise-game- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +# - ../prometheus + +patchesStrategicMerge: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +#- manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 00000000..2bf6bef8 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,39 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: ringtail/kube-rbac-proxy:v0.12.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 00000000..cb352f41 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 00000000..5ed428dd --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,8 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) \ No newline at end of file diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml new file mode 100644 index 00000000..db6430ee --- /dev/null +++ b/config/manager/controller_manager_config.yaml @@ -0,0 +1,21 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: 127.0.0.1:8080 +webhook: + port: 9443 +leaderElection: + leaderElect: true + resourceName: c637bb1e.my.domain +# leaderElectionReleaseOnCancel defines if the leader should step down volume +# when the Manager ends. This requires the binary to immediately end when the +# Manager is stopped, otherwise, this setting is unsafe. Setting this significantly +# speeds up voluntary leader transitions as the new leader don't have to wait +# LeaseDuration time first. +# In the default scaffold provided, the program ends immediately after +# the manager stops, so would be fine to enable this option. However, +# if you are doing or is intended to do any operation such as perform cleanups +# after the manager stops then its usage might be unsafe. +# leaderElectionReleaseOnCancel: true diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 00000000..29d11a06 --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: kruise-game-manager + newTag: test diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 00000000..e65c4c61 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: +# securityContext: +# runAsNonRoot: true + # TODO(user): For common cases that do not require escalating privileges + # it is recommended to ensure that all your Pods/Containers are restrictive. + # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + # Please uncomment the following code if your project does NOT have to work on old Kubernetes + # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # seccompProfile: + # type: RuntimeDefault + containers: + - command: + - /manager + args: + - --leader-elect=false + image: controller:latest + name: manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8082 + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /readyz + port: 8082 + initialDelaySeconds: 5 + periodSeconds: 5 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 1024Mi + requests: + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 00000000..ed137168 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 00000000..d19136ae --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,20 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 00000000..51a75db4 --- /dev/null +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 00000000..80e1857c --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 00000000..ec7acc0a --- /dev/null +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml new file mode 100644 index 00000000..71f17972 --- /dev/null +++ b/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/config/rbac/gameserver_editor_role.yaml b/config/rbac/gameserver_editor_role.yaml new file mode 100644 index 00000000..c1757849 --- /dev/null +++ b/config/rbac/gameserver_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit gameservers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameserver-editor-role +rules: +- apiGroups: + - game.kruise.io + resources: + - gameservers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - game.kruise.io + resources: + - gameservers/status + verbs: + - get diff --git a/config/rbac/gameserver_viewer_role.yaml b/config/rbac/gameserver_viewer_role.yaml new file mode 100644 index 00000000..713b6510 --- /dev/null +++ b/config/rbac/gameserver_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view gameservers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameserver-viewer-role +rules: +- apiGroups: + - game.kruise.io + resources: + - gameservers + verbs: + - get + - list + - watch +- apiGroups: + - game.kruise.io + resources: + - gameservers/status + verbs: + - get diff --git a/config/rbac/gameserverset_editor_role.yaml b/config/rbac/gameserverset_editor_role.yaml new file mode 100644 index 00000000..60495196 --- /dev/null +++ b/config/rbac/gameserverset_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit gameserversets. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameserverset-editor-role +rules: +- apiGroups: + - game.kruise.io + resources: + - gameserversets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - game.kruise.io + resources: + - gameserversets/status + verbs: + - get diff --git a/config/rbac/gameserverset_viewer_role.yaml b/config/rbac/gameserverset_viewer_role.yaml new file mode 100644 index 00000000..3da7cd1c --- /dev/null +++ b/config/rbac/gameserverset_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view gameserversets. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameserverset-viewer-role +rules: +- apiGroups: + - game.kruise.io + resources: + - gameserversets + verbs: + - get + - list + - watch +- apiGroups: + - game.kruise.io + resources: + - gameserversets/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 00000000..731832a6 --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 00000000..4190ec80 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 00000000..1d1321ed --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 00000000..410e7621 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,131 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.kruise.io + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.kruise.io + resources: + - statefulsets/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get + - patch + - update +- apiGroups: + - game.kruise.io + resources: + - gameservers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - game.kruise.io + resources: + - gameservers/finalizers + verbs: + - update +- apiGroups: + - game.kruise.io + resources: + - gameservers/status + verbs: + - get + - patch + - update +- apiGroups: + - game.kruise.io + resources: + - gameserversets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - game.kruise.io + resources: + - gameserversets/finalizers + verbs: + - update +- apiGroups: + - game.kruise.io + resources: + - gameserversets/status + verbs: + - get + - patch + - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 00000000..2070ede4 --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 00000000..7cd6025b --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller-manager + namespace: system diff --git a/config/samples/game.kruise.io_v1alpha1_gameserver.yaml b/config/samples/game.kruise.io_v1alpha1_gameserver.yaml new file mode 100644 index 00000000..e69de29b diff --git a/config/samples/game.kruise.io_v1alpha1_gameserverset.yaml b/config/samples/game.kruise.io_v1alpha1_gameserverset.yaml new file mode 100644 index 00000000..e69de29b diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 00000000..2da25882 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- service.yaml diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 00000000..e6801506 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: kruise-game-system +spec: + ports: + - port: 443 + targetPort: 9876 + selector: + control-plane: controller-manager \ No newline at end of file diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md new file mode 100644 index 00000000..12540195 --- /dev/null +++ b/docs/getting_started/installation.md @@ -0,0 +1,26 @@ +# Installation + +## Install manually + +### 0. Edit Makefile, changing {IMG} + +### 1. Build docker image with the kruise-game controller manager. +```shell +make docker-build +``` + +### 2. Push docker image with the kruise-game controller manager. +```shell +make docker-push +``` + +### 3. Deploy kruise-game controller manager to the K8s cluster. +```shell +make deploy +``` + +## Uninstall manually +```shell +make undeploy +``` + diff --git a/docs/getting_started/introduction.md b/docs/getting_started/introduction.md new file mode 100644 index 00000000..6e8850e5 --- /dev/null +++ b/docs/getting_started/introduction.md @@ -0,0 +1,30 @@ +# Introduction + +## What is Kruise-Game? +Kruise-Game is an open source project based on OpenKruise, to solve the problem of game server landing in Kubernetes. + +## Why is Kruise-Game? +Game servers are stateful services, and there are differences in the operation and maintenance of each game server, which also increases with time. In Kubernetes, general workloads manages a batch of game servers according to pod templates, which cannot take into account the differences in game server status. Batch management and directional management are in conflict in k8s. **Kruise-Game** was born to resolve that. Kruise-Game contains two CRDs, GameServer and GameServerSet: + +- `GameServer` is responsible for the management of game server status. Users can customize the game server status to reflect the differences between game servers; +- `GameServerSet` is responsible for batch management of game servers. Users can customize update/reduction strategies according to the status of game servers. + +## Features +- Game server status management + - Mark game servers status without effecting to its lifecycle +- Flexible scaling/deletion mechanism + - Support scaling down by user-defined status & priority + - Support specifying game server to delete directly +- Flexible update mechanism + - Support hot update (in-place update) + - Support updating game server by user-defined priority + - Can control the range of the game servers to be updated + - Can control the pace of the entire update process +- Custom service quality + - Support probing game servers‘ containers and mark game servers status automatically + +## What's Next +Here are some recommended next steps: + +- Start to [Install kruise-game](./installation.md). +- Learn Kruise-Game's [Basic Usage](../tutorials/basic_usage.md). \ No newline at end of file diff --git a/docs/images/logo.jpg b/docs/images/logo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5d60f2a4bf61e53935c8fc6244c8cf62de22be37 GIT binary patch literal 82885 zcmeFa2lV4sl{oC*wDev`LLkGW0cIl0wrqJwLDiPzqAuHnqH4>MtYTZTOR z+rD<&jkyz9nxS)B+qQBI@Gp1U&vFmqhv77w48v%5>7qgo#hKRjpXFB1iK`m9E%0yu z-`j4IUp|~pqkRxGo?N6`p%eHJTJ+o%*x$M%TFP`t42T@>8_1*KpA5ZqpFWIvOz7M2V z+0dTd>nc<4l0Dd}*WKL~zncgXdpB}XWv>aB(cRVBMR28DKnf4qT`a&g2rfVccrT3Z zE0p%ZmEHTFJ(nI0QWqzdvri=H{oR4LOZMy^SC(ZLWu=S4WB|dnS`8``p<;0_sIhml z7*F;2-tpwz4UpU6koLq(+-T~CgbMVGcL=#qn~J&`0HLQ6S93f zu*cI0pdkz$t{tGx9Ssjl_`lMq&IE(IZ#aP5VDQ)aY_;ybaS$A~P~%oEu44A%vwq@ ze|qQK73h_i`R+=3_xbu5uz0@LvK@WqPxoBfiR>|#%-o4x*8}@vXjAvjFgCe&2E9h@ zP%JJ6lDDq~W}u}{byA<&mlg|PIuuYav1GWquTa^C6!Qfz$qE-kcd2{O2-jtyF#*>n3eI9PfP2mzqn zY8vm$Tx&YKw1`yJcpa2=0IRz;8rtq)IK31u6%Vx9pLci7_RB5HQ}qMw_GjK*yS1QS z3e`qX8N&q0>$Bnj!?6K{Nf^#x7ztGcY@*bMsa3Bn^?EI1$kwcdVP(Pd+gQGUtsfi% z^;ZvtU2yIkdoU)f8iPl`a465TG|DFhxF#rGtu85Z8dL;Lu$Dy4BO^cw3cMH^QZRkC zA-H%?Q^Ns{0+a?Ay8(#xu$F*fJ*~YoDOeVL@B$u5QAwuJmc+uQzjfyAKfIx>Fj_r@ z1RoCs8Yk>W_y6j~jqqo8gpI9X^TUT<{m6TpE-MkOh zm@B3~>}xB0hHIphKD__k&F;WZ>D^w*+W_g;;aPu`Tn``5z`M78pR9lqj5SvM^?PB3 z2~oEK-?u<=GvwO3XU}M;Q+<_$Ha#1fIy6Bg99uQQJgcsbbdbQw)hpIAeNhrt<>ab- z(CWjYv<7%^X77+n07!W~e9+7&;fM*;1ic{)xOyvMNDAr5gMn06OGaHSSOhGr0RrwO zy)Nr9R68}g_|O=Gao0uPECPw!|4`#Kw+AcEw?`Dx9`&Xzw>-bSeE;7Kh9dC?(G@mQUSugeiMzBM*I3K)aECC!mD1;vPQ)KhsIg>v!9uI{<<_&;ZP3r8UqSV!UXYsjcC z;hL_gC0#?tYuOFN$g2CP;o{L6Ly)+886@Ts7`kr;DqxN>P*?M|=+F8KFnfxKr+3B3 zD3%AhRvxSY{tfBBGv@0zytu+}e}NqFWsIiL8JL~?8X#Db$tqX{aJ<#Tu(ml~O9;JQ zi8qb#7NPbqAc+`8n+{Th{a*LLv;-^t9nZl`*`LYr>%_Pn^%t8g0cI7Z-iFO_@AiiK z=cR%X1B}OT0jmp`jS;QQOyPi_VyakWMHx5 zdssdL8Li6v2KPBvU(Hi zGk7ql4>8@tCqf;;ObTBJ4H2_Eyeu>#%wh2SfFZF#M$i~zVQ!CTW=wg5p+^x?#wwU! zAo>|Q$HpNs#JCO?4Tx~SO|Ya#WJ0rvO%DW$DLjs zAM`ZbHS6sR(8l-b`k-IM!>qmrI2QC2p8!4z#t2W92A-LHd@*V8g2m%Ss$pSv9Y?%I zg4qyWV;YFylnHFYG_t{hpn8lp7|sZWVf=v`uK^OxGE*3g#soiMDa`W;Ib`L`_lT~? zj0<~)Mw8BTP?gA8nI{C)Ckqtc9n>l$(&Hx?#FP0_44LK%%5o@Yw8*G!T-E3fL0U!MnJ-7xzlF{J8l zs4HSrsz+sgE+&)uTrfKG>8M@~jma?88yI79!@1OGOw4e;Y^?z{c*SZA;YOp2S%Fw0 z8+K;XLj-FCq3z?9d}HQ0)Evz>ik{=+d8m;O2egFaOg$R}1W7SGHf#_y!*rG58ek|j zI*c=M*#)aIA>$^Crp098s6(^^1`$S?%V#W(da@*FNSlDV?vnl>YO}?^hIEC78YY}2y?+84i*W| zWlYj`rxAw~l7f`cP12jJ0UGYYZMMxRNx*os+ndIV(y%$~&19lNG!t`Hby2Sghp8t) zjV4si+A=DbSh!BLS|#WuAmXQ9PtuU>&e9-Z)16eL}tnTT4Zx2VVh z+m^hdEL-yqs~|Huv3QKCZR`{oY4ar&8!HW3n88$CQE@SV8-X&SC5LV{lzCh3!kt)w z`ikI7yxN|s9Ie8t$!a9rH`UnJYOqDA2-{s_Sf&T5)_{8p*wZ!5@6YJ40siQbpTJr{ zHGDPoyQFFk{iW8GEX$@#Ysd?**|ZqHF&N#o33 zijRb1r`yXd%H_UbE$X-VgbrtLdG2BoqT(cjyg3sUACe zY9(@UHrAoqL@h2-Ll{nb^&(;n*_mFfQpOxf?K&iy#55a@$$=?HbHCfjOmDh~#x2~0 z;pMC$DC;3x4ZN^yD52F;;WEI>il)VU!_us3yG(V5GrKiMxN*SRwp9_wv2SN$wO2|B zhoI3xcR6$vx)yfJxD&AXHNb3)j-0XxAw9e?;9Q&t>11FQi5A02gH$5ba;Gu`o6CTa zM?=M?<8rSv9CLNJY*vRzp23-61Cv=(7uKI*$| zsEh=oNri8RdE3Ku!H7uBlk;NCGNLz*r83fPdsSUwSc&s_NnHc%B3%|po;F5I&M#@* zGCOPqO}Edofe<(gU1t+DC~!uCWagL&%o+lFV~eu|wlW*fi;jyxm9QQRiX6QW?9k9% zHs-W2(??d!2NA7%^NLcA^dP9mdNWFkVQ*sq($KO>LJuaEzL2U-VzP9XNh0`Db^%jWjhhbYMcXj4Y0)i} zo5TLBT}Py3Ow2~J%G8)_0M05l$`}hH6}8%^TISMF%0ty!C55_}*&-*I=HO+Nb+Tzn zYAqf(4I0#PqBiH|s4h+na$aaOQgl#TG?$yH*O>&!Vl?8W=%l!)8m+dufR}Q&JvGb4 zWn%DcG+!=|V_`O>(1KBw_*xS$6l8fm6l#WH?Xu`t4Cz)RukJ zii*WFH3sdBE-{HkwToJ5xU|D58kH*0pgkjMWknt?JFHxurlVe-Pml)hK^=`k2BlyP z(4PUlxiSu0GZ(EWMpWoTMrGDb%uZUZGRBm$i)Pgw&8D3iRz;VY#*;qUZO$QxYoYV; zGJ=Fct<{7TzPr&JYC)-(hiop7kIOOdmGh=kQK4j{aQSE2BMLgNFkjtf1O$Ob@V8#GBQ^VWo^8T3NsdFj_zolOcq#32G6b1E91E1jqvv0WiQawgEV3iNFs)Ga@q=RH{xD zDc9^#N-L@l@wqk^Tpm}ZjbUrvcTqxI3{uJ$mZI#%+@e?FiMAq)y-H`<9$K6Au-}l$ z!i-_qu*DL?L4U{s(#29iowijDPZni}r{IoU=yz8R0u7#9UB9OFi$gHkrxV~-OM4{| zw#HW3-s~w+)oH+ar_`Z_J-y*F0q2$%K+yAqhe)a)*}b428)LR;j}dMZaas^n+$fG` zlSHqi;flUs_zZ~)3bh_G5EF_3QemtJJfujRSg9Bkt%&U*Gw9C;eQFUJjf`$J#In>T zXq9!%lv=W+NDhX&q%1VI2Nn{%#BcmIZJVVAb4&5E!FtH-s882j%d=>ugvOm^wirZ( zJii>Wk~eM13c`!N1#H#G=+dy=g+*YRTcNH+iLo&&;F(dpod$n(_@vK*Ef#V6K{PYR z{fgto9WT=Lam)pATv<$$IG;lLhRaw6!~`R#m7I|%R%_;ae9QJKc&}|&{B}j@N^N{r zokE;jTLyAuvEFi7?h(t{(ygRguf~(3KyQ@t2(kt!#t>4i#Dx(lRf#&m&6ZtUR~LNU z_36f$#R52w#4%OG(SanzN+*Ir(_B^QXSU_x=@A2OhaS(@ETxOZ4#wmz?SsJ zm@4NzABMm$+J#RD`>5@3`bZk;x zc_?wk^J-X2wY4>dc|Bb&Crz$jHH1XzE*Xim7G6S0JVH0-E#3|@EeIMTFf4g#(d~sd zZE{j)R>4_(T5XI+dZE@pT_^zC{Tg6kOPzw>tqfzi%2HJ`MCvHz808f8Mu<9}lDx^K z%OTVWDJVdY6f3c?UJ3cU9nIB>mDf@fTe?kT2rsfVz#;4`9lIU2{Z42%WUzmflF-Wf zSSb@=ywdPWZU|bkCRlEPS|2zP4g|Z1S#z` zz^0qj2vfD(WO7e_oDLLJwPvy$Ati5G@XN*ateR=_O6;{Zx((kE2w>x*dOn{m3r#_{ zJ6<7U8*vAzAqZJ^k|qKd$FUq+g29KQ2@kffIiyf|C^L|*2V4@RLA%x{uA)%M^ zQ@q{{l9p2)1q+Cl#*1!?%qy*O%^D9`1a1tWa-7a}XHl@bBjKrZ10eEb&;Yd)K+Sdt`!H`60N6d zt5z0tDjUu?77cp!bg^LjxLb5>rA-bBesc^g=$2O%=U%DVC59WOu zo>rX}=XRmya*_EcQ=B+j)0Akv?6q(;Ba&F*_<5+q&2l_PHmig2-HjYk;cpjc6A zMSSWFhGl-EW2+^^C^`WT=4xpPBOcsg`w6)~VsEJxnr@uhODV0$RHK*o;#Mg?op-p@ z;oCu~s}b$y<4P%B>U?A>@rKK&%nWo1Y-8mCu8jg~p%qoCJJCuFQPrm-L)PtPZE6IN zSXCmfV7AP(InKCF-KFaYglDx@t6r>>m z!giDN7ju;Bpv}}zm$O-O2G%{*7-w+buMEjr8Sm1Lufio5$y?1@9W7{nZ9QZ|EflMg zDON*4Vi{X3F$$`6$5K!(0Yd}mGLFXO%TN-{hyJ2mhS9W{CZ$OiYSo*7QJjf`pjf%c zS4Cvw$EHPenDHfK+F3-#f*he`(0nCdd`K>!2HOZb!5ml>u{I$VSj`qjN&_{^q)IiK zt&u-(R7-`bjE!ijF)z z%tWv%cBF8f5jCc@noTJ)PajoWa?oSDWiqU*(!xQB44% zs#2Rt^T9Mvk_64wA(pF5eRKhVIC4srT700DeZOn?6@wMIT16ErJ*8DaEBTENmT#5< zV^L|hU>{+cou%EW4rgJ6ldQ4OV98s|1hqk_`2x{WdvcT_5#-@!lSVifoojy9>)U$U z%*O?}#H}%0+A}syyoCe25G0QRCwZv_8me&G!&C2pxtkWxvlPs5CJ$5pXwAicOYoU27{ z_BVgl7_RD-**ws3!4?WIY|+bP=FQ{0uZMG8=tO#(4S3e;acTryDrD8`mz-WP0tPJ6 z(gY}FoYn|6OKu+C2sU2Ugk_AHOVmY^SxrrAQmW0W6=5Nd%RcVL*P{`y{V3GDi zepwd`mEifBT;Z0>d3BD|@&@EB@^C6u9ktCWW3A1yo-5+0>Gc_w$Tyia4ONokgHDvu z-6fJhLE4?lisqLWbFCaL-JudDzT7QWJg>kTQME^f1UE{um1`4K`)m}%c|C!9P_yPD z=^9`+r76p;zXJ5KcTgi8e0NM%auccO)#jg|LSX1#?N$Qror5bW820jVSQC z8Yr%&aupm#aE6s$A+OQ(0iiHqtdvKIo6rq3u=@_TS+OH-=GjJAYeCHlZCf?qm#Z7D zlucDA?NzyIKU=UBCkrRz=CGK6t)V;&9VCFzgoLn;2{a0TE5usNw7=P;3|{J$VzSm9 zO_5P$#MvXMGdGGfUIg392<&F}a9fGG>PBhTOIFGnT}6_MXjook3l79F`Ce>|yioQr zSCK`sZsLq0_L@sa;}nccN(c)T3sM%HYIw_a5yp`tqj}aMX%ok#>;-ghX}<{ zqblcePXL(+o9D^rQ;FUzxtWaeNgt)aT=P8tvh zzbBz9!5(ZQdc!DFQd^5k`OQp0=TJ5yyyCLDm?8~yn%5Y2=4+wWpU?dnLM{W-u8u># zuam_rlmUv zI6eZDOk|OvBdt7XcP5FHFZ9$X>V#c2<&8vv`KaJa67M+W`i3b%oH5isFf-Mz6;d4iE4M9pqcLiCaI+{G$Ip!YRoJ8$P8=Tw7FmV8cv0* z(o$+_V2&pJx;{2&wTb#r-n7!Cih}jR3OlL3j69_tXsyWztX?IB8Y*p^8H9010Zm>J)R2p0O%SZFeH(M4==ST4RLlkoz`IjDHLlOI`wF{zm!1W zOzWXOG6$@;^I*jp@sMvcz;;@b{bUT8Qp)G!fSW-Zi(Z@Obq7)q9f3+zvNC33K!yEH zGV(w{47OIFfo))+*t!7d3mghnZf~*1@Gd>2{WNdNK+a&K1TWD7Yc#dA)rl%meh!EiU~H6M+DPQ!>+*9CCTD;SDSXNanme}M?)a%!)duU4i!-&3J59@GF+SG ztpU@UcF1}M1dZAWGx2J2bv!VG*u2*JO zr)6Z_1T&)^Yl8^094}f!+(-kzsuc6evP46kz=V__p+*{2fzaZyoofeW?_Sgfw3hy>2hi{i`9rE zP!F8Bwc%1Vg?R>pva^I11%fa<+o_~!e{HbYh! zn2}1V+S4GD?MoUIV$C8cVG`vK)m9r-?S?a^^SV{b_nd&M1J)R|hVB5jjO@=+v1W$d z=9Fz$2ywI8m_;@c)esz4TB>fGHR$dvg*x>T9sF`~7`9P-nO&ez9*td&D)5!5?ewLGM>wo)eI3D%MPbV!VBzcod>3$iiP z4Rn&omGLIHpo#=#oe^SvT+?Z%ABeWmAWsYgODtz9$i+oD%6~H5Z~_QdzL;7 zEKQ7h##Au|g~%^sE?*XijU`2l4TI`COf!qpkoT)KGOz=nsD!{NfJxA}W{M~=hb%9y zH=1K`#;QUxlJAHNVf5+dL=}ork<*6}4ENGCm%-s7AEwZzEOdH^LdGg0&iKiMgMqy| zj2&>Qvy@MGB!N+_5(r{8FAYSzUYUoJJ}!c&DXCcTqzIETz2CeAt4*RYVv|H-bUqie zmIgdmkitb=yH4<>t~QzgA#nAX*5TqY#6^Y!kAMkRj?lT=&ohBv4^7QeafaS-IiPwx zMRa>j*~HVS$^zp9G0@pGw5CnWQeje6f$vhSNx)3Dc-hrBcSeb+TQ-A4OQ><`#>Q$6 zK^d`$>!rZIu#l}K*h}Vc1kD++5;dA!1t^G2b4rbXJ?BYkFY+kQpz9>w6{mqxDB%?# z5IxBW230vV8;f};@844b)GJKKpx7-${ej&p8F0lMsp(KbXA(M5)renOvuG-e(+J`q zzXFc&mF;*`lu?3?RGm;7E{vmXo$xneLyUmBHEAG|QcI1SimgJ;i9eg6isBEKDoc2J zn~SH6xHJla#wJ~7D&Xn}3P#IL+nF(au*g!n5j6#Oe~_!)9JD8^=ySWN!pMwEmgPk9 z>98}XbEyWDh&Iqx!3tG^3~mO<<5fvFsXB)>#@AwVk4*K|nu350tA(`kTH6604d39w zHxavm3G^0#@&Dv zfU%!Y6>ANUCz@`LqibTsxN()LdK}n>wb^Qr_rz~;r+R;rhLM4I=t6HIdO@lHowPct zM3Lo%c4gL~yfN0wgvO9cW{%qu9RNVl)CozvhvKqYpJW=mrb)@|A5NbdTJ(dHxuw(!9@ExQjKL=cEn_Ip64 z#HbdL;7r4C(jPWiHY+hT(Nn4(IhCv6WPmjWI}TZQ8q6r>uspdCt2*35N*jPfmt2fQ zHqV#ICf|g{@jT_5{xUAcsMT@gF3|tuc@wxY3njsg`dx$^$w{HKlyykR;~v!;yCbGc z<-3*AM&Y!VHM1>_%A+Y$ALW+|u2*aXAavmrky#T&5&LC$r1!Ie8!p881YjpFl929E z4G$NPL4Y^JW!19sM%rv|00JT@J*jJE@x-uWnVXW89_26~pg_}=U{+)XazTSN5M>N0 z7Wk|+h=;>!Z=NtohmHCo;er5hk6N_n6=N;f7HWf&D8TCKr_KDhIG9yqXei;nCbi%-Q(J`^dSbO4X!$R%f{2#l$&Yyp&1>VUfRs%Jc{@Esiz>gEF(RX ztpYVkDAh)Mv!c*Ty4kGgOFJ0FOWOlNJ*;4Gd0OO!GRLV7wvi7!?LgzMy1`IZ$j}}T zv{CB~+g541ROSL#6YJn(nuIBJO{+7nHoQ3!iHL>PY@^nJJIg8GHnb_+*GC)!j>44i zk>~nmVU6LaCTdz+8c~#r6)RrNHE_+Xce^R#+ETO8jV!Pk_NPf#1u-tK(5l)&jD;+g zNHd&j)LKKZrTOzxBmlaz1xTrF+R3eydjm4WX~Iy@X@5~Cu+ zItI=eyS+jnh3ZK5rZxz*Qgd5(n$)tZXO*-swR?Dw4t#LlbwC88x;}Dn!5so`@)}?u zO%krI4E-L^xMl=d0-IeMYy)E!IDA^35A5l3CO|c@rGc<84D8~j<5i*MNJMnFyGWu^ zz%PBig%+n9&r&-A?s5>Qh%&QP5UnE9M_5&u#UU_3ruHJKL(3RqMgr6v`vu^J#fYJm znw2JPHVu6U4ok5e#;t-QiEgm*io`?HloGX5?on}Zkt{i&2zHlMaJUZCZ-!P8jx5nm zX_jtOU}i>0D2SCIQ#xPu#k4f2&w-YPYYiZy7`z^G7ljjA*JVqp*X)uz9fMQ7^=fyN z9~(|u2Z3EKzn}u(YIn@MHq(`Mx6sKKvb5sB3%u-mQP*8IMJ=i}fWu!xAYw5e zLv5xh#wPEU5dj(#<6$QOu@DJ9-CLU zr>glZ(y+kko|>k019_oKk~Z?d7ekeP0%3U6DU<~lIJCx#PCUmAEbd{v(32KsSX9UT zhPY9dx=d#zO+Z{#9H6=c&gIRV#SD$QO(D?hJr@UO!uw+Z9BP@*sQ!$_ktQS0xZbGA z+7w4ZK>i4!s4>LC6E~hEM#M?RE5wsfGk~o@wP8&*tr^TIbJPYMbb(>jL>eQ%Gkkz6fRpg48v^Q?H`t__n(^kmK9|TvJAyz!fod<@nfx|e|iTitpjhXpJRYdburtAdY8)Fj zU$C9cMqlmVoLfQ&$Ty8)GHf#~bEM5D>0lgSHL7S7@kTM9!nTqq)d56tZN*n<4Fbxx zU}$V;XiFdm4RFG~)!Haa1vk=q;t&~d5ZNk)j8rbM7&}*qg#%8gc7dl&FHf6bf6+@e z?0Ys6z2yt}IE3vH0YTAdN);jG=(MI-9dL2tSL94sG8QDj5Ek0popn7?C`<~b8hCnZUaUji zIG)I89-PhBq@@o$`_&>>Txf_64&31V$#QdM2~7uiZq$+VnhbW?DZdQSyjFvNA~*{T z7nzK}iP^Pmn>1BIeA$La)wr5&1@EKOnBbqa7(-MFf# zaoJ6q4y{bVQLHYSg}r7>G?l?bts0K9`e4N6xke?y!LeHyoTqQcF5H;e5jY10q;>6L zzr*HQGv%I|U}+mTGCc?E%ThN0u@9Vt4hYBo2*n!UAu$6;1>u#yKY}slp^mtj%di3Y zG9NgV<+4T=7~6>oAXG`zT3y+nx=FbXA*(9j3>6`DR<!Y>naFRrq_$ z)&LLJa|%J5kNnpkuW!B;KlE$zzvsS!)tv%|{RV$^YeB2Ej~P#<`q;D?@=}m-k?UIf z5XUIhs)fQ{1DuW9TQkbVy+&Ct@3mmlFpJd^VwcP7FW&Fm`~a5wL6afSOTk?e`$1cO zC+LHk-vx9haly?gx_?le*=z-$EEdpOv0N+RG>OA-4K8CSQUhlQVWfZ(={~_b?1Wor;ax zwY{iQE9^C`a%HbxE*18oR?#XrwUSfOjfN<=~v2_@p{2kB_YJNEA;s2T~ z8|)6#{&3wSe}yv}AcvG50B}3&4r0wgcT(&Z+J-v;*Pkp7Dp(P3bs2?Z$9K-Xy#QP? z@^@WUvA(fopKh*X>(Yr1q-4cBaEr_Bc?VEFa9PEs!eN91e0NX)pP9pZ3kGWMEmh0q zy{KM7_Eu}fO2sUoNZB!OhksD>zureSw8#5zFsbYX`!TqC1S#w*mv6tpWP{1N)?WuQ z37u)CCpIIjGiO2Xbc?{O#*s{$FL>hMjkwh_}z`yH4DT54x^spET8z=`7-t&~d>X zLVV(mrx!zaQ{w=M?gVH5RX5ViIQ;!McZPXL?So)~+mQAFWUQHKC;R6I5ANFXT_e$x z#cmk6XmwlAooelGy&=Lu7iz6nrA;olt_$202HkO0*j*N^I~J7xlZgYv`acQv5Lo}{ zI$pnrTwGt*u<{*hbSGhucof<%01v5$pKwZm+-lh|lG= z<&Mhj%E7suZcd_>M9Zr?PZ^5XJpq+g=>yMizXC2n1oZ`-j-Cjdc^K4X<0Y|;-}*V&{WvPxUQ*aCG{ zI7e1st8GSK8>F{4LHfz5YiA(+C6GSPp9L;R{~n}I4Qzdq%k4OJl|Qx3AxIZM z`q)Gk36Q=xmpk&4 zt0xw?pEZc|@gkSojCs{6ck+t1y8)3aaIFGtu!{is?|AOE@Qzw{pIGsBQ{&gdnrB<5 zfA&D$fwl+o!mD#R^j1KdXCKHjUX;sS_s_Z98Q(aNcit;RSkGLpz z;^$AzZU5P4b30BsCb#|amw}S4JNVW-O3kfio_5X;H*ycspnUy%r=M-l0GHCMQt^=#I4@eBew=y08=TOZo`)YkP|U*EcE>*lRrZT)fE_HD;*J9XQAwq39d z+Jvvqc0-L-4i>ANo6Ro&It)!Q}N^{8FX*!9v~Z{795UDxmW z?yg&pIO2#ikH{Z!$q}6+^dqK6Jo$(h9r30kK5)bhM|}T?-yeC*k>?y)Jd!@Lf8_Yc zCmea@k#9Wm{YTz#>Sx3G4sP`Q8`J=vn)Sr$%>F5iO zzT{~6=+V)SKl+77zvbwU9sRANe|^j`$Lu+#c8qY0d(1x_^TK2P?U+v^30I!*juSqA!cR{;>ck69Bu>;%eC&x= zo%rq(zk1@WC!KNp?Wdf3 z3UP{g$`em{^(h}a<@=}ZI`zU+nNvrnKK;};pZd8|Z#nJc)5@o*r_E2h>a_QrcH`;W zPru-F=5+t`XPy4`)4y{1AMbUadtvvo@AcGsz2#mv-0OE|-204&oME5wv@_m%#+T3d z9MYus;kzgOPxbNBnxo(uOVd!Df8t$V(G?$PI> z=Q`&;@7xccd&_y}oYy+<(dS)r-Z#!a>U{M4;rTB({}bo`?t%+1&@T9w3$DB1M;D%P zA#>sK!fP)4*8PvW|7G`&?|=3Ezx04z4?rK_J>aDe_`(CXJrH@|@PSu7@bmd?`AXi+ zzchdSgLXcs_Mq{Du71#0_a3_!+nesaX75cGoqkdCq9j*x%3CToSI?_Hvig$hSJ6{Z9(@M-$=c2uUVB{ay|q8=tL&TZd;7j$Jb3Sey$8SX z!9TkA!i(*TUw!d+FWGa6e#t8@x$)B7mp<~+S6q7IW%s*GzwGME{`(>4KE!&+YajAM z>;c#adlU9^ynrY8JMrHU`-sO7A0)Svb@FNC=ctn@g?cgd9eNKvps%5CsU!7A)j!zS z+29(_X?&Hr4`VWKU~XZn>|@xEa>sHK_hRn9n-6Rz&G)pnwb<5kTi*P92 z=eeD4^B3|l{~y9Gffue4ejvi)qs321r$~>G-YET6#^q0=e#!HM_Ou~GD`4j6Q*7K~J?Mv({>>oOp zI9E778eB4X?%?L(Wy33nKXnQB#qKXhjnT_Tzw%7~1&3|ifWbnw~-Q$zT-uR>8 zeZzTpeUy)$9Nif2i=Q9gl5ojuCtH(;P2M#<6)01mo!x)-&$FA-OVg{;-)Cy}j`>OR z@%*!k2QHqv_~DXXzUEOo9%Vl2!;ik-qaXk1n;wHb=9Q1#daUu-4?phQ$35wB-~Xrj zKfV6(M?c=*s{#lshW`6~RXx4z`;mpt_)zk8|m(i>j(ke9vX<@b5{(_j9_SGceE+AHg? zy!Pq~uDIbh@U;T;Kps#tuYtMS^(_i~PuN%Ma#@CCl|L7a4Z+OES&wk^x zuGx0Y?3$b3boraU@MiMO?|RGLx4iPLr@!^-|Cal=*}whxZI680SKiLO{lo7--|?1z zKmXrf^3GG<`Lt`du3cPv%XP!+ZhTkoUDv;xefNjo^WgWq^FJ>7kJr8Te($~NeW$(e zS?@pk{ZD%TZ68>E;MNaDAN=Wu1|RzFhad6bZ+@ivkuQBz_~_?8#(nIQAE!V5kxyWs z_`oMG`Q&>)weM5!{xtgO>poNY%(b6IKKsu9ME>)gpR0WC+Rs-%|E@38zVM#wFTVc$ zH(YkZhrURD@#9}&zVw+dx4(S-SCp@O?W>o6^~SGRU;EM5{jdM>8|gRx^v!?z*3NG| z?b|1Q`^xW}{hh1->jD4u)*CA~zV9aTrqBMj^xxn7uKC@cd~fo-KY#y8KRE6OSN`yv zAHLy7#UH)*X8PtY{P>|i{{By*pZw{kPyX3SKYPi~@Bj0E|HVUo@!4Ctw|xJX$uDpF z)iZv5=C5D(oAPfyd@FzJcYYiE_D{ci+V9W&{pLO7BeX^@6XSQ9fVkK7n}Em((k( z+P}Q(YXCDaxE^SnsV|VV@wtWrsq@DMJPu_(IN9cP_IWm5H@zm$M@@*fw9v2hi z?dML)r&W%+<)$ZGQF`t>&YL2q-|GhAi}LwbOXdYnB!BtgSKaoF+_Bri5Kr2EQtqpi6+iX9lP~|~rL6Fz%U^xN)B4x`;rt)`CzAh7QaJx5kNNZqev*I5 zN8iSNwDJn=9bfzHdH;6FFRyv=H9u>=kN?>R&N=t=&;RptFaODPt?BbFfB3V$cEg~3 zX}j{U$KCt%Pd>E$o*B+^%72coqTh7$gFpCxmx4cftoy6%@fR;%_3M{>~viqL>^v4m_PO9PjJe&9`lvUnvcnU{7;{F z>@UzqJ?5F02(J}W>vQ{Ve)Igx5BOet-z`7>*YN{>J$y=Azx8upeB0;W@yr|Fa?WS{ zUm@q5{m{$KvtIdC_{}eR-zUF*%}>s_@cVD?os~cob>&|!efY~hW@_Jl&$plRKmYa6 zAIRqOe|-Ay=<^=_uh&2Eh)4hCSJ}63x#_5j9{#gx{o$hXl1{(;uGt^QV5w{P~vx^op|OX|L+euKh#)^ACO1YaaH+m)0I(`)_{Ev)=lW?*-SL-`n&23+f*_ zEq~_1BYOW2ZSNVCX1cb4x@XT!k}+jc>}?WT?6J3~XOgH=>#Xy8pPL`=`-5k(-gj}g>$>je8K`@J z&`_=`(d&&Lwi!6lEH5nZl~G8jc-T`CHbSP~LL#0^Y*5ADQp0rlJUg zj9GYM^gvPu*r@>N9s*nA5dhDZZULrE6okM*I<|}5BIrBdHdr@B1*HSg;2nrx2)5); zl%R&2JSYW!Oh6I8Nr(7ha%naoew!8}9f|e`$*;dtlr^uTMK~?yE5VuNHAN!~zwJFW zxi4;l1Q8wlnw_T&C-j0#GNCet9SvCM5afO{0I};f*v($iRRt|!p3^*1oOH7})8L2& zlj4kj5qw+s`*V0jCmw_f7sv?7DnuO1!ecyV_pHl6Vf%Zy@Q+#naK-5y2e5jWB%f6C zc2p?YyZc0|L&H>KaE&d7m#Cf4bXxi?2v+UJTCkzck-#-zgRsX@iDZF*zhT)&b)>=C zleU5;g{&l}jytg@);9>GFsYN$BFTE@uE3&rb_sf}A0S8 z%Aq6q`n-#i0x=&a6P$YeRWK6RpECBfM;p+5{r&HF9B(H?S3RF4Uw~KVaU5G%eui;r zO|oc(7)M@kh4)Yp(lM1Mn?`z*nu5V=$=?Gg%wfTDM?_x9iuVxc+hj`5`L5GJe3#g0 z^lT5g)TTUo)_pnG1;;_eJ8?^#TJ=`w?5;Ro3-}T6GWFhRTza9V*+Q$9Nd4M;tccJj z{sDtGxucsKv4#3_dmXpFIHtuMw|h`97L~>2=OqXntZOCT3rX~|b@!!J%s2~0YOTuU zSNuRWc1P3LHtoqv`$LO)!v;VvXR&}S9%jI0$0+J!*K*T}byWrr+>L2S^~3fLSpPfk ze{lT&KkQ#4`6~+sw+^Q(DXn|IE~U`wAUm$zBQWA(j2l5nmnwjHg_3BIC(vbaBHI(& ze?y28LkSYWBatgx6;T_jSqDv`hA%|H3E^YD$wS(t{ z^j^*-DAabS#qZ!L86o7S?)HPm!w<3%cM42k@jsC_m5Csl;I z>&u0E#79XkKh?Juivr?ty&c$M;Lo71Lhh8~4yA3Lz{|m$Nw08RR1t{td-dCSM_D7v zYuI^7Q+YA8{LP!vK6hoxwCU9M5NASdxLx*S$KZ#vVT1h=q)Io540gFn**zEau-SgJ z$7gO9RD{DASUZA72Z4zTp16THbtnGW#nTNNXi1a^)%IlYm}b}E1~3Qt<7>9d(Puf) zUdJ)lUJtHMm?}=!!#c*Z7cW0YBoANVT7wVLf8NS%5+=Tx_hy-==-;XURUFB5{ZtY!Ae5L41ni0dNOQNjis{*KapV#7N=dS z=O#2-bGQ@DMtCUP58iGGqQ5y4JtDGWMofVCvyCmMO0eZgmT$T1d%jqGBh0FrqqvCI zvry)mJ8M z+spMV7b#95+7PI4u^Ll4X{d-P^CYE)Sm!VA$ zAwzG;`5pO@yvZ$=9H{C8+hWkX{`0yBN8cNXBJaD#LQ)QaK8DwCqmh;bATTO^o!Gr$ zVG$IfYSl;cSD=oO{T;Pt{B$&1Ju>pS|OXw18m3c zlM3$mz-l3d51f+=E@cO>qD`s$7s72A`?U2EBfPU>I`7MyJyR6G>03R@_PNy~?nzz? zBkBnNU&C7`${(P{>PMfQ2j7lfsEX$}&!kw?>XhtJ7|P+YM3J>w$Fdge&gDN08&3(G zcod$i&ubHt37F13u{;(_*h~CzULf%4cyTCYqUv;7x+UGHiW0IA9Ia;p!Wt27NACwL zqezu?S zSR}^gu@2jb7FW_sb2=C5`4L3LDZZT?M<)vZX_gM#UPv9%7+{kn1XQ3SKQ1pM55R># zR#lbfDRJKo>~6TCkHU7g-sAAH-ao#R%Z$NH?v>xDqNyu-+Bu^}7wU=JsPX{;y6y=`nN8Sct*srTRW4|fe zVdaCrKK9MMHV5~JedFPyukD(auxKKj1eJ44sH!5o+|v&%unry1_f7P!X!OS1P&=$B z`ZZ@&J?zQ2(PUYXAk zid#L`sLkT>m~6K;n!2`~BXnELwub{1AWHmBp6N$K)%TOMca+1?ENhnStd%0REGCT`DRu+@JAf zMIMaz`Idr6*BgY-urA9zpv9}8ouKWVwQt`rJg&5>oxi`ee1HF5DSwahu#vLK!)}Rm z6UkzxqS=vli-xDCc^HeKZ62JTU#U%QF%8bl%Pg1CU`a9jnr138I@Zoydc#8sM8yC1 zPRH0}FPLyX8#4-hzF2|~-omcByq-*B@aJl_7#J)a-!uu5b-VlY(#vNlr-sjN1Aq2w z(4F0o{X3y=7-zn)N}S!hf_c`d}GO~iq|-&;UGCjWBctPPIZY$h)k z4wYk`*&LLJ9KoaYSqVG!e62?Y{2jj%I*@^g-;%q$zs-CpfA<;pzD0ddWjoU)_Y`7t zsH*Tt`&gG(vB~y*!RrEom{2PMOl>GJ1G`%+QtlFpa;!@;P_i=ds@qD8`tn2=he&CK+=8>er97u=b<^PIJjXPMc4+&q7i3*nd{aH*&RwG2L}m&w+oq%{kpZ zsyedl+?sk(^zuxv7&wdUo=8jmrA+f`H+)dpQ9P$)OeKMO|D?g3EK_^Ibp$xV5AAc5 z_^2_K*l<{kJJN1K6*lCbS=3rC-D3wPND64|&(j^nG zlo2;T{6i|eM}}+HBPbuTrS|h>nk2QMElGvpGU)@0^_nV2`Q0Zuj!hs*A4rREp~Zby zK=>}-=0rMxcXJx2x{zfryhIj94g!Hdy??qozcF@X|M>@X@5its1k-Hzgol2$s`K|G zt0Xu6a{mAI&63Bm{e6_FV1+47Zr=|56AzCpduirpw*aF{=!uG|GH<*_wci# z>=jv^0d}36qx&dYOhP%d&3$m};TNYX>o@Gq$#g$mbLCO3z#< zdq}pTrYOjdYUoOC({?b>n^Jh=3QeBJ^tU9EM@IB>qk5KRzp?@o9 zm9HcW^!05S>u%Z-m%?o73qYlJjAe}>C9^pugop-B`>(xLc^V4X63`d&vbWH$~)TsC~N+ECgL>U*j5#2z_WSU zMqq$*2UFh{DTRu!>}jr}ndpSh?-1hJZsZj1v&Q3UG(* zozE{qg1(AP8=KaK)Q1Dar?$RhD@BXe^*@Js-f}-!*9kk7O%L{#nT9a^4!j2`o>Q|T z;;8o_?_IN(AfwoqnipI)ZFB>|NlKJMtRKeo8vZvQ=M5VWR(AF6%hVelrTMu0vRU)+ zsAtS4+#|Zg;=fl6{K6Zi5Rmw<7-@I}o0%#f`Dd@%;Zl4keNm-exoJ zo8Y@h$vU(mrI5ZYm8jpw7Jf@e>k>|Hsd0>4bwP>bttk0fwCWDYSjHBag_2C1()@E# zRal!P%?fP7d-0l@%vbqYhWlM3#65O+3wpz!(jC62PV-A2Q*)!C%o*#$Re$PpzsqNp z$t5k0I;CpmQRBYO-hDK_UYRD~sQGBu5jT%_)oT*c4CG-2^GLjNVqS+}B-b6v16=JX zBsgJX(-4zl$6zf*B8FiXTx*J<&4oGCX!0b=T!z`)vxxh#IaFj}m>YB$F@k=!@6(f6 zRL(H?yGSlZ5guhvY}n+_>j>}rE#{V_`S^}w6+`AQy}%4pl$gQoI!Q;BO@ZK!12Erp zoa@)cX1Vy{9^+dSP!{PY$S32}zwT%aH%fOwMp(+dX+5Kcm?Wq=!9a!oAW1CO54F_% zD*Wc2JYKO{vTbT=mTwE~4;)I2@x!6%lTNI))ZWcK{*Am8aue`lw9#4QNJWl)0C@MFoyx{tzYytle)?Xy(RgvOdf-c#p9ddI}&86Ab=Nhf97y1z1 z45JoeYm*?>sQr`b@4XIUgSYzQD)v$gqF>WmH-^2nJ(#BgD=cSQrLJo7?rJ?>2R7a; zoWj=^SA%RX0%(5xu?bB~$8sw;mG0rMC=5Ng?RflZ*#-2IMr6i!MxTh%B4)>EpycxS zENN{ytJwLVgW1=~&b4WW$%Rv`?Rw;MQ!n>$7uN?T1Lf;6TU`6uQOl+6%9=Og6tFOJ z14K{auB{~x{5X0!?LWFh3w*TXcbN0<6b5JN+1Yq?)Q}WlfQ~5OUOr*Cd+v&TBwFr7 z#iO^P4R&XK`GhdS7#?dxpa16afjY0?l*nUx@aw!$iXpViV|mmD$SKA}HLMwz6)OBfOl6b$WNBn=4iiQQpvgW4w&XD-Uqz8d2@P3dA^7d(KJ|Dsh@{V zTFoldQ-%4o57R!2@-He3GTyA5T9DyRE0VtQ<+X3}%)=Oq8Th%*$-{PFb%7kM_v2^) za224bc(k6JQkh2B(zO-U0k^nhr`m)#&H}Rx-?TfIqu;D!c4qu+d{4}mvf;20YGPD9 zZ5RCH4y;35>o9JZIT-Df+emMoiM##7H%xnC20RF&mnEk-uL!m_sXSiX5MK9cX>fb( zmS2^yf6_M{v*l8N&EBNEpG$4EC-^JVj}%&~9rLy?xFYIk3Djt1qM4$eQf%ApiI|7_ zc#ch}r_uC{%3r}TG6k;HSc8T8J1W-HgX#o&EIK7Uxrg{MS}t%(y-7^|XzYL;BqY32 z{o2p4XY$)Q=@b_5T#<#JMt-d3NS3ds78I(iB@6?z-jr1xrgdQu?4(zfo}R{vH?wV; z_1svQ`_1SPW2>5eDLInA_MkO(SO|W<=g6#tO5^WEO7#2BsutkZll8D^y_nTa6b9&M zm^R&^okznOsJ7rA!&%3l1~4Bn3fpMDimJVBla-Q%bYLnCn-&#!!Ik>YtfMPOUMBj;ZD_HqvVc}p z_U7RzCg!GvHH+UxI=TsbKpY;33*flVAQkhJbr#qw2^AMy8~`rnC}faNpBwm3O?JNi zzk-+E31!;pzw9Dqj|6AACq8HSLFH6e93X-dUZyU}^4^M9Jl|f;U*m5p$P!EnNt0)v z$Q#fC{*QAvJ)NWfjI)X+L9OBI5zB@zbSU|C6VxoZCTWXM>VhEtD{pn zF=Zzu-)7frH^RuL*IP3}oVuI~Tm&~v^ih>%S;-`fpc$zfXR}&`N+=w)IgpiwBl-67 z1VAdcQ*b|y1WJp20rfu3?Maxertz!-Ht>W3%{>)x4?kVII-0#68H?W___dWc?|^|= zo0u?1tK*g^BV4%WV3X7Fsg{EBRO{LqdB-Gri~oWvJy>+Bs$tn!$ASQ8ssh)-k`wQ3 zQtNKChRd2s!hs5)v<6TE_?x2p7rVE%;-3#t+LvvKjg8J`Nq&M?Y8wAwe0Ps!y47Xt zUYe9aQ=?wCxsZ3dGEM7{;DU1jU>c?4pFTL!h3? z0idoUyP3YA?hn9pT+3jwj(}&R_U@mrC-B-be8$o|A^bh0G^}^Bo9lDr%I~4U zke{oYpb_Zg=gdK3B(4(Ti8>PN_;)RX-4*f zYav{*##gkwhaX1)Y^1aoaE#|F2cE>0uHe=dutOa5$n;4Z--t-=*14|V3VCn^A&qob zFRAt(uzW!jcg)`cV^CS2gr zqi{WxjdY6&RB27wA-?l>tNc>FH#)q{U0-i%SctYm-{Lk{11ZigSwf#|>yU&-6^YpX zF_8%gUX@UISqi~|V14ViQ#Ic#5#0>(i`+K+qF8IprVHv$K@>61j{i0^IgmuREALi! zY}TIW?9YKu_LPJ5e%jgso*1(rP14KQ;-cR+52zPhnbRKpgMnm#iK0wrQryND3=Iue6M{0`w3^MNe{H-3hxEO% z_Xa~JJTxv5Bc?AnDBVwuN>iiXqt55f=gfk=GNu@15`jz(A{PAKXZDQID6S#f*Mk%6 zb*oD^sSo|bRl4qzU3`^WEoa34>nx)|IslWOeY_8Of!pzW`rM>LyYt1+8FJ5fE!FMe zFn%jXAP*68Zmj#F_f7M|>SBc%=JZIPQYb?j6E_7 z!nU?S?mn2-8oDdnABrjDadHL>69sjf6?F(|@K~2oDX%{#{$gI8udiOO%bpZ*{Z$D+ zu^Y0Db4T)x6QXr>;H^H`zj)a)+q8i!ClQ4UE~K1$_EA#vme`DBNY(PVF8EPj z3^px)iRXfAhn{>z>81g%J~*N#WE#=7bb~DQSDpkoOW!@%Hme92q;whA4E@3O{$8Iq zX$J>jH((S&Z(1d~YZ?2>*Ixs`v$#NF?|SU;9XFN&j@JSvA>r`TIoQ%nTH#|&tqmar z)rH$XZhDrMCS#vjSk!<3rQ|pEP_h*kNn$mMcctG^4q4^L{30lhF2szkMr<5q-<}N`LA>-6X zj|Zsx28-q&H!ipi^61w9!TSsaSu`2f7ybkA7UOa^IlTmb;35=ssFQ~=xLQrGGI-C( zeurSjq^gXSPc-j&prxo??Pvi!4Ju9MSC7S|)(h3ZV{#_a z*b6R2(YEzy;sH+-CGMBRspbZ|ldQV|*t@cFGho2Xp~mvkvspe7NX8+~fq(26xiaWg zaYj0G+TpeB%J?OW*9MG?)Y=>FG_~Y7&gWWN!XRbL+Tiy^#@iY|*t!;|Z4*a>A}8o8 zF;M{jl>?QZRWzeqFR71#1m@PC=Q75Z52&6&^Zn&~sl_nJ2WWC8XwJRU--}r|?vE^v z3vp}FulsvA2mRw#rpBrDu7%mmhFy-D<9`20i$;cr?Ng1MLL69E8|Z~g2h*^|znb4W zK|i2{=&k!=gZ?^qJCmp!69cHWg;kNxi+d9Lz82=Ir!stoyk_kGfsk>xP^o zCe-eB9#F1-?;Ax}b<6Ser)%>s*tYOkTEdEtPV%K~rfn;Q9!z@I3SOdF`ZASYD|^p= z0l6!@ZK^1D{Qm&ZfBhE$Xq~0(Bc)%u9PN7_*EZX~rD($ZbVl^= zNqIj$l3*O$+41Spmhjli5_n2uN)9T4Goj3V?NcW?i-eiT{%H4-Xtyvp|8wR*)NBTS z(~#lcdOpzZ+fsnmg>02dke5{MF8FEusy%q9?wZwE&p!-zjjwfssaM6VTxn{9`)=sZ zeBDUS?TEwq9pc<*VmwYSU#4Jt%oIEnp&3wwgMpR^rDHQ{VoG~@sC{VOpAide;iY_? z@SS3nY$pUfCuO>(JKp}LOT?XdY&{`gHu$(Rw?F^yi@kMMp;BC@n1C$w=94@~r0PP? zP2!W%T4Sns $s{S0sGSfN^-v2^Atr9pW`kjR{KPlhsqnB5Ng&5Z{7tgSS|j=H*E ziBplh7Sa){8wl>9j}~TNi0X#5A=T!WI}u9Z_g+)5EQeuPPQslyOw?LZxwSXK4DXA0 z?Uo4cByZsY@7-?T7DfI&3<`cNhFcas(2f z7~pt;+!cNE=Wr3&eu{^qt%@ewmXg*xSX1lLB8ch<4XF=9DW|x+7TS#7-q;-KvTr+h z=97e5emiDsM}PkZVQCIM$G^5^B7XlkV8tLBR@bIiTK`DCum1AK{XmAd5#^yHBfry) zsUl7p!&Of#{>l@)WaN6x!Ts6Z3%P23=)3b)|NM|9|3l`RWu*7I2Pvi1a$?qN*VrKk zKb$Cw)^~WD{e*y7*4S%K&|2Kdrw{QlH88l@?8esS*0e%TGJ!TVYMI6CX?HYn`S~o{ zAxzgh<`M3|yZZ|a;XYq;6zXqM3Y%1p7Lh;r3E_S#O>cTaKH6CRaB?gByl=C}#l_9< zDDztZN{C4F2wCf~nX2)QUYG;Ntnm`RP8m$kwgQvo;GmYO4h7>6k#^}NH%|Of>#-T< zuU9tcv3HBE?)ap?UJ@k+<&0VIA-OHWcvM6^I);oNTPO+e&h z^NuXC+ba6D<9fLacrF9iW$2PssxjlQqReN}^|V@4!B}|cPH+&}L|A2x)9)m+81`4V zq+D*=THXf@VqUso@E`6Sw*`;ntli`b&@+)8O6S${$6p(I0&w-^*PTuTui-o;(*av) zZBh%1AN$T{$#u(fArDx3MC+I%TN3(5 zb<_v?FIc0;;+CRnq(yaoOUjzSY3P(xxR7X!5?IYs_zg|j6;^I;M!C6S`H+Lb?e3$J$g2vvlR{Td0F2kUL*fLW8`JNXSslawVC<= zzcEdi`%)94yt`HqX*UsMos?nqo=I~omDmsRCRY?mcbUB?-#n(XKa-w1wHk+yb=17k z`M)lJA{Zd@>SpN9U7K#!mcjrf);DI_^PdLo4pldv*lOn}X^F*3d$H@Sdn*UPtUiFOs7cKST^-O>x}W!(M!k_Eb+(lA z-^U7ZaRpw^M|FT>;zl$TQ&pL0pi!v%A<81D2Gw(z?(Y+c`|)3Evi|iOu|=39mBtiB zciWv5DP=4cthQ_aw!BzcyV4%r#}Iv8^9*WoqE|A=-D0T=7hY^T>P;jXOrMSK%%0ix zZ(aW!>guZt4~%KZ_{=hN9@N^QP)RnqtgdJ0F}jlux61jWij(#8YdL|Ykb}A}qy7zy z&LdaIT&ee-tix1?`@(=WJkT&z^iwDw6422xzk znQaat9RUe26L;7*vz1k-oP8J8h^CP0cuHxy29&J`v$X~Twd!8d@}s`H|_fkd2QPj z%?F6*b>qi7#4&)loG$Y)x!mZQgi*Mq+XZ#*-N7m;ooC8y%I>sy%?? z|5INl)T|tnTCOS8_@aED^hMLfMt9AB-LIg^!^?##Cpe}?yV|DM3IL)|Rf9e{mD-X& zAD6{{>43*S2FEztssJnU87^P^V&Z)DVU;fxa2kA<^cjxaoW+(=VBjzUh%ch z(z@@}q(yzQvvV>;JLjMS7^@8iEgYd%S(I=|Go?a@n7e}d%__5O^!3#~isV5VyaDe= z%tsK8y&rbdnhyM_sq|1P07E{h3!7x}$khkrT@qi8mD*2Ge`i0t_sgC!CgzW+q3P0p z5MI?kaNq-8*~ppasgi{gP#V+5rl~ne3UVXG;M+biBar*mxXlkSra@!$p_sz}ilA&>X_ zA0}2Qy9xBI3czQ}-R) z!hBQnvB44jv7^cBi;!Q!%2dw?T2EVe5v5}T4GB}<8bZbxi_ra?kR6+F7wG6Q5MHht z(NLH#E_XlGY(9u=$n-xS`_SNgFZ|F1J+UpoqrLm`iu0Yh$;hU1r%<06R_Ys@9vT@8_fi z414-m*caIqLMhWL+5IsUFFEL`6U`9xyk&?UL$(gW&v3|uE%#P9S!R0NqpqRGP4DDO z{^p1*_Gl)BkE`VmnJB#Wmk(q>bRYCGm)zu;(3hGm%i~*C$n-Fdwi^^JOc=06z(LL1 zZ4Iw_X1_#OBh-WFplg*>Eq%w=YGbX6WtrK(rg9U^)^E&?oF7f;iorPb+x{!b@!mq! z>DCp?way8XzziM0q`ey0Ryt9-%5v#T_dwgu8fBW?D$;pAymO+(#wv=w9A zS{RTx$AtXb+UnExB11QMdO2e=x&X8{07U>&#uteAw`S=S{INS~dwY zCl+cB?x9#Mxi7M|TKES&-0UvTdlrY^{#2UP1I|qW2tb>jiiNecNmqyE_t$~@l`b_5 z*N1TfF2ygG`?^U3Jq@=MZ#VFco8C=oz9jOi4Y5*rFKQX#@@GXsrD2mB1wvU0-=%TAB(=k{jc|7#IiW=8d<@|&6 zk_s>lwK1YI&SC!6SM&!!1kIY$v0; z&O#8T`1$FNKZPRKX2STQV8QR!Zqlwksoi}P8ya|`=HE=k2)v}Gz6fyA=8-xs7w8Mf z`AdYt7LqA0%l&^R6REE6^c)W=;`3y4EUP2#w%Gdy3evyhV5u+tS86$Dx1o?q zdof69g}n~QL*$DuWyH}!{i67{vNZPDL#w@Xov#^gtCLV@sPr1hV!s<9o;#v-nV|D!wm@#i%Ylo z_TduHp4Gd-vm!G4@_R_iR@ECFlbvk}!0YDWtNy+D=b9F>VdREvk6^g(=)9tGuS4gW zjW9`;;=(ytIq6(|PII+G^*q207!eSQY-&_C0=9avOsq|8*rr*!pcN?kx0{aE$fv?C z@2nGPog}GTSa}2Q6hY@GwA8PBpCcW$^f<9-?Ws4zH*E(Zr&V$QxZGj*_VoLJf zt`;PxsIcfzXAEDk^a`02m88)C5CJ;TobD{C$jOiN6zE!BieYa}OrI`5Xm`h1>RpRZ z1DgtG83*GUs18NzZ2a8QXGIMTq13EugP&pCXEO{@4VS1bLYI$)I5!0K88T=P*@m)m zzP6G|#p+3n%t^iS!Um{%p*)(6-&TVQE86tM75a2T0~#Wwx> zkn)OTgUuLXHRL@zir8*FXfqxCK8h9kn~nvz-B+;?=H})fmpJS_@B>%FzG-1H{MNI@Zcz8Mk_d@F2}d< zH%B%sWN)%*uEws*wvVYIW8zvGCOo60dHuWn+lTP6nl$n~$X$sdn}b^pf%>o8qTz*0 zcq*PVNQ(+}tj;XAHPU$IhYWj)weO~RT;?NffJMB35!5;p^5JwIfwf-Ncc!;&2s8|C zOp_T-&?e<%w!KwSxl@7Qlc1nMw@{_^ zgtz!Gn_h^L9E>LGL*`w}9u3sL9C&>WnHV#9sD1`7cz}6u^!LWY*_8_Ntj3>FC)}Y8 zKFFe49fe{RlP-0Ld$vYCL`02GG0DbeY3J*O2h+Uf7hEenz=lHlws_GqAwQtb5|FQ1 z5ipNlUOo2_^d>obwfpVt1&vs|r7azn&*;)AZqq<|hL7#pCheTLZ#N%rhwOf6PU z8CDHFtVCn zenuL`I=(tTNhki_^qMjuda0&UaC#r zrv^n?nY=!uDalt}kY6WEQ#ARDZm+}LlCj|sOPXx5dQhGA`{gr8lX%XW(6`t2?;G(u|fj6+NP1sJG3I70ioMY zA*pX^pI<|tN+>2LBduFdw1ZcQWHzH^V^ba>$_kmqEldkCl7lcnZL*PSVoH3E0;XJo zkJ-k!5*<4`9gTfhUw<7n(s=3VqPZXIavlF z#P`69=V0oWL=41XGlu8=M%t(Eu@uRaJV}s_5!1cl-roH15VY+~U^t2mg41j!^ypu7 zRpCSDjniT4kOsEdKzF57OK4SN&0N>4)sOUrNESo1<+yF4O=tr%G0VAbmTFXL<5`Bb zqTvAf9<+J@8J@ZF4^rKY$(oxq}v4Z zeC&p7=zltHYv-76p9HkRRsz2#yz}Qqz07yp#8J^nB_(u#mDQ@bN=)t$(tKo} z+E}{ES6NR?ZcSXPl;dTH)LG*8L7^HJ4tEa=l!&3BN+Gw)`5xWVR5H=aG*ik;i?LgQ)mTW>F61q>p8#Kka^>7- zhlQ#jt*Ez!akVJL@^8nRgm0}S>NOCT56pFU9nK)z=K)#DOVV~=MU(|10?dPvKv*F# ztY$B`X@bMMb~$W11I0{14GQ|}rm5?|CoiidzZx~Bn+UkjotExG*?iJoljYa|Uo~pu z>y?V4e_!%iQ)Bt(dc$q^@o4`=hR$L7=ZL}Ce;xRY&lpdt=iO0}UO%SzT5j2vj1&IT zoN-$7yn1SdPXyLaxI4k%Mn?5QhWb5e{vgP_KTe7npIvtLsZ}S`z{~NzSjF?%fXxf8 zO@v>=XaT)R#e?4>WO8(sZV^2N&^Z>LpZ(gjrWIWa4M=~~KwtZ->2cxRkEiMXST?|K zRMC5bD*rI~x0vQ8*dFl?dTi_bhUrGUso{uA`e3pI!zI6iFLS2X>g89yp@N(Z0)40{u~(>wEXK%Txe_xozRkHdhkv(l&insW3#s+}mx&e22*W==d8zW8qk3W`; z0*4doTWZxq2k17rW_l2?`Tzv)3ZqS(rvy-E+gw{!HELStgp&Xs4?zg=8g!=b+sx`# zVzuo0a=omEy#P?ZjF}s45Bt2X4_vqNJr5Q#2Gc`zT6893bKWIP$GJIt&aFAp&2gMQ zL{qZ+UHtWHtW3woP~)o&bD(63Tsf@+pl_vVVYcJPctmZGlMRd%u$^#uaW0tt_5-j* z3v{!kxx3oJo`NDPB88>*xYwks#FX{A;xjs2nSP9$(?a6$i8kjFki$VmhI~u8Zf!26 z89(ggqfRQRGPN~@LYCdS+p_Rh%A-C=0NvA{>wwq%r5Uyr`8 zhj4|;2lt!;x7;<~hRxPrd|oLP>0NNW+Ndpky$|qcacOaNQK0Bm5E#DIw3PCDlSxaC z8@cZ|#TH*-I8SVNOdDgQe;`gMl+7)5li!|4UOrO2v<)Ec|AKjXb+8p;=96Elg{L{F z$_)`pngG#MHh511v?j<#)4`+`E8KL-sFLZ3veP7)`1GzbaQP#OY)=D?$H$&Kh9w5U zgEb@S^^_^(B=N2dqmLbtk(f4VQN;Xtvl8cRxVZ*v_1A&Ph&}5el zyj;Frg(7u%#?NcK-)Us!akWXr27KE#w6j?(3n`v)&}sI_lGuB z)@7*mo?V%4OH>l=f8pTfE>~IGCV7Q+6R*`(A#0+MUSwOKQ^wZ3LYpnD&1=;aQ^lQJ zfeewRIKuW!MzqVW}u6S#H^9{R55DgC zq~5VxQvq3;)Ph7E5*6U;vJ*(rzJtl?Z5N$Rvfu(t$+a}8oN)Yg9DZ{}p;S}Wih^;{ zK_eJ_W#bNw(>wfXVJbA~NIy|i(8 zbuxu!H%=CZkEa?L9X$V-kZ!MMxWD#qn*4P$jw=PFU57SZhxMU-GFB}!QtA_v@`BPW z%sd1PoAN!K|8smsKs0VSRC;hK=oy|vp5`t5Yo@z(P_*gIKx?17)^C%%u9?c3s!uq@ zOV!LEKJ5}alz#j%!WI?QbE}XA85#G7BP7VaktkLM8E>sxEDJUz;;SUXM|ePY-m*6{c7I` zh0=Q$Tyg32-BFc&n&V;}mAOvxNCS*lVcrJBO^8Pgx?Ku4$~(ds%J>4TxOYEzeu1jP zS}L2LpFF4ywfI^_;XuN9nUi{yzovCxw1z9RsnjGKHs+)9Jxdj|d%z(B&`bx5clX8m zF&d;DgY;7@`V~aM%yctak8T+MvEyI4O81#7TRO?G-4G+wsjyqGV#^W=;8K_a-O}*| zv%`JORw$e@Qkxe(?}EQg$m9Y5~$v=x}3m2cbPu zEdo3S1)7~*f`?E7pUL_bC3#H8_R4F9D-Rp%5<50dJ4~`*F9Jq%FUN!SQ_@CatPFih zbC=WyZ2p&hw}h3Hl&}pA^|dM*VQ->LDp89tnV_)2whn=vvE?PXY5bekl)&{N|+ zP$RN!<8w3({s)$uf0QE*j^7t6yY)l{gM9|qSMN4b!X4nd2JO^LEmoj@9U2oyEH=>^ zKNh`Fklg3OR$RmdbZX;`lq8b8wvI2jK)V7CW!-20&s-Fb2_Yc?k#p6~QSMqA0jz9aXZQZwMz@EOUs8kg2) zpNv<~f;c;|X&K!S@&@ey2DS1=s6A*nr^|2aW4Ia0;FfD=|5OG{eWy;jrs2k$LrwXu zwm&07A*EDf1;yaPCB4ny8ID{bN48}!I2)b6qy!AkN7H~&i#cT_nXZiq%I-@t*>*WH|cZPSp4^5R!3=d7De>^{V+KMkEZTgCCY@F!x?);S| zYzwCa0P_;9B!4eEN#P4adx&KhtZD zC$2k#L|&)FiGS$rGOO%E-7_~Fg^d`Nc)9blt&E%_ZguAHjbEY{pEc(noUf3 zL0W{@zM?6X^ve|8qo|VIM$A0MjuSh=p6zaznx)qleJH|DQ?;owBon&3*s-ZRbGvqB zxk;y@)ZG{Jd-zaMTt-q@a;YRPg54jfuTUNBz-#?%ihaaED9E^*h~0Q>KKyGiPf=w8 zrmAsUN?Ojtt3blVxgK!81 zh!P@XnP`YJoVoyan}W)B)N$Zp>oVF}^9we+kasX*F2?rB_*l`;H6VbQLmqoSrq}kEjfKl|0jFz9hFwT{S9aCOp-}2 zMvZ-9iBY4*-t{K26Et?k!o(80#%>VwrWj+xSc!e?iUNa*sE8&S3)UzCDk}Dl6?@6U zy>s8Se$SupyXO7pS1cb%sxWu&0U{!N2@ck4U^K<73 z4ATx494+kpGU7SAHC$+tE|<=M^BE1Y!WQm29XfUBJlPY7ArKwbdv=0k2;v&|7eimV z^;-<~txEKpGVNrvZkpPVf?K6KU14UsfIlOh2wa5fdq87kt4^5X7^(&1et~6Ah?5lIO*Z?Q0EMl-MPF;0kf- zD)NRLU)vj1h6*yq0`?s1wC>tc%JssDsto|3Qd;qNF>CfdBX-*0b(r5S14hoBIBj!p zpH#UTYwt2UgqyzmSb5wVd0B}j00zEF2$>3~Uef;;+t8X)6~g7+XP6Z8({@##}v_jC-9q&Uazw&kbv)2F_Z#9YkZ-oHQy( z;_F$Hd%v_dy4ihc0IRWB8^NgAiYUWX2Q2= zuA6ndxf#$g=rt`Yt8vysIr%^j{@hX=b{bWAc+v3Rn_BS7+2bK~z-hSEr+A6!jd-q^ z^qsGvz1eb))BgE) zRk4ftjI3l+`(9HiVJt^M@}xnKE7LX6TJ3inNka@dR4A)wZ3=p+Rk?26z`d+vnD^d< z0K2)iOdV#MQUI9VR*U?+B4VtH`}wIrp*)OzMQ&tGn2(Q%JgldoS=+p?(JbmIP-!7E zf%xN``~1-}%#h3VWX4`3xHuS|0)VNp0lNzu>P!G1m|o(Q^|NRY@9tY*O+Z=w!(=r_ z#?>L4nM09v-Mw%7SgTv70>~Cc9vu!Ld>stL_7Ldm^>$RUJs9d3JS^UsmF7D(mM4Ee z2GRGm2sIipN}iN-CoTpP%qQlq++^YH6X3t+_5i=(G53nH&Axkl>-d^A^+-^2m`(58 zBRdGuobP}xZ=HL0-m961GcGZ!iCDql8yqBZ6eT~$uVkPE2N>k zv^2!~jKpw7Fu}VqBEAL>*Xz zHSA}1sxNn`o$>M1J2vcWtX?1|U3J{y_Lxu)rqU&=<4YEYm+J1Yxqk1S3~tSKovlTN z1uj@O;rZ0Sy?YmtIR&sASr%HrEy^w6nB*`LTk@sQvT9XDRA$l|u5?NKo zJ9U=sVENCJVcFHb1TFt0TA^j5d9wAbZsha5@Kf6Lo?fRpndIo$ZEXe-EAP35URbTgqec!7 z>^^}op*a2=`!R>pWBEr%c}?mfZEz%4#MwX(D|=$-Nuxp3=wW0sTaNnQqv=1vSLC*4 zPPDh2b3zza_OqSWOr0-9%6@}0C$(s3#zA7Pqo_Y@^|qQV@m3tM{Nu-u`PnUw+ir+e z|1bty6);ZE0(urD1j!^QlHWPr0)Pd5>vnaX780wN zJ4e|N_KbJ2g4FvD;j$6fHRBZIJEMFG*$qt{_Jaon-dcpxE!Y~svcq{6ls#}C9J|;Drys9#A{CuKM(b7xPk(TxE0_{XIh_g z6v=JtC%dP9dNX#nJDKs1j)=WgM~kIN@7j?gtLVmnz$wZQbZ%oS&85 z9YnUDgs3*V#~eOUD6$@LVpH#&R8KUMZ1xd3Nj|* zjIy8qS!QE``s3V0cIi=GrAh#)i*~qL=ALqj5o3w4-i#6KM~=b1qe2&B-Xs|uAq&9S zr}-+Zn%v}}zi%5SDYV+tX+ni!h7SKhG>uN0o4o%`y|30;NqVi(&qewjNzNgk1xH~3 zF?v>eg=!q*A0y08Cy4!ce*=YMKhkPy?S^tNTXC%qw4z^JTWN4TyKpja?+kkE%WSTC z5qAC+Stw}Cn+k$eV%MrGPSwq~ew@3!ck>NfeAJS~t5YOE+D*KWkRfY2nmist_-C>{ zB`L(GQeEk`=WQP@x!5~N>|FgI@*kBPOLhW>eX55k?;Ho-yu=v1Db2eA@PU*p><*q@ z=_>Ebj?mWvNsFXL&^jkS$e%cBYG=5AMZ2Z3gGOjrFR%y{x2R^uJFLfMD=&Tgm`AaC zJ_fMqI=1JY?63 zzD{RFk>OP2Bl=#IeW3HqnX<06UVlJb5-4{f>RL}ukukctbnNY%YtA*i{}`KHC;$wu z&7w*%28qEzkbb~^ac_XUox?eKK$2$Tws^wgn^&3z*txAgOMZ3y7oDjQ@U`)Sc2wNB zn{fp9#-&GAK3Hot$HbKM7=7Yx{YqO@ha+iev2$bN*Cs)*8q~B?b>`h5jX~BsXmO?x)uU4DFwI@!sfEslGA)6oPygG<2&iz+N5Bfw? z{H&XGv&@JF;>M)_p3ppiKxD1T16R4dKL}5DHRS5NsLKwOqBs<#=E~p}i7%%UYX(Bb z#d*!M!12(n>H_s@@6?28)%t$<_#v}2SGK6JmK<+X_7(NaJ2f#m28yjNklYsOmj@~# zDX$_uOY`GA(N`$~2LEKorg}sdCo)H*8xSZCys>E#hE3cr!K9?lxt&F5eR4eT2m3cP z&+k8Eko^US-kD(|i@&;)`n>XFjE^RjDk3kCf8%S= zt{LpRj=H;UTIHem(%>Mf^Y-?BTGLOR&#LMweR43RdY|L*Ev0yPFQwO4AvA)`Dv?6X zG67@6TM_D#g%K9qN23!r*D!{J)P{MP_<&GonZ=KDJN*?laNOofb4VxnTDW*u*ICkD z?{N!7o57?AHdeF8|FAVPo2^}=5#-L&&nBP~V*J-k&q5sTF(tl$=e1BLY ziZb5c+nsN6pKaw~^h5`j{(c_j)PHxIP4Kl&S?n}yj?j>|-zXW$9ZkAkg-X>@YZ>k6 zF)JPL=bMeke2B$&mH&r0y;t+J3(W>fF{`}SQw)h3)gvR@(3*ECWWLXYr-1IE(nXvo zHnDwVOk$2{z>E17Au-l7>_XuvvJNdgcKCpT_Bp!&_+IuW@U=!~T6Ke7Szukd>+wBy z`UGAF27sKZHuplpH)x!1_R1?jG+~wV5PdR-A@qBmBmz4=OFo%xg_j>|BFf;L4Rb6emv<=v&$PxG3=f!`hN_tFV#Em9Eyb|Ldc;;wE6{3%52w@#7>t;n{cGH)JSfbx<=W*d&ht)vd#9_{CB@9!u& z&!&!m?fN`C?6M7MqrrFgWbt+G4{KQ9z~bjcS@|w`6w_W1dQ#4N+Nmb8|H@{|cRdA- z9U__1-#7W&*&p-Q_J-l<($l;T~CY)c}ETm zo6O&?9}L(7V!AUsF-vw?<(R6K!wcevkBH9sWsA0@`nbprVz)K9$$7N3zg)+k^T1|_ zJ$Yju=c>dvyo}h)z@Era&h8|)xxa-Bsq@|J`K4-4UG4MIoc~y7q1u&V@hc*f3pr5Q zZ7VAZ_`L7!gjHw`$a}NsepA?XPwGiH$>0Ll>>luTd!6a$*|olgmY|V!yW4$L0&x26 zmBY+TN#tl%r=DVBqVH2TtXyoomLV$!2Q%ylefiaSW9viuf(_>ux6Eg{;N>GS*xA=u zf@GzhGcWEx5$a2Tlua$RRbXmFSvhK_nH~oLkoK|*2laKS7Q0`YA8+A5sShpIK0}_I zin=HZ*oY=mMned)`jz&d3R4VIXV=1_iYCy@UM9p|quU+}ak5$(J7n|%Ya{p!!&qk9 zwAR=FH>sA6oW~r4o~ma=3$toX(b>TVCgJvx%$(D>{z{#+FM&hKC54d^MgdEyPS) zow(I&g7zv(=9G^qx{F(YWa22n>Z=xw(6K>UQpl&&L9|7JQ%?328CbZ88%*|L5(G*% zJ<>{!Qv#3+i_4e_ca; z_OogycmXaoCv$(yp7v8$)`tkQuI^-IPfvraPE+HDxwexF;sefAhCNCX;}oAQxvui^ zIAaZQf{M(y_0yWBI=0C}7c;YV?yGlUD$>iOK?&2~YVn?$WJ^K`76%cI-TgPTspA@^ z!Qgd1xIis%xiYT$x?}(tcVT$y!1F;FQgcI1=*eh(Wz_fqEue5Mlw9aDin7oGcAbrE zhimN2t~C=TsZdBz^NrYF%5zhn>=BWuK#3y{X5`Emp$-3t&GV?4Ve-Mn?C$i$+_gWi zTAI;lBwjq*FK;3n=e|g&CqxA>eUBCD_CKocKjY7SP^WEyHo0M{N>oZ{FqSrY#WwZER7cqxEoU$u9^syJt*vEi36;4u6(W zwIlr)GEtY<<^z&)=N;!GGs`3l@t~QZtq6+j5`1vqoRRw7`VQgU?+BoEeqd6 z;(CK@kdhIcH&jY^^iAY}Mq0d3^7c!WYJxBr2TQ;B4aJw9qAmFycWY0EQL~vdBKd{nGm^Tl$2VC{Y4oAX zY>aCpM(B4Y`B^V|?|Y-X#YxJC4_etWi@uJVPOSO6r9JRbocsErQ0Ny_f23@hbJL=D zt5lCo`zDA~Jmu2UqFj@+ES7lVsr?b{_G1OEaRm|YMU@__Qdi2lkI1yg8 zbQF<#-0_9(;|0g$Cx`X*S6*(e*vg(4;y&WWZ|6oQ7J=e#{pK8o6xOzhaE-z*hkegC z4}Q`ItHB))#30_BAL#4q%fkc49SR#{D|dF(a{1ql48y_A-}Xd^8(xwAhkd7Rr7ISY z&UYd_w#lihnD!#wHrPhX4T?c;MwewKNK|3RHc#$GtcmttzT<89B zW0Xz=`bJK6=W4vfcv!%1GGX=0hQ>G^vt!{8?L?di9>P+jZ$7MiWA3ZjRB^ zP;yDT`I3a4ghw%qlk?WU)rOp$+(=2;2D8XC3%kQt&|Y7+PzzUNos4ojj=j(aYVrQt zhT)pFME>_z!ZpLDpXJcR&L)AFhVxaO8%eW7Qo66s%#5UEo9ptbYQ)-`)t`%dN=@1i zrTXX(%_wE+N_Lv2j`%~5wO({$TZ_Vr2QN;98fVq?JY7p%#bBR&y-Q#>D@@coOnNG5 z$Od6pwXZ*4T~L@eHKkcD8oIk=ZBVgHsu(Ddy|d~MdJK&&Vuo@f=a)dSZqd?e%- zUxX)3nO;}ATIcnaJn_1^^K@;^NWp%`YKx~aeQGcO9vGbbsjZxb3V0D^@YRcL>#VmD z^QJ{Zu*_iC-$CKkpFXQ2p=>i48;i2h^mJoWnSuM0&&q7fr`Zz)P}VWH{0Ggs&8@5` z4q5r;#8n7_fub=b*Wzry?-)HVa0krt(4C=10HvE_H~+q(bk53@7V~XX3Halj?RMw$ zc1iE?k7=xaGf;A5j}Xz>qYBa=IV7j!E=WcVfs&$h6YJ!BZ5mvwCpUg?A+%8qHv}3G zOOv}{Z0r2G5Vzk$v=zQpLm1CO7PdC>zX7tNH}uFBt4UQPwfU#yO4(4HsZF^Iv*iyl`4i&5G9*z6 {=rT7nHELXSN*W=V_ z{UPr%{18r>ccWE&$xXIscMagFx^CvtcEz4wX|2!s@}OMRn`9;RyR%KipV(D}sAA-= z_K4Iv?vt8c{Vhkz#%L$~Q+4GF>ST4=^j)@s$J!XU@(k}sA5+O#>#2NJnsV?7rZ5;t z+jVPu8e=~EkwAHR?8qrGA-^3R zh9`}xNoyy?Y?6=2dz(`)!bAc)MFS@t8~P2#Ya)CPa$gM7Q)cN3xv-3^t|tp${@QzMBN zLT)2Hig;VzB|1ozXhB|Z-d$kWgCsZQ$hQ`h*@Q5?2$xvM8 z^Xg2n+k6|ZV5l}=r{zU2QorkSLSi`;U#4q1$OQLS9`^X6@RbWsw|#PJg0QRi{Xa#K zPPgIEc&pOY+s7@kQ3u^dX(9^jn{mmykrC3gMjdLjI| z(WZ7evU%>*#cCmBDf#r6v4CMyE`H(*wW2biH2p*gW**-+CP&wN`=di{(Y z=7mno_JW*vHx0~3QO?B!7%!gR@|2iRa~x=a4cW8U2C`*Xi?R1`t|ucBRPwtmeWSvo zKrfrwZUrRZovGECV88dbE*Yt9d|rJZ4*D{?Wv(U0`{A`D>w!*tS6U(JTFhw3L)gP_ z)^lHMa1Y^j2tF}2(RC-xDpPJC^vGa&A7cLywc%HJ)D2pN$ykoh4JBF5+sDR2kww$A@rCp!l6T05g78F5(WG0zf-(FdjV+d*D{KGO&l~Q~lc^<1dB)5LtABfRsDZupwuaj#jI3~^}=^f~KbV&P? zn+f5||8_OqHe6aio;4Z578M?(CYU?iP6*$0NCD}IkzQ$N5LO;$-6a(FGfezBQLh8$ zuO}BUItN_fyA7jF+o-x~(~S6MUu-|vy&{_o>&C$dqDyiCtUyERC={h<=ty>Ib|-=> z2cwKsuFrJT$*ud8HvyZ~(PinuAnTGDYyk~gAB=bSofQOH6hh(%C}n@>S^y{$h;?yk1;-+cUwPu$&mv9bKm+wOQ! z4et1ODWSZCg~fGs)a&fr>Wt~G^FG(9!fMUB76j(} z3OrEC-kHE@W`TUzNj^JZXen=K5t=muVSLRKseCUmqdDOn-DPFmj+E{YTLkwd(*69~ z(vuSfHTxT0?x7z}uM;uau{N@*+PBeN-t3PQxeAG?qCy;Q=zv93xg)lcKJPnm&l&r_ z(<*nJdt3MT6NyX1?W?U-{`uPJTw%hjbI{15eE8XRqqtN-YMt$zOO)Z+XGlDKvMN;j zB5bdBxNdDMHI2_C*rDD&iS7PW3h8jTonjn7y&WHU?daV|1f%qHB6IZkL_G*A?P;oC z5Mr7bW`j|+zb0Cb=dKbfE)s2&{1yOu>qS-q-`Q%G;B5g2QoLB;0*EX&GFtvBe`QvV zs*#r2&B7kJ^~7B*>G!FoWAN;5uR~4ffZ^|1^L|LIsgsJ~`TzuKI|7ILc}1M&wqzP9 z@vt0)UY8HFtCUSe-nXO^V*L9mG~9WsHC^r_gr@LcF|tC60mnD{%xp6XvKqT6!JeAt zx)?86uZE9%g$?D+{S~bAbetc$t{9Oz++>xkx7_G{sk~WBT1z`u>HC>(VzLE>fElF~ z(7z8+a&6{RQTGOV87nMycmEKb`7o0%R^e&b;o=@guw8A4a_VIupR>>?7k;%9kaQUhbPk!0jHNZYz7=g6yP zGr~%P6KleTat7iIL>!1ZeL;<9uA-KHBYV=)RE`s=O-z46?3)RCsIgf$HJjL2-%?P5 znFcH?8}6=+SBv^~jCvti>&f|-759#*Mx0f;r(zB@a?ZG*B+56!XtS@<*FJ_lfW*eQ z|3S$`VP1^BFUCrx02CTTYosiSth#b}IMi8Ia8awJN~V<+*xe~su$T1OdN=Ux6k_qH z*SFa(Wum%S*hF;`^jI4bj%P+{gTy+4CZ;C1rs&4n2i7>q?&?j865|~CQG6hK0MIor zoc9x2pL2U`9#t;Et6|+zSIVpAC;sqb^X37QI}3iM4w3Y3oVeC5fe0oo1oGC#6f)OUiZllCv+ zCC)2n_R?iV=A<;2p)3={C!gypD)7vt9)qIite8KGoaTT1agNHD4?L`%M){W=KD4A= zK>u{&Z>1-Vmu@TuHX79+G;O7e;w zjJkg7gKxl2+z`*LBt8kFWi_%T(D^WvC75J${K79n@!s`gx_4@4>L6?eEuSYkKQCTY zo>u>aO2tOk85W1Sb0BDehtDoV@qB7^f0cRVb51{>v`L4KL|nJanvwaLhRBog08kku zw>+00Ye;QY!GY%Vo-+9Pd!+|1=l1y4H6^wP*NvS8(Sd$is6 z7NYZt_fu?uX5G1*-R1dQ=fMRvN^DNWknYiy4-;uyGW#uqfOjVYD<@e9R8nAspHt0Q zqWh&YTxKJ~apwfHf%c@=Ul$RQ5yoeYtbl*Ms!UV4`0ZPlaeDg{Dtc06Qwp9-;O z347H~R$>lxz)wE3(mk7=5Or+!yY056seHwNQ-&u%@%cz%hthsz(!7LgoLZ4RA2F7B z(s+EmuKrGqJ_>zwnn$iE%riVLB2F?qse{yiXOJkw!9c|;^)VeH>2RR&8qa|Z>~c%_ zd_hm~V4v)v0IbEdW@0x2zV!54Il6_a{TDszX-)C^|6oczgh(F+jW&c`T(D7|VqDNp zE$n*5RhEWc==oHi>(FmavdmLHQNMGV-?p@l&huOCN^}jL&+Z<`V`Q$B7m0kIZ0#~9 zVO9NkIGb-K!5NptI|66-`&6H(yVT7~P2bSdHf;ur`wl}0la{K37s~3)lsedkxKp0sFhUKh@j&Bq2$czaM*m3eK? zoI8i!+~Po1Ss?>CjR!>}#~`>&E+yQW8vmz0p+E8;2&s%}BV5VLo#p!`8zwUOK%U!F z3%a}bZnte)gE@AF+rhnAAaK4X-WnVR zcGqLG8S6puyKHJ7JFvezAF-EK@jzfq2T!-N0B`)U?8s&l%Vs5B#_<%rN`y^rY#n#o z38#2avF!6e0pC{=u8H=Jd%=6TK!IyQu3@9B9A5P-9T?*UN#>&9U#zs_A$6c8xk#%! zZr;>6(jNtDo~>>?YJS()nA803pBT3HPdTy$vB-AeN9SJri}?dv7Ecl zvNkwm$wEMRZQ1frgXy_>Yw`y=53Ih!0aD`-{-E=Nij6}tI_~WRCc{62AH7HF*jPU) zo8EgSM$^|MTl9Ocb(>9-V=-I|KGt%AV_=ZtcZEK~;DTVS ztjG?YrE<}a;u2i$q{OlP%^Yq6-U8kljIXNw-OtduTYM|x%YaU87itTMKT26P5&98h zr4edl6QKmx^)<$R1#ow9Ry-B^ysmdGd>Zu`O=6Q!HSWg^`@|;uCH08+3k5@jd|{aJ zWI_n5n~m8%9?m^*H21T9(GApVubbEO#5v4wJPGc$NZWO%wPctuc3(E^U+7q+6ZY6- zL}1yrnxn*nHVHlD;uN#!Et7zxOC9WfJ~=6?UL`{_j1idQ*_&koHlS|-CXaYjM~)hJ zO(I4w*q_fp9&Xv4NSDcxI2(h2KabN?2n}ejh<78Ae}slr5DBIsq1rIM52P>&>R6gs zAI&$=xZ4+Ai6n^d6qCvN+1p`+W2uJY8*CFDLe2GYEh>hRweK!;iKaL%jRt9nUow`} z$59j;5rmcdkX!K7-@B%r|D4BK@m*aZ4!Ad}*(`&dVI=OI<8&Asjj1Y8FoeF`@ zz~!qROCHA}jhOoB0_xzg-tr-=OcZCpg;NXyd0Uxl!@&b*#<2WclLK=G&nrE>V3iK& ztpXc1hv?V}wmMvHMt(bI^`N-cA+haNY>y9*L+Fj4*cxP- zdEjxshP!({MEjQvTVLxqr$nrUNOv_2`pGUB2HKl)fpoBga$jR_I7z(OJra7h<7D6r z;e-}htYAdKj5>gUY09jj09xGZK%?c#PQZ8*%l^Ag>IPxdH0-Fs(#?4&*cVlry2!Io zI|d0lu7-FQ5?dfCy*D3U^LT**-L0fy?!2d7k!4l0J-<;lX}TjV)8D$>`|D!CDxV_dH)GPW zEE|PXmTYr-t}~Kk?yoUnS}}Glir_2zrPx1*g4KM)_R37=KUl_v;&>Fq5*^yR^0JOF zjI-p8o_fN?0K!G3rK-3gOU5s|2VrDs?>c_gN|Ght3i^3Q%5(G)Yomr}I40OLscPjm zv-K)<-?tR#NdsbsLV5;DmZcT5(n3s7k>De)AOy!i2EECXsG`2L22iT=3OXs(K|HeJ zMMw3ZwPJ0qH5K0y76!mU`(=>58lkmyup^A)Y%|G}(A^jcw(?H$50e^_y%%EYSLe{! zc)A*(6jd4K_}sxQwNr0l0~vMSah}eB4`%K8Lq;;H4`fT#aGgD6(XV-Bc^iFFr^>fp zy~t^Oxtq}Z^rz|<)nj`R9B!=b`ezI3lj5=ocp=kRn|p~ZoDpejowN;keI*jBU`d9N z|Az3wrolIioQikklw=|UsvFw3)5I^#K4`hfHCQ_OSuH!%ax?65RgrQ-2{7 zL3c1*K#Q`AXI^Z>;WDvA@x3eq%D-Rr zGXS!wBgpdn1^h^U8J%MM(dusCLRjnPHiW`FoB1|cFj^$K|82l1>S0rvLnv&FP#=*X z^Uy-$Q-gBuxcS@zqM0?!)XZ8ByVfHLn7GQgj#hTc)eb;ullR@A*{dY%t9_K;El4A> z5@~bzPUQYV`D8}H+ zh5&|J*;q@i+U!;Hm2Ph8bTC&_aKey(mg~KpsOjnN+tp*FgPs1&)5q%ur0XT!wvgkO z(qygfiu1lTP7c>?xb^QJ^QO8~2gx3qOCAG{*KTacuRm0M$}!wO+0hp_wf*Q}hE|p= zkITeXWwn22>59laXV>uc%|p&ndAEn~_DPv`gp!Gy*OK=fx^lYzmJ{1LNLKBZMYmwQ z_8R&Zbh!X?{1TA!F_+BY!?D-wr=2~Azfhk^TAfSDnHjYM>N|X zm)^3|sM8-rh@HjjaE*-}`4+gk$nO3)r(aqZ#@%gzBK%UkL@yu5&an4y&R_@fkra*` z@#CBoYw7p(>6y+{DD=?2DSWrDvB=qE(YAjplnF0hI(Cn0|8SZhC1HBQ&CZ6HqLwQh zge(}k}K+MLmroXTIzLV=|tva#mak^r_R{7*rCtXP0FKUZ>w70s9DYxhI4j;io zU7LKqHOp;5D%6iaQwOEAi*x;Og8907Z(22{@AHNm?j04OO%$K@EfH7jiJ`cQm^;+e zI~6Hz%|^-Zblq{(w=zEQZ0#09r^K7b4=%AhYgwLUOwi3^kH&Zbo7U&JpoCPQ7Y_1!*po5`gQUQ1SiX@Uo^2eGfu8JiH{8-{L^HEeVbnXNnTI^aKk zIv7)G-J!lovyk)=df1E)`=w}8`C)<=SZ&_8%8e79Rqv=u-_>u>Qe86YynH4wF~&-l z7ksiaNyJzg38zr8-@+q~W%S$KSwU>%f}5GXJEj^xh=ZuveDv*%!9$J-bejK0?n;~#Ma6_O zunzbd8j)`H9#I|dg1JxQRytFIxE*-U-uZljj{P>mj1TVA<|&xdHclq=PYZN!Km6n> zS3Ni7cLmgP8>5T3^;B?P*Mw*@=HZSMQkral)bKqzj(LsecE%}g{%JEIm*ZOdDhqYu z&7LUYet#8_@sW;hJ%q6G$qi4%-Nsg1!ce8v)eN>u#nkG78YadPk;_S8DooK7l>Yz9 zPXA3(|M&2J{2jQU5VVn*CmD{v>c4$4KfEU%ZR#jJ{Kdy!NsPWRY~9VmCk$cNiFNam zktB;5T8Wg#kqRa#D?aec=svLT$GLhl-^xBc7N*qH5QEB9o*Ohcvg?pto;^_eY!|yb z%qs13b8fTPrro$kDm}QGE#028`mc78Ghw|6nFx*cgEjt}`x=;Uun(Q|eFDMEvS|Q& zTzYdgJjv8`#eY)*e-P)^FoiFy*#fX`1KUr&w)0O^(kAMIQi7e#haa{~wO0q6QQp&E zk=&}h=gO2$1n7TSxBNrvHfUSwek>z|=~Y<3-~Fw-6+g9LqmFXoh>t(HCL=hh=lT#m zvTGyu;IOP>q+RWQbCDOY>T8;4&r=J4_w|lb@Z|3YJ;2kR69A4;lHO(69(8l=Zm!L> zEzhYXH~CcP-q)N8yOzrr_ZnS;2)6J*s~0ca{M6*yTHn>&hb-S2qUp{Z*4B$M6rwYp zeAajfc^nk=u@fu^GlYt56JXg1n$X3ji(>xiyC-n6G9iJ6PjJlmT~0ZGwap_|gY`Oq zT;cRqGhVYU?h&qv9YlUaA_Us-VJKu$)A}9bM)kB1KiPPiUMuiGp|q;kdw$SzxTLgM z=a0VaF3Zo9ImG8kg@bqRX`wenUq{)_RE18Y)WlNlv;1uCrS^}P*hzyO{BD3dUn*o& z%oaeS6x?uUpqj<#6hm4c2RM*km)H}QQ%SE#uYOAfd8E)XP_^|3(jgEnHHFf(xPYNB zBRy?{XRi=Q**B90+sYiVsyba)ZA1cPzV&R+gye9glPX?%jWk-g2C6)y6H6>QWK2sM>ju@GqztcORJfLrD_KF2ygJ|A zr(a0^mFq(AGCLP}HpK;fG_k$*8A3@1vzDji6Ft29Tn4|1$o_f|56W5F&+uNJ^~qV7 zoOKoKF3~NMJ)V5ba>m(pan6AI`t{z})nXf!Lf^ArvX9_gc9;KH!!DnnJ%Fc7fi+J4 zx&7`H$wN?QXD_BZJ43|2q26_u=`^vteP=f((&9#RLuO`%2PIL0*1dM;kHDhzh&6@L z4a=(jk$qTodL)d?3V<*UoM*T7gSLUE23B^q`#psf6#!uPYaSm^++RQ9+_?+yB+}cb zw(vnbP3c#)MR-8=4G+=g^Uz2YA(IOaQ|ScA_$O)wh)cL~F1gR!OdXk1bjif{Ca|yh zQh@)GLB|lN`PW22gWX(K>rB z{&oYkno7_{BND!M`&nc^8>0=O#5%{H;&~ZUIfXyY-7?VXynMsR7Psg14_;4EX%-CpOJjJGHHoE!O}D?_Jw^ z8W4_2k~BPV>|2rK;*s{YzwaWwCGmn!L9~kGMaN32c^h2aL3c6Z_IoSb7RpW?0U!)L zUpp6?GApgRWfVjK?Q1T^pLdK}E;&HtFG-0k=Pv{MCXv~t%e~`8T?FN_wh<=$8y`b{ zvDrODHt&jr#-GI(_a?--&C7j=8n$8*WiW309tT=fHT zofD`C?EBZ^(%k1U574p(?LiOgZIu^|MwKa$Ehs&5V92)8>qOq^A9Jjv=~IiKZ4N8? zs%3T7ex z=yJ;R8VrcRR4&PEj|Mmd$7dn(D$e}BTgxKqH~PR&h2*7%hhJarmy+0~)Jdy^(`LJB z5?|92b0fo31?pB1_d%Jz+G)z7GFr!=6`|OJ3P;e{{F8c9y+>C8H~r z*>iq(5)hYUuEEMHr9YAA@>;acuLnjEHuFwGZxExKLcO`-{Xfp#trfZ#ywcY;)@R`I zbMp#7eKgx}&gaVS@t#zn7obkNiaQy?c>ORDunUi0?SQL)@*w`aKS7HwxcHJ0L@`Kk zJANMZ5}bLcI4IHH1-H69a@a;n=tj84*9aYrp!+scGXYnPd8ruvc4a32J3I((;H=0rNpF|}G}y0l6sJ!_ za;gUaU%L4`B9cDT!HN_j`r+4k_Zi&cdoP4|j1I&OCSwC!lSG7@ymIRTH=U#pMfwrh zYNB=Du>ni>Z~{$(~n_qo_3pA?`ESsAqz8^DSNd9@&8|S`OovCd5HdlpRzE zz{{?La`38w}iE~+9L~Hz2ESj zx7eCF-UEdO0z#@j@TjEf(Qk~ln3U6s*L~>7D=AxkX3^1C)maX{P{x8&>+>!W=s$nO z&zE3&C)OjHBb)TkO)iwWChg?hxq)Nip+LTl@%mGagtM#jT0)cb#n|2Q_GYl zSp2?QIvU&<=R}AI#<)#Nj5FWmNUl;PiW!z-H-uzR#y;JGCYlxTMm;7vPfd4j1uTm# z)sW6|@(lc%CK`9Fqq`1JW$){>r7-U(xYHQUPnDngHlV(Ez4WXOyY;H0SJs2`(Y^i* zoBnoFX9@B3=IxGk>BhU3WVgV?ZV0Kg)+{G@`k3*t;0wJhCaBIFsJh&?Y`VjBPi`yh zzX3)MQp+(R#IReWYl)iLH_j&#GkfQ&l|?F&+GhgN-3ZBDlQ@O!YhJ+>Uk=@pyAxe_ zk(GnjpTA$rLf5iMNJc?yL>eak@2q}FFj%{IvIP3$+)DJ^9ezN2pl`^*eJT@RG&OmP z=0-60(C3jtfmUcdK~P0wk<3g#-t?oAb6;nAX5l zysl|z^FZJ1FnwQ|Qzq+|Ro<>6K73l498~01Vc-y}q9gWImi@rrN}6O>oR}+?v)4r? z)CjG_o=*zpEan-!V+4IzFZ~cRi`zc!>FPinsfxFhs4&PebGBoPq zk{s>34&`a&bie1|iNZo|iwO3W%ENhy<#BuEuOT$C85w<&A#OLp`t~lOu@5E{N+Jm= zKMM=0wCFN*`gZLNy0K$9V z)Dv~g+Rj**i#82a9~NelyB(QcO1G#8WLSvRNONa9T-MiRiMV>F^b~CliDH#w%460; zH5A^K+^Qp|6*uEYX&WDn?I1TVbX3a}k!&^Vq+HsuHg&oosCSp%VJVzl ziRE+Fn6MyumD0@A%GWg!sB9AzbiqU` zwfpQRe4yu_`>!CfkA8okZD-xAA)e3@0idHFU#z>(msy$7DsOX2tMt&%f6cKz348u6 z9>c9A3p2LDFJzg=9n~VH-IP0BsNGG9raL;ly6!0J<@#Bz=S_7*iF9QLnB;L}-WY3X zF)&^73oXDeRwVmcxU|7l!$qP)`7m2wU5u}hD0Qrfm3SiKczXfC?D}K8L@e7GxQsmc z*e`Hp^WgO{sp)B9V1NGn#apBs!U1OmUAEUE)QCG=lB*AV5BUAsEqFc=Z47%XP$?J# z;e;^%Z*ScF{l9s0&xoa*{Wv!gP`&nzz{~Wme(tMZhl}^NY(!kMAs1K(RLX=@inxXs zvnU6r`sW|dMImL2V{~k19pjT5#Pvsa{}*HL71m_he(^Fz9Sbn1fb>zCNE><$e4_#a z0z)qWqEzX<2l6Q+NK2$kAEiVHA+*pT(g{^c2qE-@KiYaL8kbFJg(u-PQvW05zf%%vRbwS0h!?o z8i^&^w_c>EVCR`(N^q55K>zB0*~M3MUhO zs4Lf0;{Li$=uLg@n^PNhb?d#DM($P{J)=J}&K~j(X!aju<*!||$c9c^5$pB@it6j8 zp$k4{`gm1FBwhaBEvI`22N&5P;{|59t&-{L6NZaQGtwS3O_X9VE327Lu3!*tMQ`n@ z9ppn-jU6$XF9*xyI$g+08arriFFQ(h#Kl`;q?-0i@7*GWXbs!n9!JGg4$GqalcH3i zExQ^ygKH-JhIG}F3p#)UMiNm47+wj0vrYLa+I!T3S+oy5sag0IS*T*1`maAfH$n4R z(b~=lQNs>hvNX53#E7%xpgW8NmTBBN0^h!$(Jbk{JAOcnJ5PL|kCmME6`fWe8TRtR zm2U9)bZ_&ZEi^x8<~1)K?D&&+%7SvPx(z$n_XFf6$5i$C zT*VYB4;B$uQ%&Cg)obULmzr^x;|1fl=t+l_27}`rV(&x*}`>E=-IUJud z`Iv&;Ct^5me5KrT8g@s+#OScF(eFs{{Ajf4^L0mV#`KzZJ2{9TLn#N|WIxAo)q&TO z(b*$slr%uuU6grDc=%)cw9r46=Qrbx=cjVoVsBzzhvrOv%Bd>^9^%retDwr|JmI3^ z|9r=(jgc}M7@qKb&Z(&O!SQEf?y67h7QzMOFH?fctJjY3)fMTT+gMYx>^fMiM;Fd+ zwJ{U7|9*;nqbWRY^;`JlEmEd>n62+!2iD7*X}pv|GjnD~m#&$UJjgD9KGlZ9c|qvDTUrj$CK)ghOL``VCUS9yFyY)lIyF}+a1Kf6U_QtNYrgGVRg7(L{o zGno2$vE_UIl-GRzp?86bm_VsnK!Ib487-aVt$C{Z2OZIml0AP`*jnO_L~J^!r3k{) zhwRf2%rTB^)!IgP+|0|uULk+A)mhwSI4^bE&0G@hP7-wa zjF0q!lMfu7kxds$iaR6CHI9F@%8OrsN9v-VL`Mfkd?;4k>S(U?ZEIEt9@N=-06=+C zs_#g-{+>VUjC8Ef{SRqdP1<(Y_2iC80@}ZthHuZn_SF-UMdBTbv=x?Oy9?U=75kbL z;XFw#1I-iG-NSJlX1yjBj`yOQ$T9xncE^ZEGu8PxjAzDaT(H-WSMU>CdUB^xYwtg2 zMfZAtunC`CC8m_*e6I@DmD_8xF&e)ehyeofhe;}aBHZ__AR87b=sFyv>qvy2P;Seb z%#D^VT1tMv?x@zCZENi4*{!Fs2(He>y?;cQ5}Kxogz#zDbD1iX?H1jK8d`kszrE6k|5!_R)KRUN zh{I!KhdCO;+!@E37$v8U~hp?sTP_tr0Xi@VjW5W~8gE^K+`arHuZ zv9ytTg~BPzIGGkqwcJtLCi9~=yQZ$;4P*!>|c;d5=@*ze)Ibd)8O&eE{T!asw7^$b7e}x|CrTik|rjZu7 zoM)yH#yzq7h{Z-M(G9^oi5eN6KYJVkl0DS_xg_aw+@qK?mK-}6*ozyrU1#1Z1xW+> zYAM0W|6>0#7IonKP*JrLW$rM*l`_3Le~c45#>oly z;2Vg3d5V@313*Mo2(CUI*J(*ED2hP@bwb?b;#!lkJvt2%x&`2ori1T|G8h`t9{XHd93i!G?yd z^7k{LDVezKO6xWX;hB}iae+fK2`(q{_GA27Kb4a+7N$^ws@N6W zA6iIh^as8l3sG?WDQ}!Km2fYp!IB&3(l`HQn`iegzf&*W`Wt!#>56t#gfE~u>vU*Z z($``u?eBkFg={JTj**+wlM;>KlNI0_;$g#U+Cr`|nM@2gEMborhFL(K!=zSMq1H=E z?KO!1hV!Zvt~s7k3l<|2Ga|PYWWG1JkhG!QKReBkhnlizHdqV9Q(Vk(&h3+O-+NXO zMAfc^eG}CFr`lB6oJuBdJv z`%1Jkv4fftGRJKVhvo$d6zCmR!_7d^%=MD{ zX%Ib^<8HnOQ4a_W-wJ`)_fU2@O{`84Z!s!2PwM^0BBPnBL7_OL%Hy{ixz)L!gkwTd z3uV_5`iBYwdSk8=fX~*(wbzJIf}EmuvYS~?b zJCk$y8Zy|zCb@}uRT=p%|GJvdq28*HVkq|}+FR=24B}$~Tx)dvZNP?b~SbW|Y2>|1icFgC+K-t=2ON@Mq5psx{p^?TMlbn4h>5 zp#xWYh$udsQNU#iF}-dU9g07Ug}y(DXv# z;f0Wj&hgUp0+i9ids{nFy&?{eZvI`mCn4`|-epaGv-V(nw|b<1X+$ z13^bds&N;`Gk?|wmgX~$F_3W3n%(~-GANu=Z0M`mWy2>EEYS?74G4(BTAdxDWF)2h zHJ3W;kTHJ~eEco3Z}GM`_*P=F%~$^F34KpAcB%wnTqN z4m}BI+O^%M62(CumN5cW!@^%EW&+J!_l2RCr8h1sH85*s%#6W?S$N(IOU9DD0%J5{ z;=mi@;K8EBlg)L%!ZVEPSJ8JQndF&2iP!rhHo|4!l|zoB zGr_uPG_l{X%Va(bQ{WH1JaK zuYT=#%#_n^h{(E+!zqu^|}Bks(Z~U zUgR}_uT5uGqM^;#kFz?EEK5ndSQFaLxz4T^ms;ZU4G zKiF0^5|Nyh;m@Xf|9e7jnU!%`{^Pvr)aR!G-Og>jj|1Khs@c6Q{0fsk_K!kQS-EVj z(blvRNweJIs{uo4k9QphhSP+_X5p2IOf1gtxNHM{L*j(`E{=bb=#y-mkX8}A5F#vY zTzVzb8lz2c2oM5D0}NyBj%N--^0ogdU@v7s^&Dmz7M2_L3UopOlFO-7E%D0D$l;?k z^p#xayu~wnCkajAxw*z|Lwn#Ox3s}Ftq`<7v$}V1mM%VnNOdXg%_^g}dQFs|x9lss zM<-wd_0f8VTLG}|9he)nB z);uvz%$!aRFo2KHlvK$z|6DCbAtC&Sa(vWyiV8*(thhEFHDLiyU6yH(t~m6&<12?s zfw77;+j;XaDQ_n-*X!E3-B4NYIL>u!w6H&SZr^PU5()9U?Ybn}jck58^>;wYOIAUY zH;IQ4itqdQ+AVflSQUfDUP&`lv`GpzeYHaMQ4#*2}qIsY%#6}%U#t!EAw-ntvc45i;5-k21Hrb$f-=vP8r*j3ON@q{c3P- zxOgyAT;mXO6n(F^Jx&m0zRRLcZm0x^s>9>PNK#|L#U-g^9rGUdhW+QSjxh;ujJMp9^e1?~Z#l(ZXHNga?>w7TFuiGVdAqDBwO`0py?GrW zT_^0C!`IwkfZc6xy7$_DXwmZFGwG5!X}dlTVCGA`w!A0XUcR>Z@5vz&F3p`#7r1HV z_aVRo%x%)Z@7b{c!?G5n%Ifi#No0|eY~#svfeffAPT&Wdd52tQ*6_BD5shaf@3V!E z)k*Y1;xJu{(A+CPei^PNo;UMHPj`AFrS#!Bb_=&33^rT~u$2HHW}Fv`l2u)uep5Sb z`(x^~vAm|_X~5h_M^kDaI(2P12X)MqyLO{eJ}<7w@kCfuq3uwD<(zI#(acqeJi>Se zM^%f_UMU`}oM{Wm=zE`hP~IhR$wm>JPk3!Jw9q7X7|dSmouS-5u|PL)C9H=V;mJ|8 zn`}sP=-Q4Qq2r_>CA*^^+QgDptA>oOv3nOcr?GZXT=+P zIm1!Ud7M=^GPMU=9F}ZNu@grJpGc^nec*<^W}=V0BU*pQgTdOVg)OgeLcyLc??62P zECrat;JpjddU$KGK5ot7%AkZ5WnHRp{vuo@{FFooy=y3Z3->1OO}O6dch(8w2_LV> zsSB#_+i4vl>c~-4DHAG=Dw1xBr}fR2*V0~y*WFk7BW?WUN%QG)z-iagky5GSZjI(R zk=IsaMQ?k$pUTZWb7I&;xZkcOueqFP=`DMZyh;bbnw@XN`GOK}&tw|REDvioiykaZ zU{$?sY9hn`j5mY~f&GOLe(?UuS%I93>EW{xL?3iuG6KVyGO6NcO+m`icfdUqh zHJhYnxYcJ~k@9}S#;%HEO8TN~iK4=G>X!|20sde!M8r_UK0q{5e#peE?+2TrnrwU2 zS`jM5lh-h3Jtj(^FuOZuH#IY?!!b%I@wKh*J#VYi-$;Y*L3#Gse}1o;tJ%EUh^sz6 z19|Qeh~P7);@=ajUgOJ@S%hrwW8*}dLKo6$DoD$Sgft z^tX(_|EsF0{r$=}a(@58MbWu!T7&J1^?b9SuGpHVulJDr6xqc1Co}1_!WWBbG{a*(A#@p_8-{Jc)W|AYC#iU-ggE~R%7l>=8I9c6qIK`?KXeTPnQ7Y;s3b}NvouJs z1G@`z;0yPhoYqvr*~_By?HH#JAtOijH^s#iJM=HP^GUN1YL_;0??&(?x4G&v8hyVj z7jcm(t2H_kqzQIcIVU_2c^?2%Jc}wBUb1^3;c){IIw zQyy?L*a9_1KLcW=_)t*W(TsrwN?n;lOVTE`C(rK#1x5Pm<8DdC@iHH= znp||zN+T1Oui+3q2d*bOn4JO1J!AUeV>Nl|Y^?O5=y!Op+1Y<+W*1Edp2x2dpKe6{ zU@KUM^br|RcSj+|zhCvAb2(u8ZNF?oaoW1%f!?QGi`RRHqh)3V?!RPdE+ck}X1X?f z6g|V8zwz~ptmH!r4<@s@I=mf zbPy%UF={w$jdKr<5AsqjYxGmiH<9e7eyhc+-4=B*#3~=h$<#?QUNGS$jK)YJ3-9av z*AKRh&nzP3F5VBa7Yv!T{=rt}^nAvu|8w?6MvDp4)rYvFZt87S5z|tEwvN z7t4sd9RSFs^kF@3$(7>7k{s_A?kXO*aVdOodGTuH9#=U0l!Do)*CouqQ2N2P9&q;h z+~x0I^e3A+UM{HA=#X}O6k7I(P!}i$v2}69SFce>@f20^ng0B=D7?+EBHtj^#K+um zUH?m!Ct@GYpXcUg`1F&+y2P=eJdd6$AIneSfIZYp&>m#yfU`*=@tXLLniO0}MilC4 z0&|kw1$&>MrAgs6YPYFg))5TazZly@t0B3uHOP|{r>vU7En)*2A} zt^QVrv~rpvc%cKn(#Eo#%9%Xw1mc|PRr97(k!Dr>1IC1%3vI~FApl#fl@Mv*GVNfb zC|oT5U8}*XboB<0Q3tE?K1NYdee$Om9?#4Mp)bM0M$QCVX1qv#c8+7Zp7&M4!Tu7S zS^Ifkot2G=tI}cEZg)psx?ekKIar%9Kl2`^D_wS?&ZTrXvCPEtxcZX>PO*(*)Ea;} zEn9j83zEOG1Wzm#{b2JTF-}Udxd6)-K36l20+Z)R&kE0kp;`3Tj=%mIPJ8~`R#)Lu z%;KbG>_+Es6~+iPgt2VJ^4k*$D`2Wx6EmIb_|h2jafBPz`eLk=$N2@)}QMJ{O{2bE=oY(dp44S zsJoX^(`KmAa1CoGZn#{qsqvXT8 zt{`{h=hx>_t_><4qMmM|^_TU?cb2{57BR8)7Pz3moP0HSvX^Y=TBC#HQ0t!E-4UbV ze0rTj5Dm0?ZnSnVn8QRBLm&9U?r*5)x_+ZM$;=u^+P>T{{d3%}(9%FQ+EYtD#E>j& zE1i%QL=f-|o!0v>SCX1BQdT5NpEefDyQk!}E4QCN{Jt*7GTF)6MBh)21u_TQ9(=AT z)DUx!9$HX6IaqK==qN*jEJ1mjMxJRu*plKuXx#Jc*=Bl0P9#oC&&Ps8T76)=^!=L& z#`mf;!2^kPB(eFRIe!1>5t&bn9A~_q^Kd|L>}D5<+T{!gZt%c{_`rHg+LkB9E`V`3 z?pn<9i8?IssPmbiSaAt^ng^Bk*;zm+QCb ztj-(D+abI{$x3V0+gdLVOTRq#S+m}xgzfukH7UU5w)zK0O;vlY+{$aJ{x&rPL~d7` z2aV-l{<8rzl@nAx1nX*S^smA)1k=~dKF))5X~Oa4hiZ@BWH+h#*a@9E5l)NZI?i`v z#F2}mTAW{ZqMMw7^c@f9Fw`TU&mY;?M`&v#wWK+%T_l{i@iy0Od$11a=;%UMKAtr) z)}O*B-jzxV-E0LqnEK?jVX&WB$^%XGq8M^GNeW=HN8R|rR{W1Bb1@vx+>pF-2VLk* zT#(dGAJWBHXN?3AB@JRp$La<$WvIq{b;lXr>1UVbbFUCSpviOe9v=X?_1fEKZZr!W z?zps^llgcDaO=QWbgvb`;(pon1w3_aSXuG?q(G$khkTJm;~zJ*6(a-99lVL_YruM$ zb_skasjRp#^WY*(_WgPn1|0%lMUxn_wj$|qq`^X7QrDyhlD?}ZF6L<8;N2G9pV^h6 z(^y@U@8^T3*yqVR}3^R7_(j zFP$IsG108oe3(9VV@8vinbR`7L+QG++~_N)C}!eU1pg@$cmCvy*tp9uzS(g~5$VG< z<_k^NEG%Abhv*q&ar-M(U{WnRsebB;XDxytXq)0KNM>9fspH7m(;l9!=^p6i1* z_LzVpZpZ#5bu#BFX6?z`oyHcK#*&_b@LwYOz!f&Xt>>UTA{?rynkq4C1`4szKZ}Vi z2Nwcj&6anPi)?(pA2ZBAqt8ni>dtu>CND*V;V{hFXaCH? z=>}#Jbghu`{YD?AJvGvtlA3AVAsmR^F zC)6IG--XNh`!m1cqc746$?Io%r6e=SSC{>5U}F|@zBm|VOD6~f{XM)1BiMM+`Jw_G z_Qkbd(zFsAnR*ly4_y+=3|Ww&G}c_7je?~qBd5^6(GGCKyY`G?pnLba;@p*}eWs}4 zDV0P{j==|5zpr^;kM-}myEE1twiqG_d!HSrdgpF-eN8a5?SO?qDl9LZ?7AsI-^|SP zw>8)E4u7hIK1lY`g_pdOp6o3o(cghbXKK&)xetIIX+7$19)1ft{Uz~)gPS&;5coiw z+jFwAhIX}!R(X7QLAmvtP4!C;gNNJ^&p$?#l6(YoM|*aJit*0W$7U+Df_fV#*8pgP z`PO6?IHzO8Bz&)xYxWH;~f6HDzOMLW!Ru zdHwsfq+Q(ydMm5xviBm!d~=)^(@(OY3GD=d3mrWluHoz$VOv*VOOL=Kq`#&y+-Nst z+5k6TZk*F4>1-WS<`p1`SUJF_f?ynGqqVTrLgskt#z_Ww;>0rT181e~Jd5|$`;0y< zM^ro3JzS9g`Yek>6p;DJ}m< zcloH0btdCvZEj688s4zR$T_>n!AI5<`~bjAtushWg)Nq(pswB1=_G|K-2zhFxF5Q4 z>E<__2@g%0*Pzpp&mM-mZz0Zh7SUo*R-AKP71T&xCr0`m(WTNmNo_MB%Ev_hG*N9Go%~`pVW=J z(hNLtw_|X_;AqLv-06M!{+ja3z)a{|2F)Dw5gzbgXSWc^c<@9H1$vVttW6%f7)Z&T zeiWY`W`g(sT(cIEr!700O;)LMdtMv!3-RhOiko&~M5=Mm!| zT!Ych>j^lo4LHh13tu~P6vdvY;NX}&Jl?8R-l?_dA0-U+exlM4+m;GwzTVk|h^AHn zq{hvIr@H}n9W~@`ztgLy0>f$u28M>)-2h#UE!l6E;l_}aJ-%fng}Fq9_8Sf2YYm!iXE0Zu0n?nP;ZHa0kPgVyv^l_-DL%BR<2-bAQH&%xd~bblSkWP9 zohPS5zk3vm-fE>G=l=jlq-VH_elTWogJHJi)Y6#BeHmz(<TBG0a?*2#S#dDU7uz$6iuUQ~9maIV&4|cT-!qoOqDj4aFRPsA*hU-+ z#~R8BMykNM;R11sE!FM^^7vJs?(?XFLm5J2^jfsjHh)Uf54J1IZQ_0T>hnz&F06D& z!>^6;HRXV+=ICV6x#N_#xT7~Dt`5W_$ga^X1L$zG5MV_2W%ZYnwe`o3S)NjZ6K`kt z)>VCglh@VF?ekk9(P`&nBTKbRn>YNhJ3$phiK4d+F$T1a58nEtHOn~-wM2wmOvwDB zQsudP0iF?IxyD?e;P~L{x<*&mo#8pjc2yR>KJ#9qDnf3wUWDJ!h{(ba4lCeOjBVlU zZp0O|xjADLp>})5Cxz(mZ63)Bn%?MJj7bjRQ6MG%bLY~hFgR_I(k4pJG-oP8v2bvrQR=cRn&Og}NMxO9j+sP$QZ%m=r zuIl}o*Smwhr?0GCO}z(R2Bcg-`pidWNtc$|9Y5Hb_#=+3Kl3%zvp@?EN?PSfL#cb9 z>FNtxvR=Qo?}^Vbi3H>Bi2b(bY#&r34JbR13J8%1auQE&DoX3_11GhG$# zTH-fn2W%T>+6B$WWRUM?ot$OL39;yafK`HaGjCChsbP?LY8VyTC2zBkzlxO&#mzO# z%&pI5M>8HDo;H&%ZW~;1L;~v~r&qK;1~c0)n}ZK6-A01Lu?Y7kZC#d}UnnP|`Y8Ao zfo@xGS3UpD)qU9Tmv;B6*4)1yDcX^Jf?(i%l|A5IOU7*Bo3B!GrmQr$@ph~x{dlW| zw-d!Tox=EXR&nA(z{*dYWKA8y2jc>dD~~eJ>RwBOZt954AZ-+B?&^QvBE$6XuRu}O~(4Qb%Ox4e!p9U^)S%ooZggIy(s#5xO4QfLl} zTCu7tWGaczhnIq0Wh!n6D_s0wi;fL}CcYmI21yu@n#JSxYT7{}hpO^AJ7ISk9(ucp zAom{>0v0`W-(^gL4q-XoQ8#+$8!X>$&%IupsBLCQuM2v2(b7=uk5v%P{QZs7N0g2N zF{{0noNkQ~780825xD#W!zJ>TA&ywL?5j42S1+gMceFK9IGoeyz6LUBd-GR1%SwKr*yMo$TCmrM*-DRWCnTyr?pfMpm#jI{ zK`k*X>rvMJ%So}k(=M~tN&2X2OFFROc##clB0`a>qip4tjIQYSq2g?4~+55jq>L`LkTwZUUI0dYp z9e00|5ZyJ~9fM4`K^huAdj4P=Uf%0H>_hRcmX@31zpkI}%phHS1E7qO2qEPIJ>^C@ z!{YCC6U|134>oZ1jM(!P`v=J2Uggy7`Ljo45eqTAe_4Sc`7hmon>xA@>EQr zVK+=v<93QSfJ-8aitDT|V=1iL3lrCc_rxuVY`^&uQ_QdhA zM}5npAM?i(8^vWOtbm4#g7@6KteZ_5V(A!f*yd>ruji3hDVVTRkteBd!dc-gd>oeA z1uuQbJ#5!{%fk6syg!yRS)N!Gy%P=elgXc9PH4o9x|U-w;srR0p__=j8$0PFE?8 zyFb`m8E(wt7VRED-c^0++1?K}_tz2&nDwa(QO29>Er#6k{^iK>%o&844~~;$vW|5_ zUO(7UVV7FQrr4_|pfSwIU32jC$;-oErqyPSAgszR;dKn%Y0P1t-y`rrE1$SmkiE8C z3Ua$0I@znS@UHo$rzmt77owzovm)WPB16jhm|P??#z$y({iRjQyE)BGddVwmI?fB* zGSJtui6`VHau1qthnt4A*dtBU(!?;5JQz*JKaDh9gjvOz%2xKNpl+_VD8jbg90RC1 zy1xBxO$eGoNkcY3G4IHTXmV58+X=g+ErfS<5w~lOerRyNV!LHaEs`G6*JYx^(b9hJ z6wVG2hK+IKHdcv3*6ASQ>BFq96XJrl&|*hR4+Sih3#nQQ)GtZ}X4n-{-J5c7N+Pb)ozsoO0uy~^YY>TQJo1^|gtL?n z$|#q6Fd?67amsFL$V|Z0F?1b8z6h1crt)?JZ+u)zJO~t#HOv@X(MSYtjHwxR<|36( zYoo&_>wL4(X)qCsk$-a>=KQLB__TvHu!8A>_dX$Re{wC}8Ak=|gD`8GI~CSOhLOWk zVUov#&zYFJw6~V+zI2=5j117TbvpgaXRu1g5xxvwe8C)R-8@xT$o&S)vAx~4Nw77u z6=|WKSI1rjxS~D`YhFv(~LD!wUn=kj2l^ccDT58IMuUb8+H|Mx8~%$ zGufp|uAfyp`7=gm*(G({^-C$VQV1B!&HIq^`F_XFQjvd@4MHtefyT>nx7HhDy{cVN z1<@#iyN)UC`J-7~jAw}$M(YO~hgc7NBw-IKGmUs7dBahHm~z#f zhh1I{ok6sbLOyw)1R>&E{bi~rN9;f1!)&$c`1qT@Jdco?j_-2vn4PL9OxOyFE-RG} zgAY18(zhI?EShEXVMqJA09}Zt_Q>% zEvI3d(br;i(yzls>2UVDy}(yJqf0ZO|MF0!CTW|Gbt!|5#ktHoi!dMlV7nQGG3@Cj3$x_-3MeR{ko)wd z@L$+bSi37wWF4eetOK&Vk~mo(T;n(Z#j|eJy7D&y0;5Dn6-n?E z^BMFVeJscXnOo;11jG8rbXoF!=+i^t(yykwQ}Xxp^*p=jfoIi|C%b_7E>AZ-Pa9h8 z$yvEXTO-Xp*>9Un&Y+R7S3`?+-tly^nu%Z0)7y}Lzskc|QqlPq`-TT98y|G!hJ?$z zYG8^qxaUr1XB!u^iM`>wxfkRk?idHt?j)1eC_~izR_1|Y7-a4FT-=j1fl#I*O*Dh4e*)EJ z@{#@c6wSdia|<@qOZ>GG2eoK}L%U1jbbfg3LPqL)j%psK_GhE)0eS20FPLWf;G@jK}_N(D`svJ@#Gj$P}r4PCeNrc$W zPQR;e8ygRmY|?Q@3t1^U>xG9A98^W_PnSZ=;!_a z6dJAYfKA9cX4F!25OT&?nb?jW08N|zKB|?-Nch1`RAZ8Hc!HJ1VWTCK) z^K%GhgQdGV3Y#$VLp;8-#@ivk?&?G4*zYlr@{#jFhAU1Pw-y+cbb*0*<>u+`CbHC5 zYZvzHU6Gw7k)!{ukXj%w8_rUQ=iF-}jEQ43Na!m8Y3#ZPj#xv?>s@AuaX;B_i@RsB z(Ad}N<(iFCXgg-ICX-o>5$;vnpt_!XH}Q97mCL0 z-j@>X#XS~SOW$!fo+9&y_Fs}&{oSh=KfA^0dIrA?tg}4ZBnW0#WF7O7pQ?k)X3=V0WvcQ%?UeR5?{>K-Q4j6Q2j>Qt z@HSogi`L$C`-5iD4aFr5*_%k~p5{~A5c7cE$odjt4mrh?)2$Ltq~wlR`a*5FQ1sM= z<;zr*28Ij6XVJT`1~T)N=pF@QnhhE9@786Y<(~$zq4GP2@bTb>VjvUa zvwtHzMQjDPInuU|MzG2^!kvE}_IhB}`NO#M`k@jFLWQCc0! z^vI;)!CoizSmOf7YF|C-e!F`FZ%{+CS+Pm~JV)VfouZ$<8z#xtX;3cWES`%$t@Cnv zjbXJ}B(~+6ib^~~=x~(WSTly3I(Q#X6Fx_x5MKZwA3fAm2Wi86Y3X(Z%|^loXo{IFx%2ksk!+7@fTL9?O_mo{?MI+$Hdqa^3YE^NpV`8M6}8 z@(c@Tf5~l^cb9%UODtsr`{$)OgzaZBfDTHfSasc@ zM%vMv_JRdP_5C~-qxpKLFgx;Tz3Q=Py3+)BF(OIIUv0dvxJvuJ<&k-^xf!vdJXXEY z6-9Q*Jfbn=lXAqr?rTp1{(p9XdZFB_PkmQdi12f@U}ll$6y}kP7HMO(>A*FsMd=?6 zGuL$y=~v^2^7m*pti8yfOLL7FGConZ*^BO!B4sRa4a(7)fl7u_p1xRh%PBAV_(ill zB;A=cI08?2WBSK2JZ*JxLgC{#TL?AjnTD8kWqe{zanHWZM2aR!OV^WM5}x@kyvtguZs`0Q!6WVZg33!!iff&)vA&l4V?@FzeG#V_{;}#C=n^ z2lb+UB1y8Ho0p?RqHg!|CqCIi<{00v^qo72Wr$-x%~cHfXkFv;+Hq*Iu(L~Vn^j!| z-WIWuzO??k=G#KUX?qu~nW%|m-d?%NF#9LBt^$his-hn|aX-FH{vy+8sR@if@gbE>PI5m!Nf*~rJ&G_ zWVd29T~ieB=9hIHflo656E3^qrLnAbNryYki#_eFMbmX_zaV-L6|xbOaZv`{YUgZripC?I=f>Mlwymuv=X%;?UaICm-)lSbIbHVT!6br z!UM5lL1$?f1hlL_x!tnpUNeQlaLZ%&9mXjw7?DJ+S2Sdy(@#T0iBtrgVO+`ZSiJk^V2i zy&<8d@`(ZML7J7`^Wc+K{~}yxfckRiM7`o?Lu0+)29t4eQj%-q`F{r0!D&xSj7>C} zUeJW3IX-Vde_KOIsFwt;%6ZRF_d0?)zC)ogH~fJrtZEGJI|*vy&>{2uu4mkz`rNkq zz00c(4OC*dRCXh*o*abKH!XbiST*T=#A(GcSg}$oD&6ogp>UiNu5HJmJDt$SOk64k z3t%-tax{?O;q{Us$JpZXdhY^Zwfrb!iUL8?)Hz8^$@dV~`IO8V8x!W~mW{;aypV8R zOW(2Qu#G;u?ypQ!tgeYXH87NPR7V{}>fXoSr`L#!voblBHcw?u0JgjrX82ECDRCm_ z{bOpa8+nIdtZl@Q&m6l)WN6X!wfJrcEk~5BZ%paukJBOH0`8Ne7_Je`(%gj^0o^G7 zO9}q&UaPkIg`JyCCt!M|{E(vR4!A%^w#dmLQkQm@m$ZD{D^5AEStw9V$he^71&KXc z$P-xL)(Mg<$`rbm!eh5nbkP`u-nFOaIN?U$iBYHb(yw|ocSjUQcc3LO-?riuJyacw zn;~UuZC-)4QlTcVL_E|F=jx=~C|izS3h=C0Y%NVLuf%4~yh~-24iXj;a@!UW7NuU6 z9^MXe1wSsOl_I6Kzp9Dmol{U~iWiAN>fPYBn!Q%hG5-%hLPyk}&76zwy>l1I@s zZAXs}|5T_4S*M>d@Hd#c)!$HbPk^Q_eIo-z&#Qv0*Twx{YX$UPsxMQFS&3ZZE0aWq zZtwrD3wJi)20z#Y4_P1IQ!!&SiJVEcKs*H)SO`gEW)9NJ&|Q;oEdqP2>54`2YLP(l zuY3pm(!VRvG>p6Aq@u)J;ni6PSf4A`X46Gr5i=MANYEa#t zHD%ANIwA4GIVD%{`Oy^-1nUTcFEiyLUcU*zSM!gl^C97iEr58ev)X5^rA;(aK2t?t z&d4vOy+A1H{9fwWV%lgDAdTL7(`kCCs%y%}tP;>`rRgc?Bfjz2P?wq1ANPnTTX5IR zijte*b5s!`_Xk6Lsh|T`f-IR>(|{Gs7F9SR64#l}21owuHYv(;-F_gb7(S3XjDvba zRPzeC+|0JlsDFX83thiRTClfn+YX(q+He%cB(QAEMw9138nz9q-CGys$C&TwnszTG zc;+tR8f~E>_z{^V&$hk+fd7C|^g(0i@qd@c{QH^F`nQV(8Fi+t~7E zoxLAJcpHV!F&N*X%MyX@iSu*%Hqd$N95df^%zR1+QLS^@v!Hk}6oI2T3WBD2 z3PIBqtYOITR^-Xm(+z;FI=+C7GWh~bM=Km;(u4}IMZtyPLP`e@{EenI`X*TIx8jB# zbCKiK#&PgTG)m9V5n#CYptr(&AoD0N2b1g-_vr1;T-sJ4eirjrK2ttg zH99sjuh!+vS!Y%A67+D@4e5Fhb2#9_N~U?%*;cmJTMw@W5hB#g@D0`*0e#RtEnbU- z08sm*M=E#6TF+RwnYLN#P-m~Os{lD zV{U(k-{^d}EiB6C*-WdX!B}1O@K!<7FMRc{YBa@dZHwqTNFQrLa9PUouRA^-@17eB zP&&D=%W2t}iNQmC^1DxfO>5W}m2_3*9SQZN&u0?aR=!2HUYCIm zSFs!-i6xEAW^Z~6a8d@F5V)lqE7aZVMQx~Xf(N`R9|t43ZA;e&4{u04)6Zt+2q~&= zT+MAUf`a(w$SGw?&t_HOAPjDVkiMlBw8(=9wOUSnhyZO2dk*>1gEnyT<;;JQ3RnHF zt-=J4^E*Pi!JhCSk8BH1hnE~h35X<=KO=qcKH(+$!>8tks1&ZJ10%uYhWYGjZyh~- z_=|A=I1f_D(0Eh$0>kZOmE3XnRF*5Kriu#FGdG9TE<}YSsy7rL%Pc<#QGH*1X!^Ex zVD50f3!j=O2`!?9h52MpJvXGC8!5`i`h=BmS4}zjY+({`W?mi1A zp-??MQlQ~yF_uaXJqEIJ{gO?H09L-xeYF63h9i93;f=nQ;p@}t(yc?&LeG5 zPqzcW5khl^{YBqHhdJ26>ajVE;81+JY163snBA9)Onkw{#AxoCnB#A637xmD)Kr-U za~dG6J}v7a;+w_#g~U9p!H8qFLuVRXrKqmnO>Phx?+#6ibRV$?C*6K9_X)SFUuyYT z7SW{zEsR|2*?i)9?bj|ronZ$#O`oCW&IS(4aP_W=6UzC~kvKhx_h-o{)TGk%;E<`! zQWQL^rPOomkn<5`eXgkksbfWUQ~?s~KPTLjOyo_Tt0Gk@xYW$-B&aHA#&nh;Zg*a&sk-63rGJ_jce|qKJ{%ZZBW?#6p>*c){ zg7&NKGTi&F$oXn$*!!*Jd>=Pw%kKN>wCrnPm6yD~53oZewx#aUygxQI^UPf>D?in@ z3YKPnI9XL5aYoHgcl*g#CKD%Z|53GG<;D8LtC!TgOX;?~r?GSX`^HUM*S@TjTHCqz z#MAKAPhMYH_aNY9!Lj`Fi=zHB$on4&tPgsWy>9d6)1kjO`98W|ntsrq=drq%8MrX*ydC}?R=Mf^=#3iAwa0 z+h3oI*S9(PoL_m%yz~cmv%{Y9yyMpHIQZ7+{n_)6A0Ew|b#J>#rOJ_)H+71KuUo_PA>c9+F_$5T%Xfj!v#CtKb+^RK(Nvp+iE`j#!Z zQO7Mm)_B}hKU`F9y85cHFLX3)UIbTnYNa@wR9R k(L)*}1<$DB(GVC7fzc2c4S~@R7!85Z5TITN)c?N;05f+AZ2$lO literal 0 HcmV?d00001 diff --git a/docs/tutorials/basic_usage.md b/docs/tutorials/basic_usage.md new file mode 100644 index 00000000..0dfc74b8 --- /dev/null +++ b/docs/tutorials/basic_usage.md @@ -0,0 +1,222 @@ +# Basic Usage + +## Requirements +- Installation of Kruise, Reference [Install OpenKruise](https://openkruise.io/zh/docs/installation/). +- Installation of Kruise-Game, Reference [Install Kruise-Game](../getting_started/installation.md) + +## Deploy GameServerSet +This is an example of GameServerSet, which manages 3 game servers. +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: minecraft + namespace: default +spec: + replicas: 3 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + gameServerTemplate: + spec: + containers: + - image: registry.cn-hangzhou.aliyuncs.com/acs/minecraft-demo:1.12.2 + name: minecraft +``` +When the deployment is complete, the cluster will generate 1 GameServerset, 3 GameServers, and 3 Pods corresponding to GameServers +```bash +kubectl get gss +NAME AGE +minecraft 9s + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-2 Ready None 0 0 + +kubectl get pod +NAME READY STATUS RESTARTS AGE +minecraft-0 1/1 Running 0 10s +minecraft-1 1/1 Running 0 10s +minecraft-2 1/1 Running 0 10s +``` + +## Game servers scale up +Directly adjust the number of replicas to the desired number +```bash +kubectl scale gss minecraft --replicas=5 +gameserverset.game.kruise.io/minecraft scaled +``` + +The number of game servers eventually became 5 +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-2 Ready None 0 0 +minecraft-3 Ready None 0 0 +minecraft-4 Ready None 0 0 +``` + +## Game servers scale down by deletion priority +Manually set the GameServer deletionPriority (you can set the deletionPriority automatically through the ServiceQuality function) +```yaml +kubectl edit gs minecraft-2 + +... +spec: + deletionPriority: 10 #initial value is 0，turn it up to 10 + opsState: None + updatePriority: 0 +... +``` +Scale down +```bash +kubectl scale gss minecraft --replicas=4 +gameserverset.game.kruise.io/minecraft scale +``` + +The numbers of game servers eventually became 4. It can be found that the No.2 gs with the highest deletionPriority has been deleted. +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-2 Deleting None 10 0 +minecraft-3 Ready None 0 0 +minecraft-4 Ready None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-3 Ready None 0 0 +minecraft-4 Ready None 0 0 +``` + +## Game servers scale down by OpsState +Manually set the GameServer OpsState to `WaitToBeDeleted` (you can set the OpsState automatically through the ServiceQuality function) + +```yaml +kubectl edit gs minecraft-3 + +... +spec: + deletionPriority: 0 + opsState: WaitToBeDeleted #Initialization is None, will be changed to WaitToBeDeleted + updatePriority: 0 +... +``` +Scale down +```bash +kubectl scale gss minecraft --replicas=3 +gameserverset.game.kruise.io/minecraft scaled +``` + +The numbers of game servers eventually became 3. It can be found that the No.3 gs with WaitToBeDeleted OpsState has been deleted. +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-3 Deleting None 10 0 +minecraft-4 Ready None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-4 Ready None 0 0 +``` + +## Specify game server offline +Specify the game server with serial No.1 to go offline +```yaml +kubectl edit gss minecraft + +... +spec: + replicas: 2 #replicas is reduced by 1, adjusted to 2 + reserveGameServerIds: + - 1 #specify serial No.1 +... +``` + +The numbers of game servers eventually became 2. It can be found that the No.1 gs has been deleted. +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Deleting None 0 0 +minecraft-4 Ready None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-4 Ready None 0 0 +``` + +## Game servers update by update priority + +Manually set the GameServer updatePriority (you can set the updatePriority automatically through the ServiceQuality function) + +```yaml +kubectl edit gs minecraft-0 + +... +spec: + deletionPriority: 0 + opsState: None + updatePriority: 10 #initial value is 0，turn it up to 10 +... +``` + +Update game servers' image +```yaml +kubectl edit gss minecraft + +... +spec: + gameServerTemplate: + spec: + containers: + - image: registry.cn-hangzhou.aliyuncs.com/acs/minecraft-demo:1.13.0 #update images tag to 1.13.0 + name: minecraft +... + +``` + +Pay attention to the update process, you can find that the GameServer with a larger updatePriority is updated first +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Updating None 0 10 +minecraft-4 Ready None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 10 +minecraft-4 Updating None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 10 +minecraft-4 Ready None 0 0 +``` \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..65151d22 --- /dev/null +++ b/go.mod @@ -0,0 +1,89 @@ +module github.com/openkruise/kruise-game + +go 1.18 + +require ( + github.com/davecgh/go-spew v1.1.1 + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/gomega v1.18.1 + github.com/openkruise/kruise-api v1.2.0 + k8s.io/api v0.24.0 + k8s.io/apimachinery v0.24.0 + k8s.io/client-go v0.24.0 + k8s.io/code-generator v0.24.0 + k8s.io/klog/v2 v2.60.1 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 + sigs.k8s.io/controller-runtime v0.12.1 +) + +require ( + cloud.google.com/go v0.81.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-logr/logr v1.2.0 // indirect + github.com/go-logr/zapr v1.2.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-cmp v0.5.5 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.12.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.1 // indirect + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/apiextensions-apiserver v0.24.0 // indirect + k8s.io/component-base v0.24.0 // indirect + k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect + k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..351d4606 --- /dev/null +++ b/go.sum @@ -0,0 +1,1017 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/openkruise/kruise-api v1.2.0 h1:MhoQtYT2tRdjrpb51xhn3lhEDWSlRGiMYQQ0Sh3zCkk= +github.com/openkruise/kruise-api v1.2.0/go.mod h1:BKMffjLFufZkj/yVpF5TjXG9gMU3Y9A3FxrVOJ5LJUI= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 h1:hI3jKY4Hpf63ns040onEbB3dAkR/H/P83hw1TG8dD3Y= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.15/go.mod h1:X3JDf1BiTRQQ6xNAxTuhgi6yL2dHc6fSr9LGzE+Z3YU= +k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg= +k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= +k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY= +k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= +k8s.io/apimachinery v0.20.15/go.mod h1:4KFiDSxCoGviCiRk9kTXIROsIf4VSGkVYjVJjJln3pg= +k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= +k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= +k8s.io/client-go v0.20.15/go.mod h1:q/vywQFfGT3jw+lXQGA9sEJDH0QEX7XUT2PwrQ2qm/I= +k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U= +k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= +k8s.io/code-generator v0.20.15/go.mod h1:MW85KuhTjX9nzhFYpRqUOYh4et0xeEBHTEjwBzFYGaM= +k8s.io/code-generator v0.24.0 h1:7v52LjqCntfGxV9x8c57gkhDqkMHd0Z2jfRqGr6it6g= +k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= +k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= +k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 h1:TT1WdmqqXareKxZ/oNXEUSwKlLiHzPMyB0t8BaFeBYI= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= +sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLONFFoI= +sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 00000000..308f268b --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/hack/gencerts.sh b/hack/gencerts.sh new file mode 100644 index 00000000..868b528e --- /dev/null +++ b/hack/gencerts.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generates a CA certificate, a server key, and a server certificate signed by the CA. + +set -e +SCRIPT=`basename ${BASH_SOURCE[0]}` + +function usage { + cat<< EOF + Usage: $SCRIPT + Options: + -h | --help Display help information. + -n | --namespace The namespace where the Spark operator is installed. + -s | --service The name of the webhook service. + -p | --in-pod Whether the script is running inside a pod or not. +EOF +} + +function parse_arguments { + while [[ $# -gt 0 ]] + do + case "$1" in + -n|--namespace) + if [[ -n "$2" ]]; then + NAMESPACE="$2" + else + echo "-n or --namespace requires a value." + exit 1 + fi + shift 2 + continue + ;; + -s|--service) + if [[ -n "$2" ]]; then + SERVICE="$2" + else + echo "-s or --service requires a value." + exit 1 + fi + shift 2 + continue + ;; + -p|--in-pod) + export IN_POD=true + shift 1 + continue + ;; + -h|--help) + usage + exit 0 + ;; + --) # End of all options. + shift + break + ;; + '') # End of all options. + break + ;; + *) + echo "Unrecognized option: $1" + exit 1 + ;; + esac + done +} + +# Set the namespace to "kruise-game-system" by default if not provided. +# Set the webhook service name to "kruise-game-webhook" by default if not provided. +IN_POD=false +SERVICE="kruise-game-webhook" +NAMESPACE="kruise-game-system" +parse_arguments "$@" + +TMP_DIR="/tmp/kruise-pod-webhook-certs" + +echo "Generating certs for the kruise-game pod admission webhook in ${TMP_DIR}." +mkdir -p ${TMP_DIR} +cat > ${TMP_DIR}/server.conf << EOF +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = DNS:kruise-game-webhook.kruise-game-system.svc +EOF + +# Create a certificate authority. +touch ${TMP_DIR}/.rnd +export RANDFILE=${TMP_DIR}/.rnd +openssl genrsa -out ${TMP_DIR}/ca-key.pem 2048 +openssl req -x509 -new -nodes -key ${TMP_DIR}/ca-key.pem -days 100000 -out ${TMP_DIR}/ca-cert.pem -subj "/CN=${SERVICE}_ca" -addext "subjectAltName = DNS:${SERVICE}_ca" + +# Create a server certificate. +openssl genrsa -out ${TMP_DIR}/server-key.pem 2048 +# Note the CN is the DNS name of the service of the webhook. +openssl req -new -key ${TMP_DIR}/server-key.pem -out ${TMP_DIR}/server.csr -subj "/CN=${SERVICE}.${NAMESPACE}.svc" -config ${TMP_DIR}/server.conf -addext "subjectAltName = DNS:kruise-game-webhook.kruise-game-system.svc" +openssl x509 -req -in ${TMP_DIR}/server.csr -CA ${TMP_DIR}/ca-cert.pem -CAkey ${TMP_DIR}/ca-key.pem -CAcreateserial -out ${TMP_DIR}/server-cert.pem -days 100000 -extensions v3_req -extfile ${TMP_DIR}/server.conf + +if [[ "$IN_POD" == "true" ]]; then + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + + # Base64 encode secrets and then remove the trailing newline to avoid issues in the curl command + ca_cert=$(cat ${TMP_DIR}/ca-cert.pem | base64 | tr -d '\n') + ca_key=$(cat ${TMP_DIR}/ca-key.pem | base64 | tr -d '\n') + server_cert=$(cat ${TMP_DIR}/server-cert.pem | base64 | tr -d '\n') + server_key=$(cat ${TMP_DIR}/server-key.pem | base64 | tr -d '\n') + + # Create the secret resource + echo "Creating a secret for the certificate and keys" + STATUS=$(curl -ik \ + -o ${TMP_DIR}/output \ + -w "%{http_code}" \ + -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "kind": "Secret", + "apiVersion": "v1", + "metadata": { + "name": "kruise-game-webhook-certs", + "namespace": "'"$NAMESPACE"'" + }, + "data": { + "ca-cert.pem": "'"$ca_cert"'", + "ca-key.pem": "'"$ca_key"'", + "server-cert.pem": "'"$server_cert"'", + "server-key.pem": "'"$server_key"'" + } + }' \ + https://kubernetes.default.svc/api/v1/namespaces/${NAMESPACE}/secrets) + + cat ${TMP_DIR}/output + + case "$STATUS" in + 201) + printf "\nSuccess - secret created.\n" + ;; + 409) + printf "\nSuccess - secret already exists.\n" + ;; + *) + printf "\nFailed creating secret.\n" + exit 1 + ;; + esac +else + kubectl create secret --namespace=${NAMESPACE} generic kruise-game-webhook-certs --from-file=${TMP_DIR}/ca-key.pem --from-file=${TMP_DIR}/ca-cert.pem --from-file=${TMP_DIR}/server-key.pem --from-file=${TMP_DIR}/server-cert.pem +fi + +# Clean up after we're done. +printf "\nDeleting ${TMP_DIR}.\n" +rm -rf ${TMP_DIR} diff --git a/hack/tools.go b/hack/tools.go new file mode 100644 index 00000000..6d8584d1 --- /dev/null +++ b/hack/tools.go @@ -0,0 +1,25 @@ +//go:build tools +// +build tools + +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This package imports things required by build scripts, to force `go mod` to see them as dependencies +package hack + +import ( + _ "k8s.io/code-generator" +) diff --git a/main.go b/main.go new file mode 100644 index 00000000..0e410266 --- /dev/null +++ b/main.go @@ -0,0 +1,150 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + controller "github.com/openkruise/kruise-game/pkg/controllers" + "github.com/openkruise/kruise-game/pkg/webhook" + "os" + "time" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + utilclient "github.com/openkruise/kruise-game/pkg/util/client" + //+kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(gamekruiseiov1alpha1.AddToScheme(scheme)) + utilruntime.Must(kruiseV1beta1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + var namespace string + var syncPeriodStr string + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8082", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.StringVar(&namespace, "namespace", "", + "Namespace if specified restricts the manager's cache to watch objects in the desired namespace. Defaults to all namespaces.") + flag.StringVar(&syncPeriodStr, "sync-period", "", "Determines the minimum frequency at which watched resources are reconciled.") + + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // syncPeriod parsed + var syncPeriod *time.Duration + if syncPeriodStr != "" { + d, err := time.ParseDuration(syncPeriodStr) + if err != nil { + setupLog.Error(err, "invalid sync period flag") + } else { + syncPeriod = &d + } + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + Port: 9443, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "game-kruise-manager", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + Namespace: namespace, + SyncPeriod: syncPeriod, + NewClient: utilclient.NewClient, + }) + + if err != nil { + setupLog.Error(err, "unable to start kruise-game-manager") + os.Exit(1) + } + + // create webhook server + wss := webhook.NewWebhookServer(mgr) + // validate webhook server + if err := wss.SetupWithManager(mgr).Initialize(mgr.GetConfig()); err != nil { + setupLog.Error(err, "unable to set up webhook server") + os.Exit(1) + } + + //+kubebuilder:scaffold:builder + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + go func() { + setupLog.Info("setup controllers") + if err = controller.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to setup controllers") + os.Exit(1) + } + }() + + setupLog.Info("starting kruise-game-manager") + + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go new file mode 100644 index 00000000..e3ff8929 --- /dev/null +++ b/pkg/client/clientset/versioned/clientset.go @@ -0,0 +1,120 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + "fmt" + "net/http" + + gamev1alpha1 "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/typed/apis/v1alpha1" + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + GameV1alpha1() gamev1alpha1.GameV1alpha1Interface +} + +// Clientset contains the clients for groups. Each group has exactly one +// version included in a Clientset. +type Clientset struct { + *discovery.DiscoveryClient + gameV1alpha1 *gamev1alpha1.GameV1alpha1Client +} + +// GameV1alpha1 retrieves the GameV1alpha1Client +func (c *Clientset) GameV1alpha1() gamev1alpha1.GameV1alpha1Interface { + return c.gameV1alpha1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.gameV1alpha1, err = gamev1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.gameV1alpha1 = gamev1alpha1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/pkg/client/clientset/versioned/doc.go b/pkg/client/clientset/versioned/doc.go new file mode 100644 index 00000000..5f8d7a83 --- /dev/null +++ b/pkg/client/clientset/versioned/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated clientset. +package versioned diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 00000000..39394cdd --- /dev/null +++ b/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,84 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + clientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + gamev1alpha1 "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/typed/apis/v1alpha1" + fakegamev1alpha1 "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// GameV1alpha1 retrieves the GameV1alpha1Client +func (c *Clientset) GameV1alpha1() gamev1alpha1.GameV1alpha1Interface { + return &fakegamev1alpha1.FakeGameV1alpha1{Fake: &c.Fake} +} diff --git a/pkg/client/clientset/versioned/fake/doc.go b/pkg/client/clientset/versioned/fake/doc.go new file mode 100644 index 00000000..5e58521b --- /dev/null +++ b/pkg/client/clientset/versioned/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go new file mode 100644 index 00000000..8ca8c4a3 --- /dev/null +++ b/pkg/client/clientset/versioned/fake/register.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + gamev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + gamev1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/pkg/client/clientset/versioned/scheme/doc.go b/pkg/client/clientset/versioned/scheme/doc.go new file mode 100644 index 00000000..643aa824 --- /dev/null +++ b/pkg/client/clientset/versioned/scheme/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go new file mode 100644 index 00000000..494b657d --- /dev/null +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + gamev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + gamev1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/apis_client.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/apis_client.go new file mode 100644 index 00000000..c27a2833 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/apis_client.go @@ -0,0 +1,111 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "net/http" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type GameV1alpha1Interface interface { + RESTClient() rest.Interface + GameServersGetter + GameServerSetsGetter +} + +// GameV1alpha1Client is used to interact with features provided by the game.kruise.io group. +type GameV1alpha1Client struct { + restClient rest.Interface +} + +func (c *GameV1alpha1Client) GameServers(namespace string) GameServerInterface { + return newGameServers(c, namespace) +} + +func (c *GameV1alpha1Client) GameServerSets(namespace string) GameServerSetInterface { + return newGameServerSets(c, namespace) +} + +// NewForConfig creates a new GameV1alpha1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*GameV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new GameV1alpha1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*GameV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &GameV1alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new GameV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *GameV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new GameV1alpha1Client for the given RESTClient. +func New(c rest.Interface) *GameV1alpha1Client { + return &GameV1alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *GameV1alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/doc.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/doc.go new file mode 100644 index 00000000..c0415069 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/doc.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/doc.go new file mode 100644 index 00000000..4525eb5a --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go new file mode 100644 index 00000000..69d83909 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/typed/apis/v1alpha1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeGameV1alpha1 struct { + *testing.Fake +} + +func (c *FakeGameV1alpha1) GameServers(namespace string) v1alpha1.GameServerInterface { + return &FakeGameServers{c, namespace} +} + +func (c *FakeGameV1alpha1) GameServerSets(namespace string) v1alpha1.GameServerSetInterface { + return &FakeGameServerSets{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeGameV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserver.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserver.go new file mode 100644 index 00000000..8e11c857 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserver.go @@ -0,0 +1,141 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeGameServers implements GameServerInterface +type FakeGameServers struct { + Fake *FakeGameV1alpha1 + ns string +} + +var gameserversResource = schema.GroupVersionResource{Group: "game.kruise.io", Version: "v1alpha1", Resource: "gameservers"} + +var gameserversKind = schema.GroupVersionKind{Group: "game.kruise.io", Version: "v1alpha1", Kind: "GameServer"} + +// Get takes name of the gameServer, and returns the corresponding gameServer object, and an error if there is any. +func (c *FakeGameServers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.GameServer, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(gameserversResource, c.ns, name), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} + +// List takes label and field selectors, and returns the list of GameServers that match those selectors. +func (c *FakeGameServers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.GameServerList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(gameserversResource, gameserversKind, c.ns, opts), &v1alpha1.GameServerList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.GameServerList{ListMeta: obj.(*v1alpha1.GameServerList).ListMeta} + for _, item := range obj.(*v1alpha1.GameServerList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested gameServers. +func (c *FakeGameServers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(gameserversResource, c.ns, opts)) + +} + +// Create takes the representation of a gameServer and creates it. Returns the server's representation of the gameServer, and an error, if there is any. +func (c *FakeGameServers) Create(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.CreateOptions) (result *v1alpha1.GameServer, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(gameserversResource, c.ns, gameServer), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} + +// Update takes the representation of a gameServer and updates it. Returns the server's representation of the gameServer, and an error, if there is any. +func (c *FakeGameServers) Update(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (result *v1alpha1.GameServer, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(gameserversResource, c.ns, gameServer), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeGameServers) UpdateStatus(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (*v1alpha1.GameServer, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(gameserversResource, "status", c.ns, gameServer), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} + +// Delete takes name of the gameServer and deletes it. Returns an error if one occurs. +func (c *FakeGameServers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(gameserversResource, c.ns, name, opts), &v1alpha1.GameServer{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeGameServers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(gameserversResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.GameServerList{}) + return err +} + +// Patch applies the patch and returns the patched gameServer. +func (c *FakeGameServers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServer, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(gameserversResource, c.ns, name, pt, data, subresources...), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserverset.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserverset.go new file mode 100644 index 00000000..3fdf7452 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserverset.go @@ -0,0 +1,141 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeGameServerSets implements GameServerSetInterface +type FakeGameServerSets struct { + Fake *FakeGameV1alpha1 + ns string +} + +var gameserversetsResource = schema.GroupVersionResource{Group: "game.kruise.io", Version: "v1alpha1", Resource: "gameserversets"} + +var gameserversetsKind = schema.GroupVersionKind{Group: "game.kruise.io", Version: "v1alpha1", Kind: "GameServerSet"} + +// Get takes name of the gameServerSet, and returns the corresponding gameServerSet object, and an error if there is any. +func (c *FakeGameServerSets) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.GameServerSet, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(gameserversetsResource, c.ns, name), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} + +// List takes label and field selectors, and returns the list of GameServerSets that match those selectors. +func (c *FakeGameServerSets) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.GameServerSetList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(gameserversetsResource, gameserversetsKind, c.ns, opts), &v1alpha1.GameServerSetList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.GameServerSetList{ListMeta: obj.(*v1alpha1.GameServerSetList).ListMeta} + for _, item := range obj.(*v1alpha1.GameServerSetList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested gameServerSets. +func (c *FakeGameServerSets) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(gameserversetsResource, c.ns, opts)) + +} + +// Create takes the representation of a gameServerSet and creates it. Returns the server's representation of the gameServerSet, and an error, if there is any. +func (c *FakeGameServerSets) Create(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.CreateOptions) (result *v1alpha1.GameServerSet, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(gameserversetsResource, c.ns, gameServerSet), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} + +// Update takes the representation of a gameServerSet and updates it. Returns the server's representation of the gameServerSet, and an error, if there is any. +func (c *FakeGameServerSets) Update(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (result *v1alpha1.GameServerSet, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(gameserversetsResource, c.ns, gameServerSet), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeGameServerSets) UpdateStatus(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (*v1alpha1.GameServerSet, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(gameserversetsResource, "status", c.ns, gameServerSet), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} + +// Delete takes name of the gameServerSet and deletes it. Returns an error if one occurs. +func (c *FakeGameServerSets) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(gameserversetsResource, c.ns, name, opts), &v1alpha1.GameServerSet{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeGameServerSets) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(gameserversetsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.GameServerSetList{}) + return err +} + +// Patch applies the patch and returns the patched gameServerSet. +func (c *FakeGameServerSets) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServerSet, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(gameserversetsResource, c.ns, name, pt, data, subresources...), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserver.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserver.go new file mode 100644 index 00000000..2d8e60dd --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserver.go @@ -0,0 +1,194 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + scheme "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// GameServersGetter has a method to return a GameServerInterface. +// A group's client should implement this interface. +type GameServersGetter interface { + GameServers(namespace string) GameServerInterface +} + +// GameServerInterface has methods to work with GameServer resources. +type GameServerInterface interface { + Create(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.CreateOptions) (*v1alpha1.GameServer, error) + Update(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (*v1alpha1.GameServer, error) + UpdateStatus(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (*v1alpha1.GameServer, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.GameServer, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.GameServerList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServer, err error) + GameServerExpansion +} + +// gameServers implements GameServerInterface +type gameServers struct { + client rest.Interface + ns string +} + +// newGameServers returns a GameServers +func newGameServers(c *GameV1alpha1Client, namespace string) *gameServers { + return &gameServers{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the gameServer, and returns the corresponding gameServer object, and an error if there is any. +func (c *gameServers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Get(). + Namespace(c.ns). + Resource("gameservers"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of GameServers that match those selectors. +func (c *gameServers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.GameServerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.GameServerList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("gameservers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested gameServers. +func (c *gameServers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("gameservers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a gameServer and creates it. Returns the server's representation of the gameServer, and an error, if there is any. +func (c *gameServers) Create(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.CreateOptions) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Post(). + Namespace(c.ns). + Resource("gameservers"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServer). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a gameServer and updates it. Returns the server's representation of the gameServer, and an error, if there is any. +func (c *gameServers) Update(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Put(). + Namespace(c.ns). + Resource("gameservers"). + Name(gameServer.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServer). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *gameServers) UpdateStatus(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Put(). + Namespace(c.ns). + Resource("gameservers"). + Name(gameServer.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServer). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the gameServer and deletes it. Returns an error if one occurs. +func (c *gameServers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("gameservers"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *gameServers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("gameservers"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched gameServer. +func (c *gameServers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("gameservers"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserverset.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserverset.go new file mode 100644 index 00000000..09b8341b --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserverset.go @@ -0,0 +1,194 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + scheme "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// GameServerSetsGetter has a method to return a GameServerSetInterface. +// A group's client should implement this interface. +type GameServerSetsGetter interface { + GameServerSets(namespace string) GameServerSetInterface +} + +// GameServerSetInterface has methods to work with GameServerSet resources. +type GameServerSetInterface interface { + Create(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.CreateOptions) (*v1alpha1.GameServerSet, error) + Update(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (*v1alpha1.GameServerSet, error) + UpdateStatus(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (*v1alpha1.GameServerSet, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.GameServerSet, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.GameServerSetList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServerSet, err error) + GameServerSetExpansion +} + +// gameServerSets implements GameServerSetInterface +type gameServerSets struct { + client rest.Interface + ns string +} + +// newGameServerSets returns a GameServerSets +func newGameServerSets(c *GameV1alpha1Client, namespace string) *gameServerSets { + return &gameServerSets{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the gameServerSet, and returns the corresponding gameServerSet object, and an error if there is any. +func (c *gameServerSets) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Get(). + Namespace(c.ns). + Resource("gameserversets"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of GameServerSets that match those selectors. +func (c *gameServerSets) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.GameServerSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.GameServerSetList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("gameserversets"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested gameServerSets. +func (c *gameServerSets) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("gameserversets"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a gameServerSet and creates it. Returns the server's representation of the gameServerSet, and an error, if there is any. +func (c *gameServerSets) Create(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.CreateOptions) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Post(). + Namespace(c.ns). + Resource("gameserversets"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServerSet). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a gameServerSet and updates it. Returns the server's representation of the gameServerSet, and an error, if there is any. +func (c *gameServerSets) Update(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Put(). + Namespace(c.ns). + Resource("gameserversets"). + Name(gameServerSet.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServerSet). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *gameServerSets) UpdateStatus(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Put(). + Namespace(c.ns). + Resource("gameserversets"). + Name(gameServerSet.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServerSet). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the gameServerSet and deletes it. Returns an error if one occurs. +func (c *gameServerSets) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("gameserversets"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *gameServerSets) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("gameserversets"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched gameServerSet. +func (c *gameServerSets) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("gameserversets"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go new file mode 100644 index 00000000..ce93eeee --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go @@ -0,0 +1,22 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type GameServerExpansion interface{} + +type GameServerSetExpansion interface{} diff --git a/pkg/client/generic_client.go b/pkg/client/generic_client.go new file mode 100644 index 00000000..5e6ae391 --- /dev/null +++ b/pkg/client/generic_client.go @@ -0,0 +1,64 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + kruiseclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" + kubeclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// GenericClientset defines a generic client +type GenericClientset struct { + DiscoveryClient discovery.DiscoveryInterface + KubeClient kubeclientset.Interface + KruiseGameClient kruiseclientset.Interface +} + +// newForConfig creates a new Clientset for the given config. +func newForConfig(c *rest.Config) (*GenericClientset, error) { + cWithProtobuf := rest.CopyConfig(c) + cWithProtobuf.ContentType = runtime.ContentTypeProtobuf + discoveryClient, err := discovery.NewDiscoveryClientForConfig(cWithProtobuf) + if err != nil { + return nil, err + } + kubeClient, err := kubeclientset.NewForConfig(cWithProtobuf) + if err != nil { + return nil, err + } + kruiseClient, err := kruiseclientset.NewForConfig(c) + if err != nil { + return nil, err + } + return &GenericClientset{ + DiscoveryClient: discoveryClient, + KubeClient: kubeClient, + KruiseGameClient: kruiseClient, + }, nil +} + +// newForConfig creates a new Clientset for the given config. +func newForConfigOrDie(c *rest.Config) *GenericClientset { + gc, err := newForConfig(c) + if err != nil { + panic(err) + } + return gc +} diff --git a/pkg/client/informers/externalversions/apis/interface.go b/pkg/client/informers/externalversions/apis/interface.go new file mode 100644 index 00000000..22eb3d38 --- /dev/null +++ b/pkg/client/informers/externalversions/apis/interface.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package apis + +import ( + v1alpha1 "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/apis/v1alpha1" + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1alpha1 provides access to shared informers for resources in V1alpha1. + V1alpha1() v1alpha1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1alpha1 returns a new v1alpha1.Interface. +func (g *group) V1alpha1() v1alpha1.Interface { + return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/client/informers/externalversions/apis/v1alpha1/gameserver.go b/pkg/client/informers/externalversions/apis/v1alpha1/gameserver.go new file mode 100644 index 00000000..36d2d8da --- /dev/null +++ b/pkg/client/informers/externalversions/apis/v1alpha1/gameserver.go @@ -0,0 +1,89 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + apisv1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + versioned "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/openkruise/kruise-game/pkg/client/listers/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GameServerInformer provides access to a shared informer and lister for +// GameServers. +type GameServerInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.GameServerLister +} + +type gameServerInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewGameServerInformer constructs a new informer for GameServer type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGameServerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGameServerInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGameServerInformer constructs a new informer for GameServer type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGameServerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.GameV1alpha1().GameServers(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.GameV1alpha1().GameServers(namespace).Watch(context.TODO(), options) + }, + }, + &apisv1alpha1.GameServer{}, + resyncPeriod, + indexers, + ) +} + +func (f *gameServerInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGameServerInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *gameServerInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisv1alpha1.GameServer{}, f.defaultInformer) +} + +func (f *gameServerInformer) Lister() v1alpha1.GameServerLister { + return v1alpha1.NewGameServerLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/apis/v1alpha1/gameserverset.go b/pkg/client/informers/externalversions/apis/v1alpha1/gameserverset.go new file mode 100644 index 00000000..69bc0237 --- /dev/null +++ b/pkg/client/informers/externalversions/apis/v1alpha1/gameserverset.go @@ -0,0 +1,89 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + apisv1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + versioned "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/openkruise/kruise-game/pkg/client/listers/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GameServerSetInformer provides access to a shared informer and lister for +// GameServerSets. +type GameServerSetInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.GameServerSetLister +} + +type gameServerSetInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewGameServerSetInformer constructs a new informer for GameServerSet type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGameServerSetInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGameServerSetInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGameServerSetInformer constructs a new informer for GameServerSet type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGameServerSetInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.GameV1alpha1().GameServerSets(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.GameV1alpha1().GameServerSets(namespace).Watch(context.TODO(), options) + }, + }, + &apisv1alpha1.GameServerSet{}, + resyncPeriod, + indexers, + ) +} + +func (f *gameServerSetInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGameServerSetInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *gameServerSetInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisv1alpha1.GameServerSet{}, f.defaultInformer) +} + +func (f *gameServerSetInformer) Lister() v1alpha1.GameServerSetLister { + return v1alpha1.NewGameServerSetLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/apis/v1alpha1/interface.go b/pkg/client/informers/externalversions/apis/v1alpha1/interface.go new file mode 100644 index 00000000..5153b412 --- /dev/null +++ b/pkg/client/informers/externalversions/apis/v1alpha1/interface.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // GameServers returns a GameServerInformer. + GameServers() GameServerInformer + // GameServerSets returns a GameServerSetInformer. + GameServerSets() GameServerSetInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// GameServers returns a GameServerInformer. +func (v *version) GameServers() GameServerInformer { + return &gameServerInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + +// GameServerSets returns a GameServerSetInformer. +func (v *version) GameServerSets() GameServerSetInformer { + return &gameServerSetInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go new file mode 100644 index 00000000..acd3bb16 --- /dev/null +++ b/pkg/client/informers/externalversions/factory.go @@ -0,0 +1,179 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + reflect "reflect" + sync "sync" + time "time" + + versioned "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + apis "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/apis" + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +// Start initializes all requested informers. +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + go informer.Run(stopCh) + f.startedInformers[informerType] = true + } + } +} + +// WaitForCacheSync waits for all started informers' cache were synced. +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InternalInformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + Game() apis.Interface +} + +func (f *sharedInformerFactory) Game() apis.Interface { + return apis.New(f, f.namespace, f.tweakListOptions) +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go new file mode 100644 index 00000000..a6575f3e --- /dev/null +++ b/pkg/client/informers/externalversions/generic.go @@ -0,0 +1,63 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + "fmt" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=game.kruise.io, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("gameservers"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Game().V1alpha1().GameServers().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("gameserversets"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Game().V1alpha1().GameServerSets().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 00000000..ea4b1151 --- /dev/null +++ b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + versioned "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/pkg/client/listers/apis/v1alpha1/expansion_generated.go b/pkg/client/listers/apis/v1alpha1/expansion_generated.go new file mode 100644 index 00000000..2f3dea8e --- /dev/null +++ b/pkg/client/listers/apis/v1alpha1/expansion_generated.go @@ -0,0 +1,34 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +// GameServerListerExpansion allows custom methods to be added to +// GameServerLister. +type GameServerListerExpansion interface{} + +// GameServerNamespaceListerExpansion allows custom methods to be added to +// GameServerNamespaceLister. +type GameServerNamespaceListerExpansion interface{} + +// GameServerSetListerExpansion allows custom methods to be added to +// GameServerSetLister. +type GameServerSetListerExpansion interface{} + +// GameServerSetNamespaceListerExpansion allows custom methods to be added to +// GameServerSetNamespaceLister. +type GameServerSetNamespaceListerExpansion interface{} diff --git a/pkg/client/listers/apis/v1alpha1/gameserver.go b/pkg/client/listers/apis/v1alpha1/gameserver.go new file mode 100644 index 00000000..13d67161 --- /dev/null +++ b/pkg/client/listers/apis/v1alpha1/gameserver.go @@ -0,0 +1,98 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// GameServerLister helps list GameServers. +// All objects returned here must be treated as read-only. +type GameServerLister interface { + // List lists all GameServers in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.GameServer, err error) + // GameServers returns an object that can list and get GameServers. + GameServers(namespace string) GameServerNamespaceLister + GameServerListerExpansion +} + +// gameServerLister implements the GameServerLister interface. +type gameServerLister struct { + indexer cache.Indexer +} + +// NewGameServerLister returns a new GameServerLister. +func NewGameServerLister(indexer cache.Indexer) GameServerLister { + return &gameServerLister{indexer: indexer} +} + +// List lists all GameServers in the indexer. +func (s *gameServerLister) List(selector labels.Selector) (ret []*v1alpha1.GameServer, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.GameServer)) + }) + return ret, err +} + +// GameServers returns an object that can list and get GameServers. +func (s *gameServerLister) GameServers(namespace string) GameServerNamespaceLister { + return gameServerNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// GameServerNamespaceLister helps list and get GameServers. +// All objects returned here must be treated as read-only. +type GameServerNamespaceLister interface { + // List lists all GameServers in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.GameServer, err error) + // Get retrieves the GameServer from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.GameServer, error) + GameServerNamespaceListerExpansion +} + +// gameServerNamespaceLister implements the GameServerNamespaceLister +// interface. +type gameServerNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all GameServers in the indexer for a given namespace. +func (s gameServerNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.GameServer, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.GameServer)) + }) + return ret, err +} + +// Get retrieves the GameServer from the indexer for a given namespace and name. +func (s gameServerNamespaceLister) Get(name string) (*v1alpha1.GameServer, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("gameserver"), name) + } + return obj.(*v1alpha1.GameServer), nil +} diff --git a/pkg/client/listers/apis/v1alpha1/gameserverset.go b/pkg/client/listers/apis/v1alpha1/gameserverset.go new file mode 100644 index 00000000..5ab7e072 --- /dev/null +++ b/pkg/client/listers/apis/v1alpha1/gameserverset.go @@ -0,0 +1,98 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// GameServerSetLister helps list GameServerSets. +// All objects returned here must be treated as read-only. +type GameServerSetLister interface { + // List lists all GameServerSets in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.GameServerSet, err error) + // GameServerSets returns an object that can list and get GameServerSets. + GameServerSets(namespace string) GameServerSetNamespaceLister + GameServerSetListerExpansion +} + +// gameServerSetLister implements the GameServerSetLister interface. +type gameServerSetLister struct { + indexer cache.Indexer +} + +// NewGameServerSetLister returns a new GameServerSetLister. +func NewGameServerSetLister(indexer cache.Indexer) GameServerSetLister { + return &gameServerSetLister{indexer: indexer} +} + +// List lists all GameServerSets in the indexer. +func (s *gameServerSetLister) List(selector labels.Selector) (ret []*v1alpha1.GameServerSet, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.GameServerSet)) + }) + return ret, err +} + +// GameServerSets returns an object that can list and get GameServerSets. +func (s *gameServerSetLister) GameServerSets(namespace string) GameServerSetNamespaceLister { + return gameServerSetNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// GameServerSetNamespaceLister helps list and get GameServerSets. +// All objects returned here must be treated as read-only. +type GameServerSetNamespaceLister interface { + // List lists all GameServerSets in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.GameServerSet, err error) + // Get retrieves the GameServerSet from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.GameServerSet, error) + GameServerSetNamespaceListerExpansion +} + +// gameServerSetNamespaceLister implements the GameServerSetNamespaceLister +// interface. +type gameServerSetNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all GameServerSets in the indexer for a given namespace. +func (s gameServerSetNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.GameServerSet, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.GameServerSet)) + }) + return ret, err +} + +// Get retrieves the GameServerSet from the indexer for a given namespace and name. +func (s gameServerSetNamespaceLister) Get(name string) (*v1alpha1.GameServerSet, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("gameserverset"), name) + } + return obj.(*v1alpha1.GameServerSet), nil +} diff --git a/pkg/client/registry.go b/pkg/client/registry.go new file mode 100644 index 00000000..16c00c6e --- /dev/null +++ b/pkg/client/registry.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + "k8s.io/client-go/rest" +) + +var ( + cfg *rest.Config + + defaultGenericClient *GenericClientset +) + +// NewRegistry creates clientset by client-go +func NewRegistry(c *rest.Config) error { + var err error + defaultGenericClient, err = newForConfig(c) + if err != nil { + return err + } + cfgCopy := *c + cfg = &cfgCopy + return nil +} + +// GetGenericClient returns default clientset +func GetGenericClient() *GenericClientset { + return defaultGenericClient +} + +// GetGenericClientWithName returns clientset with given name as user-agent +func GetGenericClientWithName(name string) *GenericClientset { + if cfg == nil { + return nil + } + newCfg := *cfg + newCfg.UserAgent = fmt.Sprintf("%s/%s", cfg.UserAgent, name) + return newForConfigOrDie(&newCfg) +} diff --git a/pkg/client/versioned/clientset.go b/pkg/client/versioned/clientset.go new file mode 100644 index 00000000..5293c19a --- /dev/null +++ b/pkg/client/versioned/clientset.go @@ -0,0 +1,18 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package versioned diff --git a/pkg/client/versioned/doc.go b/pkg/client/versioned/doc.go new file mode 100644 index 00000000..5f8d7a83 --- /dev/null +++ b/pkg/client/versioned/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated clientset. +package versioned diff --git a/pkg/client/versioned/fake/clientset_generated.go b/pkg/client/versioned/fake/clientset_generated.go new file mode 100644 index 00000000..2c9023a6 --- /dev/null +++ b/pkg/client/versioned/fake/clientset_generated.go @@ -0,0 +1,18 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake diff --git a/pkg/client/versioned/fake/doc.go b/pkg/client/versioned/fake/doc.go new file mode 100644 index 00000000..5e58521b --- /dev/null +++ b/pkg/client/versioned/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/pkg/client/versioned/fake/register.go b/pkg/client/versioned/fake/register.go new file mode 100644 index 00000000..2c9023a6 --- /dev/null +++ b/pkg/client/versioned/fake/register.go @@ -0,0 +1,18 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake diff --git a/pkg/client/versioned/scheme/doc.go b/pkg/client/versioned/scheme/doc.go new file mode 100644 index 00000000..c9b70b54 --- /dev/null +++ b/pkg/client/versioned/scheme/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/pkg/client/versioned/scheme/register.go b/pkg/client/versioned/scheme/register.go new file mode 100644 index 00000000..fb38ef60 --- /dev/null +++ b/pkg/client/versioned/scheme/register.go @@ -0,0 +1,18 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package scheme diff --git a/pkg/controllers/controller.go b/pkg/controllers/controller.go new file mode 100644 index 00000000..c4719751 --- /dev/null +++ b/pkg/controllers/controller.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/openkruise/kruise-game/pkg/controllers/gameserver" + "github.com/openkruise/kruise-game/pkg/controllers/gameserverset" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +var controllerAddFuncs []func(manager.Manager) error + +func init() { + controllerAddFuncs = append(controllerAddFuncs, gameserver.Add) + controllerAddFuncs = append(controllerAddFuncs, gameserverset.Add) +} + +func SetupWithManager(m manager.Manager) error { + for _, f := range controllerAddFuncs { + if err := f(m); err != nil { + if kindMatchErr, ok := err.(*meta.NoKindMatchError); ok { + klog.Infof("CRD %v is not installed, its controller will perform noops!", kindMatchErr.GroupKind) + continue + } + return err + } + } + return nil +} diff --git a/pkg/controllers/gameserver/gameserver_controller.go b/pkg/controllers/gameserver/gameserver_controller.go new file mode 100644 index 00000000..62ad5fc0 --- /dev/null +++ b/pkg/controllers/gameserver/gameserver_controller.go @@ -0,0 +1,251 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gameserver + +import ( + "context" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + utildiscovery "github.com/openkruise/kruise-game/pkg/util/discovery" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var ( + controllerKind = gamekruiseiov1alpha1.SchemeGroupVersion.WithKind("GameServer") + // leave it to batch size + concurrentReconciles = 10 +) + +func Add(mgr manager.Manager) error { + if !utildiscovery.DiscoverGVK(controllerKind) { + return nil + } + return add(mgr, newReconciler(mgr)) +} + +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + recorder := mgr.GetEventRecorderFor("gameserver-controller") + return &GameServerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + recorder: recorder, + } +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + klog.Info("Starting GameServer Controller") + c, err := controller.New("gameserver-controller", mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + klog.Error(err) + return err + } + if err = c.Watch(&source.Kind{Type: &gamekruiseiov1alpha1.GameServer{}}, &handler.EnqueueRequestForOwner{ + OwnerType: &corev1.Pod{}, + IsController: true, + }); err != nil { + klog.Error(err) + return err + } + if err = watchPod(c); err != nil { + klog.Error(err) + return err + } + + return nil +} + +// GameServerReconciler reconciles a GameServer object +type GameServerReconciler struct { + client.Client + Scheme *runtime.Scheme + recorder record.EventRecorder +} + +func watchPod(c controller.Controller) error { + if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.Funcs{ + CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := createEvent.Object.(*corev1.Pod) + if _, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + }}) + } + }, + UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { + newPod := updateEvent.ObjectNew.(*corev1.Pod) + if _, exist := newPod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: newPod.GetName(), + Namespace: newPod.GetNamespace(), + }}) + } + }, + DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := deleteEvent.Object.(*corev1.Pod) + if _, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: deleteEvent.Object.GetNamespace(), + Name: deleteEvent.Object.GetName(), + }, + }) + } + }, + }); err != nil { + return err + } + return nil +} + +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the GameServer object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile +func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + namespacedName := req.NamespacedName + + // get pod + pod := &corev1.Pod{} + err := r.Get(ctx, namespacedName, pod) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + klog.Errorf("failed to find pod %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + // get GameServer + gs := &gamekruiseiov1alpha1.GameServer{} + err = r.Get(ctx, namespacedName, gs) + if err != nil { + if errors.IsNotFound(err) { + err := r.initGameServer(pod) + if err != nil && !errors.IsAlreadyExists(err) && !errors.IsNotFound(err) { + klog.Errorf("failed to create GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + klog.Errorf("failed to find GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + gsm := NewGameServerManager(gs, pod, r.Client) + + gss, err := r.getGameServerSet(pod) + if err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + klog.Errorf("failed to get GameServerSet for GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + podUpdated, err := gsm.SyncToPod() + if err != nil || podUpdated { + return reconcile.Result{Requeue: podUpdated}, err + } + + err = gsm.SyncToGs(gss.Spec.ServiceQualities) + if err != nil { + return reconcile.Result{}, err + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *GameServerReconciler) SetupWithManager(mgr ctrl.Manager) (c controller.Controller, err error) { + c, err = ctrl.NewControllerManagedBy(mgr). + For(&gamekruiseiov1alpha1.GameServer{}).Build(r) + return c, err +} + +func (r *GameServerReconciler) getGameServerSet(pod *corev1.Pod) (*gamekruiseiov1alpha1.GameServerSet, error) { + gssName := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey] + gss := &gamekruiseiov1alpha1.GameServerSet{} + err := r.Client.Get(context.Background(), types.NamespacedName{ + Namespace: pod.GetNamespace(), + Name: gssName, + }, gss) + return gss, err +} + +func (r *GameServerReconciler) initGameServer(pod *corev1.Pod) error { + gs := &gamekruiseiov1alpha1.GameServer{} + gs.Name = pod.GetName() + gs.Namespace = pod.GetNamespace() + + // set owner reference + ors := make([]metav1.OwnerReference, 0) + or := metav1.OwnerReference{ + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + ors = append(ors, or) + gs.OwnerReferences = ors + + // set NetWork + gs.Spec.NetworkDisabled = false + + // set OpsState + gs.Spec.OpsState = gamekruiseiov1alpha1.None + + // set UpdatePriority + updatePriority := intstr.FromInt(0) + gs.Spec.UpdatePriority = &updatePriority + + // set deletionPriority + deletionPriority := intstr.FromInt(0) + gs.Spec.DeletionPriority = &deletionPriority + + return r.Client.Create(context.Background(), gs) +} diff --git a/pkg/controllers/gameserver/gameserver_manager.go b/pkg/controllers/gameserver/gameserver_manager.go new file mode 100644 index 00000000..8c1ba37e --- /dev/null +++ b/pkg/controllers/gameserver/gameserver_manager.go @@ -0,0 +1,214 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gameserver + +import ( + "context" + kruisePub "github.com/openkruise/kruise-api/apps/pub" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Control interface { + SyncToPod() (bool, error) + SyncToGs(qualities []gameKruiseV1alpha1.ServiceQuality) error +} + +type GameServerManager struct { + gameServer *gameKruiseV1alpha1.GameServer + pod *corev1.Pod + client client.Client +} + +func (manager GameServerManager) SyncToPod() (bool, error) { + // compare GameServer Spec With Pod + pod := manager.pod + gs := manager.gameServer + podLabels := pod.GetLabels() + podDeletePriority := podLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey] + podUpdatePriority := podLabels[gameKruiseV1alpha1.GameServerUpdatePriorityKey] + podGsOpsState := podLabels[gameKruiseV1alpha1.GameServerOpsStateKey] + podGsState := podLabels[gameKruiseV1alpha1.GameServerStateKey] + + updated := false + newLabels := make(map[string]string) + if gs.Spec.DeletionPriority.String() != podDeletePriority { + newLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey] = gs.Spec.DeletionPriority.String() + updated = true + } + if gs.Spec.UpdatePriority.String() != podUpdatePriority { + newLabels[gameKruiseV1alpha1.GameServerUpdatePriorityKey] = gs.Spec.UpdatePriority.String() + updated = true + } + if string(gs.Spec.OpsState) != podGsOpsState { + newLabels[gameKruiseV1alpha1.GameServerOpsStateKey] = string(gs.Spec.OpsState) + updated = true + } + + var gsState gameKruiseV1alpha1.GameServerState + switch pod.Status.Phase { + case corev1.PodRunning: + // GameServer Updating + lifecycleState, exist := pod.GetLabels()[kruisePub.LifecycleStateKey] + if exist && (lifecycleState == string(kruisePub.LifecycleStateUpdating) || lifecycleState == string(kruisePub.LifecycleStatePreparingUpdate)) { + gsState = gameKruiseV1alpha1.Updating + break + } + // GameServer Deleting + if !pod.DeletionTimestamp.IsZero() { + gsState = gameKruiseV1alpha1.Deleting + break + } + // GameServer Ready / NotReady + for _, con := range pod.Status.Conditions { + if con.Type == corev1.PodReady { + if con.Status == corev1.ConditionTrue { + gsState = gameKruiseV1alpha1.Ready + } else { + gsState = gameKruiseV1alpha1.NotReady + } + break + } + } + case corev1.PodFailed: + gsState = gameKruiseV1alpha1.Crash + case corev1.PodPending: + gsState = gameKruiseV1alpha1.Creating + default: + gsState = gameKruiseV1alpha1.Unknown + } + if string(gsState) != podGsState { + newLabels[gameKruiseV1alpha1.GameServerStateKey] = string(gsState) + updated = true + } + + if updated { + patchPod := map[string]interface{}{"metadata": map[string]map[string]string{"labels": newLabels}} + patchPodBytes, err := json.Marshal(patchPod) + if err != nil { + return updated, err + } + err = manager.client.Patch(context.TODO(), pod, client.RawPatch(types.StrategicMergePatchType, patchPodBytes)) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("failed to patch Pod %s in %s,because of %s.", pod.GetName(), pod.GetNamespace(), err.Error()) + return updated, err + } + } + + return updated, nil +} + +func (manager GameServerManager) SyncToGs(qualities []gameKruiseV1alpha1.ServiceQuality) error { + gs := manager.gameServer + pod := manager.pod + podLabels := pod.GetLabels() + podDeletePriority := intstr.FromString(podLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey]) + podUpdatePriority := intstr.FromString(podLabels[gameKruiseV1alpha1.GameServerUpdatePriorityKey]) + podGsState := gameKruiseV1alpha1.GameServerState(podLabels[gameKruiseV1alpha1.GameServerStateKey]) + + gsConditions := gs.Status.ServiceQualitiesCondition + podConditions := pod.Status.Conditions + var sqNames []string + for _, sq := range qualities { + sqNames = append(sqNames, sq.Name) + } + var toExec map[string]string + var newGsConditions []gameKruiseV1alpha1.ServiceQualityCondition + for _, pc := range podConditions { + if util.IsStringInList(string(pc.Type), sqNames) { + toExecAction := true + lastActionTransitionTime := metav1.Now() + for _, gsc := range gsConditions { + if gsc.Name == string(pc.Type) && gsc.Status == string(pc.Status) { + toExecAction = false + lastActionTransitionTime = gsc.LastActionTransitionTime + break + } + } + serviceQualityCondition := gameKruiseV1alpha1.ServiceQualityCondition{ + Name: string(pc.Type), + Status: string(pc.Status), + LastProbeTime: pc.LastProbeTime, + LastTransitionTime: pc.LastTransitionTime, + LastActionTransitionTime: lastActionTransitionTime, + } + newGsConditions = append(newGsConditions, serviceQualityCondition) + if toExecAction { + toExec[string(pc.Type)] = string(pc.Status) + } + } + } + if toExec != nil { + var spec gameKruiseV1alpha1.GameServerSpec + for _, sq := range qualities { + for name := range toExec { + if sq.Name == name { + // TODO exec action + } + } + } + + patchSpec := map[string]interface{}{"spec": spec} + jsonPatchSpec, err := json.Marshal(patchSpec) + if err != nil { + return err + } + err = manager.client.Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, jsonPatchSpec)) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("failed to patch GameServer spec %s in %s,because of %s.", gs.GetName(), gs.GetNamespace(), err.Error()) + return err + } + } + + status := gameKruiseV1alpha1.GameServerStatus{ + PodStatus: pod.Status, + CurrentState: podGsState, + DesiredState: gameKruiseV1alpha1.Ready, + UpdatePriority: &podUpdatePriority, + DeletionPriority: &podDeletePriority, + ServiceQualitiesCondition: newGsConditions, + LastTransitionTime: metav1.Now(), + } + patchStatus := map[string]interface{}{"status": status} + jsonPatchStatus, err := json.Marshal(patchStatus) + if err != nil { + return err + } + err = manager.client.Status().Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, jsonPatchStatus)) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("failed to patch GameServer Status %s in %s,because of %s.", gs.GetName(), gs.GetNamespace(), err.Error()) + return err + } + + return nil +} + +func NewGameServerManager(gs *gameKruiseV1alpha1.GameServer, pod *corev1.Pod, c client.Client) Control { + return &GameServerManager{ + gameServer: gs, + pod: pod, + client: c, + } +} diff --git a/pkg/controllers/gameserverset/gameserverset_controller.go b/pkg/controllers/gameserverset/gameserverset_controller.go new file mode 100644 index 00000000..c3b017fe --- /dev/null +++ b/pkg/controllers/gameserverset/gameserverset_controller.go @@ -0,0 +1,286 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gameserverset + +import ( + "context" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + utildiscovery "github.com/openkruise/kruise-game/pkg/util/discovery" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var ( + controllerKind = gamekruiseiov1alpha1.SchemeGroupVersion.WithKind("GameServerSet") + // leave it to batch size + concurrentReconciles = 10 +) + +func Add(mgr manager.Manager) error { + if !utildiscovery.DiscoverGVK(controllerKind) { + return nil + } + return add(mgr, newReconciler(mgr)) +} + +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + recorder := mgr.GetEventRecorderFor("gameserverset-controller") + return &GameServerSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + recorder: recorder, + } +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + klog.Info("Starting GameServerSet Controller") + c, err := controller.New("gameserverset-controller", mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + klog.Error(err) + return err + } + + if err = c.Watch(&source.Kind{Type: &gamekruiseiov1alpha1.GameServerSet{}}, &handler.EnqueueRequestForObject{}); err != nil { + klog.Error(err) + return err + } + + if err = watchPod(c); err != nil { + klog.Error(err) + return err + } + + if err = watchWorkloads(c); err != nil { + klog.Error(err) + return err + } + + return nil +} + +// watch pod +func watchPod(c controller.Controller) (err error) { + + if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.Funcs{ + CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := createEvent.Object.(*corev1.Pod) + if gssName, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: gssName, + Namespace: pod.GetNamespace(), + }}) + } + }, + UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := updateEvent.ObjectNew.(*corev1.Pod) + if gssName, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: gssName, + Namespace: pod.GetNamespace(), + }}) + } + }, + DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := deleteEvent.Object.(*corev1.Pod) + if gssName, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: gssName, + Namespace: pod.GetNamespace(), + }}) + } + }, + }); err != nil { + return err + } + return nil +} + +// watch workloads +func watchWorkloads(c controller.Controller) (err error) { + if err := c.Watch(&source.Kind{Type: &kruiseV1beta1.StatefulSet{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &gamekruiseiov1alpha1.GameServerSet{}, + }); err != nil { + return err + } + return nil +} + +// GameServerSetReconciler reconciles a GameServerSet object +type GameServerSetReconciler struct { + client.Client + Scheme *runtime.Scheme + recorder record.EventRecorder +} + +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameserversets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameserversets/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameserversets/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the GameServerSet object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile +func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + namespacedName := req.NamespacedName + + // get GameServerSet + gss := &gamekruiseiov1alpha1.GameServerSet{} + err := r.Get(ctx, namespacedName, gss) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + klog.Errorf("Failed to find GameServerSet %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + // get advanced statefulset + asts := &kruiseV1beta1.StatefulSet{} + err = r.Get(ctx, namespacedName, asts) + if err != nil { + if errors.IsNotFound(err) { + err = r.initAsts(gss) + if err != nil { + klog.Errorf("failed to create advanced statefulset %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + klog.Errorf("failed to find advanced statefulset %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + // get actual Pod list + podList := &corev1.PodList{} + err = r.List(ctx, podList, &client.ListOptions{ + Namespace: gss.GetNamespace(), + LabelSelector: labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gss.GetName(), + })}) + if err != nil { + klog.Errorf("failed to list GameServers of GameServerSet %s in %s.", gss.GetName(), gss.GetNamespace()) + return reconcile.Result{}, err + } + + gsm := NewGameServerSetManager(gss, asts, podList.Items, r.Client) + + // scale game servers + if gsm.IsNeedToScale() { + err = gsm.GameServerScale() + if err != nil { + klog.Errorf("GameServerSet %s failed to scale GameServers in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + + // update workload + if gsm.IsNeedToUpdateWorkload() { + err = gsm.UpdateWorkload() + if err != nil { + klog.Errorf("GameServerSet %s failed to synchronize workload in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + + // TODO sync PodProbeMarker + err = gsm.SyncPodProbeMarker() + if err != nil { + klog.Errorf("GameServerSet %s failed to synchronize PodProbeMarker in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + // sync GameServerSet Status + err = gsm.SyncStatus() + if err != nil { + klog.Errorf("GameServerSet %s failed to synchronize its status in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *GameServerSetReconciler) SetupWithManager(mgr ctrl.Manager) (c controller.Controller, err error) { + c, err = ctrl.NewControllerManagedBy(mgr). + For(&gamekruiseiov1alpha1.GameServerSet{}).Build(r) + return c, err +} + +func (r *GameServerSetReconciler) initAsts(gss *gamekruiseiov1alpha1.GameServerSet) error { + asts := &kruiseV1beta1.StatefulSet{} + asts.Namespace = gss.GetNamespace() + asts.Name = gss.GetName() + + // set owner reference + ors := make([]metav1.OwnerReference, 0) + or := metav1.OwnerReference{ + APIVersion: gss.APIVersion, + Kind: gss.Kind, + Name: gss.GetName(), + UID: gss.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + ors = append(ors, or) + asts.SetOwnerReferences(ors) + + // set label + astsLabels := make(map[string]string) + astsLabels[gamekruiseiov1alpha1.AstsHashKey] = util.GetAstsHash(gss) + asts.SetLabels(astsLabels) + + // set label selector + asts.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{gamekruiseiov1alpha1.GameServerOwnerGssKey: gss.GetName()}, + } + + // set replicas + asts.Spec.Replicas = gss.Spec.Replicas + + asts = util.GetNewAstsFromGss(gss, asts) + + return r.Client.Create(context.Background(), asts) +} diff --git a/pkg/controllers/gameserverset/gameserverset_manager.go b/pkg/controllers/gameserverset/gameserverset_manager.go new file mode 100644 index 00000000..268ffabd --- /dev/null +++ b/pkg/controllers/gameserverset/gameserverset_manager.go @@ -0,0 +1,246 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gameserverset + +import ( + "context" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sort" +) + +type Control interface { + GameServerScale() error + UpdateWorkload() error + SyncStatus() error + IsNeedToScale() bool + IsNeedToUpdateWorkload() bool + SyncPodProbeMarker() error +} + +type GameServerSetManager struct { + gameServerSet *gameKruiseV1alpha1.GameServerSet + asts *kruiseV1beta1.StatefulSet + podList []corev1.Pod + client client.Client +} + +func NewGameServerSetManager(gss *gameKruiseV1alpha1.GameServerSet, asts *kruiseV1beta1.StatefulSet, gsList []corev1.Pod, c client.Client) Control { + return &GameServerSetManager{ + gameServerSet: gss, + asts: asts, + podList: gsList, + client: c, + } +} + +func (manager *GameServerSetManager) IsNeedToScale() bool { + gss := manager.gameServerSet + asts := manager.asts + gsList := manager.podList + + currentReplicas := len(gsList) + workloadReplicas := int(*asts.Spec.Replicas) + expectedReplicas := int(*gss.Spec.Replicas) + + // workload is reconciling its replicas, don't interrupt + if currentReplicas != workloadReplicas { + return false + } + + // no need to scale + return !(expectedReplicas == currentReplicas && util.IsSliceEqual(util.StringToIntSlice(gss.GetAnnotations()[gameKruiseV1alpha1.GameServerSetReserveIdsKey], ","), gss.Spec.ReserveGameServerIds)) +} + +func (manager *GameServerSetManager) GameServerScale() error { + gss := manager.gameServerSet + asts := manager.asts + gsList := manager.podList + c := manager.client + ctx := context.Background() + + currentReplicas := len(gsList) + expectedReplicas := int(*gss.Spec.Replicas) + as := gss.GetAnnotations() + reserveIds := util.StringToIntSlice(as[gameKruiseV1alpha1.GameServerSetReserveIdsKey], ",") + notExistIds := util.StringToIntSlice(as[gameKruiseV1alpha1.GameServerSetNotExistIdsKey], ",") + gssReserveIds := gss.Spec.ReserveGameServerIds + + klog.Infof("GameServers %s/%s already has %d replicas, expect to have %d replicas.", gss.GetNamespace(), gss.GetName(), currentReplicas, expectedReplicas) + + newNotExistIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, gsList) + + asts.Spec.ReserveOrdinals = append(gssReserveIds, newNotExistIds...) + asts.Spec.Replicas = gss.Spec.Replicas + asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{ + MaxUnavailable: gss.Spec.ScaleStrategy.MaxUnavailable, + } + err := c.Update(ctx, asts) + if err != nil { + klog.Errorf("failed to update workload replicas %s in %s,because of %s.", gss.GetName(), gss.GetNamespace(), err.Error()) + return err + } + + gssAnnotations := make(map[string]string) + gssAnnotations[gameKruiseV1alpha1.GameServerSetReserveIdsKey] = util.IntSliceToString(gssReserveIds, ",") + gssAnnotations[gameKruiseV1alpha1.GameServerSetNotExistIdsKey] = util.IntSliceToString(newNotExistIds, ",") + patchGss := map[string]interface{}{"metadata": map[string]map[string]string{"annotations": gssAnnotations}} + patchGssBytes, _ := json.Marshal(patchGss) + err = c.Patch(ctx, gss, client.RawPatch(types.MergePatchType, patchGssBytes)) + if err != nil { + klog.Errorf("failed to patch GameServerSet %s in %s,because of %s.", gss.GetName(), gss.GetNamespace(), err.Error()) + return err + } + + return nil +} + +func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod) []int { + workloadManageIds := util.GetIndexListFromPodList(pods) + + var toAdd []int + var toDelete []int + + // 1. compute reserved GameServerIds, firstly + + // 1.a. to delete those new reserved GameServers already in workloadManageIds + toDelete = util.GetSliceInAandInB(util.GetSliceInANotInB(gssReserveIds, reserveIds), workloadManageIds) + + // 1.b. to add those remove-reserved GameServers already in workloadManageIds + existLastIndex := -1 + if len(workloadManageIds) != 0 { + sort.Ints(workloadManageIds) + existLastIndex = workloadManageIds[len(workloadManageIds)-1] + } + for _, id := range util.GetSliceInANotInB(reserveIds, gssReserveIds) { + if existLastIndex > id { + toAdd = append(toAdd, id) + } + } + + // 2. compute remain GameServerIds, secondly + + numToAdd := expectedReplicas - len(pods) + len(toDelete) - len(toAdd) + if numToAdd < 0 { + + // 2.a to delete GameServers according to DeleteSequence + sortedGs := util.DeleteSequenceGs(pods) + sort.Sort(sortedGs) + toDelete = append(toDelete, util.GetIndexListFromPodList(sortedGs[:-numToAdd])...) + } else { + + // 2.b to add GameServers, firstly add those in add notExistIds, secondly add those in future sequence + numNotExist := len(notExistIds) + if numNotExist < numToAdd { + toAdd = append(toAdd, notExistIds...) + times := 0 + for i := existLastIndex + 1; times < numToAdd-numNotExist; i++ { + if !util.IsNumInList(i, gssReserveIds) { + toAdd = append(toAdd, i) + times++ + } + } + } else { + toAdd = append(toAdd, notExistIds[:numToAdd]...) + } + } + + newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...) + newManageIds = util.GetSliceInANotInB(newManageIds, toDelete) + var newNotExistIds []int + if len(newManageIds) != 0 { + sort.Ints(newManageIds) + for i := 0; i < newManageIds[len(newManageIds)-1]; i++ { + if !util.IsNumInList(i, newManageIds) && !util.IsNumInList(i, gssReserveIds) { + newNotExistIds = append(newNotExistIds, i) + } + } + } + + return newNotExistIds +} + +func (manager *GameServerSetManager) IsNeedToUpdateWorkload() bool { + return manager.asts.GetLabels()[gameKruiseV1alpha1.AstsHashKey] != util.GetAstsHash(manager.gameServerSet) +} + +func (manager *GameServerSetManager) UpdateWorkload() error { + gss := manager.gameServerSet + asts := manager.asts + + // sync with Advanced StatefulSet + asts = util.GetNewAstsFromGss(gss, asts) + astsLabels := asts.GetLabels() + astsLabels[gameKruiseV1alpha1.AstsHashKey] = util.GetAstsHash(manager.gameServerSet) + asts.SetLabels(astsLabels) + return manager.client.Update(context.Background(), asts) +} + +func (manager *GameServerSetManager) SyncPodProbeMarker() error { + return nil +} + +func (manager *GameServerSetManager) SyncStatus() error { + gss := manager.gameServerSet + asts := manager.asts + c := manager.client + ctx := context.Background() + podList := manager.podList + + maintainingGs := 0 + waitToBeDeletedGs := 0 + + for _, pod := range podList { + + podLabels := pod.GetLabels() + opsState := podLabels[gameKruiseV1alpha1.GameServerOpsStateKey] + + // ops state + switch opsState { + case string(gameKruiseV1alpha1.WaitToDelete): + waitToBeDeletedGs++ + case string(gameKruiseV1alpha1.Maintaining): + maintainingGs++ + } + } + + status := gameKruiseV1alpha1.GameServerSetStatus{ + Replicas: *gss.Spec.Replicas, + CurrentReplicas: int32(len(podList)), + AvailableReplicas: asts.Status.AvailableReplicas, + ReadyReplicas: asts.Status.ReadyReplicas, + UpdatedReplicas: asts.Status.UpdatedReplicas, + UpdatedReadyReplicas: asts.Status.UpdatedReadyReplicas, + MaintainingReplicas: pointer.Int32Ptr(int32(maintainingGs)), + WaitToBeDeletedReplicas: pointer.Int32Ptr(int32(waitToBeDeletedGs)), + LabelSelector: asts.Status.LabelSelector, + } + + patchStatus := map[string]interface{}{"status": status} + jsonPatch, err := json.Marshal(patchStatus) + if err != nil { + return err + } + return c.Status().Patch(ctx, gss, client.RawPatch(types.MergePatchType, jsonPatch)) +} diff --git a/pkg/util/client/delegating_client.go b/pkg/util/client/delegating_client.go new file mode 100644 index 00000000..29022fbf --- /dev/null +++ b/pkg/util/client/delegating_client.go @@ -0,0 +1,167 @@ +/* +Copyright 2022 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "flag" + "strings" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +var ( + disableNoDeepCopy bool +) + +func init() { + flag.BoolVar(&disableNoDeepCopy, "disable-no-deepcopy", false, "If you are going to disable NoDeepCopy List in some controllers and webhooks.") +} + +// NewClient creates the default caching client with disable deepcopy list from cache. +func NewClient(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + c, err := client.New(config, options) + if err != nil { + return nil, err + } + + uncachedGVKs := map[schema.GroupVersionKind]struct{}{} + for _, obj := range uncachedObjects { + gvk, err := apiutil.GVKForObject(obj, c.Scheme()) + if err != nil { + return nil, err + } + uncachedGVKs[gvk] = struct{}{} + } + + return &delegatingClient{ + scheme: c.Scheme(), + mapper: c.RESTMapper(), + Reader: &delegatingReader{ + CacheReader: cache, + ClientReader: c, + noDeepCopyLister: &noDeepCopyLister{cache: cache, scheme: c.Scheme()}, + scheme: c.Scheme(), + uncachedGVKs: uncachedGVKs, + }, + Writer: c, + StatusClient: c, + }, nil +} + +type delegatingClient struct { + client.Reader + client.Writer + client.StatusClient + + scheme *runtime.Scheme + mapper meta.RESTMapper +} + +// Scheme returns the scheme this client is using. +func (d *delegatingClient) Scheme() *runtime.Scheme { + return d.scheme +} + +// RESTMapper returns the rest mapper this client is using. +func (d *delegatingClient) RESTMapper() meta.RESTMapper { + return d.mapper +} + +// delegatingReader forms a Reader that will cause Get and List requests for +// unstructured types to use the ClientReader while requests for any other type +// of object with use the CacheReader. This avoids accidentally caching the +// entire cluster in the common case of loading arbitrary unstructured objects +// (e.g. from OwnerReferences). +type delegatingReader struct { + CacheReader client.Reader + ClientReader client.Reader + + noDeepCopyLister *noDeepCopyLister + + uncachedGVKs map[schema.GroupVersionKind]struct{} + scheme *runtime.Scheme + cacheUnstructured bool +} + +func (d *delegatingReader) shouldBypassCache(obj runtime.Object) (bool, error) { + gvk, err := apiutil.GVKForObject(obj, d.scheme) + if err != nil { + return false, err + } + // TODO: this is producing unsafe guesses that don't actually work, + // but it matches ~99% of the cases out there. + if meta.IsListType(obj) { + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + } + if _, isUncached := d.uncachedGVKs[gvk]; isUncached { + return true, nil + } + if !d.cacheUnstructured { + _, isUnstructured := obj.(*unstructured.Unstructured) + _, isUnstructuredList := obj.(*unstructured.UnstructuredList) + return isUnstructured || isUnstructuredList, nil + } + return false, nil +} + +// Get retrieves an obj for a given object key from the Kubernetes Cluster. +func (d *delegatingReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if isUncached, err := d.shouldBypassCache(obj); err != nil { + return err + } else if isUncached { + return d.ClientReader.Get(ctx, key, obj) + } + return d.CacheReader.Get(ctx, key, obj) +} + +// List retrieves list of objects for a given namespace and list options. +func (d *delegatingReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if isUncached, err := d.shouldBypassCache(list); err != nil { + return err + } else if isUncached { + return d.ClientReader.List(ctx, list, opts...) + } + if !disableNoDeepCopy && isDisableDeepCopy(opts) { + return d.noDeepCopyLister.List(ctx, list, opts...) + } + return d.CacheReader.List(ctx, list, opts...) +} + +var DisableDeepCopy = disableDeepCopy{} + +type disableDeepCopy struct{} + +func (_ disableDeepCopy) ApplyToList(_ *client.ListOptions) { +} + +func isDisableDeepCopy(opts []client.ListOption) bool { + for _, opt := range opts { + if opt == DisableDeepCopy { + return true + } + } + return false +} diff --git a/pkg/util/client/no_deepcopy_lister.go b/pkg/util/client/no_deepcopy_lister.go new file mode 100644 index 00000000..3272f573 --- /dev/null +++ b/pkg/util/client/no_deepcopy_lister.go @@ -0,0 +1,198 @@ +/* +Copyright 2022 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "fmt" + "reflect" + "strings" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + toolscache "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +type noDeepCopyLister struct { + cache cache.Cache + scheme *runtime.Scheme +} + +func (r *noDeepCopyLister) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error { + startTime := time.Now() + gvk, _, err := r.objectTypeForListObject(out) + if err != nil { + return err + } + indexer, err := r.getIndexerByGVK(ctx, *gvk) + if err != nil { + return err + } + + listOpts := client.ListOptions{} + listOpts.ApplyOptions(opts) + + var objs []interface{} + switch { + case listOpts.FieldSelector != nil: + field, val, requiresExact := requiresExactMatch(listOpts.FieldSelector) + if !requiresExact { + return fmt.Errorf("non-exact field matches are not supported by the cache") + } + // list all objects by the field selector. If this is namespaced and we have one, ask for the + // namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces" + // namespace. + objs, err = indexer.ByIndex(FieldIndexName(field), KeyToNamespacedKey(listOpts.Namespace, val)) + case listOpts.Namespace != "": + objs, err = indexer.ByIndex(toolscache.NamespaceIndex, listOpts.Namespace) + default: + objs = indexer.List() + } + if err != nil { + return err + } + + var labelSel labels.Selector + if listOpts.LabelSelector != nil { + labelSel = listOpts.LabelSelector + } + limitSet := listOpts.Limit > 0 + + runtimeObjs := make([]runtime.Object, 0, len(objs)) + for _, item := range objs { + // if the Limit option is set and the number of items + // listed exceeds this limit, then stop reading. + if limitSet && int64(len(runtimeObjs)) >= listOpts.Limit { + break + } + obj, isObj := item.(runtime.Object) + if !isObj { + return fmt.Errorf("cache contained %T, which is not an Object", obj) + } + meta, err := apimeta.Accessor(obj) + if err != nil { + return err + } + if labelSel != nil { + lbls := labels.Set(meta.GetLabels()) + if !labelSel.Matches(lbls) { + continue + } + } + runtimeObjs = append(runtimeObjs, obj) + } + defer func() { + klog.V(6).Infof("Listed %v %v objects %v without DeepCopy, cost %v", gvk.GroupVersion(), gvk.Kind, len(runtimeObjs), time.Since(startTime)) + }() + return apimeta.SetList(out, runtimeObjs) +} + +func (r *noDeepCopyLister) getIndexerByGVK(ctx context.Context, gvk schema.GroupVersionKind) (toolscache.Indexer, error) { + informer, err := r.cache.GetInformerForKind(ctx, gvk) + if err != nil { + return nil, err + } + sharedInformer, ok := informer.(toolscache.SharedIndexInformer) + if !ok { + return nil, fmt.Errorf("informer %T from cache is not a SharedIndexInformer", informer) + } + return sharedInformer.GetIndexer(), nil +} + +// objectTypeForListObject tries to find the runtime.Object and associated GVK +// for a single object corresponding to the passed-in list type. We need them +// because they are used as cache map key. +func (r *noDeepCopyLister) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) { + gvk, err := apiutil.GVKForObject(list, r.scheme) + if err != nil { + return nil, nil, err + } + + if !strings.HasSuffix(gvk.Kind, "List") { + return nil, nil, fmt.Errorf("non-list type %T (kind %q) passed as output", list, gvk) + } + // we need the non-list GVK, so chop off the "List" from the end of the kind + gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] + _, isUnstructured := list.(*unstructured.UnstructuredList) + var cacheTypeObj runtime.Object + if isUnstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + cacheTypeObj = u + } else { + itemsPtr, err := apimeta.GetItemsPtr(list) + if err != nil { + return nil, nil, err + } + // http://knowyourmeme.com/memes/this-is-fine + elemType := reflect.Indirect(reflect.ValueOf(itemsPtr)).Type().Elem() + if elemType.Kind() != reflect.Ptr { + elemType = reflect.PtrTo(elemType) + } + + cacheTypeValue := reflect.Zero(elemType) + var ok bool + cacheTypeObj, ok = cacheTypeValue.Interface().(runtime.Object) + if !ok { + return nil, nil, fmt.Errorf("cannot get cache for %T, its element %T is not a runtime.Object", list, cacheTypeValue.Interface()) + } + } + + return &gvk, cacheTypeObj, nil +} + +// requiresExactMatch checks if the given field selector is of the form `k=v` or `k==v`. +func requiresExactMatch(sel fields.Selector) (field, val string, required bool) { + reqs := sel.Requirements() + if len(reqs) != 1 { + return "", "", false + } + req := reqs[0] + if req.Operator != selection.Equals && req.Operator != selection.DoubleEquals { + return "", "", false + } + return req.Field, req.Value, true +} + +// FieldIndexName constructs the name of the index over the given field, +// for use with an indexer. +func FieldIndexName(field string) string { + return "field:" + field +} + +// noNamespaceNamespace is used as the "namespace" when we want to list across all namespaces. +const allNamespacesNamespace = "__all_namespaces" + +// KeyToNamespacedKey prefixes the given index key with a namespace +// for use in field selector indexes. +func KeyToNamespacedKey(ns string, baseKey string) string { + if ns != "" { + return ns + "/" + baseKey + } + return allNamespacesNamespace + "/" + baseKey +} diff --git a/pkg/util/discovery/discovery.go b/pkg/util/discovery/discovery.go new file mode 100644 index 00000000..0147c719 --- /dev/null +++ b/pkg/util/discovery/discovery.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "fmt" + "k8s.io/klog/v2" + "time" + + "github.com/openkruise/kruise-game/pkg/client" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +var ( + internalScheme = runtime.NewScheme() + + errKindNotFound = fmt.Errorf("kind not found in group version resources") + backOff = wait.Backoff{ + Steps: 4, + Duration: 500 * time.Millisecond, + Factor: 5.0, + Jitter: 0.1, + } +) + +func init() { + _ = AddToScheme(internalScheme) +} + +func DiscoverGVK(gvk schema.GroupVersionKind) bool { + genericClient := client.GetGenericClient() + if genericClient == nil { + return true + } + discoveryClient := genericClient.DiscoveryClient + + startTime := time.Now() + err := retry.OnError(backOff, func(err error) bool { return true }, func() error { + resourceList, err := discoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String()) + if err != nil { + return err + } + for _, r := range resourceList.APIResources { + if r.Kind == gvk.Kind { + return nil + } + } + return errKindNotFound + }) + + if err != nil { + if err == errKindNotFound { + klog.Warningf("Not found kind %s in group version %s, waiting time %s", gvk.Kind, gvk.GroupVersion().String(), time.Since(startTime)) + return false + } + + // This might be caused by abnormal apiserver or etcd, ignore it + klog.Errorf("Failed to find resources in group version %s: %v, waiting time %s", gvk.GroupVersion().String(), err, time.Since(startTime)) + } + + return true +} + +func DiscoverObject(obj runtime.Object) bool { + gvk, err := apiutil.GVKForObject(obj, internalScheme) + if err != nil { + klog.Warningf("Not recognized object %T in scheme: %v", obj, err) + return false + } + return DiscoverGVK(gvk) +} + +// AddToSchemes may be used to add all resources defined in the project to a Scheme +var AddToSchemes runtime.SchemeBuilder + +// AddToScheme adds all Resources to the Scheme +func AddToScheme(s *runtime.Scheme) error { + return AddToSchemes.AddToScheme(s) +} diff --git a/pkg/util/gameserver.go b/pkg/util/gameserver.go new file mode 100644 index 00000000..b69c556f --- /dev/null +++ b/pkg/util/gameserver.go @@ -0,0 +1,164 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + appspub "github.com/openkruise/kruise-api/apps/pub" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "strconv" + "strings" +) + +type DeleteSequenceGs []corev1.Pod + +func (dg DeleteSequenceGs) Len() int { + return len(dg) +} + +func (dg DeleteSequenceGs) Swap(i, j int) { + dg[i], dg[j] = dg[j], dg[i] +} + +func (dg DeleteSequenceGs) Less(i, j int) bool { + iLabels := dg[i].GetLabels() + jLabels := dg[j].GetLabels() + iOpsState := iLabels[gameKruiseV1alpha1.GameServerOpsStateKey] + jOpsState := jLabels[gameKruiseV1alpha1.GameServerOpsStateKey] + iDeletionPriority := iLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey] + jDeletionPriority := jLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey] + + // OpsState + if iOpsState != jOpsState { + return opsStateDeletePrority(iOpsState) > opsStateDeletePrority(jOpsState) + } + // Deletion Priority + if iDeletionPriority != jDeletionPriority { + iDeletionPriorityInt, _ := strconv.Atoi(iDeletionPriority) + jDeletionPriorityInt, _ := strconv.Atoi(jDeletionPriority) + return iDeletionPriorityInt > jDeletionPriorityInt + } + // Index Number + return GetIndexFromGsName(dg[i].GetName()) > GetIndexFromGsName(dg[j].GetName()) +} + +func opsStateDeletePrority(opsState string) int { + switch opsState { + case string(gameKruiseV1alpha1.WaitToDelete): + return 1 + case string(gameKruiseV1alpha1.None): + return 0 + case string(gameKruiseV1alpha1.Maintaining): + return -1 + } + return 0 +} + +func GetIndexFromGsName(gsName string) int { + temp := strings.Split(gsName, "-") + index, _ := strconv.Atoi(temp[len(temp)-1]) + return index +} + +func GetIndexListFromPodList(podList []corev1.Pod) []int { + var indexList []int + for i := 0; i < len(podList); i++ { + indexList = append(indexList, GetIndexFromGsName(podList[i].GetName())) + } + return indexList +} + +func GetIndexListFromGsList(gsList []gameKruiseV1alpha1.GameServer) []int { + var indexList []int + for i := 0; i < len(gsList); i++ { + indexList = append(indexList, GetIndexFromGsName(gsList[i].GetName())) + } + return indexList +} + +func GetNewAstsFromGss(gss *gameKruiseV1alpha1.GameServerSet, asts *kruiseV1beta1.StatefulSet) *kruiseV1beta1.StatefulSet { + // default: set ParallelPodManagement + asts.Spec.PodManagementPolicy = apps.ParallelPodManagement + + // set pod labels + podLabels := gss.Spec.GameServerTemplate.GetLabels() + if podLabels == nil { + podLabels = make(map[string]string) + } + podLabels[gameKruiseV1alpha1.GameServerOwnerGssKey] = gss.GetName() + asts.Spec.Template.SetLabels(podLabels) + + // set pod annotations + podAnnotations := gss.Spec.GameServerTemplate.GetAnnotations() + asts.Spec.Template.SetAnnotations(podAnnotations) + + // set template spec + asts.Spec.Template.Spec = gss.Spec.GameServerTemplate.Spec + // default: add InPlaceUpdateReady condition + readinessGates := gss.Spec.GameServerTemplate.Spec.ReadinessGates + readinessGates = append(readinessGates, corev1.PodReadinessGate{ConditionType: appspub.InPlaceUpdateReady}) + asts.Spec.Template.Spec.ReadinessGates = readinessGates + + // set VolumeClaimTemplates + asts.Spec.VolumeClaimTemplates = gss.Spec.GameServerTemplate.VolumeClaimTemplates + + // set ScaleStrategy + asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{ + MaxUnavailable: gss.Spec.ScaleStrategy.MaxUnavailable, + } + + // set UpdateStrategy + asts.Spec.UpdateStrategy.Type = gss.Spec.UpdateStrategy.Type + var rollingUpdateStatefulSetStrategy *kruiseV1beta1.RollingUpdateStatefulSetStrategy + if gss.Spec.UpdateStrategy.RollingUpdate != nil { + asts.Spec.UpdateStrategy.Type = apps.RollingUpdateStatefulSetStrategyType + rollingUpdateStatefulSetStrategy = &kruiseV1beta1.RollingUpdateStatefulSetStrategy{ + Partition: gss.Spec.UpdateStrategy.RollingUpdate.Partition, + MaxUnavailable: gss.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, + PodUpdatePolicy: gss.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy, + Paused: gss.Spec.UpdateStrategy.RollingUpdate.Paused, + InPlaceUpdateStrategy: gss.Spec.UpdateStrategy.RollingUpdate.InPlaceUpdateStrategy, + MinReadySeconds: gss.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds, + UnorderedUpdate: &kruiseV1beta1.UnorderedUpdateStrategy{ + PriorityStrategy: &appspub.UpdatePriorityStrategy{ + OrderPriority: []appspub.UpdatePriorityOrderTerm{ + { + OrderedKey: gameKruiseV1alpha1.GameServerUpdatePriorityKey, + }, + }, + }, + }, + } + } + asts.Spec.UpdateStrategy.RollingUpdate = rollingUpdateStatefulSetStrategy + + return asts +} + +type astsToUpdate struct { + UpdateStrategy gameKruiseV1alpha1.UpdateStrategy + Template gameKruiseV1alpha1.GameServerTemplate +} + +func GetAstsHash(gss *gameKruiseV1alpha1.GameServerSet) string { + return GetHash(astsToUpdate{ + UpdateStrategy: gss.Spec.UpdateStrategy, + Template: gss.Spec.GameServerTemplate, + }) +} diff --git a/pkg/util/gameserver_test.go b/pkg/util/gameserver_test.go new file mode 100644 index 00000000..d6014fa4 --- /dev/null +++ b/pkg/util/gameserver_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sort" + "testing" +) + +func TestGetIndexFromGsName(t *testing.T) { + tests := []struct { + name string + result int + }{ + { + name: "xxx-23-3", + result: 3, + }, + { + name: "www_3-12", + result: 12, + }, + } + + for _, test := range tests { + actual := GetIndexFromGsName(test.name) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestDeleteSequenceGs(t *testing.T) { + tests := []struct { + before []corev1.Pod + after []int + }{ + { + before: []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx-0", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.None), + gameKruiseV1alpha1.GameServerDeletePriorityKey: "10", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx-1", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.Maintaining), + gameKruiseV1alpha1.GameServerDeletePriorityKey: "0", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx-2", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.WaitToDelete), + gameKruiseV1alpha1.GameServerDeletePriorityKey: "0", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx-3", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.None), + gameKruiseV1alpha1.GameServerDeletePriorityKey: "0", + }, + }, + }, + }, + after: []int{2, 0, 3, 1}, + }, + } + + for _, test := range tests { + after := DeleteSequenceGs(test.before) + sort.Sort(after) + expect := test.after + actual := GetIndexListFromPodList(after) + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} diff --git a/pkg/util/hash.go b/pkg/util/hash.go new file mode 100644 index 00000000..8cea6bd4 --- /dev/null +++ b/pkg/util/hash.go @@ -0,0 +1,44 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "github.com/davecgh/go-spew/spew" + "hash" + "hash/fnv" +) + +func GetHash(objectToWrite interface{}) string { + hf := fnv.New32() + DeepHashObject(hf, objectToWrite) + return fmt.Sprint(hf.Sum32()) +} + +// DeepHashObject writes specified object to hash using the spew library +// which follows pointers and prints actual values of the nested objects +// ensuring the hash does not change when a pointer changes. +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + printer := spew.ConfigState{ + Indent: " ", + SortKeys: true, + DisableMethods: true, + SpewKeys: true, + } + printer.Fprintf(hasher, "%#v", objectToWrite) +} diff --git a/pkg/util/hash_test.go b/pkg/util/hash_test.go new file mode 100644 index 00000000..ca0c92b2 --- /dev/null +++ b/pkg/util/hash_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + corev1 "k8s.io/api/core/v1" + "testing" +) + +func TestGetHash(t *testing.T) { + tests := []struct { + objectA interface{} + objectB interface{} + result bool + }{ + { + objectA: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "containerA", + Image: "nginx:latest", + }, + }, + }, + }, + objectB: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "containerB", + Image: "nginx:latest", + }, + }, + }, + }, + result: false, + }, + } + + for _, test := range tests { + actual := GetHash(test.objectA) == GetHash(test.objectB) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} diff --git a/pkg/util/slice.go b/pkg/util/slice.go new file mode 100644 index 00000000..b80b3b37 --- /dev/null +++ b/pkg/util/slice.go @@ -0,0 +1,141 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +func IsNumInList(num int, list []int) bool { + for _, n := range list { + if num == n { + return true + } + } + return false +} + +func IsStringInList(str string, list []string) bool { + for _, s := range list { + if s == str { + return true + } + } + return false +} + +func GetSliceInANotInB(a, b []int) []int { + var ret []int + for _, aa := range a { + if !IsNumInList(aa, b) { + ret = append(ret, aa) + } + } + return ret +} + +func GetSliceInAandInB(a, b []int) []int { + var ret []int + for _, aa := range a { + if IsNumInList(aa, b) { + ret = append(ret, aa) + } + } + return ret +} + +func IntSliceToString(number []int, delimiter string) string { + return strings.Trim(strings.Replace(fmt.Sprint(number), " ", delimiter, -1), "[]") +} + +func StringToIntSlice(str string, delimiter string) []int { + if str == "" { + return nil + } + strList := strings.Split(str, delimiter) + if len(strList) == 0 { + return nil + } + var retSlice []int + for _, item := range strList { + if item == "" { + continue + } + val, err := strconv.Atoi(item) + if err != nil { + continue + } + retSlice = append(retSlice, val) + } + return retSlice +} + +func IsSliceEqual(a, b []int) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + sort.Ints(a) + sort.Ints(b) + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +func RemoveRepeat(nums []int) []int { + var result []int + tempMap := map[int]byte{} + for _, num := range nums { + beforeLen := len(tempMap) + tempMap[num] = 0 + if len(tempMap) == beforeLen { + continue + } + result = append(result, num) + } + return result +} + +func IsRepeat(nums []int) bool { + tempMap := map[int]byte{} + for _, num := range nums { + beforeLen := len(tempMap) + tempMap[num] = 0 + if len(tempMap) == beforeLen { + return true + } + } + return false +} + +func IsHasNegativeNum(nums []int) bool { + for _, num := range nums { + if num < 0 { + return true + } + } + return false +} diff --git a/pkg/util/slice_test.go b/pkg/util/slice_test.go new file mode 100644 index 00000000..ba1bd04f --- /dev/null +++ b/pkg/util/slice_test.go @@ -0,0 +1,257 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import "testing" + +func TestIsNumInList(t *testing.T) { + tests := []struct { + number int + list []int + result bool + }{ + { + number: 1, + list: []int{1, 2, 4}, + result: true, + }, + } + + for _, test := range tests { + actual := IsNumInList(test.number, test.list) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestIsStringInList(t *testing.T) { + tests := []struct { + str string + list []string + result bool + }{ + { + str: "", + list: []string{"", "", ""}, + result: true, + }, + } + + for _, test := range tests { + actual := IsStringInList(test.str, test.list) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestGetSliceInANotInB(t *testing.T) { + tests := []struct { + a []int + b []int + result []int + }{ + { + a: []int{4, 5, 1}, + b: []int{1, 2, 3}, + result: []int{4, 5}, + }, + } + + for _, test := range tests { + actual := GetSliceInANotInB(test.a, test.b) + expect := test.result + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} + +func TestGetSliceInAandInB(t *testing.T) { + tests := []struct { + a []int + b []int + result []int + }{ + { + a: []int{4, 5, 1}, + b: []int{1, 2, 3}, + result: []int{1}, + }, + } + + for _, test := range tests { + actual := GetSliceInAandInB(test.a, test.b) + expect := test.result + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} + +func TestIntSliceToString(t *testing.T) { + tests := []struct { + number []int + delimiter string + result string + }{ + { + number: []int{4, 5, 1}, + delimiter: ",", + result: "4,5,1", + }, + } + + for _, test := range tests { + actual := IntSliceToString(test.number, test.delimiter) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestStringToIntSlice(t *testing.T) { + tests := []struct { + str string + delimiter string + result []int + }{ + { + str: "4,5,1", + delimiter: ",", + result: []int{4, 5, 1}, + }, + } + + for _, test := range tests { + actual := StringToIntSlice(test.str, test.delimiter) + expect := test.result + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} + +func TestIsSliceEqual(t *testing.T) { + tests := []struct { + a []int + b []int + result bool + }{ + { + a: []int{1, 3, 5}, + b: []int{5, 1, 3}, + result: true, + }, + } + + for _, test := range tests { + actual := IsSliceEqual(test.a, test.b) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestIsRepeat(t *testing.T) { + tests := []struct { + nums []int + result bool + }{ + { + nums: []int{1, 2, 4, 1}, + result: true, + }, + { + nums: []int{1, 2, 3}, + result: false, + }, + } + + for _, test := range tests { + actual := IsRepeat(test.nums) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestRemoveRepeat(t *testing.T) { + tests := []struct { + nums []int + result []int + }{ + { + nums: []int{1, 2, 4, 2, 1}, + result: []int{1, 2, 4}, + }, + { + nums: []int{1, 2, 3}, + result: []int{1, 2, 3}, + }, + } + + for _, test := range tests { + actual := RemoveRepeat(test.nums) + expect := test.result + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} + +func TestIsHasNegativeNum(t *testing.T) { + tests := []struct { + nums []int + result bool + }{ + { + nums: []int{1, -2, 4, 1}, + result: true, + }, + { + nums: []int{1, 2, 3}, + result: false, + }, + { + nums: []int{}, + result: false, + }, + } + + for _, test := range tests { + actual := IsHasNegativeNum(test.nums) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} diff --git a/pkg/webhook/mutating_pod.go b/pkg/webhook/mutating_pod.go new file mode 100644 index 00000000..5eaf8195 --- /dev/null +++ b/pkg/webhook/mutating_pod.go @@ -0,0 +1,53 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "encoding/json" + corev1 "k8s.io/api/core/v1" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type PodMutatingHandler struct { + Client client.Client + decoder *admission.Decoder +} + +func (pmh *PodMutatingHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + pod := &corev1.Pod{} + err := pmh.decoder.Decode(req, pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + pod = mutatingPod(pod, pmh.Client) + // mutate the fields in pod + + marshaledPod, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} + +func mutatingPod(pod *corev1.Pod, client client.Client) *corev1.Pod { + + return pod +} diff --git a/pkg/webhook/util/generator/certgenerator.go b/pkg/webhook/util/generator/certgenerator.go new file mode 100644 index 00000000..a031fea8 --- /dev/null +++ b/pkg/webhook/util/generator/certgenerator.go @@ -0,0 +1,41 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +// Artifacts hosts a private key, its corresponding serving certificate and +// the CA certificate that signs the serving certificate. +type Artifacts struct { + // PEM encoded private key + Key []byte + // PEM encoded serving certificate + Cert []byte + // PEM encoded CA private key + CAKey []byte + // PEM encoded CA certificate + CACert []byte + // Resource version of the certs + ResourceVersion string +} + +// CertGenerator is an interface to provision the serving certificate. +type CertGenerator interface { + // Generate returns a Artifacts struct. + Generate(CommonName string) (*Artifacts, error) + // SetCA sets the PEM-encoded CA private key and CA cert for signing the generated serving cert. + SetCA(caKey, caCert []byte) +} diff --git a/pkg/webhook/util/generator/fake/certgenerator.go b/pkg/webhook/util/generator/fake/certgenerator.go new file mode 100644 index 00000000..218d51ee --- /dev/null +++ b/pkg/webhook/util/generator/fake/certgenerator.go @@ -0,0 +1,54 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "bytes" + "fmt" + + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" +) + +// CertGenerator is a certGenerator for testing. +type CertGenerator struct { + CAKey []byte + CACert []byte + DNSNameToCertArtifacts map[string]*generator.Artifacts +} + +var _ generator.CertGenerator = &CertGenerator{} + +// SetCA sets the PEM-encoded CA private key and CA cert for signing the generated serving cert. +func (cp *CertGenerator) SetCA(CAKey, CACert []byte) { + cp.CAKey = CAKey + cp.CACert = CACert +} + +// Generate generates certificates by matching a common name. +func (cp *CertGenerator) Generate(commonName string) (*generator.Artifacts, error) { + certs, found := cp.DNSNameToCertArtifacts[commonName] + if !found { + return nil, fmt.Errorf("failed to find common name %q in the certGenerator", commonName) + } + if cp.CAKey != nil && cp.CACert != nil && + !bytes.Contains(cp.CAKey, []byte("invalid")) && !bytes.Contains(cp.CACert, []byte("invalid")) { + certs.CAKey = cp.CAKey + certs.CACert = cp.CACert + } + return certs, nil +} diff --git a/pkg/webhook/util/generator/selfsigned.go b/pkg/webhook/util/generator/selfsigned.go new file mode 100644 index 00000000..e4268d1e --- /dev/null +++ b/pkg/webhook/util/generator/selfsigned.go @@ -0,0 +1,192 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "crypto" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math" + "math/big" + "net" + "time" + + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" +) + +const ( + rsaKeySize = 2048 +) + +// ServiceToCommonName generates the CommonName for the certificate when using a k8s service. +func ServiceToCommonName(serviceNamespace, serviceName string) string { + return fmt.Sprintf("%s.%s.svc", serviceName, serviceNamespace) +} + +// SelfSignedCertGenerator implements the certGenerator interface. +// It provisions self-signed certificates. +type SelfSignedCertGenerator struct { + caKey []byte + caCert []byte +} + +var _ CertGenerator = &SelfSignedCertGenerator{} + +// SetCA sets the PEM-encoded CA private key and CA cert for signing the generated serving cert. +func (cp *SelfSignedCertGenerator) SetCA(caKey, caCert []byte) { + cp.caKey = caKey + cp.caCert = caCert +} + +// Generate creates and returns a CA certificate, certificate and +// key for the server. serverKey and serverCert are used by the server +// to establish trust for clients, CA certificate is used by the +// client to verify the server authentication chain. +// The cert will be valid for 365 days. +func (cp *SelfSignedCertGenerator) Generate(commonName string) (*Artifacts, error) { + var err error + + valid, signingKey, signingCert := cp.validCACert() + if !valid { + signingKey, err = NewPrivateKey() + if err != nil { + return nil, fmt.Errorf("failed to create the CA private key: %v", err) + } + signingCert, err = cert.NewSelfSignedCACert(cert.Config{CommonName: "webhook-cert-ca"}, signingKey) + if err != nil { + return nil, fmt.Errorf("failed to create the CA cert: %v", err) + } + } + + hostIP := net.ParseIP(commonName) + var altIPs []net.IP + DNSNames := []string{"localhost"} + if hostIP.To4() != nil { + altIPs = append(altIPs, hostIP.To4()) + } else { + DNSNames = append(DNSNames, commonName) + } + + key, err := NewPrivateKey() + if err != nil { + return nil, fmt.Errorf("failed to create the private key: %v", err) + } + signedCert, err := NewSignedCert( + cert.Config{ + CommonName: commonName, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + AltNames: cert.AltNames{IPs: altIPs, DNSNames: DNSNames}, + }, + key, signingCert, signingKey, + ) + if err != nil { + return nil, fmt.Errorf("failed to create the cert: %v", err) + } + return &Artifacts{ + Key: EncodePrivateKeyPEM(key), + Cert: EncodeCertPEM(signedCert), + CAKey: EncodePrivateKeyPEM(signingKey), + CACert: EncodeCertPEM(signingCert), + }, nil +} + +func (cp *SelfSignedCertGenerator) validCACert() (bool, *rsa.PrivateKey, *x509.Certificate) { + if !ValidCACert(cp.caKey, cp.caCert, cp.caCert, "", time.Now().AddDate(1, 0, 0)) { + return false, nil, nil + } + + key, err := keyutil.ParsePrivateKeyPEM(cp.caKey) + if err != nil { + return false, nil, nil + } + privateKey, ok := key.(*rsa.PrivateKey) + if !ok { + return false, nil, nil + } + + certs, err := cert.ParseCertsPEM(cp.caCert) + if err != nil { + return false, nil, nil + } + if len(certs) != 1 { + return false, nil, nil + } + return true, privateKey, certs[0] +} + +// NewPrivateKey creates an RSA private key +func NewPrivateKey() (*rsa.PrivateKey, error) { + return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) +} + +// NewSignedCert creates a signed certificate using the given CA certificate and key +func NewSignedCert(cfg cert.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + return nil, err + } + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: caCert.NotBefore, + NotAfter: time.Now().Add(time.Hour * 24 * 365 * 10).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + } + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(certDERBytes) +} + +// EncodePrivateKeyPEM returns PEM-encoded private key data +func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte { + block := pem.Block{ + Type: keyutil.RSAPrivateKeyBlockType, + Bytes: x509.MarshalPKCS1PrivateKey(key), + } + return pem.EncodeToMemory(&block) +} + +// EncodeCertPEM returns PEM-encoded certificate data +func EncodeCertPEM(ct *x509.Certificate) []byte { + block := pem.Block{ + Type: cert.CertificateBlockType, + Bytes: ct.Raw, + } + return pem.EncodeToMemory(&block) +} diff --git a/pkg/webhook/util/generator/util.go b/pkg/webhook/util/generator/util.go new file mode 100644 index 00000000..6080a30a --- /dev/null +++ b/pkg/webhook/util/generator/util.go @@ -0,0 +1,62 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "time" +) + +// ValidCACert treats cert and key are valid if they meet the following requirements: +// - key and cert are valid pair +// - caCert is the root ca of cert +// - cert is for dnsName +// - cert won't expire before time +func ValidCACert(key, cert, caCert []byte, dnsName string, time time.Time) bool { + if len(key) == 0 || len(cert) == 0 || len(caCert) == 0 { + return false + } + // Verify key and cert are valid pair + _, err := tls.X509KeyPair(cert, key) + if err != nil { + return false + } + + // Verify cert is valid for at least 1 year. + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caCert) { + return false + } + block, _ := pem.Decode(cert) + if block == nil { + return false + } + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return false + } + ops := x509.VerifyOptions{ + DNSName: dnsName, + Roots: pool, + CurrentTime: time, + } + _, err = c.Verify(ops) + return err == nil +} diff --git a/pkg/webhook/util/writer/atomic/atomic_writer.go b/pkg/webhook/util/writer/atomic/atomic_writer.go new file mode 100644 index 00000000..0ca25204 --- /dev/null +++ b/pkg/webhook/util/writer/atomic/atomic_writer.go @@ -0,0 +1,452 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package atomic + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" +) + +const ( + maxFileNameLength = 255 + maxPathLength = 4096 +) + +// Writer handles atomically projecting content for a set of files into +// a target directory. +// +// Note: +// +// 1. Writer reserves the set of pathnames starting with `..`. +// 2. Writer offers no concurrency guarantees and must be synchronized +// by the caller. +// +// The visible files in this volume are symlinks to files in the writer's data +// directory. Actual files are stored in a hidden timestamped directory which +// is symlinked to by the data directory. The timestamped directory and +// data directory symlink are created in the writer's target dir.  This scheme +// allows the files to be atomically updated by changing the target of the +// data directory symlink. +// +// Consumers of the target directory can monitor the ..data symlink using +// inotify or fanotify to receive events when the content in the volume is +// updated. +type Writer struct { + targetDir string +} + +type FileProjection struct { + Data []byte + Mode int32 +} + +// NewAtomicWriter creates a new Writer configured to write to the given +// target directory, or returns an error if the target directory does not exist. +func NewAtomicWriter(targetDir string) (*Writer, error) { + _, err := os.Stat(targetDir) + if os.IsNotExist(err) { + return nil, err + } + + return &Writer{targetDir: targetDir}, nil +} + +const ( + dataDirName = "..data" + newDataDirName = "..data_tmp" +) + +// Write does an atomic projection of the given payload into the writer's target +// directory. Input paths must not begin with '..'. +// +// The Write algorithm is: +// +// 1. The payload is validated; if the payload is invalid, the function returns +// 2.  The current timestamped directory is detected by reading the data directory +// symlink +// 3. The old version of the volume is walked to determine whether any +// portion of the payload was deleted and is still present on disk. +// 4. The data in the current timestamped directory is compared to the projected +// data to determine if an update is required. +// 5.  A new timestamped dir is created +// 6. The payload is written to the new timestamped directory +// 7.  Symlinks and directory for new user-visible files are created (if needed). +// +// For example, consider the files: +// /podName +// /user/labels +// /k8s/annotations +// +// The user visible files are symbolic links into the internal data directory: +// /podName -> ..data/podName +// /usr -> ..data/usr +// /k8s -> ..data/k8s +// +// The data directory itself is a link to a timestamped directory with +// the real data: +// /..data -> ..2016_02_01_15_04_05.12345678/ +// 8.  A symlink to the new timestamped directory ..data_tmp is created that will +// become the new data directory +// 9.  The new data directory symlink is renamed to the data directory; rename is atomic +// 10. Old paths are removed from the user-visible portion of the target directory +// 11.  The previous timestamped directory is removed, if it exists +func (w *Writer) Write(payload map[string]FileProjection) error { + // (1) + cleanPayload, err := validatePayload(payload) + if err != nil { + klog.Error(err, "invalid payload") + return err + } + + // (2) + dataDirPath := path.Join(w.targetDir, dataDirName) + oldTsDir, err := os.Readlink(dataDirPath) + if err != nil { + if !os.IsNotExist(err) { + klog.Error(err, "unable to read link for data directory") + return err + } + // although Readlink() returns "" on err, don't be fragile by relying on it (since it's not specified in docs) + // empty oldTsDir indicates that it didn't exist + oldTsDir = "" + } + oldTsPath := path.Join(w.targetDir, oldTsDir) + + var pathsToRemove sets.String + // if there was no old version, there's nothing to remove + if len(oldTsDir) != 0 { + // (3) + pathsToRemove, err = w.pathsToRemove(cleanPayload, oldTsPath) + if err != nil { + klog.Error(err, "unable to determine user-visible files to remove") + return err + } + + // (4) + if should, err := shouldWritePayload(cleanPayload, oldTsPath); err != nil { + klog.Error(err, "unable to determine whether payload should be written to disk") + return err + } else if !should && len(pathsToRemove) == 0 { + klog.V(6).Info("no update required for target directory", "directory", w.targetDir) + return nil + } else { + klog.V(1).Info("write required for target directory", "directory", w.targetDir) + } + } + + // (5) + tsDir, err := w.newTimestampDir() + if err != nil { + klog.Error(err, "error creating new ts data directory") + return err + } + tsDirName := filepath.Base(tsDir) + + // (6) + if err = w.writePayloadToDir(cleanPayload, tsDir); err != nil { + klog.Error(err, "unable to write payload to ts data directory", "ts directory", tsDir) + return err + } + klog.V(1).Info("performed write of new data to ts data directory", "ts directory", tsDir) + + // (7) + if err = w.createUserVisibleFiles(cleanPayload); err != nil { + klog.Error(err, "unable to create visible symlinks in target directory", "target directory", w.targetDir) + return err + } + + // (8) + newDataDirPath := path.Join(w.targetDir, newDataDirName) + if err = os.Symlink(tsDirName, newDataDirPath); err != nil { + os.RemoveAll(tsDir) + klog.Error(err, "unable to create symbolic link for atomic update") + return err + } + + // (9) + if runtime.GOOS == "windows" { + os.Remove(dataDirPath) + err = os.Symlink(tsDirName, dataDirPath) + os.Remove(newDataDirPath) + } else { + err = os.Rename(newDataDirPath, dataDirPath) + } + if err != nil { + os.Remove(newDataDirPath) + os.RemoveAll(tsDir) + klog.Error(err, "unable to rename symbolic link for data directory", "data directory", newDataDirPath) + return err + } + + // (10) + if err = w.removeUserVisiblePaths(pathsToRemove); err != nil { + klog.Error(err, "unable to remove old visible symlinks") + return err + } + + // (11) + if len(oldTsDir) > 0 { + if err = os.RemoveAll(oldTsPath); err != nil { + klog.Error(err, "unable to remove old data directory", "data directory", oldTsDir) + return err + } + } + + return nil +} + +// validatePayload returns an error if any path in the payload returns a copy of the payload with the paths cleaned. +func validatePayload(payload map[string]FileProjection) (map[string]FileProjection, error) { + cleanPayload := make(map[string]FileProjection) + for k, content := range payload { + if err := validatePath(k); err != nil { + return nil, err + } + + cleanPayload[filepath.Clean(k)] = content + } + + return cleanPayload, nil +} + +// validatePath validates a single path, returning an error if the path is +// invalid. paths may not: +// +// 1. be absolute +// 2. contain '..' as an element +// 3. start with '..' +// 4. contain filenames larger than 255 characters +// 5. be longer than 4096 characters +func validatePath(targetPath string) error { + // TODO: somehow unify this with the similar api validation, + // validateVolumeSourcePath; the error semantics are just different enough + // from this that it was time-prohibitive trying to find the right + // refactoring to re-use. + if targetPath == "" { + return fmt.Errorf("invalid path: must not be empty: %q", targetPath) + } + if path.IsAbs(targetPath) { + return fmt.Errorf("invalid path: must be relative path: %s", targetPath) + } + + if len(targetPath) > maxPathLength { + return fmt.Errorf("invalid path: must be less than or equal to %d characters", maxPathLength) + } + + items := strings.Split(targetPath, string(os.PathSeparator)) + for _, item := range items { + if item == ".." { + return fmt.Errorf("invalid path: must not contain '..': %s", targetPath) + } + if len(item) > maxFileNameLength { + return fmt.Errorf("invalid path: filenames must be less than or equal to %d characters", maxFileNameLength) + } + } + if strings.HasPrefix(items[0], "..") && len(items[0]) > 2 { + return fmt.Errorf("invalid path: must not start with '..': %s", targetPath) + } + + return nil +} + +// shouldWritePayload returns whether the payload should be written to disk. +func shouldWritePayload(payload map[string]FileProjection, oldTsDir string) (bool, error) { + for userVisiblePath, fileProjection := range payload { + shouldWrite, err := shouldWriteFile(path.Join(oldTsDir, userVisiblePath), fileProjection.Data) + if err != nil { + return false, err + } + + if shouldWrite { + return true, nil + } + } + + return false, nil +} + +// shouldWriteFile returns whether a new version of a file should be written to disk. +func shouldWriteFile(path string, content []byte) (bool, error) { + _, err := os.Lstat(path) + if os.IsNotExist(err) { + return true, nil + } + + contentOnFs, err := ioutil.ReadFile(path) + if err != nil { + return false, err + } + + return !bytes.Equal(content, contentOnFs), nil +} + +// pathsToRemove walks the current version of the data directory and +// determines which paths should be removed (if any) after the payload is +// written to the target directory. +func (w *Writer) pathsToRemove(payload map[string]FileProjection, oldTsDir string) (sets.String, error) { + paths := sets.NewString() + visitor := func(path string, info os.FileInfo, err error) error { + relativePath := strings.TrimPrefix(path, oldTsDir) + relativePath = strings.TrimPrefix(relativePath, string(os.PathSeparator)) + if relativePath == "" { + return nil + } + + paths.Insert(relativePath) + return nil + } + + err := filepath.Walk(oldTsDir, visitor) + if os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + + newPaths := sets.NewString() + for file := range payload { + // add all subpaths for the payload to the set of new paths + // to avoid attempting to remove non-empty dirs + for subPath := file; subPath != ""; { + newPaths.Insert(subPath) + subPath, _ = filepath.Split(subPath) + subPath = strings.TrimSuffix(subPath, string(os.PathSeparator)) + } + } + + result := paths.Difference(newPaths) + if len(result) > 0 { + klog.V(1).Info("paths to remove", "target directory", w.targetDir, "paths", result) + } + + return result, nil +} + +// newTimestampDir creates a new timestamp directory +func (w *Writer) newTimestampDir() (string, error) { + tsDir, err := ioutil.TempDir(w.targetDir, time.Now().UTC().Format("..2006_01_02_15_04_05.")) + if err != nil { + klog.Error(err, "unable to create new temp directory") + return "", err + } + + // 0755 permissions are needed to allow 'group' and 'other' to recurse the + // directory tree. do a chmod here to ensure that permissions are set correctly + // regardless of the process' umask. + err = os.Chmod(tsDir, 0755) + if err != nil { + klog.Error(err, "unable to set mode on new temp directory") + return "", err + } + + return tsDir, nil +} + +// writePayloadToDir writes the given payload to the given directory. The +// directory must exist. +func (w *Writer) writePayloadToDir(payload map[string]FileProjection, dir string) error { + for userVisiblePath, fileProjection := range payload { + content := fileProjection.Data + mode := os.FileMode(fileProjection.Mode) + fullPath := path.Join(dir, userVisiblePath) + baseDir, _ := filepath.Split(fullPath) + + err := os.MkdirAll(baseDir, os.ModePerm) + if err != nil { + klog.Error(err, "unable to create directory", "directory", baseDir) + return err + } + + err = ioutil.WriteFile(fullPath, content, mode) + if err != nil { + klog.Error(err, "unable to write file", "file", fullPath, "mode", mode) + return err + } + // Chmod is needed because ioutil.WriteFile() ends up calling + // open(2) to create the file, so the final mode used is "mode & + // ~umask". But we want to make sure the specified mode is used + // in the file no matter what the umask is. + err = os.Chmod(fullPath, mode) + if err != nil { + klog.Error(err, "unable to write file", "file", fullPath, "mode", mode) + } + } + + return nil +} + +// createUserVisibleFiles creates the relative symlinks for all the +// files configured in the payload. If the directory in a file path does not +// exist, it is created. +// +// Viz: +// For files: "bar", "foo/bar", "baz/bar", "foo/baz/blah" +// the following symlinks are created: +// bar -> ..data/bar +// foo -> ..data/foo +// baz -> ..data/baz +func (w *Writer) createUserVisibleFiles(payload map[string]FileProjection) error { + for userVisiblePath := range payload { + slashpos := strings.Index(userVisiblePath, string(os.PathSeparator)) + if slashpos == -1 { + slashpos = len(userVisiblePath) + } + linkname := userVisiblePath[:slashpos] + _, err := os.Readlink(path.Join(w.targetDir, linkname)) + if err != nil && os.IsNotExist(err) { + // The link into the data directory for this path doesn't exist; create it + visibleFile := path.Join(w.targetDir, linkname) + dataDirFile := path.Join(dataDirName, linkname) + + err = os.Symlink(dataDirFile, visibleFile) + if err != nil { + return err + } + } + } + return nil +} + +// removeUserVisiblePaths removes the set of paths from the user-visible +// portion of the writer's target directory. +func (w *Writer) removeUserVisiblePaths(paths sets.String) error { + ps := string(os.PathSeparator) + var lasterr error + for p := range paths { + // only remove symlinks from the volume root directory (i.e. items that don't contain '/') + if strings.Contains(p, ps) { + continue + } + if err := os.Remove(path.Join(w.targetDir, p)); err != nil { + klog.Error(err, "unable to prune old user-visible path", "path", p) + lasterr = err + } + } + + return lasterr +} diff --git a/pkg/webhook/util/writer/certwriter.go b/pkg/webhook/util/writer/certwriter.go new file mode 100644 index 00000000..c52ef21e --- /dev/null +++ b/pkg/webhook/util/writer/certwriter.go @@ -0,0 +1,107 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer + +import ( + "errors" + "time" + + "k8s.io/klog/v2" + + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" +) + +const ( + // CAKeyName is the name of the CA private key + CAKeyName = "ca-key.pem" + // CACertName is the name of the CA certificate + CACertName = "ca-cert.pem" + // ServerKeyName is the name of the server private key + ServerKeyName = "key.pem" + ServerKeyName2 = "tls.key" + // ServerCertName is the name of the serving certificate + ServerCertName = "cert.pem" + ServerCertName2 = "tls.crt" +) + +// CertWriter provides method to handle webhooks. +type CertWriter interface { + // EnsureCert provisions the cert for the webhookClientConfig. + EnsureCert(dnsName string) (*generator.Artifacts, bool, error) +} + +// handleCommon ensures the given webhook has a proper certificate. +// It uses the given certReadWriter to read and (or) write the certificate. +func handleCommon(dnsName string, ch certReadWriter) (*generator.Artifacts, bool, error) { + if len(dnsName) == 0 { + return nil, false, errors.New("dnsName should not be empty") + } + if ch == nil { + return nil, false, errors.New("certReaderWriter should not be nil") + } + + certs, changed, err := createIfNotExists(ch) + if err != nil { + return nil, changed, err + } + + // Recreate the cert if it's invalid. + valid := validCert(certs, dnsName) + if !valid { + klog.Info("cert is invalid or expired, regenerating a new one") + certs, err = ch.overwrite(certs.ResourceVersion) + if err != nil { + return nil, false, err + } + changed = true + } + return certs, changed, nil +} + +func createIfNotExists(ch certReadWriter) (*generator.Artifacts, bool, error) { + // Try to read first + certs, err := ch.read() + if isNotFound(err) { + // Create if not exists + certs, err = ch.write() + // This may happen if there is another racer. + if isAlreadyExists(err) { + certs, err = ch.read() + } + return certs, true, err + } + return certs, false, err +} + +// certReadWriter provides methods for reading and writing certificates. +type certReadWriter interface { + // read a webhook name and returns the certs for it. + read() (*generator.Artifacts, error) + // write the certs and return the certs it wrote. + write() (*generator.Artifacts, error) + // overwrite the existing certs and return the certs it wrote. + overwrite(resourceVersion string) (*generator.Artifacts, error) +} + +func validCert(certs *generator.Artifacts, dnsName string) bool { + if certs == nil { + return false + } + expired := time.Now().AddDate(0, 6, 0) + return generator.ValidCACert(certs.Key, certs.Cert, certs.CACert, dnsName, expired) +} diff --git a/pkg/webhook/util/writer/error.go b/pkg/webhook/util/writer/error.go new file mode 100644 index 00000000..d7399eef --- /dev/null +++ b/pkg/webhook/util/writer/error.go @@ -0,0 +1,44 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer + +type notFoundError struct { + err error +} + +func (e notFoundError) Error() string { + return e.err.Error() +} + +func isNotFound(err error) bool { + _, ok := err.(notFoundError) + return ok +} + +type alreadyExistError struct { + err error +} + +func (e alreadyExistError) Error() string { + return e.err.Error() +} + +func isAlreadyExists(err error) bool { + _, ok := err.(alreadyExistError) + return ok +} diff --git a/pkg/webhook/util/writer/fs.go b/pkg/webhook/util/writer/fs.go new file mode 100644 index 00000000..93350e35 --- /dev/null +++ b/pkg/webhook/util/writer/fs.go @@ -0,0 +1,229 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path" + + "k8s.io/klog/v2" + + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" + "github.com/openkruise/kruise-game/pkg/webhook/util/writer/atomic" +) + +const ( + FsCertWriter = "fs" +) + +// fsCertWriter provisions the certificate by reading and writing to the filesystem. +type fsCertWriter struct { + // dnsName is the DNS name that the certificate is for. + dnsName string + + *FSCertWriterOptions +} + +// FSCertWriterOptions are options for constructing a FSCertWriter. +type FSCertWriterOptions struct { + // certGenerator generates the certificates. + CertGenerator generator.CertGenerator + // path is the directory that the certificate and private key and CA certificate will be written. + Path string +} + +var _ CertWriter = &fsCertWriter{} + +func (ops *FSCertWriterOptions) setDefaults() { + if ops.CertGenerator == nil { + ops.CertGenerator = &generator.SelfSignedCertGenerator{} + } +} + +func (ops *FSCertWriterOptions) validate() error { + if len(ops.Path) == 0 { + return errors.New("path must be set in FSCertWriterOptions") + } + return nil +} + +// NewFSCertWriter constructs a CertWriter that persists the certificate on filesystem. +func NewFSCertWriter(ops FSCertWriterOptions) (CertWriter, error) { + ops.setDefaults() + err := ops.validate() + if err != nil { + return nil, err + } + return &fsCertWriter{FSCertWriterOptions: &ops}, nil +} + +// EnsureCert provisions certificates for a webhookClientConfig by writing the certificates in the filesystem. +func (f *fsCertWriter) EnsureCert(dnsName string) (*generator.Artifacts, bool, error) { + // create or refresh cert and write it to fs + f.dnsName = dnsName + return handleCommon(f.dnsName, f) +} + +func (f *fsCertWriter) write() (*generator.Artifacts, error) { + return f.doWrite() +} + +func (f *fsCertWriter) overwrite(_ string) (*generator.Artifacts, error) { + return f.doWrite() +} + +func (f *fsCertWriter) doWrite() (*generator.Artifacts, error) { + certs, err := f.CertGenerator.Generate(f.dnsName) + if err != nil { + return nil, err + } + + if err = WriteCertsToDir(f.Path, certs); err != nil { + return nil, err + } + return certs, nil +} + +func WriteCertsToDir(path string, certs *generator.Artifacts) error { + // Writer's algorithm only manages files using symbolic link. + // If a file is not a symbolic link, will ignore the update for it. + // We want to cleanup for Writer by removing old files that are not symbolic links. + err := prepareToWrite(path) + if err != nil { + return err + } + + aw, err := atomic.NewAtomicWriter(path) + if err != nil { + return err + } + return aw.Write(certToProjectionMap(certs)) +} + +// prepareToWrite ensures it directory is compatible with the atomic.Writer library. +func prepareToWrite(dir string) error { + _, err := os.Stat(dir) + switch { + case os.IsNotExist(err): + klog.Info("cert directory doesn't exist, creating", "directory", dir) + // TODO: figure out if we can reduce the permission. (Now it's 0777) + err = os.MkdirAll(dir, 0777) + if err != nil { + return fmt.Errorf("can't create dir: %v", dir) + } + case err != nil: + return err + } + + filenames := []string{CAKeyName, CACertName, ServerCertName, ServerCertName2, ServerKeyName, ServerKeyName2} + for _, f := range filenames { + abspath := path.Join(dir, f) + _, err := os.Stat(abspath) + if os.IsNotExist(err) { + continue + } else if err != nil { + klog.Error(err, "unable to stat file", "file", abspath) + } + _, err = os.Readlink(abspath) + // if it's not a symbolic link + if err != nil { + err = os.Remove(abspath) + if err != nil { + klog.Error(err, "unable to remove old file", "file", abspath) + } + } + } + return nil +} + +func (f *fsCertWriter) read() (*generator.Artifacts, error) { + if err := ensureExist(f.Path); err != nil { + return nil, err + } + caKeyBytes, err := ioutil.ReadFile(path.Join(f.Path, CAKeyName)) + if err != nil { + return nil, err + } + caCertBytes, err := ioutil.ReadFile(path.Join(f.Path, CACertName)) + if err != nil { + return nil, err + } + certBytes, err := ioutil.ReadFile(path.Join(f.Path, ServerCertName)) + if err != nil { + return nil, err + } + keyBytes, err := ioutil.ReadFile(path.Join(f.Path, ServerKeyName)) + if err != nil { + return nil, err + } + return &generator.Artifacts{ + CAKey: caKeyBytes, + CACert: caCertBytes, + Cert: certBytes, + Key: keyBytes, + }, nil +} + +func ensureExist(dir string) error { + filenames := []string{CAKeyName, CACertName, ServerCertName, ServerCertName2, ServerKeyName, ServerKeyName2} + for _, filename := range filenames { + _, err := os.Stat(path.Join(dir, filename)) + switch { + case err == nil: + continue + case os.IsNotExist(err): + return notFoundError{err} + default: + return err + } + } + return nil +} + +func certToProjectionMap(cert *generator.Artifacts) map[string]atomic.FileProjection { + // TODO: figure out if we can reduce the permission. (Now it's 0666) + return map[string]atomic.FileProjection{ + CAKeyName: { + Data: cert.CAKey, + Mode: 0666, + }, + CACertName: { + Data: cert.CACert, + Mode: 0666, + }, + ServerCertName: { + Data: cert.Cert, + Mode: 0666, + }, + ServerCertName2: { + Data: cert.Cert, + Mode: 0666, + }, + ServerKeyName: { + Data: cert.Key, + Mode: 0666, + }, + ServerKeyName2: { + Data: cert.Key, + Mode: 0666, + }, + } +} diff --git a/pkg/webhook/util/writer/secret.go b/pkg/webhook/util/writer/secret.go new file mode 100644 index 00000000..67592261 --- /dev/null +++ b/pkg/webhook/util/writer/secret.go @@ -0,0 +1,179 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer + +import ( + "context" + "errors" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" +) + +const ( + SecretCertWriter = "secret" +) + +// secretCertWriter provisions the certificate by reading and writing to the k8s secrets. +type secretCertWriter struct { + *SecretCertWriterOptions + + // dnsName is the DNS name that the certificate is for. + dnsName string +} + +// SecretCertWriterOptions is options for constructing a secretCertWriter. +type SecretCertWriterOptions struct { + // client talks to a kubernetes cluster for creating the secret. + Clientset clientset.Interface + // certGenerator generates the certificates. + CertGenerator generator.CertGenerator + // secret points the secret that contains certificates that written by the CertWriter. + Secret *types.NamespacedName +} + +var _ CertWriter = &secretCertWriter{} + +func (ops *SecretCertWriterOptions) setDefaults() { + if ops.CertGenerator == nil { + ops.CertGenerator = &generator.SelfSignedCertGenerator{} + } +} + +func (ops *SecretCertWriterOptions) validate() error { + if ops.Clientset == nil { + return errors.New("client must be set in SecretCertWriterOptions") + } + if ops.Secret == nil { + return errors.New("secret must be set in SecretCertWriterOptions") + } + return nil +} + +// NewSecretCertWriter constructs a CertWriter that persists the certificate in a k8s secret. +func NewSecretCertWriter(ops SecretCertWriterOptions) (CertWriter, error) { + ops.setDefaults() + err := ops.validate() + if err != nil { + return nil, err + } + return &secretCertWriter{SecretCertWriterOptions: &ops}, nil +} + +// EnsureCert provisions certificates for a webhookClientConfig by writing the certificates to a k8s secret. +func (s *secretCertWriter) EnsureCert(dnsName string) (*generator.Artifacts, bool, error) { + // Create or refresh the certs based on clientConfig + s.dnsName = dnsName + return handleCommon(s.dnsName, s) +} + +var _ certReadWriter = &secretCertWriter{} + +func (s *secretCertWriter) buildSecret() (*corev1.Secret, *generator.Artifacts, error) { + certs, err := s.CertGenerator.Generate(s.dnsName) + if err != nil { + return nil, nil, err + } + secret := certsToSecret(certs, *s.Secret) + return secret, certs, err +} + +func (s *secretCertWriter) write() (*generator.Artifacts, error) { + secret, certs, err := s.buildSecret() + if err != nil { + return nil, err + } + _, err = s.Clientset.CoreV1().Secrets(secret.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil, alreadyExistError{err} + } + return certs, err +} + +func (s *secretCertWriter) overwrite(resourceVersion string) (*generator.Artifacts, error) { + secret, certs, err := s.buildSecret() + if err != nil { + return nil, err + } + secret.ResourceVersion = resourceVersion + secret, err = s.Clientset.CoreV1().Secrets(secret.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) + if err != nil { + klog.Infof("Cert writer update secret failed: %v", err) + return nil, err + } + klog.Infof("Cert writer update secret %s resourceVersion from %s to %s", + secret.Name, resourceVersion, secret.ResourceVersion, + ) + return certs, nil +} + +func (s *secretCertWriter) read() (*generator.Artifacts, error) { + secret, err := s.Clientset.CoreV1().Secrets(s.Secret.Namespace).Get(context.TODO(), s.Secret.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, notFoundError{err} + } + if err != nil { + return nil, err + } + certs := secretToCerts(secret) + if certs.CACert != nil && certs.CAKey != nil { + // Store the CA for next usage. + s.CertGenerator.SetCA(certs.CAKey, certs.CACert) + } + return certs, nil +} + +func secretToCerts(secret *corev1.Secret) *generator.Artifacts { + ret := &generator.Artifacts{ + ResourceVersion: secret.ResourceVersion, + } + if secret.Data != nil { + ret.CAKey = secret.Data[CAKeyName] + ret.CACert = secret.Data[CACertName] + ret.Cert = secret.Data[ServerCertName] + ret.Key = secret.Data[ServerKeyName] + } + return ret +} + +func certsToSecret(certs *generator.Artifacts, sec types.NamespacedName) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: sec.Namespace, + Name: sec.Name, + }, + Data: map[string][]byte{ + CAKeyName: certs.CAKey, + CACertName: certs.CACert, + ServerKeyName: certs.Key, + ServerKeyName2: certs.Key, + ServerCertName: certs.Cert, + ServerCertName2: certs.Cert, + }, + } +} diff --git a/pkg/webhook/validating_gss.go b/pkg/webhook/validating_gss.go new file mode 100644 index 00000000..9de1b2b2 --- /dev/null +++ b/pkg/webhook/validating_gss.go @@ -0,0 +1,61 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "fmt" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type GssValidaatingHandler struct { + Client client.Client + decoder *admission.Decoder +} + +func (gvh *GssValidaatingHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + gss := &gamekruiseiov1alpha1.GameServerSet{} + err := gvh.decoder.Decode(req, gss) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + var reason string + allowed, err := validatingGss(gss, gvh.Client) + if err != nil { + reason = err.Error() + } + + return admission.ValidationResponse(allowed, reason) +} + +func validatingGss(gss *gamekruiseiov1alpha1.GameServerSet, client client.Client) (bool, error) { + // validate reserveGameServerIds + rgsIds := gss.Spec.ReserveGameServerIds + if util.IsRepeat(rgsIds) { + return false, fmt.Errorf("reserveGameServerIds should not be repeat. Now it is %v", rgsIds) + } + if util.IsHasNegativeNum(rgsIds) { + return false, fmt.Errorf("reserveGameServerIds should be greater or equal to 0. Now it is %v", rgsIds) + } + + return true, nil +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go new file mode 100644 index 00000000..033a1cfb --- /dev/null +++ b/pkg/webhook/webhook.go @@ -0,0 +1,262 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "flag" + "fmt" + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" + "github.com/openkruise/kruise-game/pkg/webhook/util/writer" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +var ( + mutatePodPath = "/mutate-v1-pod" + validateGssPath = "/validate-v1alpha1-gss" + mutatingWebhookConfigurationName = "kruise-game-mutating-webhook" + validatingWebhookConfigurationName = "kruise-game-validating-webhook" +) + +var ( + webhookPort int + webhookCertDir string + webhookServiceNamespace string + webhookServiceName string +) + +func init() { + flag.IntVar(&webhookPort, "webhook-port", 9876, "The port of the MutatingWebhookConfiguration object.") + flag.StringVar(&webhookCertDir, "webhook-server-certs-dir", "/tmp/webhook-certs/", "Path to the X.509-formatted webhook certificate.") + flag.StringVar(&webhookServiceNamespace, "webhook-service-namespace", "kruise-game-system", "kruise game webhook service namespace.") + flag.StringVar(&webhookServiceName, "webhook-service-name", "kruise-game-webhook-service", "kruise game wehook service name.") +} + +// +kubebuilder:rbac:groups=apps.kruise.io,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.kruise.io,resources=statefulsets/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=create;get;list;watch;update;patch +// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=create;get;list;watch;update;patch +// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch + +type Webhook struct { + mgr manager.Manager +} + +func NewWebhookServer(mgr manager.Manager) *Webhook { + return &Webhook{ + mgr: mgr, + } +} + +func (ws *Webhook) SetupWithManager(mgr manager.Manager) *Webhook { + server := mgr.GetWebhookServer() + server.Host = "0.0.0.0" + server.Port = webhookPort + server.CertDir = webhookCertDir + decoder, err := admission.NewDecoder(runtime.NewScheme()) + if err != nil { + log.Fatalln(err) + } + server.Register(mutatePodPath, &webhook.Admission{Handler: &PodMutatingHandler{Client: mgr.GetClient(), decoder: decoder}}) + server.Register(validateGssPath, &webhook.Admission{Handler: &GssValidaatingHandler{Client: mgr.GetClient(), decoder: decoder}}) + return ws +} + +// Initialize create MutatingWebhookConfiguration before start +func (ws *Webhook) Initialize(cfg *rest.Config) error { + dnsName := generator.ServiceToCommonName(webhookServiceNamespace, webhookServiceName) + + var certWriter writer.CertWriter + var err error + + certWriter, err = writer.NewFSCertWriter(writer.FSCertWriterOptions{Path: webhookCertDir}) + + certs, _, err := certWriter.EnsureCert(dnsName) + if err != nil { + return fmt.Errorf("failed to ensure certs: %v", err) + } + + if err := writer.WriteCertsToDir(webhookCertDir, certs); err != nil { + return fmt.Errorf("failed to write certs to dir: %v", err) + } + + clientSet, err := clientset.NewForConfig(cfg) + + if err != nil { + return err + } + + if err := checkValidatingConfiguration(dnsName, clientSet, certs.CACert); err != nil { + return fmt.Errorf("failed to check mutating webhook,because of %s", err.Error()) + } + + if err := checkMutatingConfiguration(dnsName, clientSet, certs.CACert); err != nil { + return fmt.Errorf("failed to check mutating webhook,because of %s", err.Error()) + } + return nil +} + +func checkValidatingConfiguration(dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + vwc, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.TODO(), validatingWebhookConfigurationName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // create new webhook + return createValidatingWebhook(dnsName, kubeClient, caBundle) + } else { + return err + } + } + return updateValidatingWebhook(vwc, dnsName, kubeClient, caBundle) +} + +func checkMutatingConfiguration(dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + mwc, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookConfigurationName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // create new webhook + return createMutatingWebhook(dnsName, kubeClient, caBundle) + } else { + return err + } + } + return updateMutatingWebhook(mwc, dnsName, kubeClient, caBundle) +} + +func createValidatingWebhook(dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + sideEffectClassNone := admissionregistrationv1.SideEffectClassNone + fail := admissionregistrationv1.Fail + + webhookConfig := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: validatingWebhookConfigurationName, + }, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + Name: dnsName, + SideEffects: &sideEffectClassNone, + FailurePolicy: &fail, + AdmissionReviewVersions: []string{"v1"}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: webhookServiceNamespace, + Name: webhookServiceName, + Path: &validateGssPath, + }, + CABundle: caBundle, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"game.kruise.io"}, + APIVersions: []string{"v1alpha1"}, + Resources: []string{"gameserversets"}, + }, + }, + }, + }, + }, + } + + if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), webhookConfig, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create %s: %v", validatingWebhookConfigurationName, err) + } + return nil +} + +func createMutatingWebhook(dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + sideEffectClassNone := admissionregistrationv1.SideEffectClassNone + ignore := admissionregistrationv1.Ignore + + webhookConfig := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: mutatingWebhookConfigurationName, + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + { + Name: dnsName, + SideEffects: &sideEffectClassNone, + FailurePolicy: &ignore, + AdmissionReviewVersions: []string{"v1"}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: webhookServiceNamespace, + Name: webhookServiceName, + Path: &mutatePodPath, + }, + CABundle: caBundle, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }, + }, + }, + }, + } + + if _, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), webhookConfig, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create %s: %v", mutatingWebhookConfigurationName, err) + } + return nil +} + +func updateValidatingWebhook(vwc *admissionregistrationv1.ValidatingWebhookConfiguration, dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + var mutatingWHs []admissionregistrationv1.ValidatingWebhook + for _, wh := range vwc.Webhooks { + if wh.Name == dnsName { + wh.ClientConfig.CABundle = caBundle + } + mutatingWHs = append(mutatingWHs, wh) + } + vwc.Webhooks = mutatingWHs + if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), vwc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update %s: %v", validatingWebhookConfigurationName, err) + } + return nil +} + +func updateMutatingWebhook(mwc *admissionregistrationv1.MutatingWebhookConfiguration, dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + var mutatingWHs []admissionregistrationv1.MutatingWebhook + for _, wh := range mwc.Webhooks { + if wh.Name == dnsName { + wh.ClientConfig.CABundle = caBundle + } + mutatingWHs = append(mutatingWHs, wh) + } + mwc.Webhooks = mutatingWHs + if _, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), mwc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update %s: %v", mutatingWebhookConfigurationName, err) + } + return nil +} diff --git a/scripts/generate_client.sh b/scripts/generate_client.sh new file mode 100755 index 00000000..da43ad7c --- /dev/null +++ b/scripts/generate_client.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +go mod vendor + +rm -rf ./pkg/client/{clientset,informers,listers} + +/bin/bash ./vendor/k8s.io/code-generator/generate-groups.sh all \ +github.com/openkruise/kruise-game/pkg/client github.com/openkruise/kruise-game "apis:v1alpha1" -h ./hack/boilerplate.go.txt \ No newline at end of file diff --git a/test/e2e/client/client.go b/test/e2e/client/client.go new file mode 100644 index 00000000..7e863e15 --- /dev/null +++ b/test/e2e/client/client.go @@ -0,0 +1,137 @@ +package client + +import ( + "context" + "fmt" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/utils/pointer" + "time" +) + +const ( + Namespace = "e2e-test" + GameServerSet = "default-gss" + GameContainerName = "default-game" +) + +type Client struct { + kruisegameClient kruisegameclientset.Interface + kubeClint clientset.Interface +} + +func NewKubeClient(kruisegameClient kruisegameclientset.Interface, kubeClint clientset.Interface) *Client { + return &Client{ + kruisegameClient: kruisegameClient, + kubeClint: kubeClint, + } +} + +func (client *Client) CreateNamespace() error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: Namespace, + Namespace: Namespace, + }, + } + _, err := client.kubeClint.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) + return err +} + +func (client *Client) DeleteNamespace() error { + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + err = client.kubeClint.CoreV1().Namespaces().Delete(context.TODO(), Namespace, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + } + return false, nil + }) +} + +func (client *Client) DefaultGameServerSet() *gameKruiseV1alpha1.GameServerSet { + return &gameKruiseV1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: GameServerSet, + Namespace: Namespace, + }, + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + Replicas: pointer.Int32Ptr(3), + UpdateStrategy: gameKruiseV1alpha1.UpdateStrategy{ + Type: apps.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &gameKruiseV1alpha1.RollingUpdateStatefulSetStrategy{ + PodUpdatePolicy: kruiseV1beta1.InPlaceIfPossiblePodUpdateStrategyType, + }, + }, + GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: GameContainerName, + Image: "nginx:1.9.7", + }, + }, + }, + }, + }, + }, + } +} + +func (client *Client) CreateGameServerSet(gss *gameKruiseV1alpha1.GameServerSet) (*gameKruiseV1alpha1.GameServerSet, error) { + if gss == nil { + return nil, fmt.Errorf("gss is nil") + } + return client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Create(context.TODO(), gss, metav1.CreateOptions{}) +} + +func (client *Client) UpdateGameServerSet(gss *gameKruiseV1alpha1.GameServerSet) (*gameKruiseV1alpha1.GameServerSet, error) { + return client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Update(context.TODO(), gss, metav1.UpdateOptions{}) +} + +func (client *Client) DeleteGameServerSet() error { + return wait.PollImmediate(3*time.Second, time.Minute, func() (done bool, err error) { + err = client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Delete(context.TODO(), GameServerSet, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + } + return false, nil + }) +} + +func (client *Client) GetGameServerSet() (*gameKruiseV1alpha1.GameServerSet, error) { + return client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Get(context.TODO(), GameServerSet, metav1.GetOptions{}) +} + +func (client *Client) PatchGameServer(gsName string, data []byte) (*gameKruiseV1alpha1.GameServer, error) { + return client.kruisegameClient.GameV1alpha1().GameServers(Namespace).Patch(context.TODO(), gsName, types.MergePatchType, data, metav1.PatchOptions{}) +} + +func (client *Client) PatchGameServerSet(data []byte) (*gameKruiseV1alpha1.GameServerSet, error) { + return client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Patch(context.TODO(), GameServerSet, types.MergePatchType, data, metav1.PatchOptions{}) +} + +func (client *Client) GetGameServerList(labelSelector string) (*gameKruiseV1alpha1.GameServerList, error) { + return client.kruisegameClient.GameV1alpha1().GameServers(Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector}) +} + +func (client *Client) GetPodList(labelSelector string) (*corev1.PodList, error) { + return client.kubeClint.CoreV1().Pods(Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector}) +} + +func (client *Client) GetPod(podName string) (*corev1.Pod, error) { + return client.kubeClint.CoreV1().Pods(Namespace).Get(context.TODO(), podName, metav1.GetOptions{}) +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 00000000..ca47b8a9 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,35 @@ +package e2e + +import ( + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/openkruise/kruise-game/test/e2e/framework" + "github.com/openkruise/kruise-game/test/e2e/testcase" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "testing" +) + +func TestE2E(t *testing.T) { + var err error + + cfg := config.GetConfigOrDie() + f := framework.NewFrameWork(cfg) + + gomega.RegisterFailHandler(ginkgo.Fail) + + ginkgo.Describe("Run kruise game manager e2e tests", func() { + ginkgo.BeforeSuite(func() { + err = f.BeforeSuit() + gomega.Expect(err).To(gomega.BeNil()) + }) + + ginkgo.AfterSuite(func() { + err = f.AfterSuit() + gomega.Expect(err).To(gomega.BeNil()) + }) + + testcase.RunTestCases(f) + }) + + ginkgo.RunSpecs(t, "run kgm e2e test") +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go new file mode 100644 index 00000000..cfae04c8 --- /dev/null +++ b/test/e2e/framework/framework.go @@ -0,0 +1,216 @@ +package framework + +import ( + "encoding/json" + "fmt" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + "github.com/openkruise/kruise-game/pkg/util" + "github.com/openkruise/kruise-game/test/e2e/client" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "time" +) + +type Framework struct { + client *client.Client +} + +func NewFrameWork(config *restclient.Config) *Framework { + kruisegameClient := kruisegameclientset.NewForConfigOrDie(config) + kubeClient := clientset.NewForConfigOrDie(config) + return &Framework{ + client: client.NewKubeClient(kruisegameClient, kubeClient), + } +} + +func (f *Framework) BeforeSuit() error { + err := f.client.CreateNamespace() + if err != nil { + if apierrors.IsAlreadyExists(err) { + err = f.client.DeleteGameServerSet() + if err != nil { + return err + } + } else { + return err + } + } + return nil +} + +func (f *Framework) AfterSuit() error { + return f.client.DeleteNamespace() +} + +func (f *Framework) AfterEach() error { + return f.client.DeleteGameServerSet() +} + +func (f *Framework) DeployGameServerSet() (*gamekruiseiov1alpha1.GameServerSet, error) { + gss := f.client.DefaultGameServerSet() + return f.client.CreateGameServerSet(gss) +} + +func (f *Framework) GameServerScale(gss *gamekruiseiov1alpha1.GameServerSet, desireNum int, reserveGsId *int) (*gamekruiseiov1alpha1.GameServerSet, error) { + // TODO: change patch type + newReserves := gss.Spec.ReserveGameServerIds + if reserveGsId != nil { + newReserves = append(newReserves, *reserveGsId) + } + + numJson := map[string]interface{}{"spec": map[string]interface{}{"replicas": desireNum, "reserveGameServerIds": newReserves}} + data, err := json.Marshal(numJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServerSet(data) +} + +func (f *Framework) ImageUpdate(gss *gamekruiseiov1alpha1.GameServerSet, name, image string) (*gamekruiseiov1alpha1.GameServerSet, error) { + var newContainers []corev1.Container + for _, c := range gss.Spec.GameServerTemplate.Spec.Containers { + newContainer := c + if c.Name == name { + newContainer.Image = image + } + newContainers = append(newContainers, newContainer) + } + + conJson := map[string]interface{}{"spec": map[string]interface{}{"gameServerTemplate": map[string]interface{}{"spec": map[string]interface{}{"containers": newContainers}}}} + data, err := json.Marshal(conJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServerSet(data) +} + +func (f *Framework) MarkGameServerOpsState(gsName string, opsState string) (*gamekruiseiov1alpha1.GameServer, error) { + osJson := map[string]interface{}{"spec": map[string]string{"opsState": opsState}} + data, err := json.Marshal(osJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServer(gsName, data) +} + +func (f *Framework) ChangeGameServerDeletionPriority(gsName string, deletionPriority string) (*gamekruiseiov1alpha1.GameServer, error) { + dpJson := map[string]interface{}{"spec": map[string]string{"deletionPriority": deletionPriority}} + data, err := json.Marshal(dpJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServer(gsName, data) +} + +func (f *Framework) WaitForGsCreated(gss *gamekruiseiov1alpha1.GameServerSet) error { + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + gssName := gss.GetName() + labelSelector := labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gssName, + }).String() + podList, err := f.client.GetPodList(labelSelector) + if err != nil { + return false, err + } + if len(podList.Items) != int(*gss.Spec.Replicas) { + return false, nil + } + return true, nil + }) +} + +func (f *Framework) WaitForUpdated(gss *gamekruiseiov1alpha1.GameServerSet, name, image string) error { + return wait.PollImmediate(10*time.Second, 10*time.Minute, + func() (done bool, err error) { + gssName := gss.GetName() + labelSelector := labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gssName, + }).String() + podList, err := f.client.GetPodList(labelSelector) + if err != nil { + return false, err + } + updated := 0 + + for _, pod := range podList.Items { + for _, c := range pod.Status.ContainerStatuses { + if name == c.Name && image == c.Image { + updated++ + break + } + } + } + + if gss.Spec.UpdateStrategy.RollingUpdate == nil || gss.Spec.UpdateStrategy.RollingUpdate.Partition == nil { + if int32(updated) != *gss.Spec.Replicas { + return false, nil + } + } else { + if int32(updated) != *gss.Spec.Replicas-*gss.Spec.UpdateStrategy.RollingUpdate.Partition { + return false, nil + } + } + return true, nil + }) +} + +func (f *Framework) ExpectGssCorrect(gss *gamekruiseiov1alpha1.GameServerSet, expectIndex []int) error { + + if err := f.WaitForGsCreated(gss); err != nil { + return err + } + + gssName := gss.GetName() + labelSelector := labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gssName, + }).String() + + podList, err := f.client.GetPodList(labelSelector) + if err != nil { + return err + } + + podIndexList := util.GetIndexListFromPodList(podList.Items) + + if !util.IsSliceEqual(expectIndex, podIndexList) { + return fmt.Errorf("current pods and expected pods do not correspond") + } + + return nil +} + +func (f *Framework) WaitForGsOpsStateUpdate(gsName string, opsState string) error { + return wait.PollImmediate(5*time.Second, 1*time.Minute, + func() (done bool, err error) { + pod, err := f.client.GetPod(gsName) + if err != nil { + return false, err + } + currentOpsState := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOpsStateKey] + if currentOpsState == opsState { + return true, nil + } + return false, nil + }) +} + +func (f *Framework) WaitForGsDeletionPriorityUpdated(gsName string, deletionPriority string) error { + return wait.PollImmediate(5*time.Second, 1*time.Minute, + func() (done bool, err error) { + pod, err := f.client.GetPod(gsName) + if err != nil { + return false, err + } + currentPriority := pod.GetLabels()[gamekruiseiov1alpha1.GameServerDeletePriorityKey] + if currentPriority == deletionPriority { + return true, nil + } + return false, nil + }) +} diff --git a/test/e2e/testcase/testcase.go b/test/e2e/testcase/testcase.go new file mode 100644 index 00000000..700d4c28 --- /dev/null +++ b/test/e2e/testcase/testcase.go @@ -0,0 +1,78 @@ +package testcase + +import ( + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/test/e2e/client" + "github.com/openkruise/kruise-game/test/e2e/framework" +) + +func RunTestCases(f *framework.Framework) { + ginkgo.Describe("kruise game controllers", func() { + + ginkgo.AfterEach(func() { + err := f.AfterEach() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("scale", func() { + + // deploy + gss, err := f.DeployGameServerSet() + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2}) + gomega.Expect(err).To(gomega.BeNil()) + + // scale up + gss, err = f.GameServerScale(gss, 5, nil) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2, 3, 4}) + gomega.Expect(err).To(gomega.BeNil()) + + // scale down when setting WaitToDelete + _, err = f.MarkGameServerOpsState(gss.GetName()+"-2", string(gameKruiseV1alpha1.WaitToDelete)) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForGsOpsStateUpdate(gss.GetName()+"-2", string(gameKruiseV1alpha1.WaitToDelete)) + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.GameServerScale(gss, 4, nil) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 3, 4}) + gomega.Expect(err).To(gomega.BeNil()) + + // scale down when setting deletion priority + _, err = f.ChangeGameServerDeletionPriority(gss.GetName()+"-3", "100") + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForGsDeletionPriorityUpdated(gss.GetName()+"-3", "100") + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.GameServerScale(gss, 3, nil) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 4}) + gomega.Expect(err).To(gomega.BeNil()) + }) + + ginkgo.It("update pod", func() { + + // deploy + gss, err := f.DeployGameServerSet() + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2}) + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.ImageUpdate(gss, client.GameContainerName, "nginx:latest") + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForUpdated(gss, client.GameContainerName, "nginx:latest") + gomega.Expect(err).To(gomega.BeNil()) + }) + }) +}