diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..0f046820 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ +testbin/ diff --git a/.gitignore b/.gitignore index cfe9bae1..c0a7a54c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,15 @@ + # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib -bin/ -testbin/ -.temp +bin +testbin/* # Test binary, build with `go test -c` *.test -test/e2e/generated/bindata.go # Output of the go coverage tool, specifically when used with LiteIDE *.out @@ -24,6 +23,3 @@ test/e2e/generated/bindata.go *.swp *.swo *~ -.vscode - -.DS_Store diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b2ce32b3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +# Build the manager binary +FROM golang:1.18 as builder + +WORKDIR /workspace +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY apis/ apis/ +COPY pkg/ pkg/ + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go + +# Use distroless as minimal base images to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM alpine:3.14 +WORKDIR / +COPY --from=builder /workspace/manager . + +ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..19496598 --- /dev/null +++ b/Makefile @@ -0,0 +1,133 @@ + +# Image URL to use all building/pushing images targets +IMG ?= kruise-game-manager:test +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.24.1 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +.PHONY: test +test: manifests generate fmt vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out + +##@ Build + +.PHONY: build +build: generate fmt vet ## Build manager binary. + go build -o bin/manager main.go + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./main.go + +.PHONY: docker-build +docker-build: ## Build docker images with the manager. + docker build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker images with the manager. + docker push ${IMG} + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +.PHONY: undeploy +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +##@ Build Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest + +## Tool Versions +KUSTOMIZE_VERSION ?= v4.5.5 +CONTROLLER_TOOLS_VERSION ?= v0.9.0 + +KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest diff --git a/PROJECT b/PROJECT new file mode 100644 index 00000000..2bf8d697 --- /dev/null +++ b/PROJECT @@ -0,0 +1,25 @@ +domain: my.domain +layout: +- go.kubebuilder.io/v3 +projectName: kruise-game +repo: github.com/openkruise/kruise-game +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: kruise.io + group: game + kind: GameServerSet + path: github.com/openkruise/kruise-game/apis/v1alpha1 + version: v1alpha1 +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: kruise.io + group: game + kind: GameServer + path: github.com/openkruise/kruise-game/apis/v1alpha1 + version: v1alpha1 +version: "3" diff --git a/README.md b/README.md index 2dce58b5..edeed89d 100644 --- a/README.md +++ b/README.md @@ -1 +1,48 @@ -# kruise-game \ No newline at end of file +# kruise-game + +## Introduction +`Kruise-Game` is an open source project based on OpenKruise, to solve the problem of game server landing in Kubernetes. + +OpenKruiseGame logo + + +## Why is Kruise-Game? +Game servers are stateful services, and there are differences in the operation and maintenance of each game server, which also increases with time. In Kubernetes, general workloads manages a batch of game servers according to pod templates, which cannot take into account the differences in game server status. Batch management and directional management are in conflict in k8s. **Kruise-Game** was born to resolve that. Kruise-Game contains two CRDs, GameServer and GameServerSet: + +- `GameServer` is responsible for the management of game server status. Users can customize the game server status to reflect the differences between game servers; +- `GameServerSet` is responsible for batch management of game servers. Users can customize update/reduction strategies according to the status of game servers. + +Features: +- Game server status management + - Mark game servers status without effecting to its lifecycle +- Flexible scaling/deletion mechanism + - Support scaling down by user-defined status & priority + - Support specifying game server to delete directly +- Flexible update mechanism + - Support hot update (in-place update) + - Support updating game server by user-defined priority + - Can control the range of the game servers to be updated + - Can control the pace of the entire update process +- Custom service quality + - Support probing game servers‘ containers and mark game servers status automatically + +## Quick Start + +- [Installation](./docs/getting_started/installation.md) +- [Basic Usage](./docs/tutorials/basic_usage.md) + +## License + +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/apis/v1alpha1/doc.go b/apis/v1alpha1/doc.go new file mode 100644 index 00000000..4f1b3af6 --- /dev/null +++ b/apis/v1alpha1/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:openapi-gen=true +// +groupName=game.kruise.io +package v1alpha1 diff --git a/apis/v1alpha1/gameserver_types.go b/apis/v1alpha1/gameserver_types.go new file mode 100644 index 00000000..108ed317 --- /dev/null +++ b/apis/v1alpha1/gameserver_types.go @@ -0,0 +1,156 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + GameServerStateKey = "game.kruise.io/gs-state" + GameServerOpsStateKey = "game.kruise.io/gs-opsState" + GameServerUpdatePriorityKey = "game.kruise.io/gs-update-priority" + GameServerDeletePriorityKey = "game.kruise.io/gs-delete-priority" +) + +// GameServerSpec defines the desired state of GameServer +type GameServerSpec struct { + OpsState OpsState `json:"opsState,omitempty"` + UpdatePriority *intstr.IntOrString `json:"updatePriority,omitempty"` + DeletionPriority *intstr.IntOrString `json:"deletionPriority,omitempty"` + NetworkDisabled bool `json:"networkDisabled,omitempty"` +} + +type GameServerState string + +const ( + Unknown GameServerState = "Unknown" + Creating GameServerState = "Creating" + Ready GameServerState = "Ready" + NotReady GameServerState = "NotReady" + Crash GameServerState = "Crash" + Updating GameServerState = "Updating" + Deleting GameServerState = "Deleting" +) + +type OpsState string + +const ( + Maintaining OpsState = "Maintaining" + WaitToDelete OpsState = "WaitToBeDeleted" + None OpsState = "None" +) + +type ServiceQuality struct { + corev1.Probe `json:",inline"` + Name string `json:"name"` + ContainerName string `json:"containerName,omitempty"` + ServiceQualityAction []ServiceQualityAction `json:"serviceQualityAction,omitempty"` +} + +type ServiceQualityCondition struct { + Name string `json:"name"` + Status string `json:"status,omitempty"` + LastProbeTime metav1.Time `json:"lastProbeTime,omitempty"` + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + LastActionTransitionTime metav1.Time `json:"lastActionTransitionTime,omitempty"` +} + +type ServiceQualityAction struct { + Permanent bool `json:"permanent,omitempty"` // default: true + State bool `json:"state"` + GameServerSpec `json:",inline"` +} + +// GameServerStatus defines the observed state of GameServer +type GameServerStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + DesiredState GameServerState `json:"desiredState,omitempty"` + CurrentState GameServerState `json:"currentState,omitempty"` + NetworkStatus NetworkStatus `json:"networkStatus,omitempty"` + PodStatus corev1.PodStatus `json:"podStatus,omitempty"` + ServiceQualitiesCondition []ServiceQualityCondition `json:"serviceQualitiesConditions,omitempty"` + // Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update. + UpdatePriority *intstr.IntOrString `json:"updatePriority,omitempty"` + DeletionPriority *intstr.IntOrString `json:"deletionPriority,omitempty"` + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` +} + +type NetworkStatus struct { + NetworkType string `json:"networkType,omitempty"` + InternalAddresses []NetworkAddress `json:"internalAddresses,omitempty"` + ExternalAddresses []NetworkAddress `json:"externalAddresses,omitempty"` + DesiredNetworkState NetworkState `json:"desiredNetworkState,omitempty"` + CurrentNetworkState NetworkState `json:"currentNetworkState,omitempty"` + CreateTime metav1.Time `json:"createTime,omitempty"` + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` +} + +type NetworkState string + +type NetworkAddress struct { + IP string `json:"ip"` + // TODO add IPv6 + Ports []NetworkPort `json:"ports,omitempty"` + PortRange NetworkPortRange `json:"portRange,omitempty"` + EndPoint string `json:"endPoint,omitempty"` +} + +type NetworkPort struct { + Name string `json:"name"` + Protocol corev1.Protocol `json:"protocol,omitempty"` + Port *intstr.IntOrString `json:"port,omitempty"` +} + +type NetworkPortRange struct { + Protocol corev1.Protocol `json:"protocol,omitempty"` + PortRange string `json:"portRange,omitempty"` +} + +//+genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="STATE",type="string",JSONPath=".status.currentState",description="The current state of GameServer" +//+kubebuilder:printcolumn:name="OPSSTATE",type="string",JSONPath=".spec.opsState",description="The operations state of GameServer" +//+kubebuilder:printcolumn:name="DP",type="string",JSONPath=".status.deletionPriority",description="The current deletionPriority of GameServer" +//+kubebuilder:printcolumn:name="UP",type="string",JSONPath=".status.updatePriority",description="The current updatePriority of GameServer" +//+kubebuilder:resource:shortName=gs + +// GameServer is the Schema for the gameservers API +type GameServer struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GameServerSpec `json:"spec,omitempty"` + Status GameServerStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// GameServerList contains a list of GameServer +type GameServerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GameServer `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GameServer{}, &GameServerList{}) +} diff --git a/apis/v1alpha1/gameserverset_types.go b/apis/v1alpha1/gameserverset_types.go new file mode 100644 index 00000000..f32a30ea --- /dev/null +++ b/apis/v1alpha1/gameserverset_types.go @@ -0,0 +1,170 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + appspub "github.com/openkruise/kruise-api/apps/pub" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +const ( + GameServerOwnerGssKey = "game.kruise.io/owner-gss" + GameServerSetReserveIdsKey = "game.kruise.io/reserve-ids" + GameServerSetNotExistIdsKey = "game.kruise.io/not-exist-ids" + AstsHashKey = "game.kruise.io/asts-hash" +) + +// GameServerSetSpec defines the desired state of GameServerSet +type GameServerSetSpec struct { + // replicas is the desired number of replicas of the given Template. + // These are replicas in the sense that they are instantiations of the + // same Template, but individual replicas also have a consistent identity. + //+kubebuilder:validation:Required + //+kubebuilder:validation:Minimum=0 + Replicas *int32 `json:"replicas"` + // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + // Important: Run "make" to regenerate code after modifying this file + GameServerTemplate GameServerTemplate `json:"gameServerTemplate,omitempty"` + ReserveGameServerIds []int `json:"reserveGameServerIds,omitempty"` + ServiceQualities []ServiceQuality `json:"serviceQualities,omitempty"` + UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"` + ScaleStrategy ScaleStrategy `json:"scaleStrategy,omitempty"` + Network *Network `json:"network,omitempty"` +} + +type GameServerTemplate struct { + // +kubebuilder:pruning:PreserveUnknownFields + // +kubebuilder:validation:Schemaless + corev1.PodTemplateSpec `json:",inline"` + VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` +} + +type Network struct { + NetworkType string `json:"networkType,omitempty"` + NetworkConf []NetworkConfParams `json:"networkConf,omitempty"` +} + +type NetworkConfParams KVParams + +type KVParams struct { + Name string `json:"name,omitempty"` + Value string `json:"value,omitempty"` +} + +type UpdateStrategy struct { + // Type indicates the type of the StatefulSetUpdateStrategy. + // Default is RollingUpdate. + // +optional + Type apps.StatefulSetUpdateStrategyType `json:"type,omitempty"` + // RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType. + // +optional + RollingUpdate *RollingUpdateStatefulSetStrategy `json:"rollingUpdate,omitempty"` +} + +type RollingUpdateStatefulSetStrategy struct { + // Partition indicates the ordinal at which the StatefulSet should be partitioned by default. + // But if unorderedUpdate has been set: + // - Partition indicates the number of pods with non-updated revisions when rolling update. + // - It means controller will update $(replicas - partition) number of pod. + // Default value is 0. + // +optional + Partition *int32 `json:"partition,omitempty"` + // The maximum number of pods that can be unavailable during the update. + // Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). + // Absolute number is calculated from percentage by rounding down. + // Also, maxUnavailable can just be allowed to work with Parallel podManagementPolicy. + // Defaults to 1. + // +optional + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + // PodUpdatePolicy indicates how pods should be updated + // Default value is "ReCreate" + // +optional + PodUpdatePolicy kruiseV1beta1.PodUpdateStrategyType `json:"podUpdatePolicy,omitempty"` + // Paused indicates that the StatefulSet is paused. + // Default value is false + // +optional + Paused bool `json:"paused,omitempty"` + // UnorderedUpdate contains strategies for non-ordered update. + // If it is not nil, pods will be updated with non-ordered sequence. + // Noted that UnorderedUpdate can only be allowed to work with Parallel podManagementPolicy + // +optional + // UnorderedUpdate *kruiseV1beta1.UnorderedUpdateStrategy `json:"unorderedUpdate,omitempty"` + // InPlaceUpdateStrategy contains strategies for in-place update. + // +optional + InPlaceUpdateStrategy *appspub.InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"` + // MinReadySeconds indicates how long will the pod be considered ready after it's updated. + // MinReadySeconds works with both OrderedReady and Parallel podManagementPolicy. + // It affects the pod scale up speed when the podManagementPolicy is set to be OrderedReady. + // Combined with MaxUnavailable, it affects the pod update speed regardless of podManagementPolicy. + // Default value is 0, max is 300. + // +optional + MinReadySeconds *int32 `json:"minReadySeconds,omitempty"` +} + +type ScaleStrategy struct { + kruiseV1beta1.StatefulSetScaleStrategy `json:",inline"` +} + +// GameServerSetStatus defines the observed state of GameServerSet +type GameServerSetStatus struct { + // replicas from advancedStatefulSet + Replicas int32 `json:"replicas"` + ReadyReplicas int32 `json:"readyReplicas"` + AvailableReplicas int32 `json:"availableReplicas"` + CurrentReplicas int32 `json:"currentReplicas"` + UpdatedReplicas int32 `json:"updatedReplicas"` + UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"` + MaintainingReplicas *int32 `json:"maintainingReplicas,omitempty"` + WaitToBeDeletedReplicas *int32 `json:"waitToBeDeletedReplicas,omitempty"` + // LabelSelector is label selectors for query over pods that should match the replica count used by HPA. + LabelSelector string `json:"labelSelector,omitempty"` +} + +//+genclient +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector +//+kubebuilder:resource:shortName=gss + +// GameServerSet is the Schema for the gameserversets API +type GameServerSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec GameServerSetSpec `json:"spec,omitempty"` + Status GameServerSetStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// GameServerSetList contains a list of GameServerSet +type GameServerSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []GameServerSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&GameServerSet{}, &GameServerSetList{}) +} diff --git a/apis/v1alpha1/groupversion_info.go b/apis/v1alpha1/groupversion_info.go new file mode 100644 index 00000000..0726d726 --- /dev/null +++ b/apis/v1alpha1/groupversion_info.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the game.kruise.io v1alpha1 API group +//+kubebuilder:object:generate=true +//+groupName=game.kruise.io +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "game.kruise.io", Version: "v1alpha1"} + + SchemeGroupVersion = GroupVersion + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) + +// Resource is required by pkg/client/listers/... +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 00000000..dfa22766 --- /dev/null +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,562 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "github.com/openkruise/kruise-api/apps/pub" + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServer) DeepCopyInto(out *GameServer) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServer. +func (in *GameServer) DeepCopy() *GameServer { + if in == nil { + return nil + } + out := new(GameServer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GameServer) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerList) DeepCopyInto(out *GameServerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GameServer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerList. +func (in *GameServerList) DeepCopy() *GameServerList { + if in == nil { + return nil + } + out := new(GameServerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GameServerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSet) DeepCopyInto(out *GameServerSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSet. +func (in *GameServerSet) DeepCopy() *GameServerSet { + if in == nil { + return nil + } + out := new(GameServerSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GameServerSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSetList) DeepCopyInto(out *GameServerSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]GameServerSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSetList. +func (in *GameServerSetList) DeepCopy() *GameServerSetList { + if in == nil { + return nil + } + out := new(GameServerSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *GameServerSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSetSpec) DeepCopyInto(out *GameServerSetSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + in.GameServerTemplate.DeepCopyInto(&out.GameServerTemplate) + if in.ReserveGameServerIds != nil { + in, out := &in.ReserveGameServerIds, &out.ReserveGameServerIds + *out = make([]int, len(*in)) + copy(*out, *in) + } + if in.ServiceQualities != nil { + in, out := &in.ServiceQualities, &out.ServiceQualities + *out = make([]ServiceQuality, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) + in.ScaleStrategy.DeepCopyInto(&out.ScaleStrategy) + if in.Network != nil { + in, out := &in.Network, &out.Network + *out = new(Network) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSetSpec. +func (in *GameServerSetSpec) DeepCopy() *GameServerSetSpec { + if in == nil { + return nil + } + out := new(GameServerSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSetStatus) DeepCopyInto(out *GameServerSetStatus) { + *out = *in + if in.MaintainingReplicas != nil { + in, out := &in.MaintainingReplicas, &out.MaintainingReplicas + *out = new(int32) + **out = **in + } + if in.WaitToBeDeletedReplicas != nil { + in, out := &in.WaitToBeDeletedReplicas, &out.WaitToBeDeletedReplicas + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSetStatus. +func (in *GameServerSetStatus) DeepCopy() *GameServerSetStatus { + if in == nil { + return nil + } + out := new(GameServerSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerSpec) DeepCopyInto(out *GameServerSpec) { + *out = *in + if in.UpdatePriority != nil { + in, out := &in.UpdatePriority, &out.UpdatePriority + *out = new(intstr.IntOrString) + **out = **in + } + if in.DeletionPriority != nil { + in, out := &in.DeletionPriority, &out.DeletionPriority + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerSpec. +func (in *GameServerSpec) DeepCopy() *GameServerSpec { + if in == nil { + return nil + } + out := new(GameServerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerStatus) DeepCopyInto(out *GameServerStatus) { + *out = *in + in.NetworkStatus.DeepCopyInto(&out.NetworkStatus) + in.PodStatus.DeepCopyInto(&out.PodStatus) + if in.ServiceQualitiesCondition != nil { + in, out := &in.ServiceQualitiesCondition, &out.ServiceQualitiesCondition + *out = make([]ServiceQualityCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.UpdatePriority != nil { + in, out := &in.UpdatePriority, &out.UpdatePriority + *out = new(intstr.IntOrString) + **out = **in + } + if in.DeletionPriority != nil { + in, out := &in.DeletionPriority, &out.DeletionPriority + *out = new(intstr.IntOrString) + **out = **in + } + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerStatus. +func (in *GameServerStatus) DeepCopy() *GameServerStatus { + if in == nil { + return nil + } + out := new(GameServerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GameServerTemplate) DeepCopyInto(out *GameServerTemplate) { + *out = *in + in.PodTemplateSpec.DeepCopyInto(&out.PodTemplateSpec) + if in.VolumeClaimTemplates != nil { + in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates + *out = make([]v1.PersistentVolumeClaim, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GameServerTemplate. +func (in *GameServerTemplate) DeepCopy() *GameServerTemplate { + if in == nil { + return nil + } + out := new(GameServerTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KVParams) DeepCopyInto(out *KVParams) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KVParams. +func (in *KVParams) DeepCopy() *KVParams { + if in == nil { + return nil + } + out := new(KVParams) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Network) DeepCopyInto(out *Network) { + *out = *in + if in.NetworkConf != nil { + in, out := &in.NetworkConf, &out.NetworkConf + *out = make([]NetworkConfParams, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Network. +func (in *Network) DeepCopy() *Network { + if in == nil { + return nil + } + out := new(Network) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkAddress) DeepCopyInto(out *NetworkAddress) { + *out = *in + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]NetworkPort, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + out.PortRange = in.PortRange +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkAddress. +func (in *NetworkAddress) DeepCopy() *NetworkAddress { + if in == nil { + return nil + } + out := new(NetworkAddress) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkConfParams) DeepCopyInto(out *NetworkConfParams) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkConfParams. +func (in *NetworkConfParams) DeepCopy() *NetworkConfParams { + if in == nil { + return nil + } + out := new(NetworkConfParams) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkPort) DeepCopyInto(out *NetworkPort) { + *out = *in + if in.Port != nil { + in, out := &in.Port, &out.Port + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkPort. +func (in *NetworkPort) DeepCopy() *NetworkPort { + if in == nil { + return nil + } + out := new(NetworkPort) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkPortRange) DeepCopyInto(out *NetworkPortRange) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkPortRange. +func (in *NetworkPortRange) DeepCopy() *NetworkPortRange { + if in == nil { + return nil + } + out := new(NetworkPortRange) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkStatus) DeepCopyInto(out *NetworkStatus) { + *out = *in + if in.InternalAddresses != nil { + in, out := &in.InternalAddresses, &out.InternalAddresses + *out = make([]NetworkAddress, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ExternalAddresses != nil { + in, out := &in.ExternalAddresses, &out.ExternalAddresses + *out = make([]NetworkAddress, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.CreateTime.DeepCopyInto(&out.CreateTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkStatus. +func (in *NetworkStatus) DeepCopy() *NetworkStatus { + if in == nil { + return nil + } + out := new(NetworkStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpdateStatefulSetStrategy) DeepCopyInto(out *RollingUpdateStatefulSetStrategy) { + *out = *in + if in.Partition != nil { + in, out := &in.Partition, &out.Partition + *out = new(int32) + **out = **in + } + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(intstr.IntOrString) + **out = **in + } + if in.InPlaceUpdateStrategy != nil { + in, out := &in.InPlaceUpdateStrategy, &out.InPlaceUpdateStrategy + *out = new(pub.InPlaceUpdateStrategy) + **out = **in + } + if in.MinReadySeconds != nil { + in, out := &in.MinReadySeconds, &out.MinReadySeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateStatefulSetStrategy. +func (in *RollingUpdateStatefulSetStrategy) DeepCopy() *RollingUpdateStatefulSetStrategy { + if in == nil { + return nil + } + out := new(RollingUpdateStatefulSetStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScaleStrategy) DeepCopyInto(out *ScaleStrategy) { + *out = *in + in.StatefulSetScaleStrategy.DeepCopyInto(&out.StatefulSetScaleStrategy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleStrategy. +func (in *ScaleStrategy) DeepCopy() *ScaleStrategy { + if in == nil { + return nil + } + out := new(ScaleStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceQuality) DeepCopyInto(out *ServiceQuality) { + *out = *in + in.Probe.DeepCopyInto(&out.Probe) + if in.ServiceQualityAction != nil { + in, out := &in.ServiceQualityAction, &out.ServiceQualityAction + *out = make([]ServiceQualityAction, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceQuality. +func (in *ServiceQuality) DeepCopy() *ServiceQuality { + if in == nil { + return nil + } + out := new(ServiceQuality) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceQualityAction) DeepCopyInto(out *ServiceQualityAction) { + *out = *in + in.GameServerSpec.DeepCopyInto(&out.GameServerSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceQualityAction. +func (in *ServiceQualityAction) DeepCopy() *ServiceQualityAction { + if in == nil { + return nil + } + out := new(ServiceQualityAction) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceQualityCondition) DeepCopyInto(out *ServiceQualityCondition) { + *out = *in + in.LastProbeTime.DeepCopyInto(&out.LastProbeTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + in.LastActionTransitionTime.DeepCopyInto(&out.LastActionTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceQualityCondition. +func (in *ServiceQualityCondition) DeepCopy() *ServiceQualityCondition { + if in == nil { + return nil + } + out := new(ServiceQualityCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateStrategy) DeepCopyInto(out *UpdateStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdateStatefulSetStrategy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStrategy. +func (in *UpdateStrategy) DeepCopy() *UpdateStrategy { + if in == nil { + return nil + } + out := new(UpdateStrategy) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/game.kruise.io_gameservers.yaml b/config/crd/bases/game.kruise.io_gameservers.yaml new file mode 100644 index 00000000..abd95b3d --- /dev/null +++ b/config/crd/bases/game.kruise.io_gameservers.yaml @@ -0,0 +1,838 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: gameservers.game.kruise.io +spec: + group: game.kruise.io + names: + kind: GameServer + listKind: GameServerList + plural: gameservers + shortNames: + - gs + singular: gameserver + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The current state of GameServer + jsonPath: .status.currentState + name: STATE + type: string + - description: The operations state of GameServer + jsonPath: .spec.opsState + name: OPSSTATE + type: string + - description: The current deletionPriority of GameServer + jsonPath: .status.deletionPriority + name: DP + type: string + - description: The current updatePriority of GameServer + jsonPath: .status.updatePriority + name: UP + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: GameServer is the Schema for the gameservers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GameServerSpec defines the desired state of GameServer + properties: + deletionPriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + networkDisabled: + type: boolean + opsState: + type: string + updatePriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + status: + description: GameServerStatus defines the observed state of GameServer + properties: + currentState: + type: string + deletionPriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + desiredState: + description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + of cluster Important: Run "make" to regenerate code after modifying + this file' + type: string + lastTransitionTime: + format: date-time + type: string + networkStatus: + properties: + createTime: + format: date-time + type: string + currentNetworkState: + type: string + desiredNetworkState: + type: string + externalAddresses: + items: + properties: + endPoint: + type: string + ip: + type: string + portRange: + properties: + portRange: + type: string + protocol: + default: TCP + type: string + type: object + ports: + description: TODO add IPv6 + items: + properties: + name: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + default: TCP + type: string + required: + - name + type: object + type: array + required: + - ip + type: object + type: array + internalAddresses: + items: + properties: + endPoint: + type: string + ip: + type: string + portRange: + properties: + portRange: + type: string + protocol: + default: TCP + type: string + type: object + ports: + description: TODO add IPv6 + items: + properties: + name: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + protocol: + default: TCP + type: string + required: + - name + type: object + type: array + required: + - ip + type: object + type: array + lastTransitionTime: + format: date-time + type: string + networkType: + type: string + type: object + podStatus: + description: PodStatus represents information about the status of + a pod. Status may trail the actual state of a system, especially + if the node that hosts the pod cannot contact the control plane. + properties: + conditions: + description: 'Current service state of pod. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + items: + description: PodCondition contains details for the current condition + of this pod. + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + format: date-time + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the + condition's last transition. + type: string + status: + description: 'Status is the status of the condition. Can + be True, False, Unknown. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + type: + description: 'Type is the type of the condition. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-conditions' + type: string + required: + - status + - type + type: object + type: array + containerStatuses: + description: 'The list has one entry per container in the manifest. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-and-container-status' + items: + description: ContainerStatus contains details for the current + status of this container. + properties: + containerID: + description: Container's ID in the format '://'. + type: string + image: + description: 'The image the container is running. More info: + https://kubernetes.io/docs/concepts/containers/images.' + type: string + imageID: + description: ImageID of the container's image. + type: string + lastState: + description: Details about the container's last termination + condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + name: + description: This must be a DNS_LABEL. Each container in + a pod must have a unique name. Cannot be updated. + type: string + ready: + description: Specifies whether the container has passed + its readiness probe. + type: boolean + restartCount: + description: The number of times the container has been + restarted. + format: int32 + type: integer + started: + description: Specifies whether the container has passed + its startup probe. Initialized as false, becomes true + after startupProbe is considered successful. Resets to + false when the container is restarted, or if kubelet loses + state temporarily. Is always true when no startupProbe + is defined. + type: boolean + state: + description: Details about the container's current condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + required: + - image + - imageID + - name + - ready + - restartCount + type: object + type: array + ephemeralContainerStatuses: + description: Status for any ephemeral containers that have run + in this pod. This field is beta-level and available on clusters + that haven't disabled the EphemeralContainers feature gate. + items: + description: ContainerStatus contains details for the current + status of this container. + properties: + containerID: + description: Container's ID in the format '://'. + type: string + image: + description: 'The image the container is running. More info: + https://kubernetes.io/docs/concepts/containers/images.' + type: string + imageID: + description: ImageID of the container's image. + type: string + lastState: + description: Details about the container's last termination + condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + name: + description: This must be a DNS_LABEL. Each container in + a pod must have a unique name. Cannot be updated. + type: string + ready: + description: Specifies whether the container has passed + its readiness probe. + type: boolean + restartCount: + description: The number of times the container has been + restarted. + format: int32 + type: integer + started: + description: Specifies whether the container has passed + its startup probe. Initialized as false, becomes true + after startupProbe is considered successful. Resets to + false when the container is restarted, or if kubelet loses + state temporarily. Is always true when no startupProbe + is defined. + type: boolean + state: + description: Details about the container's current condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + required: + - image + - imageID + - name + - ready + - restartCount + type: object + type: array + hostIP: + description: IP address of the host to which the pod is assigned. + Empty if not yet scheduled. + type: string + initContainerStatuses: + description: 'The list has one entry per init container in the + manifest. The most recent successful init container will have + ready = true, the most recently started container will have + startTime set. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-and-container-status' + items: + description: ContainerStatus contains details for the current + status of this container. + properties: + containerID: + description: Container's ID in the format '://'. + type: string + image: + description: 'The image the container is running. More info: + https://kubernetes.io/docs/concepts/containers/images.' + type: string + imageID: + description: ImageID of the container's image. + type: string + lastState: + description: Details about the container's last termination + condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + name: + description: This must be a DNS_LABEL. Each container in + a pod must have a unique name. Cannot be updated. + type: string + ready: + description: Specifies whether the container has passed + its readiness probe. + type: boolean + restartCount: + description: The number of times the container has been + restarted. + format: int32 + type: integer + started: + description: Specifies whether the container has passed + its startup probe. Initialized as false, becomes true + after startupProbe is considered successful. Resets to + false when the container is restarted, or if kubelet loses + state temporarily. Is always true when no startupProbe + is defined. + type: boolean + state: + description: Details about the container's current condition. + properties: + running: + description: Details about a running container + properties: + startedAt: + description: Time at which the container was last + (re-)started + format: date-time + type: string + type: object + terminated: + description: Details about a terminated container + properties: + containerID: + description: Container's ID in the format '://' + type: string + exitCode: + description: Exit status from the last termination + of the container + format: int32 + type: integer + finishedAt: + description: Time at which the container last terminated + format: date-time + type: string + message: + description: Message regarding the last termination + of the container + type: string + reason: + description: (brief) reason from the last termination + of the container + type: string + signal: + description: Signal from the last termination of + the container + format: int32 + type: integer + startedAt: + description: Time at which previous execution of + the container started + format: date-time + type: string + required: + - exitCode + type: object + waiting: + description: Details about a waiting container + properties: + message: + description: Message regarding why the container + is not yet running. + type: string + reason: + description: (brief) reason the container is not + yet running. + type: string + type: object + type: object + required: + - image + - imageID + - name + - ready + - restartCount + type: object + type: array + message: + description: A human readable message indicating details about + why the pod is in this condition. + type: string + nominatedNodeName: + description: nominatedNodeName is set only when this pod preempts + other pods on the node, but it cannot be scheduled right away + as preemption victims receive their graceful termination periods. + This field does not guarantee that the pod will be scheduled + on this node. Scheduler may decide to place the pod elsewhere + if other nodes become available sooner. Scheduler may also decide + to give the resources on this node to a higher priority pod + that is created after preemption. As a result, this field may + be different than PodSpec.nodeName when the pod is scheduled. + type: string + phase: + description: "The phase of a Pod is a simple, high-level summary + of where the Pod is in its lifecycle. The conditions array, + the reason and message fields, and the individual container + status arrays contain more detail about the pod's status. There + are five possible phase values: \n Pending: The pod has been + accepted by the Kubernetes system, but one or more of the container + images has not been created. This includes time before being + scheduled as well as time spent downloading images over the + network, which could take a while. Running: The pod has been + bound to a node, and all of the containers have been created. + At least one container is still running, or is in the process + of starting or restarting. Succeeded: All containers in the + pod have terminated in success, and will not be restarted. Failed: + All containers in the pod have terminated, and at least one + container has terminated in failure. The container either exited + with non-zero status or was terminated by the system. Unknown: + For some reason the state of the pod could not be obtained, + typically due to an error in communicating with the host of + the pod. \n More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#pod-phase" + type: string + podIP: + description: IP address allocated to the pod. Routable at least + within the cluster. Empty if not yet allocated. + type: string + podIPs: + description: podIPs holds the IP addresses allocated to the pod. + If this field is specified, the 0th entry must match the podIP + field. Pods may be allocated at most 1 value for each of IPv4 + and IPv6. This list is empty if no IPs have been allocated yet. + items: + description: 'IP address information for entries in the (plural) + PodIPs field. Each entry includes: IP: An IP address allocated + to the pod. Routable at least within the cluster.' + properties: + ip: + description: ip is an IP address (IPv4 or IPv6) assigned + to the pod + type: string + type: object + type: array + qosClass: + description: 'The Quality of Service (QOS) classification assigned + to the pod based on resource requirements See PodQOSClass type + for available QOS classes More info: https://git.k8s.io/community/contributors/design-proposals/node/resource-qos.md' + type: string + reason: + description: A brief CamelCase message indicating details about + why the pod is in this state. e.g. 'Evicted' + type: string + startTime: + description: RFC 3339 date and time at which the object was acknowledged + by the Kubelet. This is before the Kubelet pulled the container + image(s) for the pod. + format: date-time + type: string + type: object + serviceQualitiesConditions: + items: + properties: + lastActionTransitionTime: + format: date-time + type: string + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + name: + type: string + status: + type: string + required: + - name + type: object + type: array + updatePriority: + anyOf: + - type: integer + - type: string + description: Lifecycle defines the lifecycle hooks for Pods pre-delete, + in-place update. + x-kubernetes-int-or-string: true + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/game.kruise.io_gameserversets.yaml b/config/crd/bases/game.kruise.io_gameserversets.yaml new file mode 100644 index 00000000..ea9002c1 --- /dev/null +++ b/config/crd/bases/game.kruise.io_gameserversets.yaml @@ -0,0 +1,665 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: gameserversets.game.kruise.io +spec: + group: game.kruise.io + names: + kind: GameServerSet + listKind: GameServerSetList + plural: gameserversets + shortNames: + - gss + singular: gameserverset + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: GameServerSet is the Schema for the gameserversets API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GameServerSetSpec defines the desired state of GameServerSet + properties: + gameServerTemplate: + description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + Important: Run "make" to regenerate code after modifying this file' + properties: + volumeClaimTemplates: + items: + description: PersistentVolumeClaim is a user's request for and + claim to a persistent volume + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of + this representation of an object. Servers should convert + recognized schemas to the latest internal value, and may + reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST + resource this object represents. Servers may infer this + from the endpoint the client submits requests to. Cannot + be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata' + type: object + spec: + description: 'spec defines the desired characteristics of + a volume requested by a pod author. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'accessModes contains the desired access + modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify + either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) If the provisioner + or an external controller can support the specified + data source, it will create a new volume based on + the contents of the specified data source. If the + AnyVolumeDataSource feature gate is enabled, this + field will always have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'dataSourceRef specifies the object from + which to populate the volume with data, if a non-empty + volume is desired. This may be any local object from + a non-empty API group (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume binding + will only succeed if the type of the specified object + matches some installed volume populator or dynamic + provisioner. This field will replace the functionality + of the DataSource field and as such if both fields + are non-empty, they must have the same value. For + backwards compatibility, both fields (DataSource and + DataSourceRef) will be set to the same value automatically + if one of them is empty and the other is non-empty. + There are two important differences between DataSource + and DataSourceRef: * While DataSource only allows + two specific types of objects, DataSourceRef allows + any non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed values + (dropping them), DataSourceRef preserves all values, + and generates an error if a disallowed value is specified. + (Beta) Using this field requires the AnyVolumeDataSource + feature gate to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the resource + being referenced. If APIGroup is not specified, + the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum resources + the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify resource + requirements that are lower than previous value but + must still be higher than capacity recorded in the + status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is + omitted for a container, it defaults to Limits + if that is explicitly specified, otherwise to + an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over volumes + to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a + selector that contains values, a key, and an + operator that relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. If the + operator is Exists or DoesNotExist, the + values array must be empty. This array is + replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is "In", + and the values array contains only "value". The + requirements are ANDed. + type: object + type: object + storageClassName: + description: 'storageClassName is the name of the StorageClass + required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of volume + is required by the claim. Value of Filesystem is implied + when not included in claim spec. + type: string + volumeName: + description: volumeName is the binding reference to + the PersistentVolume backing this claim. + type: string + type: object + status: + description: 'status represents the current information/status + of a persistent volume claim. Read-only. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + accessModes: + description: 'accessModes contains the actual access + modes the volume backing the PVC has. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: allocatedResources is the storage resource + within AllocatedResources tracks the capacity allocated + to a PVC. It may be larger than the actual capacity + when a volume expansion operation is requested. For + storage quota, the larger value from allocatedResources + and PVC.spec.resources is used. If allocatedResources + is not set, PVC.spec.resources alone is used for quota + calculation. If a volume expansion capacity request + is lowered, allocatedResources is only lowered if + there are no expansion operations in progress and + if the actual volume capacity is equal or lower than + the requested capacity. This is an alpha field and + requires enabling RecoverVolumeExpansionFailure feature. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: capacity represents the actual resources + of the underlying volume. + type: object + conditions: + description: conditions is the current Condition of + persistent volume claim. If underlying persistent + volume is being resized then the Condition will be + set to 'ResizeStarted'. + items: + description: PersistentVolumeClaimCondition contails + details about state of pvc + properties: + lastProbeTime: + description: lastProbeTime is the time we probed + the condition. + format: date-time + type: string + lastTransitionTime: + description: lastTransitionTime is the time the + condition transitioned from one status to another. + format: date-time + type: string + message: + description: message is the human-readable message + indicating details about last transition. + type: string + reason: + description: reason is a unique, this should be + a short, machine understandable string that + gives the reason for condition's last transition. + If it reports "ResizeStarted" that means the + underlying persistent volume is being resized. + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType + is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: phase represents the current phase of PersistentVolumeClaim. + type: string + resizeStatus: + description: resizeStatus stores status of resize operation. + ResizeStatus is not set by default but when expansion + is complete resizeStatus is set to empty string by + resize controller or kubelet. This is an alpha field + and requires enabling RecoverVolumeExpansionFailure + feature. + type: string + type: object + type: object + type: array + type: object + x-kubernetes-preserve-unknown-fields: true + network: + properties: + networkConf: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + networkType: + type: string + type: object + replicas: + description: replicas is the desired number of replicas of the given + Template. These are replicas in the sense that they are instantiations + of the same Template, but individual replicas also have a consistent + identity. + format: int32 + minimum: 0 + type: integer + reserveGameServerIds: + items: + type: integer + type: array + scaleStrategy: + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can be unavailable + during scaling. Value can be an absolute number (ex: 5) or a + percentage of desired pods (ex: 10%). Absolute number is calculated + from percentage by rounding down. It can just be allowed to + work with Parallel podManagementPolicy.' + x-kubernetes-int-or-string: true + type: object + serviceQualities: + items: + properties: + containerName: + type: string + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit status + of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be + considered failed after having succeeded. Defaults to 3. Minimum + value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + This is a beta field and requires enabling GRPCContainerProbe + feature gate. + properties: + port: + description: Port number of the gRPC service. Number must + be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place + in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). + \n If this is not specified, the default behavior is defined + by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod + IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to be + used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults + to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + name: + type: string + periodSeconds: + description: How often (in seconds) to perform the probe. Default + to 10 seconds. Minimum value is 1. + format: int32 + type: integer + serviceQualityAction: + items: + properties: + deletionPriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + networkDisabled: + type: boolean + opsState: + type: string + permanent: + type: boolean + state: + type: boolean + updatePriority: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - state + type: object + type: array + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate + gracefully upon probe failure. The grace period is the duration + in seconds after the processes running in the pod are sent + a termination signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer than the + expected cleanup time for your process. If this value is nil, + the pod's terminationGracePeriodSeconds will be used. Otherwise, + this value overrides the value provided by the pod spec. Value + must be non-negative integer. The value zero indicates stop + immediately via the kill signal (no opportunity to shut down). + This is a beta field and requires enabling ProbeTerminationGracePeriod + feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds + is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + required: + - name + type: object + type: array + updateStrategy: + properties: + rollingUpdate: + description: RollingUpdate is used to communicate parameters when + Type is RollingUpdateStatefulSetStrategyType. + properties: + inPlaceUpdateStrategy: + description: UnorderedUpdate contains strategies for non-ordered + update. If it is not nil, pods will be updated with non-ordered + sequence. Noted that UnorderedUpdate can only be allowed + to work with Parallel podManagementPolicy UnorderedUpdate + *kruiseV1beta1.UnorderedUpdateStrategy `json:"unorderedUpdate,omitempty"` + InPlaceUpdateStrategy contains strategies for in-place update. + properties: + gracePeriodSeconds: + description: GracePeriodSeconds is the timespan between + set Pod status to not-ready and update images in Pod + spec when in-place update a Pod. + format: int32 + type: integer + type: object + maxUnavailable: + anyOf: + - type: integer + - type: string + description: 'The maximum number of pods that can be unavailable + during the update. Value can be an absolute number (ex: + 5) or a percentage of desired pods (ex: 10%). Absolute number + is calculated from percentage by rounding down. Also, maxUnavailable + can just be allowed to work with Parallel podManagementPolicy. + Defaults to 1.' + x-kubernetes-int-or-string: true + minReadySeconds: + description: MinReadySeconds indicates how long will the pod + be considered ready after it's updated. MinReadySeconds + works with both OrderedReady and Parallel podManagementPolicy. + It affects the pod scale up speed when the podManagementPolicy + is set to be OrderedReady. Combined with MaxUnavailable, + it affects the pod update speed regardless of podManagementPolicy. + Default value is 0, max is 300. + format: int32 + type: integer + partition: + description: 'Partition indicates the ordinal at which the + StatefulSet should be partitioned by default. But if unorderedUpdate + has been set: - Partition indicates the number of pods with + non-updated revisions when rolling update. - It means controller + will update $(replicas - partition) number of pod. Default + value is 0.' + format: int32 + type: integer + paused: + description: Paused indicates that the StatefulSet is paused. + Default value is false + type: boolean + podUpdatePolicy: + description: PodUpdatePolicy indicates how pods should be + updated Default value is "ReCreate" + type: string + type: object + type: + description: Type indicates the type of the StatefulSetUpdateStrategy. + Default is RollingUpdate. + type: string + type: object + required: + - replicas + type: object + status: + description: GameServerSetStatus defines the observed state of GameServerSet + properties: + availableReplicas: + format: int32 + type: integer + currentReplicas: + format: int32 + type: integer + labelSelector: + description: LabelSelector is label selectors for query over pods + that should match the replica count used by HPA. + type: string + maintainingReplicas: + format: int32 + type: integer + readyReplicas: + format: int32 + type: integer + replicas: + description: replicas from advancedStatefulSet + format: int32 + type: integer + updatedReadyReplicas: + format: int32 + type: integer + updatedReplicas: + format: int32 + type: integer + waitToBeDeletedReplicas: + format: int32 + type: integer + required: + - availableReplicas + - currentReplicas + - readyReplicas + - replicas + - updatedReplicas + type: object + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.labelSelector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 00000000..33f766b1 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,23 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/game.kruise.io_gameserversets.yaml +- bases/game.kruise.io_gameservers.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_gameserversets.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_gameserversets.yaml +#- patches/cainjection_in_gameservers.yaml +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 00000000..ec5c150a --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 00000000..a4607e42 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,74 @@ +# Adds namespace to all resources. +namespace: kruise-game-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kruise-game- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +# - ../prometheus + +patchesStrategicMerge: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +#- manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 00000000..2bf6bef8 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,39 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + image: ringtail/kube-rbac-proxy:v0.12.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 00000000..cb352f41 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 00000000..5ed428dd --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,8 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) \ No newline at end of file diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml new file mode 100644 index 00000000..db6430ee --- /dev/null +++ b/config/manager/controller_manager_config.yaml @@ -0,0 +1,21 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: 127.0.0.1:8080 +webhook: + port: 9443 +leaderElection: + leaderElect: true + resourceName: c637bb1e.my.domain +# leaderElectionReleaseOnCancel defines if the leader should step down volume +# when the Manager ends. This requires the binary to immediately end when the +# Manager is stopped, otherwise, this setting is unsafe. Setting this significantly +# speeds up voluntary leader transitions as the new leader don't have to wait +# LeaseDuration time first. +# In the default scaffold provided, the program ends immediately after +# the manager stops, so would be fine to enable this option. However, +# if you are doing or is intended to do any operation such as perform cleanups +# after the manager stops then its usage might be unsafe. +# leaderElectionReleaseOnCancel: true diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 00000000..29d11a06 --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: kruise-game-manager + newTag: test diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 00000000..e65c4c61 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: +# securityContext: +# runAsNonRoot: true + # TODO(user): For common cases that do not require escalating privileges + # it is recommended to ensure that all your Pods/Containers are restrictive. + # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + # Please uncomment the following code if your project does NOT have to work on old Kubernetes + # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ). + # seccompProfile: + # type: RuntimeDefault + containers: + - command: + - /manager + args: + - --leader-elect=false + image: controller:latest + name: manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8082 + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /readyz + port: 8082 + initialDelaySeconds: 5 + periodSeconds: 5 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 1024Mi + requests: + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 00000000..ed137168 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 00000000..d19136ae --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,20 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 00000000..51a75db4 --- /dev/null +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 00000000..80e1857c --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 00000000..ec7acc0a --- /dev/null +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml new file mode 100644 index 00000000..71f17972 --- /dev/null +++ b/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/config/rbac/gameserver_editor_role.yaml b/config/rbac/gameserver_editor_role.yaml new file mode 100644 index 00000000..c1757849 --- /dev/null +++ b/config/rbac/gameserver_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit gameservers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameserver-editor-role +rules: +- apiGroups: + - game.kruise.io + resources: + - gameservers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - game.kruise.io + resources: + - gameservers/status + verbs: + - get diff --git a/config/rbac/gameserver_viewer_role.yaml b/config/rbac/gameserver_viewer_role.yaml new file mode 100644 index 00000000..713b6510 --- /dev/null +++ b/config/rbac/gameserver_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view gameservers. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameserver-viewer-role +rules: +- apiGroups: + - game.kruise.io + resources: + - gameservers + verbs: + - get + - list + - watch +- apiGroups: + - game.kruise.io + resources: + - gameservers/status + verbs: + - get diff --git a/config/rbac/gameserverset_editor_role.yaml b/config/rbac/gameserverset_editor_role.yaml new file mode 100644 index 00000000..60495196 --- /dev/null +++ b/config/rbac/gameserverset_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit gameserversets. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameserverset-editor-role +rules: +- apiGroups: + - game.kruise.io + resources: + - gameserversets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - game.kruise.io + resources: + - gameserversets/status + verbs: + - get diff --git a/config/rbac/gameserverset_viewer_role.yaml b/config/rbac/gameserverset_viewer_role.yaml new file mode 100644 index 00000000..3da7cd1c --- /dev/null +++ b/config/rbac/gameserverset_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view gameserversets. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameserverset-viewer-role +rules: +- apiGroups: + - game.kruise.io + resources: + - gameserversets + verbs: + - get + - list + - watch +- apiGroups: + - game.kruise.io + resources: + - gameserversets/status + verbs: + - get diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 00000000..731832a6 --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 00000000..4190ec80 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 00000000..1d1321ed --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 00000000..410e7621 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,131 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - create + - get + - list + - patch + - update + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.kruise.io + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps.kruise.io + resources: + - statefulsets/status + verbs: + - get + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/status + verbs: + - get + - patch + - update +- apiGroups: + - game.kruise.io + resources: + - gameservers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - game.kruise.io + resources: + - gameservers/finalizers + verbs: + - update +- apiGroups: + - game.kruise.io + resources: + - gameservers/status + verbs: + - get + - patch + - update +- apiGroups: + - game.kruise.io + resources: + - gameserversets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - game.kruise.io + resources: + - gameserversets/finalizers + verbs: + - update +- apiGroups: + - game.kruise.io + resources: + - gameserversets/status + verbs: + - get + - patch + - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 00000000..2070ede4 --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 00000000..7cd6025b --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller-manager + namespace: system diff --git a/config/samples/game.kruise.io_v1alpha1_gameserver.yaml b/config/samples/game.kruise.io_v1alpha1_gameserver.yaml new file mode 100644 index 00000000..e69de29b diff --git a/config/samples/game.kruise.io_v1alpha1_gameserverset.yaml b/config/samples/game.kruise.io_v1alpha1_gameserverset.yaml new file mode 100644 index 00000000..e69de29b diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 00000000..2da25882 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- service.yaml diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 00000000..e6801506 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: kruise-game-system +spec: + ports: + - port: 443 + targetPort: 9876 + selector: + control-plane: controller-manager \ No newline at end of file diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md new file mode 100644 index 00000000..12540195 --- /dev/null +++ b/docs/getting_started/installation.md @@ -0,0 +1,26 @@ +# Installation + +## Install manually + +### 0. Edit Makefile, changing {IMG} + +### 1. Build docker image with the kruise-game controller manager. +```shell +make docker-build +``` + +### 2. Push docker image with the kruise-game controller manager. +```shell +make docker-push +``` + +### 3. Deploy kruise-game controller manager to the K8s cluster. +```shell +make deploy +``` + +## Uninstall manually +```shell +make undeploy +``` + diff --git a/docs/getting_started/introduction.md b/docs/getting_started/introduction.md new file mode 100644 index 00000000..6e8850e5 --- /dev/null +++ b/docs/getting_started/introduction.md @@ -0,0 +1,30 @@ +# Introduction + +## What is Kruise-Game? +Kruise-Game is an open source project based on OpenKruise, to solve the problem of game server landing in Kubernetes. + +## Why is Kruise-Game? +Game servers are stateful services, and there are differences in the operation and maintenance of each game server, which also increases with time. In Kubernetes, general workloads manages a batch of game servers according to pod templates, which cannot take into account the differences in game server status. Batch management and directional management are in conflict in k8s. **Kruise-Game** was born to resolve that. Kruise-Game contains two CRDs, GameServer and GameServerSet: + +- `GameServer` is responsible for the management of game server status. Users can customize the game server status to reflect the differences between game servers; +- `GameServerSet` is responsible for batch management of game servers. Users can customize update/reduction strategies according to the status of game servers. + +## Features +- Game server status management + - Mark game servers status without effecting to its lifecycle +- Flexible scaling/deletion mechanism + - Support scaling down by user-defined status & priority + - Support specifying game server to delete directly +- Flexible update mechanism + - Support hot update (in-place update) + - Support updating game server by user-defined priority + - Can control the range of the game servers to be updated + - Can control the pace of the entire update process +- Custom service quality + - Support probing game servers‘ containers and mark game servers status automatically + +## What's Next +Here are some recommended next steps: + +- Start to [Install kruise-game](./installation.md). +- Learn Kruise-Game's [Basic Usage](../tutorials/basic_usage.md). \ No newline at end of file diff --git a/docs/images/logo.jpg b/docs/images/logo.jpg new file mode 100644 index 00000000..5d60f2a4 Binary files /dev/null and b/docs/images/logo.jpg differ diff --git a/docs/tutorials/basic_usage.md b/docs/tutorials/basic_usage.md new file mode 100644 index 00000000..0dfc74b8 --- /dev/null +++ b/docs/tutorials/basic_usage.md @@ -0,0 +1,222 @@ +# Basic Usage + +## Requirements +- Installation of Kruise, Reference [Install OpenKruise](https://openkruise.io/zh/docs/installation/). +- Installation of Kruise-Game, Reference [Install Kruise-Game](../getting_started/installation.md) + +## Deploy GameServerSet +This is an example of GameServerSet, which manages 3 game servers. +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: minecraft + namespace: default +spec: + replicas: 3 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + gameServerTemplate: + spec: + containers: + - image: registry.cn-hangzhou.aliyuncs.com/acs/minecraft-demo:1.12.2 + name: minecraft +``` +When the deployment is complete, the cluster will generate 1 GameServerset, 3 GameServers, and 3 Pods corresponding to GameServers +```bash +kubectl get gss +NAME AGE +minecraft 9s + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-2 Ready None 0 0 + +kubectl get pod +NAME READY STATUS RESTARTS AGE +minecraft-0 1/1 Running 0 10s +minecraft-1 1/1 Running 0 10s +minecraft-2 1/1 Running 0 10s +``` + +## Game servers scale up +Directly adjust the number of replicas to the desired number +```bash +kubectl scale gss minecraft --replicas=5 +gameserverset.game.kruise.io/minecraft scaled +``` + +The number of game servers eventually became 5 +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-2 Ready None 0 0 +minecraft-3 Ready None 0 0 +minecraft-4 Ready None 0 0 +``` + +## Game servers scale down by deletion priority +Manually set the GameServer deletionPriority (you can set the deletionPriority automatically through the ServiceQuality function) +```yaml +kubectl edit gs minecraft-2 + +... +spec: + deletionPriority: 10 #initial value is 0，turn it up to 10 + opsState: None + updatePriority: 0 +... +``` +Scale down +```bash +kubectl scale gss minecraft --replicas=4 +gameserverset.game.kruise.io/minecraft scale +``` + +The numbers of game servers eventually became 4. It can be found that the No.2 gs with the highest deletionPriority has been deleted. +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-2 Deleting None 10 0 +minecraft-3 Ready None 0 0 +minecraft-4 Ready None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-3 Ready None 0 0 +minecraft-4 Ready None 0 0 +``` + +## Game servers scale down by OpsState +Manually set the GameServer OpsState to `WaitToBeDeleted` (you can set the OpsState automatically through the ServiceQuality function) + +```yaml +kubectl edit gs minecraft-3 + +... +spec: + deletionPriority: 0 + opsState: WaitToBeDeleted #Initialization is None, will be changed to WaitToBeDeleted + updatePriority: 0 +... +``` +Scale down +```bash +kubectl scale gss minecraft --replicas=3 +gameserverset.game.kruise.io/minecraft scaled +``` + +The numbers of game servers eventually became 3. It can be found that the No.3 gs with WaitToBeDeleted OpsState has been deleted. +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-3 Deleting None 10 0 +minecraft-4 Ready None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Ready None 0 0 +minecraft-4 Ready None 0 0 +``` + +## Specify game server offline +Specify the game server with serial No.1 to go offline +```yaml +kubectl edit gss minecraft + +... +spec: + replicas: 2 #replicas is reduced by 1, adjusted to 2 + reserveGameServerIds: + - 1 #specify serial No.1 +... +``` + +The numbers of game servers eventually became 2. It can be found that the No.1 gs has been deleted. +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-1 Deleting None 0 0 +minecraft-4 Ready None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 0 +minecraft-4 Ready None 0 0 +``` + +## Game servers update by update priority + +Manually set the GameServer updatePriority (you can set the updatePriority automatically through the ServiceQuality function) + +```yaml +kubectl edit gs minecraft-0 + +... +spec: + deletionPriority: 0 + opsState: None + updatePriority: 10 #initial value is 0，turn it up to 10 +... +``` + +Update game servers' image +```yaml +kubectl edit gss minecraft + +... +spec: + gameServerTemplate: + spec: + containers: + - image: registry.cn-hangzhou.aliyuncs.com/acs/minecraft-demo:1.13.0 #update images tag to 1.13.0 + name: minecraft +... + +``` + +Pay attention to the update process, you can find that the GameServer with a larger updatePriority is updated first +```bash +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Updating None 0 10 +minecraft-4 Ready None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 10 +minecraft-4 Updating None 0 0 + +# After a while +... + +kubectl get gs +NAME STATE OPSSTATE DP UP +minecraft-0 Ready None 0 10 +minecraft-4 Ready None 0 0 +``` \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..65151d22 --- /dev/null +++ b/go.mod @@ -0,0 +1,89 @@ +module github.com/openkruise/kruise-game + +go 1.18 + +require ( + github.com/davecgh/go-spew v1.1.1 + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/gomega v1.18.1 + github.com/openkruise/kruise-api v1.2.0 + k8s.io/api v0.24.0 + k8s.io/apimachinery v0.24.0 + k8s.io/client-go v0.24.0 + k8s.io/code-generator v0.24.0 + k8s.io/klog/v2 v2.60.1 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 + sigs.k8s.io/controller-runtime v0.12.1 +) + +require ( + cloud.google.com/go v0.81.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/emicklei/go-restful v2.9.5+incompatible // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-logr/logr v1.2.0 // indirect + github.com/go-logr/zapr v1.2.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/swag v0.19.14 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/go-cmp v0.5.5 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.2 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.12.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.1 // indirect + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect + golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/text v0.3.7 // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/apiextensions-apiserver v0.24.0 // indirect + k8s.io/component-base v0.24.0 // indirect + k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect + k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..351d4606 --- /dev/null +++ b/go.sum @@ -0,0 +1,1017 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/openkruise/kruise-api v1.2.0 h1:MhoQtYT2tRdjrpb51xhn3lhEDWSlRGiMYQQ0Sh3zCkk= +github.com/openkruise/kruise-api v1.2.0/go.mod h1:BKMffjLFufZkj/yVpF5TjXG9gMU3Y9A3FxrVOJ5LJUI= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717 h1:hI3jKY4Hpf63ns040onEbB3dAkR/H/P83hw1TG8dD3Y= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.15/go.mod h1:X3JDf1BiTRQQ6xNAxTuhgi6yL2dHc6fSr9LGzE+Z3YU= +k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg= +k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= +k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY= +k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= +k8s.io/apimachinery v0.20.15/go.mod h1:4KFiDSxCoGviCiRk9kTXIROsIf4VSGkVYjVJjJln3pg= +k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= +k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= +k8s.io/client-go v0.20.15/go.mod h1:q/vywQFfGT3jw+lXQGA9sEJDH0QEX7XUT2PwrQ2qm/I= +k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U= +k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= +k8s.io/code-generator v0.20.15/go.mod h1:MW85KuhTjX9nzhFYpRqUOYh4et0xeEBHTEjwBzFYGaM= +k8s.io/code-generator v0.24.0 h1:7v52LjqCntfGxV9x8c57gkhDqkMHd0Z2jfRqGr6it6g= +k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= +k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= +k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 h1:TT1WdmqqXareKxZ/oNXEUSwKlLiHzPMyB0t8BaFeBYI= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20211110013926-83f114cd0513/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= +sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLONFFoI= +sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt new file mode 100644 index 00000000..308f268b --- /dev/null +++ b/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ \ No newline at end of file diff --git a/hack/gencerts.sh b/hack/gencerts.sh new file mode 100644 index 00000000..868b528e --- /dev/null +++ b/hack/gencerts.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generates a CA certificate, a server key, and a server certificate signed by the CA. + +set -e +SCRIPT=`basename ${BASH_SOURCE[0]}` + +function usage { + cat<< EOF + Usage: $SCRIPT + Options: + -h | --help Display help information. + -n | --namespace The namespace where the Spark operator is installed. + -s | --service The name of the webhook service. + -p | --in-pod Whether the script is running inside a pod or not. +EOF +} + +function parse_arguments { + while [[ $# -gt 0 ]] + do + case "$1" in + -n|--namespace) + if [[ -n "$2" ]]; then + NAMESPACE="$2" + else + echo "-n or --namespace requires a value." + exit 1 + fi + shift 2 + continue + ;; + -s|--service) + if [[ -n "$2" ]]; then + SERVICE="$2" + else + echo "-s or --service requires a value." + exit 1 + fi + shift 2 + continue + ;; + -p|--in-pod) + export IN_POD=true + shift 1 + continue + ;; + -h|--help) + usage + exit 0 + ;; + --) # End of all options. + shift + break + ;; + '') # End of all options. + break + ;; + *) + echo "Unrecognized option: $1" + exit 1 + ;; + esac + done +} + +# Set the namespace to "kruise-game-system" by default if not provided. +# Set the webhook service name to "kruise-game-webhook" by default if not provided. +IN_POD=false +SERVICE="kruise-game-webhook" +NAMESPACE="kruise-game-system" +parse_arguments "$@" + +TMP_DIR="/tmp/kruise-pod-webhook-certs" + +echo "Generating certs for the kruise-game pod admission webhook in ${TMP_DIR}." +mkdir -p ${TMP_DIR} +cat > ${TMP_DIR}/server.conf << EOF +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth +subjectAltName = DNS:kruise-game-webhook.kruise-game-system.svc +EOF + +# Create a certificate authority. +touch ${TMP_DIR}/.rnd +export RANDFILE=${TMP_DIR}/.rnd +openssl genrsa -out ${TMP_DIR}/ca-key.pem 2048 +openssl req -x509 -new -nodes -key ${TMP_DIR}/ca-key.pem -days 100000 -out ${TMP_DIR}/ca-cert.pem -subj "/CN=${SERVICE}_ca" -addext "subjectAltName = DNS:${SERVICE}_ca" + +# Create a server certificate. +openssl genrsa -out ${TMP_DIR}/server-key.pem 2048 +# Note the CN is the DNS name of the service of the webhook. +openssl req -new -key ${TMP_DIR}/server-key.pem -out ${TMP_DIR}/server.csr -subj "/CN=${SERVICE}.${NAMESPACE}.svc" -config ${TMP_DIR}/server.conf -addext "subjectAltName = DNS:kruise-game-webhook.kruise-game-system.svc" +openssl x509 -req -in ${TMP_DIR}/server.csr -CA ${TMP_DIR}/ca-cert.pem -CAkey ${TMP_DIR}/ca-key.pem -CAcreateserial -out ${TMP_DIR}/server-cert.pem -days 100000 -extensions v3_req -extfile ${TMP_DIR}/server.conf + +if [[ "$IN_POD" == "true" ]]; then + TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) + + # Base64 encode secrets and then remove the trailing newline to avoid issues in the curl command + ca_cert=$(cat ${TMP_DIR}/ca-cert.pem | base64 | tr -d '\n') + ca_key=$(cat ${TMP_DIR}/ca-key.pem | base64 | tr -d '\n') + server_cert=$(cat ${TMP_DIR}/server-cert.pem | base64 | tr -d '\n') + server_key=$(cat ${TMP_DIR}/server-key.pem | base64 | tr -d '\n') + + # Create the secret resource + echo "Creating a secret for the certificate and keys" + STATUS=$(curl -ik \ + -o ${TMP_DIR}/output \ + -w "%{http_code}" \ + -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "kind": "Secret", + "apiVersion": "v1", + "metadata": { + "name": "kruise-game-webhook-certs", + "namespace": "'"$NAMESPACE"'" + }, + "data": { + "ca-cert.pem": "'"$ca_cert"'", + "ca-key.pem": "'"$ca_key"'", + "server-cert.pem": "'"$server_cert"'", + "server-key.pem": "'"$server_key"'" + } + }' \ + https://kubernetes.default.svc/api/v1/namespaces/${NAMESPACE}/secrets) + + cat ${TMP_DIR}/output + + case "$STATUS" in + 201) + printf "\nSuccess - secret created.\n" + ;; + 409) + printf "\nSuccess - secret already exists.\n" + ;; + *) + printf "\nFailed creating secret.\n" + exit 1 + ;; + esac +else + kubectl create secret --namespace=${NAMESPACE} generic kruise-game-webhook-certs --from-file=${TMP_DIR}/ca-key.pem --from-file=${TMP_DIR}/ca-cert.pem --from-file=${TMP_DIR}/server-key.pem --from-file=${TMP_DIR}/server-cert.pem +fi + +# Clean up after we're done. +printf "\nDeleting ${TMP_DIR}.\n" +rm -rf ${TMP_DIR} diff --git a/hack/tools.go b/hack/tools.go new file mode 100644 index 00000000..6d8584d1 --- /dev/null +++ b/hack/tools.go @@ -0,0 +1,25 @@ +//go:build tools +// +build tools + +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This package imports things required by build scripts, to force `go mod` to see them as dependencies +package hack + +import ( + _ "k8s.io/code-generator" +) diff --git a/main.go b/main.go new file mode 100644 index 00000000..0e410266 --- /dev/null +++ b/main.go @@ -0,0 +1,150 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + controller "github.com/openkruise/kruise-game/pkg/controllers" + "github.com/openkruise/kruise-game/pkg/webhook" + "os" + "time" + + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + utilclient "github.com/openkruise/kruise-game/pkg/util/client" + //+kubebuilder:scaffold:imports +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(gamekruiseiov1alpha1.AddToScheme(scheme)) + utilruntime.Must(kruiseV1beta1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +func main() { + var metricsAddr string + var enableLeaderElection bool + var probeAddr string + var namespace string + var syncPeriodStr string + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8082", "The address the probe endpoint binds to.") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.StringVar(&namespace, "namespace", "", + "Namespace if specified restricts the manager's cache to watch objects in the desired namespace. Defaults to all namespaces.") + flag.StringVar(&syncPeriodStr, "sync-period", "", "Determines the minimum frequency at which watched resources are reconciled.") + + opts := zap.Options{ + Development: true, + } + opts.BindFlags(flag.CommandLine) + flag.Parse() + + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + + // syncPeriod parsed + var syncPeriod *time.Duration + if syncPeriodStr != "" { + d, err := time.ParseDuration(syncPeriodStr) + if err != nil { + setupLog.Error(err, "invalid sync period flag") + } else { + syncPeriod = &d + } + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: metricsAddr, + Port: 9443, + HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "game-kruise-manager", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, + Namespace: namespace, + SyncPeriod: syncPeriod, + NewClient: utilclient.NewClient, + }) + + if err != nil { + setupLog.Error(err, "unable to start kruise-game-manager") + os.Exit(1) + } + + // create webhook server + wss := webhook.NewWebhookServer(mgr) + // validate webhook server + if err := wss.SetupWithManager(mgr).Initialize(mgr.GetConfig()); err != nil { + setupLog.Error(err, "unable to set up webhook server") + os.Exit(1) + } + + //+kubebuilder:scaffold:builder + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } + + go func() { + setupLog.Info("setup controllers") + if err = controller.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to setup controllers") + os.Exit(1) + } + }() + + setupLog.Info("starting kruise-game-manager") + + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go new file mode 100644 index 00000000..e3ff8929 --- /dev/null +++ b/pkg/client/clientset/versioned/clientset.go @@ -0,0 +1,120 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package versioned + +import ( + "fmt" + "net/http" + + gamev1alpha1 "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/typed/apis/v1alpha1" + discovery "k8s.io/client-go/discovery" + rest "k8s.io/client-go/rest" + flowcontrol "k8s.io/client-go/util/flowcontrol" +) + +type Interface interface { + Discovery() discovery.DiscoveryInterface + GameV1alpha1() gamev1alpha1.GameV1alpha1Interface +} + +// Clientset contains the clients for groups. Each group has exactly one +// version included in a Clientset. +type Clientset struct { + *discovery.DiscoveryClient + gameV1alpha1 *gamev1alpha1.GameV1alpha1Client +} + +// GameV1alpha1 retrieves the GameV1alpha1Client +func (c *Clientset) GameV1alpha1() gamev1alpha1.GameV1alpha1Interface { + return c.gameV1alpha1 +} + +// Discovery retrieves the DiscoveryClient +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + if c == nil { + return nil + } + return c.DiscoveryClient +} + +// NewForConfig creates a new Clientset for the given config. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfig will generate a rate-limiter in configShallowCopy. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*Clientset, error) { + configShallowCopy := *c + + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + + // share the transport between all clients + httpClient, err := rest.HTTPClientFor(&configShallowCopy) + if err != nil { + return nil, err + } + + return NewForConfigAndClient(&configShallowCopy, httpClient) +} + +// NewForConfigAndClient creates a new Clientset for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +// If config's RateLimiter is not set and QPS and Burst are acceptable, +// NewForConfigAndClient will generate a rate-limiter in configShallowCopy. +func NewForConfigAndClient(c *rest.Config, httpClient *http.Client) (*Clientset, error) { + configShallowCopy := *c + if configShallowCopy.RateLimiter == nil && configShallowCopy.QPS > 0 { + if configShallowCopy.Burst <= 0 { + return nil, fmt.Errorf("burst is required to be greater than 0 when RateLimiter is not set and QPS is set to greater than 0") + } + configShallowCopy.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(configShallowCopy.QPS, configShallowCopy.Burst) + } + + var cs Clientset + var err error + cs.gameV1alpha1, err = gamev1alpha1.NewForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + + cs.DiscoveryClient, err = discovery.NewDiscoveryClientForConfigAndClient(&configShallowCopy, httpClient) + if err != nil { + return nil, err + } + return &cs, nil +} + +// NewForConfigOrDie creates a new Clientset for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *Clientset { + cs, err := NewForConfig(c) + if err != nil { + panic(err) + } + return cs +} + +// New creates a new Clientset for the given RESTClient. +func New(c rest.Interface) *Clientset { + var cs Clientset + cs.gameV1alpha1 = gamev1alpha1.New(c) + + cs.DiscoveryClient = discovery.NewDiscoveryClient(c) + return &cs +} diff --git a/pkg/client/clientset/versioned/doc.go b/pkg/client/clientset/versioned/doc.go new file mode 100644 index 00000000..5f8d7a83 --- /dev/null +++ b/pkg/client/clientset/versioned/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated clientset. +package versioned diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go new file mode 100644 index 00000000..39394cdd --- /dev/null +++ b/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -0,0 +1,84 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + clientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + gamev1alpha1 "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/typed/apis/v1alpha1" + fakegamev1alpha1 "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/discovery" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/testing" +) + +// NewSimpleClientset returns a clientset that will respond with the provided objects. +// It's backed by a very simple object tracker that processes creates, updates and deletions as-is, +// without applying any validations and/or defaults. It shouldn't be considered a replacement +// for a real clientset and is mostly useful in simple unit tests. +func NewSimpleClientset(objects ...runtime.Object) *Clientset { + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &Clientset{tracker: o} + cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type Clientset struct { + testing.Fake + discovery *fakediscovery.FakeDiscovery + tracker testing.ObjectTracker +} + +func (c *Clientset) Discovery() discovery.DiscoveryInterface { + return c.discovery +} + +func (c *Clientset) Tracker() testing.ObjectTracker { + return c.tracker +} + +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) + +// GameV1alpha1 retrieves the GameV1alpha1Client +func (c *Clientset) GameV1alpha1() gamev1alpha1.GameV1alpha1Interface { + return &fakegamev1alpha1.FakeGameV1alpha1{Fake: &c.Fake} +} diff --git a/pkg/client/clientset/versioned/fake/doc.go b/pkg/client/clientset/versioned/fake/doc.go new file mode 100644 index 00000000..5e58521b --- /dev/null +++ b/pkg/client/clientset/versioned/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go new file mode 100644 index 00000000..8ca8c4a3 --- /dev/null +++ b/pkg/client/clientset/versioned/fake/register.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + gamev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var scheme = runtime.NewScheme() +var codecs = serializer.NewCodecFactory(scheme) + +var localSchemeBuilder = runtime.SchemeBuilder{ + gamev1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(scheme)) +} diff --git a/pkg/client/clientset/versioned/scheme/doc.go b/pkg/client/clientset/versioned/scheme/doc.go new file mode 100644 index 00000000..643aa824 --- /dev/null +++ b/pkg/client/clientset/versioned/scheme/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go new file mode 100644 index 00000000..494b657d --- /dev/null +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package scheme + +import ( + gamev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +var Scheme = runtime.NewScheme() +var Codecs = serializer.NewCodecFactory(Scheme) +var ParameterCodec = runtime.NewParameterCodec(Scheme) +var localSchemeBuilder = runtime.SchemeBuilder{ + gamev1alpha1.AddToScheme, +} + +// AddToScheme adds all types of this clientset into the given scheme. This allows composition +// of clientsets, like in: +// +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) +// +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// +// After this, RawExtensions in Kubernetes types will serialize kube-aggregator types +// correctly. +var AddToScheme = localSchemeBuilder.AddToScheme + +func init() { + v1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"}) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/apis_client.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/apis_client.go new file mode 100644 index 00000000..c27a2833 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/apis_client.go @@ -0,0 +1,111 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "net/http" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/scheme" + rest "k8s.io/client-go/rest" +) + +type GameV1alpha1Interface interface { + RESTClient() rest.Interface + GameServersGetter + GameServerSetsGetter +} + +// GameV1alpha1Client is used to interact with features provided by the game.kruise.io group. +type GameV1alpha1Client struct { + restClient rest.Interface +} + +func (c *GameV1alpha1Client) GameServers(namespace string) GameServerInterface { + return newGameServers(c, namespace) +} + +func (c *GameV1alpha1Client) GameServerSets(namespace string) GameServerSetInterface { + return newGameServerSets(c, namespace) +} + +// NewForConfig creates a new GameV1alpha1Client for the given config. +// NewForConfig is equivalent to NewForConfigAndClient(c, httpClient), +// where httpClient was generated with rest.HTTPClientFor(c). +func NewForConfig(c *rest.Config) (*GameV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + httpClient, err := rest.HTTPClientFor(&config) + if err != nil { + return nil, err + } + return NewForConfigAndClient(&config, httpClient) +} + +// NewForConfigAndClient creates a new GameV1alpha1Client for the given config and http client. +// Note the http client provided takes precedence over the configured transport values. +func NewForConfigAndClient(c *rest.Config, h *http.Client) (*GameV1alpha1Client, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientForConfigAndClient(&config, h) + if err != nil { + return nil, err + } + return &GameV1alpha1Client{client}, nil +} + +// NewForConfigOrDie creates a new GameV1alpha1Client for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *GameV1alpha1Client { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new GameV1alpha1Client for the given RESTClient. +func New(c rest.Interface) *GameV1alpha1Client { + return &GameV1alpha1Client{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1alpha1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = scheme.Codecs.WithoutConversion() + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *GameV1alpha1Client) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/doc.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/doc.go new file mode 100644 index 00000000..c0415069 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated typed clients. +package v1alpha1 diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/doc.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/doc.go new file mode 100644 index 00000000..4525eb5a --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go new file mode 100644 index 00000000..69d83909 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_apis_client.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/typed/apis/v1alpha1" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeGameV1alpha1 struct { + *testing.Fake +} + +func (c *FakeGameV1alpha1) GameServers(namespace string) v1alpha1.GameServerInterface { + return &FakeGameServers{c, namespace} +} + +func (c *FakeGameV1alpha1) GameServerSets(namespace string) v1alpha1.GameServerSetInterface { + return &FakeGameServerSets{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeGameV1alpha1) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserver.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserver.go new file mode 100644 index 00000000..8e11c857 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserver.go @@ -0,0 +1,141 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeGameServers implements GameServerInterface +type FakeGameServers struct { + Fake *FakeGameV1alpha1 + ns string +} + +var gameserversResource = schema.GroupVersionResource{Group: "game.kruise.io", Version: "v1alpha1", Resource: "gameservers"} + +var gameserversKind = schema.GroupVersionKind{Group: "game.kruise.io", Version: "v1alpha1", Kind: "GameServer"} + +// Get takes name of the gameServer, and returns the corresponding gameServer object, and an error if there is any. +func (c *FakeGameServers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.GameServer, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(gameserversResource, c.ns, name), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} + +// List takes label and field selectors, and returns the list of GameServers that match those selectors. +func (c *FakeGameServers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.GameServerList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(gameserversResource, gameserversKind, c.ns, opts), &v1alpha1.GameServerList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.GameServerList{ListMeta: obj.(*v1alpha1.GameServerList).ListMeta} + for _, item := range obj.(*v1alpha1.GameServerList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested gameServers. +func (c *FakeGameServers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(gameserversResource, c.ns, opts)) + +} + +// Create takes the representation of a gameServer and creates it. Returns the server's representation of the gameServer, and an error, if there is any. +func (c *FakeGameServers) Create(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.CreateOptions) (result *v1alpha1.GameServer, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(gameserversResource, c.ns, gameServer), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} + +// Update takes the representation of a gameServer and updates it. Returns the server's representation of the gameServer, and an error, if there is any. +func (c *FakeGameServers) Update(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (result *v1alpha1.GameServer, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(gameserversResource, c.ns, gameServer), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeGameServers) UpdateStatus(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (*v1alpha1.GameServer, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(gameserversResource, "status", c.ns, gameServer), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} + +// Delete takes name of the gameServer and deletes it. Returns an error if one occurs. +func (c *FakeGameServers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(gameserversResource, c.ns, name, opts), &v1alpha1.GameServer{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeGameServers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(gameserversResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.GameServerList{}) + return err +} + +// Patch applies the patch and returns the patched gameServer. +func (c *FakeGameServers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServer, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(gameserversResource, c.ns, name, pt, data, subresources...), &v1alpha1.GameServer{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServer), err +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserverset.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserverset.go new file mode 100644 index 00000000..3fdf7452 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/fake/fake_gameserverset.go @@ -0,0 +1,141 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeGameServerSets implements GameServerSetInterface +type FakeGameServerSets struct { + Fake *FakeGameV1alpha1 + ns string +} + +var gameserversetsResource = schema.GroupVersionResource{Group: "game.kruise.io", Version: "v1alpha1", Resource: "gameserversets"} + +var gameserversetsKind = schema.GroupVersionKind{Group: "game.kruise.io", Version: "v1alpha1", Kind: "GameServerSet"} + +// Get takes name of the gameServerSet, and returns the corresponding gameServerSet object, and an error if there is any. +func (c *FakeGameServerSets) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.GameServerSet, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(gameserversetsResource, c.ns, name), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} + +// List takes label and field selectors, and returns the list of GameServerSets that match those selectors. +func (c *FakeGameServerSets) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.GameServerSetList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(gameserversetsResource, gameserversetsKind, c.ns, opts), &v1alpha1.GameServerSetList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.GameServerSetList{ListMeta: obj.(*v1alpha1.GameServerSetList).ListMeta} + for _, item := range obj.(*v1alpha1.GameServerSetList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested gameServerSets. +func (c *FakeGameServerSets) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(gameserversetsResource, c.ns, opts)) + +} + +// Create takes the representation of a gameServerSet and creates it. Returns the server's representation of the gameServerSet, and an error, if there is any. +func (c *FakeGameServerSets) Create(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.CreateOptions) (result *v1alpha1.GameServerSet, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(gameserversetsResource, c.ns, gameServerSet), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} + +// Update takes the representation of a gameServerSet and updates it. Returns the server's representation of the gameServerSet, and an error, if there is any. +func (c *FakeGameServerSets) Update(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (result *v1alpha1.GameServerSet, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(gameserversetsResource, c.ns, gameServerSet), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeGameServerSets) UpdateStatus(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (*v1alpha1.GameServerSet, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(gameserversetsResource, "status", c.ns, gameServerSet), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} + +// Delete takes name of the gameServerSet and deletes it. Returns an error if one occurs. +func (c *FakeGameServerSets) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(gameserversetsResource, c.ns, name, opts), &v1alpha1.GameServerSet{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeGameServerSets) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(gameserversetsResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.GameServerSetList{}) + return err +} + +// Patch applies the patch and returns the patched gameServerSet. +func (c *FakeGameServerSets) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServerSet, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(gameserversetsResource, c.ns, name, pt, data, subresources...), &v1alpha1.GameServerSet{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.GameServerSet), err +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserver.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserver.go new file mode 100644 index 00000000..2d8e60dd --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserver.go @@ -0,0 +1,194 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + scheme "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// GameServersGetter has a method to return a GameServerInterface. +// A group's client should implement this interface. +type GameServersGetter interface { + GameServers(namespace string) GameServerInterface +} + +// GameServerInterface has methods to work with GameServer resources. +type GameServerInterface interface { + Create(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.CreateOptions) (*v1alpha1.GameServer, error) + Update(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (*v1alpha1.GameServer, error) + UpdateStatus(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (*v1alpha1.GameServer, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.GameServer, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.GameServerList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServer, err error) + GameServerExpansion +} + +// gameServers implements GameServerInterface +type gameServers struct { + client rest.Interface + ns string +} + +// newGameServers returns a GameServers +func newGameServers(c *GameV1alpha1Client, namespace string) *gameServers { + return &gameServers{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the gameServer, and returns the corresponding gameServer object, and an error if there is any. +func (c *gameServers) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Get(). + Namespace(c.ns). + Resource("gameservers"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of GameServers that match those selectors. +func (c *gameServers) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.GameServerList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.GameServerList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("gameservers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested gameServers. +func (c *gameServers) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("gameservers"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a gameServer and creates it. Returns the server's representation of the gameServer, and an error, if there is any. +func (c *gameServers) Create(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.CreateOptions) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Post(). + Namespace(c.ns). + Resource("gameservers"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServer). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a gameServer and updates it. Returns the server's representation of the gameServer, and an error, if there is any. +func (c *gameServers) Update(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Put(). + Namespace(c.ns). + Resource("gameservers"). + Name(gameServer.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServer). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *gameServers) UpdateStatus(ctx context.Context, gameServer *v1alpha1.GameServer, opts v1.UpdateOptions) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Put(). + Namespace(c.ns). + Resource("gameservers"). + Name(gameServer.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServer). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the gameServer and deletes it. Returns an error if one occurs. +func (c *gameServers) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("gameservers"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *gameServers) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("gameservers"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched gameServer. +func (c *gameServers) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServer, err error) { + result = &v1alpha1.GameServer{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("gameservers"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserverset.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserverset.go new file mode 100644 index 00000000..09b8341b --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/gameserverset.go @@ -0,0 +1,194 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + scheme "github.com/openkruise/kruise-game/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// GameServerSetsGetter has a method to return a GameServerSetInterface. +// A group's client should implement this interface. +type GameServerSetsGetter interface { + GameServerSets(namespace string) GameServerSetInterface +} + +// GameServerSetInterface has methods to work with GameServerSet resources. +type GameServerSetInterface interface { + Create(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.CreateOptions) (*v1alpha1.GameServerSet, error) + Update(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (*v1alpha1.GameServerSet, error) + UpdateStatus(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (*v1alpha1.GameServerSet, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.GameServerSet, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.GameServerSetList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServerSet, err error) + GameServerSetExpansion +} + +// gameServerSets implements GameServerSetInterface +type gameServerSets struct { + client rest.Interface + ns string +} + +// newGameServerSets returns a GameServerSets +func newGameServerSets(c *GameV1alpha1Client, namespace string) *gameServerSets { + return &gameServerSets{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the gameServerSet, and returns the corresponding gameServerSet object, and an error if there is any. +func (c *gameServerSets) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Get(). + Namespace(c.ns). + Resource("gameserversets"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of GameServerSets that match those selectors. +func (c *gameServerSets) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.GameServerSetList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.GameServerSetList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("gameserversets"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested gameServerSets. +func (c *gameServerSets) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("gameserversets"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a gameServerSet and creates it. Returns the server's representation of the gameServerSet, and an error, if there is any. +func (c *gameServerSets) Create(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.CreateOptions) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Post(). + Namespace(c.ns). + Resource("gameserversets"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServerSet). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a gameServerSet and updates it. Returns the server's representation of the gameServerSet, and an error, if there is any. +func (c *gameServerSets) Update(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Put(). + Namespace(c.ns). + Resource("gameserversets"). + Name(gameServerSet.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServerSet). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *gameServerSets) UpdateStatus(ctx context.Context, gameServerSet *v1alpha1.GameServerSet, opts v1.UpdateOptions) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Put(). + Namespace(c.ns). + Resource("gameserversets"). + Name(gameServerSet.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(gameServerSet). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the gameServerSet and deletes it. Returns an error if one occurs. +func (c *gameServerSets) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("gameserversets"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *gameServerSets) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("gameserversets"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched gameServerSet. +func (c *gameServerSets) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.GameServerSet, err error) { + result = &v1alpha1.GameServerSet{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("gameserversets"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go new file mode 100644 index 00000000..ce93eeee --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apis/v1alpha1/generated_expansion.go @@ -0,0 +1,22 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +type GameServerExpansion interface{} + +type GameServerSetExpansion interface{} diff --git a/pkg/client/generic_client.go b/pkg/client/generic_client.go new file mode 100644 index 00000000..5e6ae391 --- /dev/null +++ b/pkg/client/generic_client.go @@ -0,0 +1,64 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + kruiseclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/discovery" + kubeclientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// GenericClientset defines a generic client +type GenericClientset struct { + DiscoveryClient discovery.DiscoveryInterface + KubeClient kubeclientset.Interface + KruiseGameClient kruiseclientset.Interface +} + +// newForConfig creates a new Clientset for the given config. +func newForConfig(c *rest.Config) (*GenericClientset, error) { + cWithProtobuf := rest.CopyConfig(c) + cWithProtobuf.ContentType = runtime.ContentTypeProtobuf + discoveryClient, err := discovery.NewDiscoveryClientForConfig(cWithProtobuf) + if err != nil { + return nil, err + } + kubeClient, err := kubeclientset.NewForConfig(cWithProtobuf) + if err != nil { + return nil, err + } + kruiseClient, err := kruiseclientset.NewForConfig(c) + if err != nil { + return nil, err + } + return &GenericClientset{ + DiscoveryClient: discoveryClient, + KubeClient: kubeClient, + KruiseGameClient: kruiseClient, + }, nil +} + +// newForConfig creates a new Clientset for the given config. +func newForConfigOrDie(c *rest.Config) *GenericClientset { + gc, err := newForConfig(c) + if err != nil { + panic(err) + } + return gc +} diff --git a/pkg/client/informers/externalversions/apis/interface.go b/pkg/client/informers/externalversions/apis/interface.go new file mode 100644 index 00000000..22eb3d38 --- /dev/null +++ b/pkg/client/informers/externalversions/apis/interface.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package apis + +import ( + v1alpha1 "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/apis/v1alpha1" + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1alpha1 provides access to shared informers for resources in V1alpha1. + V1alpha1() v1alpha1.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1alpha1 returns a new v1alpha1.Interface. +func (g *group) V1alpha1() v1alpha1.Interface { + return v1alpha1.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/client/informers/externalversions/apis/v1alpha1/gameserver.go b/pkg/client/informers/externalversions/apis/v1alpha1/gameserver.go new file mode 100644 index 00000000..36d2d8da --- /dev/null +++ b/pkg/client/informers/externalversions/apis/v1alpha1/gameserver.go @@ -0,0 +1,89 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + apisv1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + versioned "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/openkruise/kruise-game/pkg/client/listers/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GameServerInformer provides access to a shared informer and lister for +// GameServers. +type GameServerInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.GameServerLister +} + +type gameServerInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewGameServerInformer constructs a new informer for GameServer type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGameServerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGameServerInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGameServerInformer constructs a new informer for GameServer type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGameServerInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.GameV1alpha1().GameServers(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.GameV1alpha1().GameServers(namespace).Watch(context.TODO(), options) + }, + }, + &apisv1alpha1.GameServer{}, + resyncPeriod, + indexers, + ) +} + +func (f *gameServerInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGameServerInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *gameServerInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisv1alpha1.GameServer{}, f.defaultInformer) +} + +func (f *gameServerInformer) Lister() v1alpha1.GameServerLister { + return v1alpha1.NewGameServerLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/apis/v1alpha1/gameserverset.go b/pkg/client/informers/externalversions/apis/v1alpha1/gameserverset.go new file mode 100644 index 00000000..69bc0237 --- /dev/null +++ b/pkg/client/informers/externalversions/apis/v1alpha1/gameserverset.go @@ -0,0 +1,89 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + apisv1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + versioned "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "github.com/openkruise/kruise-game/pkg/client/listers/apis/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// GameServerSetInformer provides access to a shared informer and lister for +// GameServerSets. +type GameServerSetInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.GameServerSetLister +} + +type gameServerSetInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewGameServerSetInformer constructs a new informer for GameServerSet type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewGameServerSetInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredGameServerSetInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredGameServerSetInformer constructs a new informer for GameServerSet type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredGameServerSetInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.GameV1alpha1().GameServerSets(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.GameV1alpha1().GameServerSets(namespace).Watch(context.TODO(), options) + }, + }, + &apisv1alpha1.GameServerSet{}, + resyncPeriod, + indexers, + ) +} + +func (f *gameServerSetInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredGameServerSetInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *gameServerSetInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisv1alpha1.GameServerSet{}, f.defaultInformer) +} + +func (f *gameServerSetInformer) Lister() v1alpha1.GameServerSetLister { + return v1alpha1.NewGameServerSetLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/apis/v1alpha1/interface.go b/pkg/client/informers/externalversions/apis/v1alpha1/interface.go new file mode 100644 index 00000000..5153b412 --- /dev/null +++ b/pkg/client/informers/externalversions/apis/v1alpha1/interface.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // GameServers returns a GameServerInformer. + GameServers() GameServerInformer + // GameServerSets returns a GameServerSetInformer. + GameServerSets() GameServerSetInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// GameServers returns a GameServerInformer. +func (v *version) GameServers() GameServerInformer { + return &gameServerInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + +// GameServerSets returns a GameServerSetInformer. +func (v *version) GameServerSets() GameServerSetInformer { + return &gameServerSetInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go new file mode 100644 index 00000000..acd3bb16 --- /dev/null +++ b/pkg/client/informers/externalversions/factory.go @@ -0,0 +1,179 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + reflect "reflect" + sync "sync" + time "time" + + versioned "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + apis "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/apis" + internalinterfaces "github.com/openkruise/kruise-game/pkg/client/informers/externalversions/internalinterfaces" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// SharedInformerOption defines the functional option type for SharedInformerFactory. +type SharedInformerOption func(*sharedInformerFactory) *sharedInformerFactory + +type sharedInformerFactory struct { + client versioned.Interface + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc + lock sync.Mutex + defaultResync time.Duration + customResync map[reflect.Type]time.Duration + + informers map[reflect.Type]cache.SharedIndexInformer + // startedInformers is used for tracking which informers have been started. + // This allows Start() to be called multiple times safely. + startedInformers map[reflect.Type]bool +} + +// WithCustomResyncConfig sets a custom resync period for the specified informer types. +func WithCustomResyncConfig(resyncConfig map[v1.Object]time.Duration) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + for k, v := range resyncConfig { + factory.customResync[reflect.TypeOf(k)] = v + } + return factory + } +} + +// WithTweakListOptions sets a custom filter on all listers of the configured SharedInformerFactory. +func WithTweakListOptions(tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.tweakListOptions = tweakListOptions + return factory + } +} + +// WithNamespace limits the SharedInformerFactory to the specified namespace. +func WithNamespace(namespace string) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.namespace = namespace + return factory + } +} + +// NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. +func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync) +} + +// NewFilteredSharedInformerFactory constructs a new instance of sharedInformerFactory. +// Listers obtained via this SharedInformerFactory will be subject to the same filters +// as specified here. +// Deprecated: Please use NewSharedInformerFactoryWithOptions instead +func NewFilteredSharedInformerFactory(client versioned.Interface, defaultResync time.Duration, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) SharedInformerFactory { + return NewSharedInformerFactoryWithOptions(client, defaultResync, WithNamespace(namespace), WithTweakListOptions(tweakListOptions)) +} + +// NewSharedInformerFactoryWithOptions constructs a new instance of a SharedInformerFactory with additional options. +func NewSharedInformerFactoryWithOptions(client versioned.Interface, defaultResync time.Duration, options ...SharedInformerOption) SharedInformerFactory { + factory := &sharedInformerFactory{ + client: client, + namespace: v1.NamespaceAll, + defaultResync: defaultResync, + informers: make(map[reflect.Type]cache.SharedIndexInformer), + startedInformers: make(map[reflect.Type]bool), + customResync: make(map[reflect.Type]time.Duration), + } + + // Apply all options + for _, opt := range options { + factory = opt(factory) + } + + return factory +} + +// Start initializes all requested informers. +func (f *sharedInformerFactory) Start(stopCh <-chan struct{}) { + f.lock.Lock() + defer f.lock.Unlock() + + for informerType, informer := range f.informers { + if !f.startedInformers[informerType] { + go informer.Run(stopCh) + f.startedInformers[informerType] = true + } + } +} + +// WaitForCacheSync waits for all started informers' cache were synced. +func (f *sharedInformerFactory) WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool { + informers := func() map[reflect.Type]cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informers := map[reflect.Type]cache.SharedIndexInformer{} + for informerType, informer := range f.informers { + if f.startedInformers[informerType] { + informers[informerType] = informer + } + } + return informers + }() + + res := map[reflect.Type]bool{} + for informType, informer := range informers { + res[informType] = cache.WaitForCacheSync(stopCh, informer.HasSynced) + } + return res +} + +// InternalInformerFor returns the SharedIndexInformer for obj using an internal +// client. +func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internalinterfaces.NewInformerFunc) cache.SharedIndexInformer { + f.lock.Lock() + defer f.lock.Unlock() + + informerType := reflect.TypeOf(obj) + informer, exists := f.informers[informerType] + if exists { + return informer + } + + resyncPeriod, exists := f.customResync[informerType] + if !exists { + resyncPeriod = f.defaultResync + } + + informer = newFunc(f.client, resyncPeriod) + f.informers[informerType] = informer + + return informer +} + +// SharedInformerFactory provides shared informers for resources in all known +// API group versions. +type SharedInformerFactory interface { + internalinterfaces.SharedInformerFactory + ForResource(resource schema.GroupVersionResource) (GenericInformer, error) + WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool + + Game() apis.Interface +} + +func (f *sharedInformerFactory) Game() apis.Interface { + return apis.New(f, f.namespace, f.tweakListOptions) +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go new file mode 100644 index 00000000..a6575f3e --- /dev/null +++ b/pkg/client/informers/externalversions/generic.go @@ -0,0 +1,63 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package externalversions + +import ( + "fmt" + + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + schema "k8s.io/apimachinery/pkg/runtime/schema" + cache "k8s.io/client-go/tools/cache" +) + +// GenericInformer is type of SharedIndexInformer which will locate and delegate to other +// sharedInformers based on type +type GenericInformer interface { + Informer() cache.SharedIndexInformer + Lister() cache.GenericLister +} + +type genericInformer struct { + informer cache.SharedIndexInformer + resource schema.GroupResource +} + +// Informer returns the SharedIndexInformer. +func (f *genericInformer) Informer() cache.SharedIndexInformer { + return f.informer +} + +// Lister returns the GenericLister. +func (f *genericInformer) Lister() cache.GenericLister { + return cache.NewGenericLister(f.Informer().GetIndexer(), f.resource) +} + +// ForResource gives generic access to a shared informer of the matching type +// TODO extend this to unknown resources with a client pool +func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource) (GenericInformer, error) { + switch resource { + // Group=game.kruise.io, Version=v1alpha1 + case v1alpha1.SchemeGroupVersion.WithResource("gameservers"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Game().V1alpha1().GameServers().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("gameserversets"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Game().V1alpha1().GameServerSets().Informer()}, nil + + } + + return nil, fmt.Errorf("no informer found for %v", resource) +} diff --git a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go new file mode 100644 index 00000000..ea4b1151 --- /dev/null +++ b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package internalinterfaces + +import ( + time "time" + + versioned "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + cache "k8s.io/client-go/tools/cache" +) + +// NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. +type NewInformerFunc func(versioned.Interface, time.Duration) cache.SharedIndexInformer + +// SharedInformerFactory a small interface to allow for adding an informer without an import cycle +type SharedInformerFactory interface { + Start(stopCh <-chan struct{}) + InformerFor(obj runtime.Object, newFunc NewInformerFunc) cache.SharedIndexInformer +} + +// TweakListOptionsFunc is a function that transforms a v1.ListOptions. +type TweakListOptionsFunc func(*v1.ListOptions) diff --git a/pkg/client/listers/apis/v1alpha1/expansion_generated.go b/pkg/client/listers/apis/v1alpha1/expansion_generated.go new file mode 100644 index 00000000..2f3dea8e --- /dev/null +++ b/pkg/client/listers/apis/v1alpha1/expansion_generated.go @@ -0,0 +1,34 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +// GameServerListerExpansion allows custom methods to be added to +// GameServerLister. +type GameServerListerExpansion interface{} + +// GameServerNamespaceListerExpansion allows custom methods to be added to +// GameServerNamespaceLister. +type GameServerNamespaceListerExpansion interface{} + +// GameServerSetListerExpansion allows custom methods to be added to +// GameServerSetLister. +type GameServerSetListerExpansion interface{} + +// GameServerSetNamespaceListerExpansion allows custom methods to be added to +// GameServerSetNamespaceLister. +type GameServerSetNamespaceListerExpansion interface{} diff --git a/pkg/client/listers/apis/v1alpha1/gameserver.go b/pkg/client/listers/apis/v1alpha1/gameserver.go new file mode 100644 index 00000000..13d67161 --- /dev/null +++ b/pkg/client/listers/apis/v1alpha1/gameserver.go @@ -0,0 +1,98 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// GameServerLister helps list GameServers. +// All objects returned here must be treated as read-only. +type GameServerLister interface { + // List lists all GameServers in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.GameServer, err error) + // GameServers returns an object that can list and get GameServers. + GameServers(namespace string) GameServerNamespaceLister + GameServerListerExpansion +} + +// gameServerLister implements the GameServerLister interface. +type gameServerLister struct { + indexer cache.Indexer +} + +// NewGameServerLister returns a new GameServerLister. +func NewGameServerLister(indexer cache.Indexer) GameServerLister { + return &gameServerLister{indexer: indexer} +} + +// List lists all GameServers in the indexer. +func (s *gameServerLister) List(selector labels.Selector) (ret []*v1alpha1.GameServer, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.GameServer)) + }) + return ret, err +} + +// GameServers returns an object that can list and get GameServers. +func (s *gameServerLister) GameServers(namespace string) GameServerNamespaceLister { + return gameServerNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// GameServerNamespaceLister helps list and get GameServers. +// All objects returned here must be treated as read-only. +type GameServerNamespaceLister interface { + // List lists all GameServers in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.GameServer, err error) + // Get retrieves the GameServer from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.GameServer, error) + GameServerNamespaceListerExpansion +} + +// gameServerNamespaceLister implements the GameServerNamespaceLister +// interface. +type gameServerNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all GameServers in the indexer for a given namespace. +func (s gameServerNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.GameServer, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.GameServer)) + }) + return ret, err +} + +// Get retrieves the GameServer from the indexer for a given namespace and name. +func (s gameServerNamespaceLister) Get(name string) (*v1alpha1.GameServer, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("gameserver"), name) + } + return obj.(*v1alpha1.GameServer), nil +} diff --git a/pkg/client/listers/apis/v1alpha1/gameserverset.go b/pkg/client/listers/apis/v1alpha1/gameserverset.go new file mode 100644 index 00000000..5ab7e072 --- /dev/null +++ b/pkg/client/listers/apis/v1alpha1/gameserverset.go @@ -0,0 +1,98 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// GameServerSetLister helps list GameServerSets. +// All objects returned here must be treated as read-only. +type GameServerSetLister interface { + // List lists all GameServerSets in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.GameServerSet, err error) + // GameServerSets returns an object that can list and get GameServerSets. + GameServerSets(namespace string) GameServerSetNamespaceLister + GameServerSetListerExpansion +} + +// gameServerSetLister implements the GameServerSetLister interface. +type gameServerSetLister struct { + indexer cache.Indexer +} + +// NewGameServerSetLister returns a new GameServerSetLister. +func NewGameServerSetLister(indexer cache.Indexer) GameServerSetLister { + return &gameServerSetLister{indexer: indexer} +} + +// List lists all GameServerSets in the indexer. +func (s *gameServerSetLister) List(selector labels.Selector) (ret []*v1alpha1.GameServerSet, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.GameServerSet)) + }) + return ret, err +} + +// GameServerSets returns an object that can list and get GameServerSets. +func (s *gameServerSetLister) GameServerSets(namespace string) GameServerSetNamespaceLister { + return gameServerSetNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// GameServerSetNamespaceLister helps list and get GameServerSets. +// All objects returned here must be treated as read-only. +type GameServerSetNamespaceLister interface { + // List lists all GameServerSets in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.GameServerSet, err error) + // Get retrieves the GameServerSet from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.GameServerSet, error) + GameServerSetNamespaceListerExpansion +} + +// gameServerSetNamespaceLister implements the GameServerSetNamespaceLister +// interface. +type gameServerSetNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all GameServerSets in the indexer for a given namespace. +func (s gameServerSetNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.GameServerSet, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.GameServerSet)) + }) + return ret, err +} + +// Get retrieves the GameServerSet from the indexer for a given namespace and name. +func (s gameServerSetNamespaceLister) Get(name string) (*v1alpha1.GameServerSet, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("gameserverset"), name) + } + return obj.(*v1alpha1.GameServerSet), nil +} diff --git a/pkg/client/registry.go b/pkg/client/registry.go new file mode 100644 index 00000000..16c00c6e --- /dev/null +++ b/pkg/client/registry.go @@ -0,0 +1,56 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "fmt" + + "k8s.io/client-go/rest" +) + +var ( + cfg *rest.Config + + defaultGenericClient *GenericClientset +) + +// NewRegistry creates clientset by client-go +func NewRegistry(c *rest.Config) error { + var err error + defaultGenericClient, err = newForConfig(c) + if err != nil { + return err + } + cfgCopy := *c + cfg = &cfgCopy + return nil +} + +// GetGenericClient returns default clientset +func GetGenericClient() *GenericClientset { + return defaultGenericClient +} + +// GetGenericClientWithName returns clientset with given name as user-agent +func GetGenericClientWithName(name string) *GenericClientset { + if cfg == nil { + return nil + } + newCfg := *cfg + newCfg.UserAgent = fmt.Sprintf("%s/%s", cfg.UserAgent, name) + return newForConfigOrDie(&newCfg) +} diff --git a/pkg/client/versioned/clientset.go b/pkg/client/versioned/clientset.go new file mode 100644 index 00000000..5293c19a --- /dev/null +++ b/pkg/client/versioned/clientset.go @@ -0,0 +1,18 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package versioned diff --git a/pkg/client/versioned/doc.go b/pkg/client/versioned/doc.go new file mode 100644 index 00000000..5f8d7a83 --- /dev/null +++ b/pkg/client/versioned/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated clientset. +package versioned diff --git a/pkg/client/versioned/fake/clientset_generated.go b/pkg/client/versioned/fake/clientset_generated.go new file mode 100644 index 00000000..2c9023a6 --- /dev/null +++ b/pkg/client/versioned/fake/clientset_generated.go @@ -0,0 +1,18 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake diff --git a/pkg/client/versioned/fake/doc.go b/pkg/client/versioned/fake/doc.go new file mode 100644 index 00000000..5e58521b --- /dev/null +++ b/pkg/client/versioned/fake/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package has the automatically generated fake clientset. +package fake diff --git a/pkg/client/versioned/fake/register.go b/pkg/client/versioned/fake/register.go new file mode 100644 index 00000000..2c9023a6 --- /dev/null +++ b/pkg/client/versioned/fake/register.go @@ -0,0 +1,18 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake diff --git a/pkg/client/versioned/scheme/doc.go b/pkg/client/versioned/scheme/doc.go new file mode 100644 index 00000000..c9b70b54 --- /dev/null +++ b/pkg/client/versioned/scheme/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +// This package contains the scheme of the automatically generated clientset. +package scheme diff --git a/pkg/client/versioned/scheme/register.go b/pkg/client/versioned/scheme/register.go new file mode 100644 index 00000000..fb38ef60 --- /dev/null +++ b/pkg/client/versioned/scheme/register.go @@ -0,0 +1,18 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package scheme diff --git a/pkg/controllers/controller.go b/pkg/controllers/controller.go new file mode 100644 index 00000000..c4719751 --- /dev/null +++ b/pkg/controllers/controller.go @@ -0,0 +1,45 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "github.com/openkruise/kruise-game/pkg/controllers/gameserver" + "github.com/openkruise/kruise-game/pkg/controllers/gameserverset" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +var controllerAddFuncs []func(manager.Manager) error + +func init() { + controllerAddFuncs = append(controllerAddFuncs, gameserver.Add) + controllerAddFuncs = append(controllerAddFuncs, gameserverset.Add) +} + +func SetupWithManager(m manager.Manager) error { + for _, f := range controllerAddFuncs { + if err := f(m); err != nil { + if kindMatchErr, ok := err.(*meta.NoKindMatchError); ok { + klog.Infof("CRD %v is not installed, its controller will perform noops!", kindMatchErr.GroupKind) + continue + } + return err + } + } + return nil +} diff --git a/pkg/controllers/gameserver/gameserver_controller.go b/pkg/controllers/gameserver/gameserver_controller.go new file mode 100644 index 00000000..62ad5fc0 --- /dev/null +++ b/pkg/controllers/gameserver/gameserver_controller.go @@ -0,0 +1,251 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gameserver + +import ( + "context" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + utildiscovery "github.com/openkruise/kruise-game/pkg/util/discovery" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var ( + controllerKind = gamekruiseiov1alpha1.SchemeGroupVersion.WithKind("GameServer") + // leave it to batch size + concurrentReconciles = 10 +) + +func Add(mgr manager.Manager) error { + if !utildiscovery.DiscoverGVK(controllerKind) { + return nil + } + return add(mgr, newReconciler(mgr)) +} + +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + recorder := mgr.GetEventRecorderFor("gameserver-controller") + return &GameServerReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + recorder: recorder, + } +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + klog.Info("Starting GameServer Controller") + c, err := controller.New("gameserver-controller", mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + klog.Error(err) + return err + } + if err = c.Watch(&source.Kind{Type: &gamekruiseiov1alpha1.GameServer{}}, &handler.EnqueueRequestForOwner{ + OwnerType: &corev1.Pod{}, + IsController: true, + }); err != nil { + klog.Error(err) + return err + } + if err = watchPod(c); err != nil { + klog.Error(err) + return err + } + + return nil +} + +// GameServerReconciler reconciles a GameServer object +type GameServerReconciler struct { + client.Client + Scheme *runtime.Scheme + recorder record.EventRecorder +} + +func watchPod(c controller.Controller) error { + if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.Funcs{ + CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := createEvent.Object.(*corev1.Pod) + if _, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + }}) + } + }, + UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { + newPod := updateEvent.ObjectNew.(*corev1.Pod) + if _, exist := newPod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: newPod.GetName(), + Namespace: newPod.GetNamespace(), + }}) + } + }, + DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := deleteEvent.Object.(*corev1.Pod) + if _, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: deleteEvent.Object.GetNamespace(), + Name: deleteEvent.Object.GetName(), + }, + }) + } + }, + }); err != nil { + return err + } + return nil +} + +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameservers/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the GameServer object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile +func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + namespacedName := req.NamespacedName + + // get pod + pod := &corev1.Pod{} + err := r.Get(ctx, namespacedName, pod) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + klog.Errorf("failed to find pod %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + // get GameServer + gs := &gamekruiseiov1alpha1.GameServer{} + err = r.Get(ctx, namespacedName, gs) + if err != nil { + if errors.IsNotFound(err) { + err := r.initGameServer(pod) + if err != nil && !errors.IsAlreadyExists(err) && !errors.IsNotFound(err) { + klog.Errorf("failed to create GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + klog.Errorf("failed to find GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + gsm := NewGameServerManager(gs, pod, r.Client) + + gss, err := r.getGameServerSet(pod) + if err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + klog.Errorf("failed to get GameServerSet for GameServer %s in %s, because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + podUpdated, err := gsm.SyncToPod() + if err != nil || podUpdated { + return reconcile.Result{Requeue: podUpdated}, err + } + + err = gsm.SyncToGs(gss.Spec.ServiceQualities) + if err != nil { + return reconcile.Result{}, err + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *GameServerReconciler) SetupWithManager(mgr ctrl.Manager) (c controller.Controller, err error) { + c, err = ctrl.NewControllerManagedBy(mgr). + For(&gamekruiseiov1alpha1.GameServer{}).Build(r) + return c, err +} + +func (r *GameServerReconciler) getGameServerSet(pod *corev1.Pod) (*gamekruiseiov1alpha1.GameServerSet, error) { + gssName := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey] + gss := &gamekruiseiov1alpha1.GameServerSet{} + err := r.Client.Get(context.Background(), types.NamespacedName{ + Namespace: pod.GetNamespace(), + Name: gssName, + }, gss) + return gss, err +} + +func (r *GameServerReconciler) initGameServer(pod *corev1.Pod) error { + gs := &gamekruiseiov1alpha1.GameServer{} + gs.Name = pod.GetName() + gs.Namespace = pod.GetNamespace() + + // set owner reference + ors := make([]metav1.OwnerReference, 0) + or := metav1.OwnerReference{ + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + ors = append(ors, or) + gs.OwnerReferences = ors + + // set NetWork + gs.Spec.NetworkDisabled = false + + // set OpsState + gs.Spec.OpsState = gamekruiseiov1alpha1.None + + // set UpdatePriority + updatePriority := intstr.FromInt(0) + gs.Spec.UpdatePriority = &updatePriority + + // set deletionPriority + deletionPriority := intstr.FromInt(0) + gs.Spec.DeletionPriority = &deletionPriority + + return r.Client.Create(context.Background(), gs) +} diff --git a/pkg/controllers/gameserver/gameserver_manager.go b/pkg/controllers/gameserver/gameserver_manager.go new file mode 100644 index 00000000..8c1ba37e --- /dev/null +++ b/pkg/controllers/gameserver/gameserver_manager.go @@ -0,0 +1,214 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gameserver + +import ( + "context" + kruisePub "github.com/openkruise/kruise-api/apps/pub" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Control interface { + SyncToPod() (bool, error) + SyncToGs(qualities []gameKruiseV1alpha1.ServiceQuality) error +} + +type GameServerManager struct { + gameServer *gameKruiseV1alpha1.GameServer + pod *corev1.Pod + client client.Client +} + +func (manager GameServerManager) SyncToPod() (bool, error) { + // compare GameServer Spec With Pod + pod := manager.pod + gs := manager.gameServer + podLabels := pod.GetLabels() + podDeletePriority := podLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey] + podUpdatePriority := podLabels[gameKruiseV1alpha1.GameServerUpdatePriorityKey] + podGsOpsState := podLabels[gameKruiseV1alpha1.GameServerOpsStateKey] + podGsState := podLabels[gameKruiseV1alpha1.GameServerStateKey] + + updated := false + newLabels := make(map[string]string) + if gs.Spec.DeletionPriority.String() != podDeletePriority { + newLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey] = gs.Spec.DeletionPriority.String() + updated = true + } + if gs.Spec.UpdatePriority.String() != podUpdatePriority { + newLabels[gameKruiseV1alpha1.GameServerUpdatePriorityKey] = gs.Spec.UpdatePriority.String() + updated = true + } + if string(gs.Spec.OpsState) != podGsOpsState { + newLabels[gameKruiseV1alpha1.GameServerOpsStateKey] = string(gs.Spec.OpsState) + updated = true + } + + var gsState gameKruiseV1alpha1.GameServerState + switch pod.Status.Phase { + case corev1.PodRunning: + // GameServer Updating + lifecycleState, exist := pod.GetLabels()[kruisePub.LifecycleStateKey] + if exist && (lifecycleState == string(kruisePub.LifecycleStateUpdating) || lifecycleState == string(kruisePub.LifecycleStatePreparingUpdate)) { + gsState = gameKruiseV1alpha1.Updating + break + } + // GameServer Deleting + if !pod.DeletionTimestamp.IsZero() { + gsState = gameKruiseV1alpha1.Deleting + break + } + // GameServer Ready / NotReady + for _, con := range pod.Status.Conditions { + if con.Type == corev1.PodReady { + if con.Status == corev1.ConditionTrue { + gsState = gameKruiseV1alpha1.Ready + } else { + gsState = gameKruiseV1alpha1.NotReady + } + break + } + } + case corev1.PodFailed: + gsState = gameKruiseV1alpha1.Crash + case corev1.PodPending: + gsState = gameKruiseV1alpha1.Creating + default: + gsState = gameKruiseV1alpha1.Unknown + } + if string(gsState) != podGsState { + newLabels[gameKruiseV1alpha1.GameServerStateKey] = string(gsState) + updated = true + } + + if updated { + patchPod := map[string]interface{}{"metadata": map[string]map[string]string{"labels": newLabels}} + patchPodBytes, err := json.Marshal(patchPod) + if err != nil { + return updated, err + } + err = manager.client.Patch(context.TODO(), pod, client.RawPatch(types.StrategicMergePatchType, patchPodBytes)) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("failed to patch Pod %s in %s,because of %s.", pod.GetName(), pod.GetNamespace(), err.Error()) + return updated, err + } + } + + return updated, nil +} + +func (manager GameServerManager) SyncToGs(qualities []gameKruiseV1alpha1.ServiceQuality) error { + gs := manager.gameServer + pod := manager.pod + podLabels := pod.GetLabels() + podDeletePriority := intstr.FromString(podLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey]) + podUpdatePriority := intstr.FromString(podLabels[gameKruiseV1alpha1.GameServerUpdatePriorityKey]) + podGsState := gameKruiseV1alpha1.GameServerState(podLabels[gameKruiseV1alpha1.GameServerStateKey]) + + gsConditions := gs.Status.ServiceQualitiesCondition + podConditions := pod.Status.Conditions + var sqNames []string + for _, sq := range qualities { + sqNames = append(sqNames, sq.Name) + } + var toExec map[string]string + var newGsConditions []gameKruiseV1alpha1.ServiceQualityCondition + for _, pc := range podConditions { + if util.IsStringInList(string(pc.Type), sqNames) { + toExecAction := true + lastActionTransitionTime := metav1.Now() + for _, gsc := range gsConditions { + if gsc.Name == string(pc.Type) && gsc.Status == string(pc.Status) { + toExecAction = false + lastActionTransitionTime = gsc.LastActionTransitionTime + break + } + } + serviceQualityCondition := gameKruiseV1alpha1.ServiceQualityCondition{ + Name: string(pc.Type), + Status: string(pc.Status), + LastProbeTime: pc.LastProbeTime, + LastTransitionTime: pc.LastTransitionTime, + LastActionTransitionTime: lastActionTransitionTime, + } + newGsConditions = append(newGsConditions, serviceQualityCondition) + if toExecAction { + toExec[string(pc.Type)] = string(pc.Status) + } + } + } + if toExec != nil { + var spec gameKruiseV1alpha1.GameServerSpec + for _, sq := range qualities { + for name := range toExec { + if sq.Name == name { + // TODO exec action + } + } + } + + patchSpec := map[string]interface{}{"spec": spec} + jsonPatchSpec, err := json.Marshal(patchSpec) + if err != nil { + return err + } + err = manager.client.Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, jsonPatchSpec)) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("failed to patch GameServer spec %s in %s,because of %s.", gs.GetName(), gs.GetNamespace(), err.Error()) + return err + } + } + + status := gameKruiseV1alpha1.GameServerStatus{ + PodStatus: pod.Status, + CurrentState: podGsState, + DesiredState: gameKruiseV1alpha1.Ready, + UpdatePriority: &podUpdatePriority, + DeletionPriority: &podDeletePriority, + ServiceQualitiesCondition: newGsConditions, + LastTransitionTime: metav1.Now(), + } + patchStatus := map[string]interface{}{"status": status} + jsonPatchStatus, err := json.Marshal(patchStatus) + if err != nil { + return err + } + err = manager.client.Status().Patch(context.TODO(), gs, client.RawPatch(types.MergePatchType, jsonPatchStatus)) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("failed to patch GameServer Status %s in %s,because of %s.", gs.GetName(), gs.GetNamespace(), err.Error()) + return err + } + + return nil +} + +func NewGameServerManager(gs *gameKruiseV1alpha1.GameServer, pod *corev1.Pod, c client.Client) Control { + return &GameServerManager{ + gameServer: gs, + pod: pod, + client: c, + } +} diff --git a/pkg/controllers/gameserverset/gameserverset_controller.go b/pkg/controllers/gameserverset/gameserverset_controller.go new file mode 100644 index 00000000..c3b017fe --- /dev/null +++ b/pkg/controllers/gameserverset/gameserverset_controller.go @@ -0,0 +1,286 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gameserverset + +import ( + "context" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + utildiscovery "github.com/openkruise/kruise-game/pkg/util/discovery" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var ( + controllerKind = gamekruiseiov1alpha1.SchemeGroupVersion.WithKind("GameServerSet") + // leave it to batch size + concurrentReconciles = 10 +) + +func Add(mgr manager.Manager) error { + if !utildiscovery.DiscoverGVK(controllerKind) { + return nil + } + return add(mgr, newReconciler(mgr)) +} + +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + recorder := mgr.GetEventRecorderFor("gameserverset-controller") + return &GameServerSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + recorder: recorder, + } +} + +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + klog.Info("Starting GameServerSet Controller") + c, err := controller.New("gameserverset-controller", mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: concurrentReconciles}) + if err != nil { + klog.Error(err) + return err + } + + if err = c.Watch(&source.Kind{Type: &gamekruiseiov1alpha1.GameServerSet{}}, &handler.EnqueueRequestForObject{}); err != nil { + klog.Error(err) + return err + } + + if err = watchPod(c); err != nil { + klog.Error(err) + return err + } + + if err = watchWorkloads(c); err != nil { + klog.Error(err) + return err + } + + return nil +} + +// watch pod +func watchPod(c controller.Controller) (err error) { + + if err := c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.Funcs{ + CreateFunc: func(createEvent event.CreateEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := createEvent.Object.(*corev1.Pod) + if gssName, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: gssName, + Namespace: pod.GetNamespace(), + }}) + } + }, + UpdateFunc: func(updateEvent event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := updateEvent.ObjectNew.(*corev1.Pod) + if gssName, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: gssName, + Namespace: pod.GetNamespace(), + }}) + } + }, + DeleteFunc: func(deleteEvent event.DeleteEvent, limitingInterface workqueue.RateLimitingInterface) { + pod := deleteEvent.Object.(*corev1.Pod) + if gssName, exist := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]; exist { + limitingInterface.Add(reconcile.Request{NamespacedName: types.NamespacedName{ + Name: gssName, + Namespace: pod.GetNamespace(), + }}) + } + }, + }); err != nil { + return err + } + return nil +} + +// watch workloads +func watchWorkloads(c controller.Controller) (err error) { + if err := c.Watch(&source.Kind{Type: &kruiseV1beta1.StatefulSet{}}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: &gamekruiseiov1alpha1.GameServerSet{}, + }); err != nil { + return err + } + return nil +} + +// GameServerSetReconciler reconciles a GameServerSet object +type GameServerSetReconciler struct { + client.Client + Scheme *runtime.Scheme + recorder record.EventRecorder +} + +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameserversets,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameserversets/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=game.kruise.io,resources=gameserversets/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// TODO(user): Modify the Reconcile function to compare the state specified by +// the GameServerSet object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile +func (r *GameServerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = log.FromContext(ctx) + namespacedName := req.NamespacedName + + // get GameServerSet + gss := &gamekruiseiov1alpha1.GameServerSet{} + err := r.Get(ctx, namespacedName, gss) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + klog.Errorf("Failed to find GameServerSet %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + // get advanced statefulset + asts := &kruiseV1beta1.StatefulSet{} + err = r.Get(ctx, namespacedName, asts) + if err != nil { + if errors.IsNotFound(err) { + err = r.initAsts(gss) + if err != nil { + klog.Errorf("failed to create advanced statefulset %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + klog.Errorf("failed to find advanced statefulset %s in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + // get actual Pod list + podList := &corev1.PodList{} + err = r.List(ctx, podList, &client.ListOptions{ + Namespace: gss.GetNamespace(), + LabelSelector: labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gss.GetName(), + })}) + if err != nil { + klog.Errorf("failed to list GameServers of GameServerSet %s in %s.", gss.GetName(), gss.GetNamespace()) + return reconcile.Result{}, err + } + + gsm := NewGameServerSetManager(gss, asts, podList.Items, r.Client) + + // scale game servers + if gsm.IsNeedToScale() { + err = gsm.GameServerScale() + if err != nil { + klog.Errorf("GameServerSet %s failed to scale GameServers in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + + // update workload + if gsm.IsNeedToUpdateWorkload() { + err = gsm.UpdateWorkload() + if err != nil { + klog.Errorf("GameServerSet %s failed to synchronize workload in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + return reconcile.Result{}, nil + } + + // TODO sync PodProbeMarker + err = gsm.SyncPodProbeMarker() + if err != nil { + klog.Errorf("GameServerSet %s failed to synchronize PodProbeMarker in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + // sync GameServerSet Status + err = gsm.SyncStatus() + if err != nil { + klog.Errorf("GameServerSet %s failed to synchronize its status in %s,because of %s.", namespacedName.Name, namespacedName.Namespace, err.Error()) + return reconcile.Result{}, err + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *GameServerSetReconciler) SetupWithManager(mgr ctrl.Manager) (c controller.Controller, err error) { + c, err = ctrl.NewControllerManagedBy(mgr). + For(&gamekruiseiov1alpha1.GameServerSet{}).Build(r) + return c, err +} + +func (r *GameServerSetReconciler) initAsts(gss *gamekruiseiov1alpha1.GameServerSet) error { + asts := &kruiseV1beta1.StatefulSet{} + asts.Namespace = gss.GetNamespace() + asts.Name = gss.GetName() + + // set owner reference + ors := make([]metav1.OwnerReference, 0) + or := metav1.OwnerReference{ + APIVersion: gss.APIVersion, + Kind: gss.Kind, + Name: gss.GetName(), + UID: gss.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + ors = append(ors, or) + asts.SetOwnerReferences(ors) + + // set label + astsLabels := make(map[string]string) + astsLabels[gamekruiseiov1alpha1.AstsHashKey] = util.GetAstsHash(gss) + asts.SetLabels(astsLabels) + + // set label selector + asts.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{gamekruiseiov1alpha1.GameServerOwnerGssKey: gss.GetName()}, + } + + // set replicas + asts.Spec.Replicas = gss.Spec.Replicas + + asts = util.GetNewAstsFromGss(gss, asts) + + return r.Client.Create(context.Background(), asts) +} diff --git a/pkg/controllers/gameserverset/gameserverset_manager.go b/pkg/controllers/gameserverset/gameserverset_manager.go new file mode 100644 index 00000000..268ffabd --- /dev/null +++ b/pkg/controllers/gameserverset/gameserverset_manager.go @@ -0,0 +1,246 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gameserverset + +import ( + "context" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/klog/v2" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sort" +) + +type Control interface { + GameServerScale() error + UpdateWorkload() error + SyncStatus() error + IsNeedToScale() bool + IsNeedToUpdateWorkload() bool + SyncPodProbeMarker() error +} + +type GameServerSetManager struct { + gameServerSet *gameKruiseV1alpha1.GameServerSet + asts *kruiseV1beta1.StatefulSet + podList []corev1.Pod + client client.Client +} + +func NewGameServerSetManager(gss *gameKruiseV1alpha1.GameServerSet, asts *kruiseV1beta1.StatefulSet, gsList []corev1.Pod, c client.Client) Control { + return &GameServerSetManager{ + gameServerSet: gss, + asts: asts, + podList: gsList, + client: c, + } +} + +func (manager *GameServerSetManager) IsNeedToScale() bool { + gss := manager.gameServerSet + asts := manager.asts + gsList := manager.podList + + currentReplicas := len(gsList) + workloadReplicas := int(*asts.Spec.Replicas) + expectedReplicas := int(*gss.Spec.Replicas) + + // workload is reconciling its replicas, don't interrupt + if currentReplicas != workloadReplicas { + return false + } + + // no need to scale + return !(expectedReplicas == currentReplicas && util.IsSliceEqual(util.StringToIntSlice(gss.GetAnnotations()[gameKruiseV1alpha1.GameServerSetReserveIdsKey], ","), gss.Spec.ReserveGameServerIds)) +} + +func (manager *GameServerSetManager) GameServerScale() error { + gss := manager.gameServerSet + asts := manager.asts + gsList := manager.podList + c := manager.client + ctx := context.Background() + + currentReplicas := len(gsList) + expectedReplicas := int(*gss.Spec.Replicas) + as := gss.GetAnnotations() + reserveIds := util.StringToIntSlice(as[gameKruiseV1alpha1.GameServerSetReserveIdsKey], ",") + notExistIds := util.StringToIntSlice(as[gameKruiseV1alpha1.GameServerSetNotExistIdsKey], ",") + gssReserveIds := gss.Spec.ReserveGameServerIds + + klog.Infof("GameServers %s/%s already has %d replicas, expect to have %d replicas.", gss.GetNamespace(), gss.GetName(), currentReplicas, expectedReplicas) + + newNotExistIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, gsList) + + asts.Spec.ReserveOrdinals = append(gssReserveIds, newNotExistIds...) + asts.Spec.Replicas = gss.Spec.Replicas + asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{ + MaxUnavailable: gss.Spec.ScaleStrategy.MaxUnavailable, + } + err := c.Update(ctx, asts) + if err != nil { + klog.Errorf("failed to update workload replicas %s in %s,because of %s.", gss.GetName(), gss.GetNamespace(), err.Error()) + return err + } + + gssAnnotations := make(map[string]string) + gssAnnotations[gameKruiseV1alpha1.GameServerSetReserveIdsKey] = util.IntSliceToString(gssReserveIds, ",") + gssAnnotations[gameKruiseV1alpha1.GameServerSetNotExistIdsKey] = util.IntSliceToString(newNotExistIds, ",") + patchGss := map[string]interface{}{"metadata": map[string]map[string]string{"annotations": gssAnnotations}} + patchGssBytes, _ := json.Marshal(patchGss) + err = c.Patch(ctx, gss, client.RawPatch(types.MergePatchType, patchGssBytes)) + if err != nil { + klog.Errorf("failed to patch GameServerSet %s in %s,because of %s.", gss.GetName(), gss.GetNamespace(), err.Error()) + return err + } + + return nil +} + +func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod) []int { + workloadManageIds := util.GetIndexListFromPodList(pods) + + var toAdd []int + var toDelete []int + + // 1. compute reserved GameServerIds, firstly + + // 1.a. to delete those new reserved GameServers already in workloadManageIds + toDelete = util.GetSliceInAandInB(util.GetSliceInANotInB(gssReserveIds, reserveIds), workloadManageIds) + + // 1.b. to add those remove-reserved GameServers already in workloadManageIds + existLastIndex := -1 + if len(workloadManageIds) != 0 { + sort.Ints(workloadManageIds) + existLastIndex = workloadManageIds[len(workloadManageIds)-1] + } + for _, id := range util.GetSliceInANotInB(reserveIds, gssReserveIds) { + if existLastIndex > id { + toAdd = append(toAdd, id) + } + } + + // 2. compute remain GameServerIds, secondly + + numToAdd := expectedReplicas - len(pods) + len(toDelete) - len(toAdd) + if numToAdd < 0 { + + // 2.a to delete GameServers according to DeleteSequence + sortedGs := util.DeleteSequenceGs(pods) + sort.Sort(sortedGs) + toDelete = append(toDelete, util.GetIndexListFromPodList(sortedGs[:-numToAdd])...) + } else { + + // 2.b to add GameServers, firstly add those in add notExistIds, secondly add those in future sequence + numNotExist := len(notExistIds) + if numNotExist < numToAdd { + toAdd = append(toAdd, notExistIds...) + times := 0 + for i := existLastIndex + 1; times < numToAdd-numNotExist; i++ { + if !util.IsNumInList(i, gssReserveIds) { + toAdd = append(toAdd, i) + times++ + } + } + } else { + toAdd = append(toAdd, notExistIds[:numToAdd]...) + } + } + + newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...) + newManageIds = util.GetSliceInANotInB(newManageIds, toDelete) + var newNotExistIds []int + if len(newManageIds) != 0 { + sort.Ints(newManageIds) + for i := 0; i < newManageIds[len(newManageIds)-1]; i++ { + if !util.IsNumInList(i, newManageIds) && !util.IsNumInList(i, gssReserveIds) { + newNotExistIds = append(newNotExistIds, i) + } + } + } + + return newNotExistIds +} + +func (manager *GameServerSetManager) IsNeedToUpdateWorkload() bool { + return manager.asts.GetLabels()[gameKruiseV1alpha1.AstsHashKey] != util.GetAstsHash(manager.gameServerSet) +} + +func (manager *GameServerSetManager) UpdateWorkload() error { + gss := manager.gameServerSet + asts := manager.asts + + // sync with Advanced StatefulSet + asts = util.GetNewAstsFromGss(gss, asts) + astsLabels := asts.GetLabels() + astsLabels[gameKruiseV1alpha1.AstsHashKey] = util.GetAstsHash(manager.gameServerSet) + asts.SetLabels(astsLabels) + return manager.client.Update(context.Background(), asts) +} + +func (manager *GameServerSetManager) SyncPodProbeMarker() error { + return nil +} + +func (manager *GameServerSetManager) SyncStatus() error { + gss := manager.gameServerSet + asts := manager.asts + c := manager.client + ctx := context.Background() + podList := manager.podList + + maintainingGs := 0 + waitToBeDeletedGs := 0 + + for _, pod := range podList { + + podLabels := pod.GetLabels() + opsState := podLabels[gameKruiseV1alpha1.GameServerOpsStateKey] + + // ops state + switch opsState { + case string(gameKruiseV1alpha1.WaitToDelete): + waitToBeDeletedGs++ + case string(gameKruiseV1alpha1.Maintaining): + maintainingGs++ + } + } + + status := gameKruiseV1alpha1.GameServerSetStatus{ + Replicas: *gss.Spec.Replicas, + CurrentReplicas: int32(len(podList)), + AvailableReplicas: asts.Status.AvailableReplicas, + ReadyReplicas: asts.Status.ReadyReplicas, + UpdatedReplicas: asts.Status.UpdatedReplicas, + UpdatedReadyReplicas: asts.Status.UpdatedReadyReplicas, + MaintainingReplicas: pointer.Int32Ptr(int32(maintainingGs)), + WaitToBeDeletedReplicas: pointer.Int32Ptr(int32(waitToBeDeletedGs)), + LabelSelector: asts.Status.LabelSelector, + } + + patchStatus := map[string]interface{}{"status": status} + jsonPatch, err := json.Marshal(patchStatus) + if err != nil { + return err + } + return c.Status().Patch(ctx, gss, client.RawPatch(types.MergePatchType, jsonPatch)) +} diff --git a/pkg/util/client/delegating_client.go b/pkg/util/client/delegating_client.go new file mode 100644 index 00000000..29022fbf --- /dev/null +++ b/pkg/util/client/delegating_client.go @@ -0,0 +1,167 @@ +/* +Copyright 2022 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "flag" + "strings" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +var ( + disableNoDeepCopy bool +) + +func init() { + flag.BoolVar(&disableNoDeepCopy, "disable-no-deepcopy", false, "If you are going to disable NoDeepCopy List in some controllers and webhooks.") +} + +// NewClient creates the default caching client with disable deepcopy list from cache. +func NewClient(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) { + c, err := client.New(config, options) + if err != nil { + return nil, err + } + + uncachedGVKs := map[schema.GroupVersionKind]struct{}{} + for _, obj := range uncachedObjects { + gvk, err := apiutil.GVKForObject(obj, c.Scheme()) + if err != nil { + return nil, err + } + uncachedGVKs[gvk] = struct{}{} + } + + return &delegatingClient{ + scheme: c.Scheme(), + mapper: c.RESTMapper(), + Reader: &delegatingReader{ + CacheReader: cache, + ClientReader: c, + noDeepCopyLister: &noDeepCopyLister{cache: cache, scheme: c.Scheme()}, + scheme: c.Scheme(), + uncachedGVKs: uncachedGVKs, + }, + Writer: c, + StatusClient: c, + }, nil +} + +type delegatingClient struct { + client.Reader + client.Writer + client.StatusClient + + scheme *runtime.Scheme + mapper meta.RESTMapper +} + +// Scheme returns the scheme this client is using. +func (d *delegatingClient) Scheme() *runtime.Scheme { + return d.scheme +} + +// RESTMapper returns the rest mapper this client is using. +func (d *delegatingClient) RESTMapper() meta.RESTMapper { + return d.mapper +} + +// delegatingReader forms a Reader that will cause Get and List requests for +// unstructured types to use the ClientReader while requests for any other type +// of object with use the CacheReader. This avoids accidentally caching the +// entire cluster in the common case of loading arbitrary unstructured objects +// (e.g. from OwnerReferences). +type delegatingReader struct { + CacheReader client.Reader + ClientReader client.Reader + + noDeepCopyLister *noDeepCopyLister + + uncachedGVKs map[schema.GroupVersionKind]struct{} + scheme *runtime.Scheme + cacheUnstructured bool +} + +func (d *delegatingReader) shouldBypassCache(obj runtime.Object) (bool, error) { + gvk, err := apiutil.GVKForObject(obj, d.scheme) + if err != nil { + return false, err + } + // TODO: this is producing unsafe guesses that don't actually work, + // but it matches ~99% of the cases out there. + if meta.IsListType(obj) { + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + } + if _, isUncached := d.uncachedGVKs[gvk]; isUncached { + return true, nil + } + if !d.cacheUnstructured { + _, isUnstructured := obj.(*unstructured.Unstructured) + _, isUnstructuredList := obj.(*unstructured.UnstructuredList) + return isUnstructured || isUnstructuredList, nil + } + return false, nil +} + +// Get retrieves an obj for a given object key from the Kubernetes Cluster. +func (d *delegatingReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error { + if isUncached, err := d.shouldBypassCache(obj); err != nil { + return err + } else if isUncached { + return d.ClientReader.Get(ctx, key, obj) + } + return d.CacheReader.Get(ctx, key, obj) +} + +// List retrieves list of objects for a given namespace and list options. +func (d *delegatingReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if isUncached, err := d.shouldBypassCache(list); err != nil { + return err + } else if isUncached { + return d.ClientReader.List(ctx, list, opts...) + } + if !disableNoDeepCopy && isDisableDeepCopy(opts) { + return d.noDeepCopyLister.List(ctx, list, opts...) + } + return d.CacheReader.List(ctx, list, opts...) +} + +var DisableDeepCopy = disableDeepCopy{} + +type disableDeepCopy struct{} + +func (_ disableDeepCopy) ApplyToList(_ *client.ListOptions) { +} + +func isDisableDeepCopy(opts []client.ListOption) bool { + for _, opt := range opts { + if opt == DisableDeepCopy { + return true + } + } + return false +} diff --git a/pkg/util/client/no_deepcopy_lister.go b/pkg/util/client/no_deepcopy_lister.go new file mode 100644 index 00000000..3272f573 --- /dev/null +++ b/pkg/util/client/no_deepcopy_lister.go @@ -0,0 +1,198 @@ +/* +Copyright 2022 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "context" + "fmt" + "reflect" + "strings" + "time" + + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" + toolscache "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +type noDeepCopyLister struct { + cache cache.Cache + scheme *runtime.Scheme +} + +func (r *noDeepCopyLister) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error { + startTime := time.Now() + gvk, _, err := r.objectTypeForListObject(out) + if err != nil { + return err + } + indexer, err := r.getIndexerByGVK(ctx, *gvk) + if err != nil { + return err + } + + listOpts := client.ListOptions{} + listOpts.ApplyOptions(opts) + + var objs []interface{} + switch { + case listOpts.FieldSelector != nil: + field, val, requiresExact := requiresExactMatch(listOpts.FieldSelector) + if !requiresExact { + return fmt.Errorf("non-exact field matches are not supported by the cache") + } + // list all objects by the field selector. If this is namespaced and we have one, ask for the + // namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces" + // namespace. + objs, err = indexer.ByIndex(FieldIndexName(field), KeyToNamespacedKey(listOpts.Namespace, val)) + case listOpts.Namespace != "": + objs, err = indexer.ByIndex(toolscache.NamespaceIndex, listOpts.Namespace) + default: + objs = indexer.List() + } + if err != nil { + return err + } + + var labelSel labels.Selector + if listOpts.LabelSelector != nil { + labelSel = listOpts.LabelSelector + } + limitSet := listOpts.Limit > 0 + + runtimeObjs := make([]runtime.Object, 0, len(objs)) + for _, item := range objs { + // if the Limit option is set and the number of items + // listed exceeds this limit, then stop reading. + if limitSet && int64(len(runtimeObjs)) >= listOpts.Limit { + break + } + obj, isObj := item.(runtime.Object) + if !isObj { + return fmt.Errorf("cache contained %T, which is not an Object", obj) + } + meta, err := apimeta.Accessor(obj) + if err != nil { + return err + } + if labelSel != nil { + lbls := labels.Set(meta.GetLabels()) + if !labelSel.Matches(lbls) { + continue + } + } + runtimeObjs = append(runtimeObjs, obj) + } + defer func() { + klog.V(6).Infof("Listed %v %v objects %v without DeepCopy, cost %v", gvk.GroupVersion(), gvk.Kind, len(runtimeObjs), time.Since(startTime)) + }() + return apimeta.SetList(out, runtimeObjs) +} + +func (r *noDeepCopyLister) getIndexerByGVK(ctx context.Context, gvk schema.GroupVersionKind) (toolscache.Indexer, error) { + informer, err := r.cache.GetInformerForKind(ctx, gvk) + if err != nil { + return nil, err + } + sharedInformer, ok := informer.(toolscache.SharedIndexInformer) + if !ok { + return nil, fmt.Errorf("informer %T from cache is not a SharedIndexInformer", informer) + } + return sharedInformer.GetIndexer(), nil +} + +// objectTypeForListObject tries to find the runtime.Object and associated GVK +// for a single object corresponding to the passed-in list type. We need them +// because they are used as cache map key. +func (r *noDeepCopyLister) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) { + gvk, err := apiutil.GVKForObject(list, r.scheme) + if err != nil { + return nil, nil, err + } + + if !strings.HasSuffix(gvk.Kind, "List") { + return nil, nil, fmt.Errorf("non-list type %T (kind %q) passed as output", list, gvk) + } + // we need the non-list GVK, so chop off the "List" from the end of the kind + gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] + _, isUnstructured := list.(*unstructured.UnstructuredList) + var cacheTypeObj runtime.Object + if isUnstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + cacheTypeObj = u + } else { + itemsPtr, err := apimeta.GetItemsPtr(list) + if err != nil { + return nil, nil, err + } + // http://knowyourmeme.com/memes/this-is-fine + elemType := reflect.Indirect(reflect.ValueOf(itemsPtr)).Type().Elem() + if elemType.Kind() != reflect.Ptr { + elemType = reflect.PtrTo(elemType) + } + + cacheTypeValue := reflect.Zero(elemType) + var ok bool + cacheTypeObj, ok = cacheTypeValue.Interface().(runtime.Object) + if !ok { + return nil, nil, fmt.Errorf("cannot get cache for %T, its element %T is not a runtime.Object", list, cacheTypeValue.Interface()) + } + } + + return &gvk, cacheTypeObj, nil +} + +// requiresExactMatch checks if the given field selector is of the form `k=v` or `k==v`. +func requiresExactMatch(sel fields.Selector) (field, val string, required bool) { + reqs := sel.Requirements() + if len(reqs) != 1 { + return "", "", false + } + req := reqs[0] + if req.Operator != selection.Equals && req.Operator != selection.DoubleEquals { + return "", "", false + } + return req.Field, req.Value, true +} + +// FieldIndexName constructs the name of the index over the given field, +// for use with an indexer. +func FieldIndexName(field string) string { + return "field:" + field +} + +// noNamespaceNamespace is used as the "namespace" when we want to list across all namespaces. +const allNamespacesNamespace = "__all_namespaces" + +// KeyToNamespacedKey prefixes the given index key with a namespace +// for use in field selector indexes. +func KeyToNamespacedKey(ns string, baseKey string) string { + if ns != "" { + return ns + "/" + baseKey + } + return allNamespacesNamespace + "/" + baseKey +} diff --git a/pkg/util/discovery/discovery.go b/pkg/util/discovery/discovery.go new file mode 100644 index 00000000..0147c719 --- /dev/null +++ b/pkg/util/discovery/discovery.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package discovery + +import ( + "fmt" + "k8s.io/klog/v2" + "time" + + "github.com/openkruise/kruise-game/pkg/client" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" +) + +var ( + internalScheme = runtime.NewScheme() + + errKindNotFound = fmt.Errorf("kind not found in group version resources") + backOff = wait.Backoff{ + Steps: 4, + Duration: 500 * time.Millisecond, + Factor: 5.0, + Jitter: 0.1, + } +) + +func init() { + _ = AddToScheme(internalScheme) +} + +func DiscoverGVK(gvk schema.GroupVersionKind) bool { + genericClient := client.GetGenericClient() + if genericClient == nil { + return true + } + discoveryClient := genericClient.DiscoveryClient + + startTime := time.Now() + err := retry.OnError(backOff, func(err error) bool { return true }, func() error { + resourceList, err := discoveryClient.ServerResourcesForGroupVersion(gvk.GroupVersion().String()) + if err != nil { + return err + } + for _, r := range resourceList.APIResources { + if r.Kind == gvk.Kind { + return nil + } + } + return errKindNotFound + }) + + if err != nil { + if err == errKindNotFound { + klog.Warningf("Not found kind %s in group version %s, waiting time %s", gvk.Kind, gvk.GroupVersion().String(), time.Since(startTime)) + return false + } + + // This might be caused by abnormal apiserver or etcd, ignore it + klog.Errorf("Failed to find resources in group version %s: %v, waiting time %s", gvk.GroupVersion().String(), err, time.Since(startTime)) + } + + return true +} + +func DiscoverObject(obj runtime.Object) bool { + gvk, err := apiutil.GVKForObject(obj, internalScheme) + if err != nil { + klog.Warningf("Not recognized object %T in scheme: %v", obj, err) + return false + } + return DiscoverGVK(gvk) +} + +// AddToSchemes may be used to add all resources defined in the project to a Scheme +var AddToSchemes runtime.SchemeBuilder + +// AddToScheme adds all Resources to the Scheme +func AddToScheme(s *runtime.Scheme) error { + return AddToSchemes.AddToScheme(s) +} diff --git a/pkg/util/gameserver.go b/pkg/util/gameserver.go new file mode 100644 index 00000000..b69c556f --- /dev/null +++ b/pkg/util/gameserver.go @@ -0,0 +1,164 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + appspub "github.com/openkruise/kruise-api/apps/pub" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "strconv" + "strings" +) + +type DeleteSequenceGs []corev1.Pod + +func (dg DeleteSequenceGs) Len() int { + return len(dg) +} + +func (dg DeleteSequenceGs) Swap(i, j int) { + dg[i], dg[j] = dg[j], dg[i] +} + +func (dg DeleteSequenceGs) Less(i, j int) bool { + iLabels := dg[i].GetLabels() + jLabels := dg[j].GetLabels() + iOpsState := iLabels[gameKruiseV1alpha1.GameServerOpsStateKey] + jOpsState := jLabels[gameKruiseV1alpha1.GameServerOpsStateKey] + iDeletionPriority := iLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey] + jDeletionPriority := jLabels[gameKruiseV1alpha1.GameServerDeletePriorityKey] + + // OpsState + if iOpsState != jOpsState { + return opsStateDeletePrority(iOpsState) > opsStateDeletePrority(jOpsState) + } + // Deletion Priority + if iDeletionPriority != jDeletionPriority { + iDeletionPriorityInt, _ := strconv.Atoi(iDeletionPriority) + jDeletionPriorityInt, _ := strconv.Atoi(jDeletionPriority) + return iDeletionPriorityInt > jDeletionPriorityInt + } + // Index Number + return GetIndexFromGsName(dg[i].GetName()) > GetIndexFromGsName(dg[j].GetName()) +} + +func opsStateDeletePrority(opsState string) int { + switch opsState { + case string(gameKruiseV1alpha1.WaitToDelete): + return 1 + case string(gameKruiseV1alpha1.None): + return 0 + case string(gameKruiseV1alpha1.Maintaining): + return -1 + } + return 0 +} + +func GetIndexFromGsName(gsName string) int { + temp := strings.Split(gsName, "-") + index, _ := strconv.Atoi(temp[len(temp)-1]) + return index +} + +func GetIndexListFromPodList(podList []corev1.Pod) []int { + var indexList []int + for i := 0; i < len(podList); i++ { + indexList = append(indexList, GetIndexFromGsName(podList[i].GetName())) + } + return indexList +} + +func GetIndexListFromGsList(gsList []gameKruiseV1alpha1.GameServer) []int { + var indexList []int + for i := 0; i < len(gsList); i++ { + indexList = append(indexList, GetIndexFromGsName(gsList[i].GetName())) + } + return indexList +} + +func GetNewAstsFromGss(gss *gameKruiseV1alpha1.GameServerSet, asts *kruiseV1beta1.StatefulSet) *kruiseV1beta1.StatefulSet { + // default: set ParallelPodManagement + asts.Spec.PodManagementPolicy = apps.ParallelPodManagement + + // set pod labels + podLabels := gss.Spec.GameServerTemplate.GetLabels() + if podLabels == nil { + podLabels = make(map[string]string) + } + podLabels[gameKruiseV1alpha1.GameServerOwnerGssKey] = gss.GetName() + asts.Spec.Template.SetLabels(podLabels) + + // set pod annotations + podAnnotations := gss.Spec.GameServerTemplate.GetAnnotations() + asts.Spec.Template.SetAnnotations(podAnnotations) + + // set template spec + asts.Spec.Template.Spec = gss.Spec.GameServerTemplate.Spec + // default: add InPlaceUpdateReady condition + readinessGates := gss.Spec.GameServerTemplate.Spec.ReadinessGates + readinessGates = append(readinessGates, corev1.PodReadinessGate{ConditionType: appspub.InPlaceUpdateReady}) + asts.Spec.Template.Spec.ReadinessGates = readinessGates + + // set VolumeClaimTemplates + asts.Spec.VolumeClaimTemplates = gss.Spec.GameServerTemplate.VolumeClaimTemplates + + // set ScaleStrategy + asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{ + MaxUnavailable: gss.Spec.ScaleStrategy.MaxUnavailable, + } + + // set UpdateStrategy + asts.Spec.UpdateStrategy.Type = gss.Spec.UpdateStrategy.Type + var rollingUpdateStatefulSetStrategy *kruiseV1beta1.RollingUpdateStatefulSetStrategy + if gss.Spec.UpdateStrategy.RollingUpdate != nil { + asts.Spec.UpdateStrategy.Type = apps.RollingUpdateStatefulSetStrategyType + rollingUpdateStatefulSetStrategy = &kruiseV1beta1.RollingUpdateStatefulSetStrategy{ + Partition: gss.Spec.UpdateStrategy.RollingUpdate.Partition, + MaxUnavailable: gss.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable, + PodUpdatePolicy: gss.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy, + Paused: gss.Spec.UpdateStrategy.RollingUpdate.Paused, + InPlaceUpdateStrategy: gss.Spec.UpdateStrategy.RollingUpdate.InPlaceUpdateStrategy, + MinReadySeconds: gss.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds, + UnorderedUpdate: &kruiseV1beta1.UnorderedUpdateStrategy{ + PriorityStrategy: &appspub.UpdatePriorityStrategy{ + OrderPriority: []appspub.UpdatePriorityOrderTerm{ + { + OrderedKey: gameKruiseV1alpha1.GameServerUpdatePriorityKey, + }, + }, + }, + }, + } + } + asts.Spec.UpdateStrategy.RollingUpdate = rollingUpdateStatefulSetStrategy + + return asts +} + +type astsToUpdate struct { + UpdateStrategy gameKruiseV1alpha1.UpdateStrategy + Template gameKruiseV1alpha1.GameServerTemplate +} + +func GetAstsHash(gss *gameKruiseV1alpha1.GameServerSet) string { + return GetHash(astsToUpdate{ + UpdateStrategy: gss.Spec.UpdateStrategy, + Template: gss.Spec.GameServerTemplate, + }) +} diff --git a/pkg/util/gameserver_test.go b/pkg/util/gameserver_test.go new file mode 100644 index 00000000..d6014fa4 --- /dev/null +++ b/pkg/util/gameserver_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sort" + "testing" +) + +func TestGetIndexFromGsName(t *testing.T) { + tests := []struct { + name string + result int + }{ + { + name: "xxx-23-3", + result: 3, + }, + { + name: "www_3-12", + result: 12, + }, + } + + for _, test := range tests { + actual := GetIndexFromGsName(test.name) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestDeleteSequenceGs(t *testing.T) { + tests := []struct { + before []corev1.Pod + after []int + }{ + { + before: []corev1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx-0", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.None), + gameKruiseV1alpha1.GameServerDeletePriorityKey: "10", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx-1", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.Maintaining), + gameKruiseV1alpha1.GameServerDeletePriorityKey: "0", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx-2", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.WaitToDelete), + gameKruiseV1alpha1.GameServerDeletePriorityKey: "0", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "xxx-3", + Labels: map[string]string{ + gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.None), + gameKruiseV1alpha1.GameServerDeletePriorityKey: "0", + }, + }, + }, + }, + after: []int{2, 0, 3, 1}, + }, + } + + for _, test := range tests { + after := DeleteSequenceGs(test.before) + sort.Sort(after) + expect := test.after + actual := GetIndexListFromPodList(after) + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} diff --git a/pkg/util/hash.go b/pkg/util/hash.go new file mode 100644 index 00000000..8cea6bd4 --- /dev/null +++ b/pkg/util/hash.go @@ -0,0 +1,44 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "github.com/davecgh/go-spew/spew" + "hash" + "hash/fnv" +) + +func GetHash(objectToWrite interface{}) string { + hf := fnv.New32() + DeepHashObject(hf, objectToWrite) + return fmt.Sprint(hf.Sum32()) +} + +// DeepHashObject writes specified object to hash using the spew library +// which follows pointers and prints actual values of the nested objects +// ensuring the hash does not change when a pointer changes. +func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { + hasher.Reset() + printer := spew.ConfigState{ + Indent: " ", + SortKeys: true, + DisableMethods: true, + SpewKeys: true, + } + printer.Fprintf(hasher, "%#v", objectToWrite) +} diff --git a/pkg/util/hash_test.go b/pkg/util/hash_test.go new file mode 100644 index 00000000..ca0c92b2 --- /dev/null +++ b/pkg/util/hash_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + corev1 "k8s.io/api/core/v1" + "testing" +) + +func TestGetHash(t *testing.T) { + tests := []struct { + objectA interface{} + objectB interface{} + result bool + }{ + { + objectA: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "containerA", + Image: "nginx:latest", + }, + }, + }, + }, + objectB: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "containerB", + Image: "nginx:latest", + }, + }, + }, + }, + result: false, + }, + } + + for _, test := range tests { + actual := GetHash(test.objectA) == GetHash(test.objectB) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} diff --git a/pkg/util/slice.go b/pkg/util/slice.go new file mode 100644 index 00000000..b80b3b37 --- /dev/null +++ b/pkg/util/slice.go @@ -0,0 +1,141 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +func IsNumInList(num int, list []int) bool { + for _, n := range list { + if num == n { + return true + } + } + return false +} + +func IsStringInList(str string, list []string) bool { + for _, s := range list { + if s == str { + return true + } + } + return false +} + +func GetSliceInANotInB(a, b []int) []int { + var ret []int + for _, aa := range a { + if !IsNumInList(aa, b) { + ret = append(ret, aa) + } + } + return ret +} + +func GetSliceInAandInB(a, b []int) []int { + var ret []int + for _, aa := range a { + if IsNumInList(aa, b) { + ret = append(ret, aa) + } + } + return ret +} + +func IntSliceToString(number []int, delimiter string) string { + return strings.Trim(strings.Replace(fmt.Sprint(number), " ", delimiter, -1), "[]") +} + +func StringToIntSlice(str string, delimiter string) []int { + if str == "" { + return nil + } + strList := strings.Split(str, delimiter) + if len(strList) == 0 { + return nil + } + var retSlice []int + for _, item := range strList { + if item == "" { + continue + } + val, err := strconv.Atoi(item) + if err != nil { + continue + } + retSlice = append(retSlice, val) + } + return retSlice +} + +func IsSliceEqual(a, b []int) bool { + if (a == nil) != (b == nil) { + return false + } + if len(a) != len(b) { + return false + } + sort.Ints(a) + sort.Ints(b) + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +func RemoveRepeat(nums []int) []int { + var result []int + tempMap := map[int]byte{} + for _, num := range nums { + beforeLen := len(tempMap) + tempMap[num] = 0 + if len(tempMap) == beforeLen { + continue + } + result = append(result, num) + } + return result +} + +func IsRepeat(nums []int) bool { + tempMap := map[int]byte{} + for _, num := range nums { + beforeLen := len(tempMap) + tempMap[num] = 0 + if len(tempMap) == beforeLen { + return true + } + } + return false +} + +func IsHasNegativeNum(nums []int) bool { + for _, num := range nums { + if num < 0 { + return true + } + } + return false +} diff --git a/pkg/util/slice_test.go b/pkg/util/slice_test.go new file mode 100644 index 00000000..ba1bd04f --- /dev/null +++ b/pkg/util/slice_test.go @@ -0,0 +1,257 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import "testing" + +func TestIsNumInList(t *testing.T) { + tests := []struct { + number int + list []int + result bool + }{ + { + number: 1, + list: []int{1, 2, 4}, + result: true, + }, + } + + for _, test := range tests { + actual := IsNumInList(test.number, test.list) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestIsStringInList(t *testing.T) { + tests := []struct { + str string + list []string + result bool + }{ + { + str: "", + list: []string{"", "", ""}, + result: true, + }, + } + + for _, test := range tests { + actual := IsStringInList(test.str, test.list) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestGetSliceInANotInB(t *testing.T) { + tests := []struct { + a []int + b []int + result []int + }{ + { + a: []int{4, 5, 1}, + b: []int{1, 2, 3}, + result: []int{4, 5}, + }, + } + + for _, test := range tests { + actual := GetSliceInANotInB(test.a, test.b) + expect := test.result + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} + +func TestGetSliceInAandInB(t *testing.T) { + tests := []struct { + a []int + b []int + result []int + }{ + { + a: []int{4, 5, 1}, + b: []int{1, 2, 3}, + result: []int{1}, + }, + } + + for _, test := range tests { + actual := GetSliceInAandInB(test.a, test.b) + expect := test.result + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} + +func TestIntSliceToString(t *testing.T) { + tests := []struct { + number []int + delimiter string + result string + }{ + { + number: []int{4, 5, 1}, + delimiter: ",", + result: "4,5,1", + }, + } + + for _, test := range tests { + actual := IntSliceToString(test.number, test.delimiter) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestStringToIntSlice(t *testing.T) { + tests := []struct { + str string + delimiter string + result []int + }{ + { + str: "4,5,1", + delimiter: ",", + result: []int{4, 5, 1}, + }, + } + + for _, test := range tests { + actual := StringToIntSlice(test.str, test.delimiter) + expect := test.result + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} + +func TestIsSliceEqual(t *testing.T) { + tests := []struct { + a []int + b []int + result bool + }{ + { + a: []int{1, 3, 5}, + b: []int{5, 1, 3}, + result: true, + }, + } + + for _, test := range tests { + actual := IsSliceEqual(test.a, test.b) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestIsRepeat(t *testing.T) { + tests := []struct { + nums []int + result bool + }{ + { + nums: []int{1, 2, 4, 1}, + result: true, + }, + { + nums: []int{1, 2, 3}, + result: false, + }, + } + + for _, test := range tests { + actual := IsRepeat(test.nums) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} + +func TestRemoveRepeat(t *testing.T) { + tests := []struct { + nums []int + result []int + }{ + { + nums: []int{1, 2, 4, 2, 1}, + result: []int{1, 2, 4}, + }, + { + nums: []int{1, 2, 3}, + result: []int{1, 2, 3}, + }, + } + + for _, test := range tests { + actual := RemoveRepeat(test.nums) + expect := test.result + for i := 0; i < len(actual); i++ { + if expect[i] != actual[i] { + t.Errorf("expect %v but got %v", expect, actual) + } + } + } +} + +func TestIsHasNegativeNum(t *testing.T) { + tests := []struct { + nums []int + result bool + }{ + { + nums: []int{1, -2, 4, 1}, + result: true, + }, + { + nums: []int{1, 2, 3}, + result: false, + }, + { + nums: []int{}, + result: false, + }, + } + + for _, test := range tests { + actual := IsHasNegativeNum(test.nums) + expect := test.result + if expect != actual { + t.Errorf("expect %v but got %v", expect, actual) + } + } +} diff --git a/pkg/webhook/mutating_pod.go b/pkg/webhook/mutating_pod.go new file mode 100644 index 00000000..5eaf8195 --- /dev/null +++ b/pkg/webhook/mutating_pod.go @@ -0,0 +1,53 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "encoding/json" + corev1 "k8s.io/api/core/v1" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type PodMutatingHandler struct { + Client client.Client + decoder *admission.Decoder +} + +func (pmh *PodMutatingHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + pod := &corev1.Pod{} + err := pmh.decoder.Decode(req, pod) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + pod = mutatingPod(pod, pmh.Client) + // mutate the fields in pod + + marshaledPod, err := json.Marshal(pod) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} + +func mutatingPod(pod *corev1.Pod, client client.Client) *corev1.Pod { + + return pod +} diff --git a/pkg/webhook/util/generator/certgenerator.go b/pkg/webhook/util/generator/certgenerator.go new file mode 100644 index 00000000..a031fea8 --- /dev/null +++ b/pkg/webhook/util/generator/certgenerator.go @@ -0,0 +1,41 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +// Artifacts hosts a private key, its corresponding serving certificate and +// the CA certificate that signs the serving certificate. +type Artifacts struct { + // PEM encoded private key + Key []byte + // PEM encoded serving certificate + Cert []byte + // PEM encoded CA private key + CAKey []byte + // PEM encoded CA certificate + CACert []byte + // Resource version of the certs + ResourceVersion string +} + +// CertGenerator is an interface to provision the serving certificate. +type CertGenerator interface { + // Generate returns a Artifacts struct. + Generate(CommonName string) (*Artifacts, error) + // SetCA sets the PEM-encoded CA private key and CA cert for signing the generated serving cert. + SetCA(caKey, caCert []byte) +} diff --git a/pkg/webhook/util/generator/fake/certgenerator.go b/pkg/webhook/util/generator/fake/certgenerator.go new file mode 100644 index 00000000..218d51ee --- /dev/null +++ b/pkg/webhook/util/generator/fake/certgenerator.go @@ -0,0 +1,54 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "bytes" + "fmt" + + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" +) + +// CertGenerator is a certGenerator for testing. +type CertGenerator struct { + CAKey []byte + CACert []byte + DNSNameToCertArtifacts map[string]*generator.Artifacts +} + +var _ generator.CertGenerator = &CertGenerator{} + +// SetCA sets the PEM-encoded CA private key and CA cert for signing the generated serving cert. +func (cp *CertGenerator) SetCA(CAKey, CACert []byte) { + cp.CAKey = CAKey + cp.CACert = CACert +} + +// Generate generates certificates by matching a common name. +func (cp *CertGenerator) Generate(commonName string) (*generator.Artifacts, error) { + certs, found := cp.DNSNameToCertArtifacts[commonName] + if !found { + return nil, fmt.Errorf("failed to find common name %q in the certGenerator", commonName) + } + if cp.CAKey != nil && cp.CACert != nil && + !bytes.Contains(cp.CAKey, []byte("invalid")) && !bytes.Contains(cp.CACert, []byte("invalid")) { + certs.CAKey = cp.CAKey + certs.CACert = cp.CACert + } + return certs, nil +} diff --git a/pkg/webhook/util/generator/selfsigned.go b/pkg/webhook/util/generator/selfsigned.go new file mode 100644 index 00000000..e4268d1e --- /dev/null +++ b/pkg/webhook/util/generator/selfsigned.go @@ -0,0 +1,192 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "crypto" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "math" + "math/big" + "net" + "time" + + "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" +) + +const ( + rsaKeySize = 2048 +) + +// ServiceToCommonName generates the CommonName for the certificate when using a k8s service. +func ServiceToCommonName(serviceNamespace, serviceName string) string { + return fmt.Sprintf("%s.%s.svc", serviceName, serviceNamespace) +} + +// SelfSignedCertGenerator implements the certGenerator interface. +// It provisions self-signed certificates. +type SelfSignedCertGenerator struct { + caKey []byte + caCert []byte +} + +var _ CertGenerator = &SelfSignedCertGenerator{} + +// SetCA sets the PEM-encoded CA private key and CA cert for signing the generated serving cert. +func (cp *SelfSignedCertGenerator) SetCA(caKey, caCert []byte) { + cp.caKey = caKey + cp.caCert = caCert +} + +// Generate creates and returns a CA certificate, certificate and +// key for the server. serverKey and serverCert are used by the server +// to establish trust for clients, CA certificate is used by the +// client to verify the server authentication chain. +// The cert will be valid for 365 days. +func (cp *SelfSignedCertGenerator) Generate(commonName string) (*Artifacts, error) { + var err error + + valid, signingKey, signingCert := cp.validCACert() + if !valid { + signingKey, err = NewPrivateKey() + if err != nil { + return nil, fmt.Errorf("failed to create the CA private key: %v", err) + } + signingCert, err = cert.NewSelfSignedCACert(cert.Config{CommonName: "webhook-cert-ca"}, signingKey) + if err != nil { + return nil, fmt.Errorf("failed to create the CA cert: %v", err) + } + } + + hostIP := net.ParseIP(commonName) + var altIPs []net.IP + DNSNames := []string{"localhost"} + if hostIP.To4() != nil { + altIPs = append(altIPs, hostIP.To4()) + } else { + DNSNames = append(DNSNames, commonName) + } + + key, err := NewPrivateKey() + if err != nil { + return nil, fmt.Errorf("failed to create the private key: %v", err) + } + signedCert, err := NewSignedCert( + cert.Config{ + CommonName: commonName, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + AltNames: cert.AltNames{IPs: altIPs, DNSNames: DNSNames}, + }, + key, signingCert, signingKey, + ) + if err != nil { + return nil, fmt.Errorf("failed to create the cert: %v", err) + } + return &Artifacts{ + Key: EncodePrivateKeyPEM(key), + Cert: EncodeCertPEM(signedCert), + CAKey: EncodePrivateKeyPEM(signingKey), + CACert: EncodeCertPEM(signingCert), + }, nil +} + +func (cp *SelfSignedCertGenerator) validCACert() (bool, *rsa.PrivateKey, *x509.Certificate) { + if !ValidCACert(cp.caKey, cp.caCert, cp.caCert, "", time.Now().AddDate(1, 0, 0)) { + return false, nil, nil + } + + key, err := keyutil.ParsePrivateKeyPEM(cp.caKey) + if err != nil { + return false, nil, nil + } + privateKey, ok := key.(*rsa.PrivateKey) + if !ok { + return false, nil, nil + } + + certs, err := cert.ParseCertsPEM(cp.caCert) + if err != nil { + return false, nil, nil + } + if len(certs) != 1 { + return false, nil, nil + } + return true, privateKey, certs[0] +} + +// NewPrivateKey creates an RSA private key +func NewPrivateKey() (*rsa.PrivateKey, error) { + return rsa.GenerateKey(cryptorand.Reader, rsaKeySize) +} + +// NewSignedCert creates a signed certificate using the given CA certificate and key +func NewSignedCert(cfg cert.Config, key crypto.Signer, caCert *x509.Certificate, caKey crypto.Signer) (*x509.Certificate, error) { + serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64)) + if err != nil { + return nil, err + } + if len(cfg.CommonName) == 0 { + return nil, errors.New("must specify a CommonName") + } + if len(cfg.Usages) == 0 { + return nil, errors.New("must specify at least one ExtKeyUsage") + } + + certTmpl := x509.Certificate{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + SerialNumber: serial, + NotBefore: caCert.NotBefore, + NotAfter: time.Now().Add(time.Hour * 24 * 365 * 10).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: cfg.Usages, + } + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &certTmpl, caCert, key.Public(), caKey) + if err != nil { + return nil, err + } + return x509.ParseCertificate(certDERBytes) +} + +// EncodePrivateKeyPEM returns PEM-encoded private key data +func EncodePrivateKeyPEM(key *rsa.PrivateKey) []byte { + block := pem.Block{ + Type: keyutil.RSAPrivateKeyBlockType, + Bytes: x509.MarshalPKCS1PrivateKey(key), + } + return pem.EncodeToMemory(&block) +} + +// EncodeCertPEM returns PEM-encoded certificate data +func EncodeCertPEM(ct *x509.Certificate) []byte { + block := pem.Block{ + Type: cert.CertificateBlockType, + Bytes: ct.Raw, + } + return pem.EncodeToMemory(&block) +} diff --git a/pkg/webhook/util/generator/util.go b/pkg/webhook/util/generator/util.go new file mode 100644 index 00000000..6080a30a --- /dev/null +++ b/pkg/webhook/util/generator/util.go @@ -0,0 +1,62 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generator + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "time" +) + +// ValidCACert treats cert and key are valid if they meet the following requirements: +// - key and cert are valid pair +// - caCert is the root ca of cert +// - cert is for dnsName +// - cert won't expire before time +func ValidCACert(key, cert, caCert []byte, dnsName string, time time.Time) bool { + if len(key) == 0 || len(cert) == 0 || len(caCert) == 0 { + return false + } + // Verify key and cert are valid pair + _, err := tls.X509KeyPair(cert, key) + if err != nil { + return false + } + + // Verify cert is valid for at least 1 year. + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(caCert) { + return false + } + block, _ := pem.Decode(cert) + if block == nil { + return false + } + c, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return false + } + ops := x509.VerifyOptions{ + DNSName: dnsName, + Roots: pool, + CurrentTime: time, + } + _, err = c.Verify(ops) + return err == nil +} diff --git a/pkg/webhook/util/writer/atomic/atomic_writer.go b/pkg/webhook/util/writer/atomic/atomic_writer.go new file mode 100644 index 00000000..0ca25204 --- /dev/null +++ b/pkg/webhook/util/writer/atomic/atomic_writer.go @@ -0,0 +1,452 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package atomic + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog/v2" +) + +const ( + maxFileNameLength = 255 + maxPathLength = 4096 +) + +// Writer handles atomically projecting content for a set of files into +// a target directory. +// +// Note: +// +// 1. Writer reserves the set of pathnames starting with `..`. +// 2. Writer offers no concurrency guarantees and must be synchronized +// by the caller. +// +// The visible files in this volume are symlinks to files in the writer's data +// directory. Actual files are stored in a hidden timestamped directory which +// is symlinked to by the data directory. The timestamped directory and +// data directory symlink are created in the writer's target dir.  This scheme +// allows the files to be atomically updated by changing the target of the +// data directory symlink. +// +// Consumers of the target directory can monitor the ..data symlink using +// inotify or fanotify to receive events when the content in the volume is +// updated. +type Writer struct { + targetDir string +} + +type FileProjection struct { + Data []byte + Mode int32 +} + +// NewAtomicWriter creates a new Writer configured to write to the given +// target directory, or returns an error if the target directory does not exist. +func NewAtomicWriter(targetDir string) (*Writer, error) { + _, err := os.Stat(targetDir) + if os.IsNotExist(err) { + return nil, err + } + + return &Writer{targetDir: targetDir}, nil +} + +const ( + dataDirName = "..data" + newDataDirName = "..data_tmp" +) + +// Write does an atomic projection of the given payload into the writer's target +// directory. Input paths must not begin with '..'. +// +// The Write algorithm is: +// +// 1. The payload is validated; if the payload is invalid, the function returns +// 2.  The current timestamped directory is detected by reading the data directory +// symlink +// 3. The old version of the volume is walked to determine whether any +// portion of the payload was deleted and is still present on disk. +// 4. The data in the current timestamped directory is compared to the projected +// data to determine if an update is required. +// 5.  A new timestamped dir is created +// 6. The payload is written to the new timestamped directory +// 7.  Symlinks and directory for new user-visible files are created (if needed). +// +// For example, consider the files: +// /podName +// /user/labels +// /k8s/annotations +// +// The user visible files are symbolic links into the internal data directory: +// /podName -> ..data/podName +// /usr -> ..data/usr +// /k8s -> ..data/k8s +// +// The data directory itself is a link to a timestamped directory with +// the real data: +// /..data -> ..2016_02_01_15_04_05.12345678/ +// 8.  A symlink to the new timestamped directory ..data_tmp is created that will +// become the new data directory +// 9.  The new data directory symlink is renamed to the data directory; rename is atomic +// 10. Old paths are removed from the user-visible portion of the target directory +// 11.  The previous timestamped directory is removed, if it exists +func (w *Writer) Write(payload map[string]FileProjection) error { + // (1) + cleanPayload, err := validatePayload(payload) + if err != nil { + klog.Error(err, "invalid payload") + return err + } + + // (2) + dataDirPath := path.Join(w.targetDir, dataDirName) + oldTsDir, err := os.Readlink(dataDirPath) + if err != nil { + if !os.IsNotExist(err) { + klog.Error(err, "unable to read link for data directory") + return err + } + // although Readlink() returns "" on err, don't be fragile by relying on it (since it's not specified in docs) + // empty oldTsDir indicates that it didn't exist + oldTsDir = "" + } + oldTsPath := path.Join(w.targetDir, oldTsDir) + + var pathsToRemove sets.String + // if there was no old version, there's nothing to remove + if len(oldTsDir) != 0 { + // (3) + pathsToRemove, err = w.pathsToRemove(cleanPayload, oldTsPath) + if err != nil { + klog.Error(err, "unable to determine user-visible files to remove") + return err + } + + // (4) + if should, err := shouldWritePayload(cleanPayload, oldTsPath); err != nil { + klog.Error(err, "unable to determine whether payload should be written to disk") + return err + } else if !should && len(pathsToRemove) == 0 { + klog.V(6).Info("no update required for target directory", "directory", w.targetDir) + return nil + } else { + klog.V(1).Info("write required for target directory", "directory", w.targetDir) + } + } + + // (5) + tsDir, err := w.newTimestampDir() + if err != nil { + klog.Error(err, "error creating new ts data directory") + return err + } + tsDirName := filepath.Base(tsDir) + + // (6) + if err = w.writePayloadToDir(cleanPayload, tsDir); err != nil { + klog.Error(err, "unable to write payload to ts data directory", "ts directory", tsDir) + return err + } + klog.V(1).Info("performed write of new data to ts data directory", "ts directory", tsDir) + + // (7) + if err = w.createUserVisibleFiles(cleanPayload); err != nil { + klog.Error(err, "unable to create visible symlinks in target directory", "target directory", w.targetDir) + return err + } + + // (8) + newDataDirPath := path.Join(w.targetDir, newDataDirName) + if err = os.Symlink(tsDirName, newDataDirPath); err != nil { + os.RemoveAll(tsDir) + klog.Error(err, "unable to create symbolic link for atomic update") + return err + } + + // (9) + if runtime.GOOS == "windows" { + os.Remove(dataDirPath) + err = os.Symlink(tsDirName, dataDirPath) + os.Remove(newDataDirPath) + } else { + err = os.Rename(newDataDirPath, dataDirPath) + } + if err != nil { + os.Remove(newDataDirPath) + os.RemoveAll(tsDir) + klog.Error(err, "unable to rename symbolic link for data directory", "data directory", newDataDirPath) + return err + } + + // (10) + if err = w.removeUserVisiblePaths(pathsToRemove); err != nil { + klog.Error(err, "unable to remove old visible symlinks") + return err + } + + // (11) + if len(oldTsDir) > 0 { + if err = os.RemoveAll(oldTsPath); err != nil { + klog.Error(err, "unable to remove old data directory", "data directory", oldTsDir) + return err + } + } + + return nil +} + +// validatePayload returns an error if any path in the payload returns a copy of the payload with the paths cleaned. +func validatePayload(payload map[string]FileProjection) (map[string]FileProjection, error) { + cleanPayload := make(map[string]FileProjection) + for k, content := range payload { + if err := validatePath(k); err != nil { + return nil, err + } + + cleanPayload[filepath.Clean(k)] = content + } + + return cleanPayload, nil +} + +// validatePath validates a single path, returning an error if the path is +// invalid. paths may not: +// +// 1. be absolute +// 2. contain '..' as an element +// 3. start with '..' +// 4. contain filenames larger than 255 characters +// 5. be longer than 4096 characters +func validatePath(targetPath string) error { + // TODO: somehow unify this with the similar api validation, + // validateVolumeSourcePath; the error semantics are just different enough + // from this that it was time-prohibitive trying to find the right + // refactoring to re-use. + if targetPath == "" { + return fmt.Errorf("invalid path: must not be empty: %q", targetPath) + } + if path.IsAbs(targetPath) { + return fmt.Errorf("invalid path: must be relative path: %s", targetPath) + } + + if len(targetPath) > maxPathLength { + return fmt.Errorf("invalid path: must be less than or equal to %d characters", maxPathLength) + } + + items := strings.Split(targetPath, string(os.PathSeparator)) + for _, item := range items { + if item == ".." { + return fmt.Errorf("invalid path: must not contain '..': %s", targetPath) + } + if len(item) > maxFileNameLength { + return fmt.Errorf("invalid path: filenames must be less than or equal to %d characters", maxFileNameLength) + } + } + if strings.HasPrefix(items[0], "..") && len(items[0]) > 2 { + return fmt.Errorf("invalid path: must not start with '..': %s", targetPath) + } + + return nil +} + +// shouldWritePayload returns whether the payload should be written to disk. +func shouldWritePayload(payload map[string]FileProjection, oldTsDir string) (bool, error) { + for userVisiblePath, fileProjection := range payload { + shouldWrite, err := shouldWriteFile(path.Join(oldTsDir, userVisiblePath), fileProjection.Data) + if err != nil { + return false, err + } + + if shouldWrite { + return true, nil + } + } + + return false, nil +} + +// shouldWriteFile returns whether a new version of a file should be written to disk. +func shouldWriteFile(path string, content []byte) (bool, error) { + _, err := os.Lstat(path) + if os.IsNotExist(err) { + return true, nil + } + + contentOnFs, err := ioutil.ReadFile(path) + if err != nil { + return false, err + } + + return !bytes.Equal(content, contentOnFs), nil +} + +// pathsToRemove walks the current version of the data directory and +// determines which paths should be removed (if any) after the payload is +// written to the target directory. +func (w *Writer) pathsToRemove(payload map[string]FileProjection, oldTsDir string) (sets.String, error) { + paths := sets.NewString() + visitor := func(path string, info os.FileInfo, err error) error { + relativePath := strings.TrimPrefix(path, oldTsDir) + relativePath = strings.TrimPrefix(relativePath, string(os.PathSeparator)) + if relativePath == "" { + return nil + } + + paths.Insert(relativePath) + return nil + } + + err := filepath.Walk(oldTsDir, visitor) + if os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + + newPaths := sets.NewString() + for file := range payload { + // add all subpaths for the payload to the set of new paths + // to avoid attempting to remove non-empty dirs + for subPath := file; subPath != ""; { + newPaths.Insert(subPath) + subPath, _ = filepath.Split(subPath) + subPath = strings.TrimSuffix(subPath, string(os.PathSeparator)) + } + } + + result := paths.Difference(newPaths) + if len(result) > 0 { + klog.V(1).Info("paths to remove", "target directory", w.targetDir, "paths", result) + } + + return result, nil +} + +// newTimestampDir creates a new timestamp directory +func (w *Writer) newTimestampDir() (string, error) { + tsDir, err := ioutil.TempDir(w.targetDir, time.Now().UTC().Format("..2006_01_02_15_04_05.")) + if err != nil { + klog.Error(err, "unable to create new temp directory") + return "", err + } + + // 0755 permissions are needed to allow 'group' and 'other' to recurse the + // directory tree. do a chmod here to ensure that permissions are set correctly + // regardless of the process' umask. + err = os.Chmod(tsDir, 0755) + if err != nil { + klog.Error(err, "unable to set mode on new temp directory") + return "", err + } + + return tsDir, nil +} + +// writePayloadToDir writes the given payload to the given directory. The +// directory must exist. +func (w *Writer) writePayloadToDir(payload map[string]FileProjection, dir string) error { + for userVisiblePath, fileProjection := range payload { + content := fileProjection.Data + mode := os.FileMode(fileProjection.Mode) + fullPath := path.Join(dir, userVisiblePath) + baseDir, _ := filepath.Split(fullPath) + + err := os.MkdirAll(baseDir, os.ModePerm) + if err != nil { + klog.Error(err, "unable to create directory", "directory", baseDir) + return err + } + + err = ioutil.WriteFile(fullPath, content, mode) + if err != nil { + klog.Error(err, "unable to write file", "file", fullPath, "mode", mode) + return err + } + // Chmod is needed because ioutil.WriteFile() ends up calling + // open(2) to create the file, so the final mode used is "mode & + // ~umask". But we want to make sure the specified mode is used + // in the file no matter what the umask is. + err = os.Chmod(fullPath, mode) + if err != nil { + klog.Error(err, "unable to write file", "file", fullPath, "mode", mode) + } + } + + return nil +} + +// createUserVisibleFiles creates the relative symlinks for all the +// files configured in the payload. If the directory in a file path does not +// exist, it is created. +// +// Viz: +// For files: "bar", "foo/bar", "baz/bar", "foo/baz/blah" +// the following symlinks are created: +// bar -> ..data/bar +// foo -> ..data/foo +// baz -> ..data/baz +func (w *Writer) createUserVisibleFiles(payload map[string]FileProjection) error { + for userVisiblePath := range payload { + slashpos := strings.Index(userVisiblePath, string(os.PathSeparator)) + if slashpos == -1 { + slashpos = len(userVisiblePath) + } + linkname := userVisiblePath[:slashpos] + _, err := os.Readlink(path.Join(w.targetDir, linkname)) + if err != nil && os.IsNotExist(err) { + // The link into the data directory for this path doesn't exist; create it + visibleFile := path.Join(w.targetDir, linkname) + dataDirFile := path.Join(dataDirName, linkname) + + err = os.Symlink(dataDirFile, visibleFile) + if err != nil { + return err + } + } + } + return nil +} + +// removeUserVisiblePaths removes the set of paths from the user-visible +// portion of the writer's target directory. +func (w *Writer) removeUserVisiblePaths(paths sets.String) error { + ps := string(os.PathSeparator) + var lasterr error + for p := range paths { + // only remove symlinks from the volume root directory (i.e. items that don't contain '/') + if strings.Contains(p, ps) { + continue + } + if err := os.Remove(path.Join(w.targetDir, p)); err != nil { + klog.Error(err, "unable to prune old user-visible path", "path", p) + lasterr = err + } + } + + return lasterr +} diff --git a/pkg/webhook/util/writer/certwriter.go b/pkg/webhook/util/writer/certwriter.go new file mode 100644 index 00000000..c52ef21e --- /dev/null +++ b/pkg/webhook/util/writer/certwriter.go @@ -0,0 +1,107 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer + +import ( + "errors" + "time" + + "k8s.io/klog/v2" + + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" +) + +const ( + // CAKeyName is the name of the CA private key + CAKeyName = "ca-key.pem" + // CACertName is the name of the CA certificate + CACertName = "ca-cert.pem" + // ServerKeyName is the name of the server private key + ServerKeyName = "key.pem" + ServerKeyName2 = "tls.key" + // ServerCertName is the name of the serving certificate + ServerCertName = "cert.pem" + ServerCertName2 = "tls.crt" +) + +// CertWriter provides method to handle webhooks. +type CertWriter interface { + // EnsureCert provisions the cert for the webhookClientConfig. + EnsureCert(dnsName string) (*generator.Artifacts, bool, error) +} + +// handleCommon ensures the given webhook has a proper certificate. +// It uses the given certReadWriter to read and (or) write the certificate. +func handleCommon(dnsName string, ch certReadWriter) (*generator.Artifacts, bool, error) { + if len(dnsName) == 0 { + return nil, false, errors.New("dnsName should not be empty") + } + if ch == nil { + return nil, false, errors.New("certReaderWriter should not be nil") + } + + certs, changed, err := createIfNotExists(ch) + if err != nil { + return nil, changed, err + } + + // Recreate the cert if it's invalid. + valid := validCert(certs, dnsName) + if !valid { + klog.Info("cert is invalid or expired, regenerating a new one") + certs, err = ch.overwrite(certs.ResourceVersion) + if err != nil { + return nil, false, err + } + changed = true + } + return certs, changed, nil +} + +func createIfNotExists(ch certReadWriter) (*generator.Artifacts, bool, error) { + // Try to read first + certs, err := ch.read() + if isNotFound(err) { + // Create if not exists + certs, err = ch.write() + // This may happen if there is another racer. + if isAlreadyExists(err) { + certs, err = ch.read() + } + return certs, true, err + } + return certs, false, err +} + +// certReadWriter provides methods for reading and writing certificates. +type certReadWriter interface { + // read a webhook name and returns the certs for it. + read() (*generator.Artifacts, error) + // write the certs and return the certs it wrote. + write() (*generator.Artifacts, error) + // overwrite the existing certs and return the certs it wrote. + overwrite(resourceVersion string) (*generator.Artifacts, error) +} + +func validCert(certs *generator.Artifacts, dnsName string) bool { + if certs == nil { + return false + } + expired := time.Now().AddDate(0, 6, 0) + return generator.ValidCACert(certs.Key, certs.Cert, certs.CACert, dnsName, expired) +} diff --git a/pkg/webhook/util/writer/error.go b/pkg/webhook/util/writer/error.go new file mode 100644 index 00000000..d7399eef --- /dev/null +++ b/pkg/webhook/util/writer/error.go @@ -0,0 +1,44 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer + +type notFoundError struct { + err error +} + +func (e notFoundError) Error() string { + return e.err.Error() +} + +func isNotFound(err error) bool { + _, ok := err.(notFoundError) + return ok +} + +type alreadyExistError struct { + err error +} + +func (e alreadyExistError) Error() string { + return e.err.Error() +} + +func isAlreadyExists(err error) bool { + _, ok := err.(alreadyExistError) + return ok +} diff --git a/pkg/webhook/util/writer/fs.go b/pkg/webhook/util/writer/fs.go new file mode 100644 index 00000000..93350e35 --- /dev/null +++ b/pkg/webhook/util/writer/fs.go @@ -0,0 +1,229 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path" + + "k8s.io/klog/v2" + + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" + "github.com/openkruise/kruise-game/pkg/webhook/util/writer/atomic" +) + +const ( + FsCertWriter = "fs" +) + +// fsCertWriter provisions the certificate by reading and writing to the filesystem. +type fsCertWriter struct { + // dnsName is the DNS name that the certificate is for. + dnsName string + + *FSCertWriterOptions +} + +// FSCertWriterOptions are options for constructing a FSCertWriter. +type FSCertWriterOptions struct { + // certGenerator generates the certificates. + CertGenerator generator.CertGenerator + // path is the directory that the certificate and private key and CA certificate will be written. + Path string +} + +var _ CertWriter = &fsCertWriter{} + +func (ops *FSCertWriterOptions) setDefaults() { + if ops.CertGenerator == nil { + ops.CertGenerator = &generator.SelfSignedCertGenerator{} + } +} + +func (ops *FSCertWriterOptions) validate() error { + if len(ops.Path) == 0 { + return errors.New("path must be set in FSCertWriterOptions") + } + return nil +} + +// NewFSCertWriter constructs a CertWriter that persists the certificate on filesystem. +func NewFSCertWriter(ops FSCertWriterOptions) (CertWriter, error) { + ops.setDefaults() + err := ops.validate() + if err != nil { + return nil, err + } + return &fsCertWriter{FSCertWriterOptions: &ops}, nil +} + +// EnsureCert provisions certificates for a webhookClientConfig by writing the certificates in the filesystem. +func (f *fsCertWriter) EnsureCert(dnsName string) (*generator.Artifacts, bool, error) { + // create or refresh cert and write it to fs + f.dnsName = dnsName + return handleCommon(f.dnsName, f) +} + +func (f *fsCertWriter) write() (*generator.Artifacts, error) { + return f.doWrite() +} + +func (f *fsCertWriter) overwrite(_ string) (*generator.Artifacts, error) { + return f.doWrite() +} + +func (f *fsCertWriter) doWrite() (*generator.Artifacts, error) { + certs, err := f.CertGenerator.Generate(f.dnsName) + if err != nil { + return nil, err + } + + if err = WriteCertsToDir(f.Path, certs); err != nil { + return nil, err + } + return certs, nil +} + +func WriteCertsToDir(path string, certs *generator.Artifacts) error { + // Writer's algorithm only manages files using symbolic link. + // If a file is not a symbolic link, will ignore the update for it. + // We want to cleanup for Writer by removing old files that are not symbolic links. + err := prepareToWrite(path) + if err != nil { + return err + } + + aw, err := atomic.NewAtomicWriter(path) + if err != nil { + return err + } + return aw.Write(certToProjectionMap(certs)) +} + +// prepareToWrite ensures it directory is compatible with the atomic.Writer library. +func prepareToWrite(dir string) error { + _, err := os.Stat(dir) + switch { + case os.IsNotExist(err): + klog.Info("cert directory doesn't exist, creating", "directory", dir) + // TODO: figure out if we can reduce the permission. (Now it's 0777) + err = os.MkdirAll(dir, 0777) + if err != nil { + return fmt.Errorf("can't create dir: %v", dir) + } + case err != nil: + return err + } + + filenames := []string{CAKeyName, CACertName, ServerCertName, ServerCertName2, ServerKeyName, ServerKeyName2} + for _, f := range filenames { + abspath := path.Join(dir, f) + _, err := os.Stat(abspath) + if os.IsNotExist(err) { + continue + } else if err != nil { + klog.Error(err, "unable to stat file", "file", abspath) + } + _, err = os.Readlink(abspath) + // if it's not a symbolic link + if err != nil { + err = os.Remove(abspath) + if err != nil { + klog.Error(err, "unable to remove old file", "file", abspath) + } + } + } + return nil +} + +func (f *fsCertWriter) read() (*generator.Artifacts, error) { + if err := ensureExist(f.Path); err != nil { + return nil, err + } + caKeyBytes, err := ioutil.ReadFile(path.Join(f.Path, CAKeyName)) + if err != nil { + return nil, err + } + caCertBytes, err := ioutil.ReadFile(path.Join(f.Path, CACertName)) + if err != nil { + return nil, err + } + certBytes, err := ioutil.ReadFile(path.Join(f.Path, ServerCertName)) + if err != nil { + return nil, err + } + keyBytes, err := ioutil.ReadFile(path.Join(f.Path, ServerKeyName)) + if err != nil { + return nil, err + } + return &generator.Artifacts{ + CAKey: caKeyBytes, + CACert: caCertBytes, + Cert: certBytes, + Key: keyBytes, + }, nil +} + +func ensureExist(dir string) error { + filenames := []string{CAKeyName, CACertName, ServerCertName, ServerCertName2, ServerKeyName, ServerKeyName2} + for _, filename := range filenames { + _, err := os.Stat(path.Join(dir, filename)) + switch { + case err == nil: + continue + case os.IsNotExist(err): + return notFoundError{err} + default: + return err + } + } + return nil +} + +func certToProjectionMap(cert *generator.Artifacts) map[string]atomic.FileProjection { + // TODO: figure out if we can reduce the permission. (Now it's 0666) + return map[string]atomic.FileProjection{ + CAKeyName: { + Data: cert.CAKey, + Mode: 0666, + }, + CACertName: { + Data: cert.CACert, + Mode: 0666, + }, + ServerCertName: { + Data: cert.Cert, + Mode: 0666, + }, + ServerCertName2: { + Data: cert.Cert, + Mode: 0666, + }, + ServerKeyName: { + Data: cert.Key, + Mode: 0666, + }, + ServerKeyName2: { + Data: cert.Key, + Mode: 0666, + }, + } +} diff --git a/pkg/webhook/util/writer/secret.go b/pkg/webhook/util/writer/secret.go new file mode 100644 index 00000000..67592261 --- /dev/null +++ b/pkg/webhook/util/writer/secret.go @@ -0,0 +1,179 @@ +/* +Copyright 2020 The Kruise Authors. +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package writer + +import ( + "context" + "errors" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" +) + +const ( + SecretCertWriter = "secret" +) + +// secretCertWriter provisions the certificate by reading and writing to the k8s secrets. +type secretCertWriter struct { + *SecretCertWriterOptions + + // dnsName is the DNS name that the certificate is for. + dnsName string +} + +// SecretCertWriterOptions is options for constructing a secretCertWriter. +type SecretCertWriterOptions struct { + // client talks to a kubernetes cluster for creating the secret. + Clientset clientset.Interface + // certGenerator generates the certificates. + CertGenerator generator.CertGenerator + // secret points the secret that contains certificates that written by the CertWriter. + Secret *types.NamespacedName +} + +var _ CertWriter = &secretCertWriter{} + +func (ops *SecretCertWriterOptions) setDefaults() { + if ops.CertGenerator == nil { + ops.CertGenerator = &generator.SelfSignedCertGenerator{} + } +} + +func (ops *SecretCertWriterOptions) validate() error { + if ops.Clientset == nil { + return errors.New("client must be set in SecretCertWriterOptions") + } + if ops.Secret == nil { + return errors.New("secret must be set in SecretCertWriterOptions") + } + return nil +} + +// NewSecretCertWriter constructs a CertWriter that persists the certificate in a k8s secret. +func NewSecretCertWriter(ops SecretCertWriterOptions) (CertWriter, error) { + ops.setDefaults() + err := ops.validate() + if err != nil { + return nil, err + } + return &secretCertWriter{SecretCertWriterOptions: &ops}, nil +} + +// EnsureCert provisions certificates for a webhookClientConfig by writing the certificates to a k8s secret. +func (s *secretCertWriter) EnsureCert(dnsName string) (*generator.Artifacts, bool, error) { + // Create or refresh the certs based on clientConfig + s.dnsName = dnsName + return handleCommon(s.dnsName, s) +} + +var _ certReadWriter = &secretCertWriter{} + +func (s *secretCertWriter) buildSecret() (*corev1.Secret, *generator.Artifacts, error) { + certs, err := s.CertGenerator.Generate(s.dnsName) + if err != nil { + return nil, nil, err + } + secret := certsToSecret(certs, *s.Secret) + return secret, certs, err +} + +func (s *secretCertWriter) write() (*generator.Artifacts, error) { + secret, certs, err := s.buildSecret() + if err != nil { + return nil, err + } + _, err = s.Clientset.CoreV1().Secrets(secret.Namespace).Create(context.TODO(), secret, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + return nil, alreadyExistError{err} + } + return certs, err +} + +func (s *secretCertWriter) overwrite(resourceVersion string) (*generator.Artifacts, error) { + secret, certs, err := s.buildSecret() + if err != nil { + return nil, err + } + secret.ResourceVersion = resourceVersion + secret, err = s.Clientset.CoreV1().Secrets(secret.Namespace).Update(context.TODO(), secret, metav1.UpdateOptions{}) + if err != nil { + klog.Infof("Cert writer update secret failed: %v", err) + return nil, err + } + klog.Infof("Cert writer update secret %s resourceVersion from %s to %s", + secret.Name, resourceVersion, secret.ResourceVersion, + ) + return certs, nil +} + +func (s *secretCertWriter) read() (*generator.Artifacts, error) { + secret, err := s.Clientset.CoreV1().Secrets(s.Secret.Namespace).Get(context.TODO(), s.Secret.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, notFoundError{err} + } + if err != nil { + return nil, err + } + certs := secretToCerts(secret) + if certs.CACert != nil && certs.CAKey != nil { + // Store the CA for next usage. + s.CertGenerator.SetCA(certs.CAKey, certs.CACert) + } + return certs, nil +} + +func secretToCerts(secret *corev1.Secret) *generator.Artifacts { + ret := &generator.Artifacts{ + ResourceVersion: secret.ResourceVersion, + } + if secret.Data != nil { + ret.CAKey = secret.Data[CAKeyName] + ret.CACert = secret.Data[CACertName] + ret.Cert = secret.Data[ServerCertName] + ret.Key = secret.Data[ServerKeyName] + } + return ret +} + +func certsToSecret(certs *generator.Artifacts, sec types.NamespacedName) *corev1.Secret { + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: sec.Namespace, + Name: sec.Name, + }, + Data: map[string][]byte{ + CAKeyName: certs.CAKey, + CACertName: certs.CACert, + ServerKeyName: certs.Key, + ServerKeyName2: certs.Key, + ServerCertName: certs.Cert, + ServerCertName2: certs.Cert, + }, + } +} diff --git a/pkg/webhook/validating_gss.go b/pkg/webhook/validating_gss.go new file mode 100644 index 00000000..9de1b2b2 --- /dev/null +++ b/pkg/webhook/validating_gss.go @@ -0,0 +1,61 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "fmt" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +type GssValidaatingHandler struct { + Client client.Client + decoder *admission.Decoder +} + +func (gvh *GssValidaatingHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + gss := &gamekruiseiov1alpha1.GameServerSet{} + err := gvh.decoder.Decode(req, gss) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + var reason string + allowed, err := validatingGss(gss, gvh.Client) + if err != nil { + reason = err.Error() + } + + return admission.ValidationResponse(allowed, reason) +} + +func validatingGss(gss *gamekruiseiov1alpha1.GameServerSet, client client.Client) (bool, error) { + // validate reserveGameServerIds + rgsIds := gss.Spec.ReserveGameServerIds + if util.IsRepeat(rgsIds) { + return false, fmt.Errorf("reserveGameServerIds should not be repeat. Now it is %v", rgsIds) + } + if util.IsHasNegativeNum(rgsIds) { + return false, fmt.Errorf("reserveGameServerIds should be greater or equal to 0. Now it is %v", rgsIds) + } + + return true, nil +} diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go new file mode 100644 index 00000000..033a1cfb --- /dev/null +++ b/pkg/webhook/webhook.go @@ -0,0 +1,262 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "context" + "flag" + "fmt" + "github.com/openkruise/kruise-game/pkg/webhook/util/generator" + "github.com/openkruise/kruise-game/pkg/webhook/util/writer" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +var ( + mutatePodPath = "/mutate-v1-pod" + validateGssPath = "/validate-v1alpha1-gss" + mutatingWebhookConfigurationName = "kruise-game-mutating-webhook" + validatingWebhookConfigurationName = "kruise-game-validating-webhook" +) + +var ( + webhookPort int + webhookCertDir string + webhookServiceNamespace string + webhookServiceName string +) + +func init() { + flag.IntVar(&webhookPort, "webhook-port", 9876, "The port of the MutatingWebhookConfiguration object.") + flag.StringVar(&webhookCertDir, "webhook-server-certs-dir", "/tmp/webhook-certs/", "Path to the X.509-formatted webhook certificate.") + flag.StringVar(&webhookServiceNamespace, "webhook-service-namespace", "kruise-game-system", "kruise game webhook service namespace.") + flag.StringVar(&webhookServiceName, "webhook-service-name", "kruise-game-webhook-service", "kruise game wehook service name.") +} + +// +kubebuilder:rbac:groups=apps.kruise.io,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.kruise.io,resources=statefulsets/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=create;get;list;watch;update;patch +// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations,verbs=create;get;list;watch;update;patch +// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch + +type Webhook struct { + mgr manager.Manager +} + +func NewWebhookServer(mgr manager.Manager) *Webhook { + return &Webhook{ + mgr: mgr, + } +} + +func (ws *Webhook) SetupWithManager(mgr manager.Manager) *Webhook { + server := mgr.GetWebhookServer() + server.Host = "0.0.0.0" + server.Port = webhookPort + server.CertDir = webhookCertDir + decoder, err := admission.NewDecoder(runtime.NewScheme()) + if err != nil { + log.Fatalln(err) + } + server.Register(mutatePodPath, &webhook.Admission{Handler: &PodMutatingHandler{Client: mgr.GetClient(), decoder: decoder}}) + server.Register(validateGssPath, &webhook.Admission{Handler: &GssValidaatingHandler{Client: mgr.GetClient(), decoder: decoder}}) + return ws +} + +// Initialize create MutatingWebhookConfiguration before start +func (ws *Webhook) Initialize(cfg *rest.Config) error { + dnsName := generator.ServiceToCommonName(webhookServiceNamespace, webhookServiceName) + + var certWriter writer.CertWriter + var err error + + certWriter, err = writer.NewFSCertWriter(writer.FSCertWriterOptions{Path: webhookCertDir}) + + certs, _, err := certWriter.EnsureCert(dnsName) + if err != nil { + return fmt.Errorf("failed to ensure certs: %v", err) + } + + if err := writer.WriteCertsToDir(webhookCertDir, certs); err != nil { + return fmt.Errorf("failed to write certs to dir: %v", err) + } + + clientSet, err := clientset.NewForConfig(cfg) + + if err != nil { + return err + } + + if err := checkValidatingConfiguration(dnsName, clientSet, certs.CACert); err != nil { + return fmt.Errorf("failed to check mutating webhook,because of %s", err.Error()) + } + + if err := checkMutatingConfiguration(dnsName, clientSet, certs.CACert); err != nil { + return fmt.Errorf("failed to check mutating webhook,because of %s", err.Error()) + } + return nil +} + +func checkValidatingConfiguration(dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + vwc, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.TODO(), validatingWebhookConfigurationName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // create new webhook + return createValidatingWebhook(dnsName, kubeClient, caBundle) + } else { + return err + } + } + return updateValidatingWebhook(vwc, dnsName, kubeClient, caBundle) +} + +func checkMutatingConfiguration(dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + mwc, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookConfigurationName, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + // create new webhook + return createMutatingWebhook(dnsName, kubeClient, caBundle) + } else { + return err + } + } + return updateMutatingWebhook(mwc, dnsName, kubeClient, caBundle) +} + +func createValidatingWebhook(dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + sideEffectClassNone := admissionregistrationv1.SideEffectClassNone + fail := admissionregistrationv1.Fail + + webhookConfig := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: validatingWebhookConfigurationName, + }, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + Name: dnsName, + SideEffects: &sideEffectClassNone, + FailurePolicy: &fail, + AdmissionReviewVersions: []string{"v1"}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: webhookServiceNamespace, + Name: webhookServiceName, + Path: &validateGssPath, + }, + CABundle: caBundle, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{"game.kruise.io"}, + APIVersions: []string{"v1alpha1"}, + Resources: []string{"gameserversets"}, + }, + }, + }, + }, + }, + } + + if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), webhookConfig, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create %s: %v", validatingWebhookConfigurationName, err) + } + return nil +} + +func createMutatingWebhook(dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + sideEffectClassNone := admissionregistrationv1.SideEffectClassNone + ignore := admissionregistrationv1.Ignore + + webhookConfig := &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: mutatingWebhookConfigurationName, + }, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + { + Name: dnsName, + SideEffects: &sideEffectClassNone, + FailurePolicy: &ignore, + AdmissionReviewVersions: []string{"v1"}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Namespace: webhookServiceNamespace, + Name: webhookServiceName, + Path: &mutatePodPath, + }, + CABundle: caBundle, + }, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }, + }, + }, + }, + } + + if _, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.TODO(), webhookConfig, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("failed to create %s: %v", mutatingWebhookConfigurationName, err) + } + return nil +} + +func updateValidatingWebhook(vwc *admissionregistrationv1.ValidatingWebhookConfiguration, dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + var mutatingWHs []admissionregistrationv1.ValidatingWebhook + for _, wh := range vwc.Webhooks { + if wh.Name == dnsName { + wh.ClientConfig.CABundle = caBundle + } + mutatingWHs = append(mutatingWHs, wh) + } + vwc.Webhooks = mutatingWHs + if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), vwc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update %s: %v", validatingWebhookConfigurationName, err) + } + return nil +} + +func updateMutatingWebhook(mwc *admissionregistrationv1.MutatingWebhookConfiguration, dnsName string, kubeClient clientset.Interface, caBundle []byte) error { + var mutatingWHs []admissionregistrationv1.MutatingWebhook + for _, wh := range mwc.Webhooks { + if wh.Name == dnsName { + wh.ClientConfig.CABundle = caBundle + } + mutatingWHs = append(mutatingWHs, wh) + } + mwc.Webhooks = mutatingWHs + if _, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), mwc, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update %s: %v", mutatingWebhookConfigurationName, err) + } + return nil +} diff --git a/scripts/generate_client.sh b/scripts/generate_client.sh new file mode 100755 index 00000000..da43ad7c --- /dev/null +++ b/scripts/generate_client.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +go mod vendor + +rm -rf ./pkg/client/{clientset,informers,listers} + +/bin/bash ./vendor/k8s.io/code-generator/generate-groups.sh all \ +github.com/openkruise/kruise-game/pkg/client github.com/openkruise/kruise-game "apis:v1alpha1" -h ./hack/boilerplate.go.txt \ No newline at end of file diff --git a/test/e2e/client/client.go b/test/e2e/client/client.go new file mode 100644 index 00000000..7e863e15 --- /dev/null +++ b/test/e2e/client/client.go @@ -0,0 +1,137 @@ +package client + +import ( + "context" + "fmt" + kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + apps "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/utils/pointer" + "time" +) + +const ( + Namespace = "e2e-test" + GameServerSet = "default-gss" + GameContainerName = "default-game" +) + +type Client struct { + kruisegameClient kruisegameclientset.Interface + kubeClint clientset.Interface +} + +func NewKubeClient(kruisegameClient kruisegameclientset.Interface, kubeClint clientset.Interface) *Client { + return &Client{ + kruisegameClient: kruisegameClient, + kubeClint: kubeClint, + } +} + +func (client *Client) CreateNamespace() error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: Namespace, + Namespace: Namespace, + }, + } + _, err := client.kubeClint.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) + return err +} + +func (client *Client) DeleteNamespace() error { + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + err = client.kubeClint.CoreV1().Namespaces().Delete(context.TODO(), Namespace, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + } + return false, nil + }) +} + +func (client *Client) DefaultGameServerSet() *gameKruiseV1alpha1.GameServerSet { + return &gameKruiseV1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: GameServerSet, + Namespace: Namespace, + }, + Spec: gameKruiseV1alpha1.GameServerSetSpec{ + Replicas: pointer.Int32Ptr(3), + UpdateStrategy: gameKruiseV1alpha1.UpdateStrategy{ + Type: apps.RollingUpdateStatefulSetStrategyType, + RollingUpdate: &gameKruiseV1alpha1.RollingUpdateStatefulSetStrategy{ + PodUpdatePolicy: kruiseV1beta1.InPlaceIfPossiblePodUpdateStrategyType, + }, + }, + GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{ + PodTemplateSpec: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: GameContainerName, + Image: "nginx:1.9.7", + }, + }, + }, + }, + }, + }, + } +} + +func (client *Client) CreateGameServerSet(gss *gameKruiseV1alpha1.GameServerSet) (*gameKruiseV1alpha1.GameServerSet, error) { + if gss == nil { + return nil, fmt.Errorf("gss is nil") + } + return client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Create(context.TODO(), gss, metav1.CreateOptions{}) +} + +func (client *Client) UpdateGameServerSet(gss *gameKruiseV1alpha1.GameServerSet) (*gameKruiseV1alpha1.GameServerSet, error) { + return client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Update(context.TODO(), gss, metav1.UpdateOptions{}) +} + +func (client *Client) DeleteGameServerSet() error { + return wait.PollImmediate(3*time.Second, time.Minute, func() (done bool, err error) { + err = client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Delete(context.TODO(), GameServerSet, metav1.DeleteOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + } + return false, nil + }) +} + +func (client *Client) GetGameServerSet() (*gameKruiseV1alpha1.GameServerSet, error) { + return client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Get(context.TODO(), GameServerSet, metav1.GetOptions{}) +} + +func (client *Client) PatchGameServer(gsName string, data []byte) (*gameKruiseV1alpha1.GameServer, error) { + return client.kruisegameClient.GameV1alpha1().GameServers(Namespace).Patch(context.TODO(), gsName, types.MergePatchType, data, metav1.PatchOptions{}) +} + +func (client *Client) PatchGameServerSet(data []byte) (*gameKruiseV1alpha1.GameServerSet, error) { + return client.kruisegameClient.GameV1alpha1().GameServerSets(Namespace).Patch(context.TODO(), GameServerSet, types.MergePatchType, data, metav1.PatchOptions{}) +} + +func (client *Client) GetGameServerList(labelSelector string) (*gameKruiseV1alpha1.GameServerList, error) { + return client.kruisegameClient.GameV1alpha1().GameServers(Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector}) +} + +func (client *Client) GetPodList(labelSelector string) (*corev1.PodList, error) { + return client.kubeClint.CoreV1().Pods(Namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector}) +} + +func (client *Client) GetPod(podName string) (*corev1.Pod, error) { + return client.kubeClint.CoreV1().Pods(Namespace).Get(context.TODO(), podName, metav1.GetOptions{}) +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 00000000..ca47b8a9 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,35 @@ +package e2e + +import ( + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/openkruise/kruise-game/test/e2e/framework" + "github.com/openkruise/kruise-game/test/e2e/testcase" + "sigs.k8s.io/controller-runtime/pkg/client/config" + "testing" +) + +func TestE2E(t *testing.T) { + var err error + + cfg := config.GetConfigOrDie() + f := framework.NewFrameWork(cfg) + + gomega.RegisterFailHandler(ginkgo.Fail) + + ginkgo.Describe("Run kruise game manager e2e tests", func() { + ginkgo.BeforeSuite(func() { + err = f.BeforeSuit() + gomega.Expect(err).To(gomega.BeNil()) + }) + + ginkgo.AfterSuite(func() { + err = f.AfterSuit() + gomega.Expect(err).To(gomega.BeNil()) + }) + + testcase.RunTestCases(f) + }) + + ginkgo.RunSpecs(t, "run kgm e2e test") +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go new file mode 100644 index 00000000..cfae04c8 --- /dev/null +++ b/test/e2e/framework/framework.go @@ -0,0 +1,216 @@ +package framework + +import ( + "encoding/json" + "fmt" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" + "github.com/openkruise/kruise-game/pkg/util" + "github.com/openkruise/kruise-game/test/e2e/client" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "time" +) + +type Framework struct { + client *client.Client +} + +func NewFrameWork(config *restclient.Config) *Framework { + kruisegameClient := kruisegameclientset.NewForConfigOrDie(config) + kubeClient := clientset.NewForConfigOrDie(config) + return &Framework{ + client: client.NewKubeClient(kruisegameClient, kubeClient), + } +} + +func (f *Framework) BeforeSuit() error { + err := f.client.CreateNamespace() + if err != nil { + if apierrors.IsAlreadyExists(err) { + err = f.client.DeleteGameServerSet() + if err != nil { + return err + } + } else { + return err + } + } + return nil +} + +func (f *Framework) AfterSuit() error { + return f.client.DeleteNamespace() +} + +func (f *Framework) AfterEach() error { + return f.client.DeleteGameServerSet() +} + +func (f *Framework) DeployGameServerSet() (*gamekruiseiov1alpha1.GameServerSet, error) { + gss := f.client.DefaultGameServerSet() + return f.client.CreateGameServerSet(gss) +} + +func (f *Framework) GameServerScale(gss *gamekruiseiov1alpha1.GameServerSet, desireNum int, reserveGsId *int) (*gamekruiseiov1alpha1.GameServerSet, error) { + // TODO: change patch type + newReserves := gss.Spec.ReserveGameServerIds + if reserveGsId != nil { + newReserves = append(newReserves, *reserveGsId) + } + + numJson := map[string]interface{}{"spec": map[string]interface{}{"replicas": desireNum, "reserveGameServerIds": newReserves}} + data, err := json.Marshal(numJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServerSet(data) +} + +func (f *Framework) ImageUpdate(gss *gamekruiseiov1alpha1.GameServerSet, name, image string) (*gamekruiseiov1alpha1.GameServerSet, error) { + var newContainers []corev1.Container + for _, c := range gss.Spec.GameServerTemplate.Spec.Containers { + newContainer := c + if c.Name == name { + newContainer.Image = image + } + newContainers = append(newContainers, newContainer) + } + + conJson := map[string]interface{}{"spec": map[string]interface{}{"gameServerTemplate": map[string]interface{}{"spec": map[string]interface{}{"containers": newContainers}}}} + data, err := json.Marshal(conJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServerSet(data) +} + +func (f *Framework) MarkGameServerOpsState(gsName string, opsState string) (*gamekruiseiov1alpha1.GameServer, error) { + osJson := map[string]interface{}{"spec": map[string]string{"opsState": opsState}} + data, err := json.Marshal(osJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServer(gsName, data) +} + +func (f *Framework) ChangeGameServerDeletionPriority(gsName string, deletionPriority string) (*gamekruiseiov1alpha1.GameServer, error) { + dpJson := map[string]interface{}{"spec": map[string]string{"deletionPriority": deletionPriority}} + data, err := json.Marshal(dpJson) + if err != nil { + return nil, err + } + return f.client.PatchGameServer(gsName, data) +} + +func (f *Framework) WaitForGsCreated(gss *gamekruiseiov1alpha1.GameServerSet) error { + return wait.PollImmediate(5*time.Second, 3*time.Minute, + func() (done bool, err error) { + gssName := gss.GetName() + labelSelector := labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gssName, + }).String() + podList, err := f.client.GetPodList(labelSelector) + if err != nil { + return false, err + } + if len(podList.Items) != int(*gss.Spec.Replicas) { + return false, nil + } + return true, nil + }) +} + +func (f *Framework) WaitForUpdated(gss *gamekruiseiov1alpha1.GameServerSet, name, image string) error { + return wait.PollImmediate(10*time.Second, 10*time.Minute, + func() (done bool, err error) { + gssName := gss.GetName() + labelSelector := labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gssName, + }).String() + podList, err := f.client.GetPodList(labelSelector) + if err != nil { + return false, err + } + updated := 0 + + for _, pod := range podList.Items { + for _, c := range pod.Status.ContainerStatuses { + if name == c.Name && image == c.Image { + updated++ + break + } + } + } + + if gss.Spec.UpdateStrategy.RollingUpdate == nil || gss.Spec.UpdateStrategy.RollingUpdate.Partition == nil { + if int32(updated) != *gss.Spec.Replicas { + return false, nil + } + } else { + if int32(updated) != *gss.Spec.Replicas-*gss.Spec.UpdateStrategy.RollingUpdate.Partition { + return false, nil + } + } + return true, nil + }) +} + +func (f *Framework) ExpectGssCorrect(gss *gamekruiseiov1alpha1.GameServerSet, expectIndex []int) error { + + if err := f.WaitForGsCreated(gss); err != nil { + return err + } + + gssName := gss.GetName() + labelSelector := labels.SelectorFromSet(map[string]string{ + gamekruiseiov1alpha1.GameServerOwnerGssKey: gssName, + }).String() + + podList, err := f.client.GetPodList(labelSelector) + if err != nil { + return err + } + + podIndexList := util.GetIndexListFromPodList(podList.Items) + + if !util.IsSliceEqual(expectIndex, podIndexList) { + return fmt.Errorf("current pods and expected pods do not correspond") + } + + return nil +} + +func (f *Framework) WaitForGsOpsStateUpdate(gsName string, opsState string) error { + return wait.PollImmediate(5*time.Second, 1*time.Minute, + func() (done bool, err error) { + pod, err := f.client.GetPod(gsName) + if err != nil { + return false, err + } + currentOpsState := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOpsStateKey] + if currentOpsState == opsState { + return true, nil + } + return false, nil + }) +} + +func (f *Framework) WaitForGsDeletionPriorityUpdated(gsName string, deletionPriority string) error { + return wait.PollImmediate(5*time.Second, 1*time.Minute, + func() (done bool, err error) { + pod, err := f.client.GetPod(gsName) + if err != nil { + return false, err + } + currentPriority := pod.GetLabels()[gamekruiseiov1alpha1.GameServerDeletePriorityKey] + if currentPriority == deletionPriority { + return true, nil + } + return false, nil + }) +} diff --git a/test/e2e/testcase/testcase.go b/test/e2e/testcase/testcase.go new file mode 100644 index 00000000..700d4c28 --- /dev/null +++ b/test/e2e/testcase/testcase.go @@ -0,0 +1,78 @@ +package testcase + +import ( + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/test/e2e/client" + "github.com/openkruise/kruise-game/test/e2e/framework" +) + +func RunTestCases(f *framework.Framework) { + ginkgo.Describe("kruise game controllers", func() { + + ginkgo.AfterEach(func() { + err := f.AfterEach() + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + }) + + ginkgo.It("scale", func() { + + // deploy + gss, err := f.DeployGameServerSet() + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2}) + gomega.Expect(err).To(gomega.BeNil()) + + // scale up + gss, err = f.GameServerScale(gss, 5, nil) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2, 3, 4}) + gomega.Expect(err).To(gomega.BeNil()) + + // scale down when setting WaitToDelete + _, err = f.MarkGameServerOpsState(gss.GetName()+"-2", string(gameKruiseV1alpha1.WaitToDelete)) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForGsOpsStateUpdate(gss.GetName()+"-2", string(gameKruiseV1alpha1.WaitToDelete)) + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.GameServerScale(gss, 4, nil) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 3, 4}) + gomega.Expect(err).To(gomega.BeNil()) + + // scale down when setting deletion priority + _, err = f.ChangeGameServerDeletionPriority(gss.GetName()+"-3", "100") + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForGsDeletionPriorityUpdated(gss.GetName()+"-3", "100") + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.GameServerScale(gss, 3, nil) + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 4}) + gomega.Expect(err).To(gomega.BeNil()) + }) + + ginkgo.It("update pod", func() { + + // deploy + gss, err := f.DeployGameServerSet() + gomega.Expect(err).To(gomega.BeNil()) + + err = f.ExpectGssCorrect(gss, []int{0, 1, 2}) + gomega.Expect(err).To(gomega.BeNil()) + + gss, err = f.ImageUpdate(gss, client.GameContainerName, "nginx:latest") + gomega.Expect(err).To(gomega.BeNil()) + + err = f.WaitForUpdated(gss, client.GameContainerName, "nginx:latest") + gomega.Expect(err).To(gomega.BeNil()) + }) + }) +}