From 0e9bec9d487dd102ef5274b39959ecefc3a2cfe9 Mon Sep 17 00:00:00 2001 From: Yusan Kurban Date: Wed, 15 Mar 2023 10:16:00 +0800 Subject: [PATCH] feat: add trace and pprof (#14) Signed-off-by: Yusan Kurban --- README.md | 10 +- cmd/app/options/options.go | 121 +++++++++-- cmd/app/options/options_test.go | 194 +++++++++++++++--- cmd/app/webhook.go | 5 + go.mod | 5 +- go.sum | 4 +- .../preferredimports/preferredimports.go | 5 +- pkg/webhook/mutating.go | 15 +- pkg/webhook/validateing.go | 8 +- 9 files changed, 303 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 4edfc18..b3a5fab 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,12 @@ If you want to use it in clientside with client-go, please use [pidalio](https:/ ## Quick Start -### Apply crd files to your cluster +### Add Helm source ```shell -kubectl apply -f https://raw.githubusercontent.com/k-cloud-labs/pkg/main/charts/_crds/bases/policy.kcloudlabs.io_overridepolicies.yaml -kubectl apply -f https://raw.githubusercontent.com/k-cloud-labs/pkg/main/charts/_crds/bases/policy.kcloudlabs.io_clusteroverridepolicies.yaml -kubectl apply -f https://raw.githubusercontent.com/k-cloud-labs/pkg/main/charts/_crds/bases/policy.kcloudlabs.io_clustervalidatepolicies.yaml +helm repo add k-cloud-labs https://k-cloud-labs.github.io/helm-charts ``` -### Deploy webhook to cluster +### Install All resources will be applied to `kinitiras-system` namespace by default. You can modify the deployment files as your expect. Pay attention to the deploy/webhook-configuration.yaml file. The default config will mutate and validate all kubernetes resources filtered by label `kinitiras.kcloudlabs.io/webhook: enabled`. @@ -31,7 +29,7 @@ Pay attention to the deploy/webhook-configuration.yaml file. The default config After all changes done, just apply it to your cluster. ```shell -kubectl apply -f deploy/ +helm install kinitiras-webhook k-cloud-labs/kinitiras --namespace kinitiras-system --create-namespace ``` ### Create policy diff --git a/cmd/app/options/options.go b/cmd/app/options/options.go index e5bfcb8..d77e855 100644 --- a/cmd/app/options/options.go +++ b/cmd/app/options/options.go @@ -1,6 +1,7 @@ package options import ( + "fmt" "strings" "github.com/spf13/pflag" @@ -42,7 +43,9 @@ type Options struct { // KubeAPIBurst is the burst to allow while talking with kube-apiserver. KubeAPIBurst int // PreCacheResources is a list of resources name to pre-cache when start up. - PreCacheResources string + PreCacheResources *ResourceSlice + // EnablePProf is switch to enable/disable net/http/pprof. Default value as false. + EnablePProf bool } // NewOptions builds an empty options. @@ -52,6 +55,7 @@ func NewOptions() *Options { // AddFlags adds flags to the specified FlagSet. func (o *Options) AddFlags(flags *pflag.FlagSet) { + o.PreCacheResources = NewPreCacheResources([]string{"Deployment/apps/v1", "ReplicaSet/apps/v1"}) flags.StringVar(&o.BindAddress, "bind-address", defaultBindAddress, "The IP address on which to listen for the --secure-port port.") flags.IntVar(&o.SecurePort, "secure-port", defaultPort, @@ -63,8 +67,9 @@ func (o *Options) AddFlags(flags *pflag.FlagSet) { flags.StringVar(&o.TLSMinVersion, "tls-min-version", defaultTLSMinVersion, "Minimum TLS version supported. Possible values: 1.0, 1.1, 1.2, 1.3.") flags.Float32Var(&o.KubeAPIQPS, "kube-api-qps", 40.0, "QPS to use while talking with kube-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") flags.IntVar(&o.KubeAPIBurst, "kube-api-burst", 60, "Burst to use while talking with kube-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") - flags.StringVar(&o.PreCacheResources, "pre-cache-resources", "Deployment/apps/v1,Replicas/apps/v1", "Resources list separate by comma, for example: Pod/v1,Deployment/apps/v1"+ + flags.VarP(o.PreCacheResources, "pre-cache-resources", "", "Resources list separate by comma, for example: Pod/v1,Deployment/apps/v1"+ ". Will pre cache those resources to get it quicker when policies refer resources from cluster.") + flags.BoolVar(&o.EnablePProf, "enable-pprof", false, "EnablePProf is switch to enable/disable net/http/pprof. Default value as false.") globalflag.AddGlobalFlags(flags, "global") } @@ -76,31 +81,103 @@ func PrintFlags(flags *pflag.FlagSet) { }) } -func (o *Options) PreCacheResourcesToGVKList() []schema.GroupVersionKind { - var ( - resourceList = strings.Split(o.PreCacheResources, ",") - gvkList = make([]schema.GroupVersionKind, 0, len(resourceList)) - ) - - for _, resource := range resourceList { - items := strings.Split(resource, "/") - if len(items) <= 1 { - // ignore it - continue - } +type ResourceSlice struct { + value *[]schema.GroupVersionKind + changed bool +} + +func NewPreCacheResources(slice []string) *ResourceSlice { + value := make([]schema.GroupVersionKind, 0) + s := &ResourceSlice{value: &value} + _ = s.Replace(slice) + return s +} - gvk := schema.GroupVersionKind{ - Kind: items[0], - Version: items[1], +func (s *ResourceSlice) String() string { + sb := &strings.Builder{} + + for i, gvk := range *s.value { + if i != 0 { + sb.WriteString(",") } + sb.WriteString(gvk.Kind + "/" + gvk.GroupVersion().String()) + } + + return "[" + sb.String() + "]" +} + +func (s *ResourceSlice) Set(val string) error { + if s.value == nil { + return fmt.Errorf("no target (nil pointer to []string)") + } + if !s.changed { + *s.value = make([]schema.GroupVersionKind, 0) + } + gvk, err := s.readResource(val) + if err != nil { + return err + } + *s.value = append(*s.value, gvk) + s.changed = true + return nil +} + +func (s *ResourceSlice) Type() string { + return "resourceSlice" +} + +func (s *ResourceSlice) Append(val string) error { + gvk, err := s.readResource(val) + if err != nil { + return err + } + + *s.value = append(*s.value, gvk) + return nil +} - if len(items) == 3 { - gvk.Group = items[1] - gvk.Version = items[2] +func (s *ResourceSlice) Replace(slice []string) error { + value := make([]schema.GroupVersionKind, 0, len(slice)) + for _, str := range slice { + gvk, err := s.readResource(str) + if err != nil { + return err } - gvkList = append(gvkList, gvk) + value = append(value, gvk) } - return gvkList + *s.value = value + return nil +} + +func (s *ResourceSlice) GetSlice() []string { + var slice = make([]string, 0, len(*s.value)) + for _, gvk := range *s.value { + slice = append(slice, gvk.Kind+"/"+gvk.GroupVersion().String()) + } + + return slice +} + +func (s *ResourceSlice) readResource(val string) (schema.GroupVersionKind, error) { + var gvk schema.GroupVersionKind + items := strings.Split(val, "/") + if len(items) <= 1 { + return gvk, fmt.Errorf("invalid gvk(%v)", val) + } + + gvk.Kind = items[0] + gvk.Version = items[1] + + if len(items) == 3 { + gvk.Group = items[1] + gvk.Version = items[2] + } + + return gvk, nil +} + +func (o *Options) PreCacheResourcesToGVKList() []schema.GroupVersionKind { + return *o.PreCacheResources.value } diff --git a/cmd/app/options/options_test.go b/cmd/app/options/options_test.go index 8d9e149..6a67b5f 100644 --- a/cmd/app/options/options_test.go +++ b/cmd/app/options/options_test.go @@ -4,56 +4,196 @@ import ( "reflect" "testing" - "k8s.io/apimachinery/pkg/runtime/schema" + "github.com/spf13/pflag" ) -func TestOptions_PreCacheResourcesToGVKList(t *testing.T) { +func TestPrintFlags(t *testing.T) { + type args struct { + flags *pflag.FlagSet + } + tests := []struct { + name string + args args + }{ + { + name: "1", + args: args{flags: pflag.NewFlagSet("test1", pflag.ExitOnError)}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opt := NewOptions() + opt.AddFlags(tt.args.flags) + PrintFlags(tt.args.flags) + opt.PreCacheResourcesToGVKList() + }) + } +} + +func TestResourceSlice_Set(t *testing.T) { type fields struct { - PreCacheResources string + s *ResourceSlice + } + type args struct { + val string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "1", + fields: fields{ + s: NewPreCacheResources([]string{"Pod/v1"}), + }, + args: args{ + val: "Node/v1", + }, + wantErr: false, + }, + { + name: "error", + fields: fields{ + s: NewPreCacheResources([]string{"Pod/v1"}), + }, + args: args{ + val: "v1", + }, + wantErr: true, + }, + { + name: "error2", + fields: fields{ + s: &ResourceSlice{}, + }, + args: args{ + val: "v1", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.fields.s.Set(tt.args.val); (err != nil) != tt.wantErr { + t.Errorf("Set() error = %v, wantErr %v", err, tt.wantErr) + } + t.Log(tt.fields.s.Type()) + }) + } +} + +func TestResourceSlice_Append(t *testing.T) { + type fields struct { + s *ResourceSlice + } + type args struct { + val string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "1", + fields: fields{ + s: NewPreCacheResources([]string{"Pod/v1"}), + }, + args: args{ + val: "Node/v1", + }, + wantErr: false, + }, + { + name: "error", + fields: fields{ + s: NewPreCacheResources([]string{"Pod/v1"}), + }, + args: args{ + val: "Node", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.fields.s + if err := s.Append(tt.args.val); (err != nil) != tt.wantErr { + t.Errorf("Append() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestResourceSlice_GetSlice(t *testing.T) { + type fields struct { + s *ResourceSlice } tests := []struct { name string fields fields - want []schema.GroupVersionKind + want []string }{ { - name: "empty", - fields: fields{}, - want: make([]schema.GroupVersionKind, 0), + name: "1", + fields: fields{ + s: NewPreCacheResources([]string{"Pod/v1", "Deployment/apps/v1"}), + }, + want: []string{"Pod/v1", "Deployment/apps/v1"}, }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.fields.s + if got := s.GetSlice(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetSlice() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestResourceSlice_Replace(t *testing.T) { + type fields struct { + s *ResourceSlice + } + type args struct { + slice []string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ { - name: "pod", + name: "1", fields: fields{ - PreCacheResources: "Pod/v1", + s: NewPreCacheResources([]string{"Pod/v1"}), }, - want: []schema.GroupVersionKind{ - { - Version: "v1", - Kind: "Pod", - }, + args: args{ + slice: []string{"Node/v1"}, }, + wantErr: false, }, { - name: "Deployment", + name: "error", fields: fields{ - PreCacheResources: "Pod/apps/v1", + s: NewPreCacheResources([]string{"Pod/v1"}), }, - want: []schema.GroupVersionKind{ - { - Group: "apps", - Version: "v1", - Kind: "Pod", - }, + args: args{ + slice: []string{"Node"}, }, + wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - o := &Options{ - PreCacheResources: tt.fields.PreCacheResources, - } - if got := o.PreCacheResourcesToGVKList(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("PreCacheResourcesToGVKList() = %v, want %v", got, tt.want) + s := tt.fields.s + if err := s.Replace(tt.args.slice); (err != nil) != tt.wantErr { + t.Errorf("Replace() error = %v, wantErr %v", err, tt.wantErr) } }) } diff --git a/cmd/app/webhook.go b/cmd/app/webhook.go index 75a256a..a0af392 100644 --- a/cmd/app/webhook.go +++ b/cmd/app/webhook.go @@ -9,6 +9,7 @@ import ( "os" "github.com/spf13/cobra" + "go.etcd.io/etcd/pkg/v3/debugutil" "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" @@ -156,6 +157,10 @@ func Run(ctx context.Context, opts *options.Options) error { hookServer.Register("/mutate", &webhook.Admission{Handler: pkgwebhook.NewMutatingAdmissionHandler(sm.overrideManager, sm.policyInterrupterManager)}) hookServer.Register("/validate", &webhook.Admission{Handler: pkgwebhook.NewValidatingAdmissionHandler(sm.validateManager, sm.policyInterrupterManager)}) hookServer.WebhookMux.Handle("/readyz", http.StripPrefix("/readyz", &healthz.Handler{})) + // pprof + for s, handler := range debugutil.PProfHandlers() { + hookServer.Register(s, handler) + } }() // blocks until the context is done. diff --git a/go.mod b/go.mod index 4fbde5e..16152a8 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,11 @@ go 1.18 require ( github.com/go-logr/logr v1.2.3 - github.com/k-cloud-labs/pkg v0.4.3 + github.com/k-cloud-labs/pkg v0.4.4 github.com/open-policy-agent/cert-controller v0.3.0 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 + go.etcd.io/etcd/pkg/v3 v3.5.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b k8s.io/api v0.23.6 @@ -17,6 +18,7 @@ require ( k8s.io/component-base v0.23.6 k8s.io/klog v1.0.0 k8s.io/klog/v2 v2.60.1 + k8s.io/utils v0.0.0-20211116205334-6203023598ed sigs.k8s.io/controller-runtime v0.11.2 ) @@ -95,7 +97,6 @@ require ( gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/apiextensions-apiserver v0.23.5 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 // indirect sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect diff --git a/go.sum b/go.sum index f8fd7aa..5ee2216 100644 --- a/go.sum +++ b/go.sum @@ -323,8 +323,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k-cloud-labs/pkg v0.4.3 h1:rhuvfvqlfppMuDTxcZU/mnq9Aj9InhIoNqRFCrYIUuY= -github.com/k-cloud-labs/pkg v0.4.3/go.mod h1:IlPsCdDrC9lTY5eX241KsVvN9FKGMUTm2MEJA1vEQ/I= +github.com/k-cloud-labs/pkg v0.4.4 h1:3pSJ1woTqTUxw+JPtT2z69j3L/rgLwZuCS7mLEdlfhc= +github.com/k-cloud-labs/pkg v0.4.4/go.mod h1:IlPsCdDrC9lTY5eX241KsVvN9FKGMUTm2MEJA1vEQ/I= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/hack/tools/preferredimports/preferredimports.go b/hack/tools/preferredimports/preferredimports.go index 60d0d0d..1d3d7be 100644 --- a/hack/tools/preferredimports/preferredimports.go +++ b/hack/tools/preferredimports/preferredimports.go @@ -30,7 +30,6 @@ import ( "go/format" "go/parser" "go/token" - "io/ioutil" "log" "os" "path/filepath" @@ -132,7 +131,7 @@ func (a *analyzer) collect(dir string) { panic(fmt.Sprintf("Error stat'ing file: %s\n%s\n", pathToFile, err.Error())) } - err = ioutil.WriteFile(pathToFile, buffer.Bytes(), fileInfo.Mode()) + err = os.WriteFile(pathToFile, buffer.Bytes(), fileInfo.Mode()) if err != nil { panic(fmt.Sprintf("Error writing file: %s\n%s\n", pathToFile, err.Error())) } @@ -238,7 +237,7 @@ func main() { sort.Strings(c.dirs) if len(*importAliases) > 0 { - bytes, err := ioutil.ReadFile(*importAliases) + bytes, err := os.ReadFile(*importAliases) if err != nil { log.Fatalf("Error reading import aliases: %v", err) } diff --git a/pkg/webhook/mutating.go b/pkg/webhook/mutating.go index 79c6b4a..bad0019 100644 --- a/pkg/webhook/mutating.go +++ b/pkg/webhook/mutating.go @@ -6,10 +6,12 @@ import ( "encoding/json" "errors" "net/http" + "time" admissionv1 "k8s.io/api/admission/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/klog/v2" + utiltrace "k8s.io/utils/trace" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" @@ -29,6 +31,9 @@ var _ admission.Handler = &MutatingAdmission{} var _ admission.DecoderInjector = &MutatingAdmission{} func (a *MutatingAdmission) Handle(ctx context.Context, req admission.Request) admission.Response { + trace := utiltrace.New("Mutating", traceFields(req, "mutating")...) + defer trace.LogIfLong(500 * time.Millisecond) + obj, oldObj, err := decodeObj(a.decoder, req) if err != nil { return admission.Errored(http.StatusBadRequest, err) @@ -73,7 +78,7 @@ func (a *MutatingAdmission) Handle(ctx context.Context, req admission.Request) a klog.V(6).InfoS("override obj", "obj", buf.String()) } - cops, ops, err := a.overrideManager.ApplyOverridePolicies(newObj, oldObj, req.Operation) + cops, ops, err := a.overrideManager.ApplyOverridePolicies(utils.ContextWithTrace(ctx, trace), newObj, oldObj, req.Operation) if err != nil { return admission.Errored(http.StatusInternalServerError, err) } @@ -158,3 +163,11 @@ func decodeObj(decoder *admission.Decoder, req admission.Request) (*unstructured return obj, oldObj, nil } + +func traceFields(req admission.Request, stage string) []utiltrace.Field { + return []utiltrace.Field{ + {Key: "op", Value: req.Operation}, + {Key: "stage", Value: stage}, + {Key: "name", Value: req.Name}, + {Key: "gvk", Value: req.Kind.String()}} +} diff --git a/pkg/webhook/validateing.go b/pkg/webhook/validateing.go index 7602917..ac357c2 100644 --- a/pkg/webhook/validateing.go +++ b/pkg/webhook/validateing.go @@ -3,10 +3,13 @@ package webhook import ( "context" "net/http" + "time" + utiltrace "k8s.io/utils/trace" "sigs.k8s.io/controller-runtime/pkg/webhook" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "github.com/k-cloud-labs/pkg/utils" "github.com/k-cloud-labs/pkg/utils/interrupter" "github.com/k-cloud-labs/pkg/utils/validatemanager" @@ -24,6 +27,9 @@ var _ admission.Handler = &ValidatingAdmission{} var _ admission.DecoderInjector = &ValidatingAdmission{} func (v *ValidatingAdmission) Handle(ctx context.Context, req admission.Request) admission.Response { + trace := utiltrace.New("Mutating", traceFields(req, "validating")...) + defer trace.LogIfLong(500 * time.Millisecond) + obj, oldObj, err := decodeObj(v.decoder, req) if err != nil { return admission.Errored(http.StatusBadRequest, err) @@ -35,7 +41,7 @@ func (v *ValidatingAdmission) Handle(ctx context.Context, req admission.Request) return admission.Denied(err.Error()) } - result, err := v.validateManager.ApplyValidatePolicies(obj, oldObj, req.Operation) + result, err := v.validateManager.ApplyValidatePolicies(utils.ContextWithTrace(ctx, trace), obj, oldObj, req.Operation) if err != nil { return admission.Errored(http.StatusInternalServerError, err) }