From 7dc487a855dc5612ea31d62de42e594bb1cfec16 Mon Sep 17 00:00:00 2001 From: Abhijit Das <31204670+abhijit-dev82@users.noreply.github.com> Date: Fri, 23 Sep 2022 00:58:19 +0530 Subject: [PATCH] Feature : Detect depcreated API's on a live cluster resources (#367) Feature : Detect depcreated API's on a live cluster resources Feature : Detect depcreated API's on a live cluster resources Feature : Detect deprecated API's on a live cluster resources Feature : Detect deprecated API's on a live cluster resources Feature : Detect deprecated API's on a live cluster resources Feature : Detect deprecated API's on a live cluster resources Feature : Detect deprecated API's on a live cluster resources Feature : Detect deprecated API's on a live cluster resources Feature : Detect deprecated API's on a live cluster resources Feature : Detect deprecated API's on a live cluster resources Co-authored-by: Andrew Suderman --- cmd/root.go | 31 ++++ docs/faq.md | 10 +- docs/quickstart.md | 9 ++ pkg/api/versions.go | 2 +- pkg/discovery-api/discovery_api.go | 182 ++++++++++++++++++++++++ pkg/discovery-api/discovery_api_test.go | 54 +++++++ 6 files changed, 282 insertions(+), 6 deletions(-) create mode 100644 pkg/discovery-api/discovery_api.go create mode 100644 pkg/discovery-api/discovery_api_test.go diff --git a/cmd/root.go b/cmd/root.go index b2533fdc..d1db33b7 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -36,6 +36,7 @@ import ( "strings" "github.com/fairwindsops/pluto/v5/pkg/api" + discoveryapi "github.com/fairwindsops/pluto/v5/pkg/discovery-api" "github.com/fairwindsops/pluto/v5/pkg/finder" "github.com/fairwindsops/pluto/v5/pkg/helm" "github.com/rogpeppe/go-internal/semver" @@ -98,6 +99,8 @@ func init() { rootCmd.AddCommand(listVersionsCmd) rootCmd.AddCommand(detectCmd) + rootCmd.AddCommand(detectApiResourceCmd) + klog.InitFlags(nil) pflag.CommandLine.AddGoFlag(flag.CommandLine.Lookup("v")) } @@ -395,6 +398,34 @@ var listVersionsCmd = &cobra.Command{ }, } +var detectApiResourceCmd = &cobra.Command{ + Use: "detect-api-resources", + Short: "detect-api-resources", + Long: `Detect Kubernetes apiVersions from an active cluster.`, + Run: func(cmd *cobra.Command, args []string) { + + disCl, err := discoveryapi.NewDiscoveryClient(apiInstance) + if err != nil { + fmt.Println("Error creating Discovery REST Client: ", err) + os.Exit(1) + } + err = disCl.GetApiResources() + if err != nil { + fmt.Println("Error getting API resources using discovery client:", err) + os.Exit(1) + } + + err = apiInstance.DisplayOutput() + if err != nil { + fmt.Println("Error Parsing Output:", err) + os.Exit(1) + } + retCode := apiInstance.GetReturnCode() + klog.V(5).Infof("retCode: %d", retCode) + os.Exit(retCode) + }, +} + // Execute the stuff func Execute(VERSION string, COMMIT string, versionsFile []byte) { version = VERSION diff --git a/docs/faq.md b/docs/faq.md index 3832bc09..0bbdab8c 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -9,14 +9,14 @@ meta: See above in the [Purpose](/#purpose) section of this doc. Kubectl is likely lying to you because it only tells you what the default is for the given kubernetes version even if an object was deployed with a newer API version. -### Why doesn't Pluto check the last-applied-configuration annotation? - -If you see the annotation `kubectl.kubernetes.io/last-applied-configuration` on an object in your cluster it means that object was updated with `kubectl apply`. We don't consider this an entirely reliable solution for checking. In fact, others have pointed out that updating the same object with `kubectl patch` will **remove** the annotation. Due to the flaky behavior here, we will not plan on supporting this. - ### I don't use helm, how can I do in cluster checks? Currently, the only in-cluster check we are confident in supporting is helm. If your deployment method can generate yaml manifests for kubernetes, you should be able to use the `detect` or `detect-files` functionality described below after the manifest files have been generated. ### I updated the API version of an object, but pluto still reports that the apiVersion needs to be updated. -Pluto looks at the API Versions of objects in releases that are in a `Deployed` state, and Helm has an issue where it might list old revisions of a release as still being in a `Deployed` state. To fix this, look at the release revision history with `helm history `, and determine if older releases still show a `Deployed` state. If so, delete the Helm release secret(s) associated with the revision number(s). For example, `kubectl delete secret sh.helm.release.v1.my-release.v10` where `10` corresponds to the release number. Then run Pluto again to see if the object has been removed from the report. \ No newline at end of file +Pluto looks at the API Versions of objects in releases that are in a `Deployed` state, and Helm has an issue where it might list old revisions of a release as still being in a `Deployed` state. To fix this, look at the release revision history with `helm history `, and determine if older releases still show a `Deployed` state. If so, delete the Helm release secret(s) associated with the revision number(s). For example, `kubectl delete secret sh.helm.release.v1.my-release.v10` where `10` corresponds to the release number. Then run Pluto again to see if the object has been removed from the report. + +### Why API version check on a live cluster using the "last-applied-configuration" annotation is not reliable? + +The annotation `kubectl.kubernetes.io/last-applied-configuration` on an object in your cluster holds the API version by which it was created. In fact, others have pointed out that updating the same object with `kubectl patch` will **remove** the annotation. Hence this is not a reliable method to detect deprecated API's from a live cluster. \ No newline at end of file diff --git a/docs/quickstart.md b/docs/quickstart.md index 9708f1ec..6bab69f7 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -52,3 +52,12 @@ $ helm template e2e/tests/assets/helm3chart | pluto detect - KIND VERSION DEPRECATED DEPRECATED IN RESOURCE NAME Deployment extensions/v1beta1 true v1.16.0 RELEASE-NAME-helm3chart-v1beta1 ``` + +### API resources (in-cluster) +``` +$ pluto detect-api-resources -owide +NAME NAMESPACE KIND VERSION REPLACEMENT DEPRECATED DEPRECATED IN REMOVED REMOVED IN +psp PodSecurityPolicy policy/v1beta1 true v1.21.0 false v1.25.0 +``` + +This indicates that the PodSecurityPolicy was deployed with apps/v1beta1 which is deprecated in 1.21 diff --git a/pkg/api/versions.go b/pkg/api/versions.go index b9095e79..f6d26753 100644 --- a/pkg/api/versions.go +++ b/pkg/api/versions.go @@ -181,7 +181,7 @@ func yamlToStub(data []byte) ([]*Stub, error) { // expandList checks if we have a List manifest. // If it is the case, the manifests inside are expanded, otherwise we just return the single manifest func expandList(stubs *[]*Stub, currentStub *Stub) { - if currentStub.Items != nil { + if len(currentStub.Items) > 0 { klog.V(5).Infof("found a list with %d items, attempting to expand", len(currentStub.Items)) for _, stub := range currentStub.Items { currentItem := stub diff --git a/pkg/discovery-api/discovery_api.go b/pkg/discovery-api/discovery_api.go new file mode 100644 index 00000000..e1ca36c1 --- /dev/null +++ b/pkg/discovery-api/discovery_api.go @@ -0,0 +1,182 @@ +// Copyright 2022 FairwindsOps Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2020 Fairwinds +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +package discoveryapi + +import ( + "context" + "encoding/json" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + + "github.com/fairwindsops/pluto/v5/pkg/api" +) + +// DiscoveryClient is the declaration to hold objects needed for client-go/discovery. +type DiscoveryClient struct { + ClientSet dynamic.Interface + restConfig *rest.Config + DiscoveryClient discovery.DiscoveryInterface + Instance *api.Instance +} + +// NewDiscoveryClient returns a new struct with config portions complete. +func NewDiscoveryClient(instance *api.Instance) (*DiscoveryClient, error) { + cl := &DiscoveryClient{ + Instance: instance, + } + + var err error + cl.restConfig, err = NewRestClientConfig(rest.InClusterConfig) + if err != nil { + return nil, err + } + + if cl.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(cl.restConfig); err != nil { + return nil, err + } + + cl.ClientSet, err = dynamic.NewForConfig(cl.restConfig) + if err != nil { + return nil, err + } + return cl, nil +} + +// NewRestClientConfig returns a new Rest Client config portions complete. +func NewRestClientConfig(inClusterFn func() (*rest.Config, error)) (*rest.Config, error) { + + if restConfig, err := inClusterFn(); err == nil { + return restConfig, nil + } + + pathOptions := clientcmd.NewDefaultPathOptions() + + config, err := pathOptions.GetStartingConfig() + if err != nil { + return nil, err + } + + configOverrides := clientcmd.ConfigOverrides{} + + clientConfig := clientcmd.NewDefaultClientConfig(*config, &configOverrides) + restConfig, err := clientConfig.ClientConfig() + if err != nil { + return nil, err + } + + return restConfig, nil +} + +// GetApiResources discovers the api-resources for a cluster +func (cl *DiscoveryClient) GetApiResources() error { + + resourcelist, err := cl.DiscoveryClient.ServerPreferredResources() + if err != nil { + if apierrors.IsNotFound(err) { + return err + } + if apierrors.IsForbidden(err) { + klog.Error("Failed to list objects for Name discovery. Permission denied! Please check if you have the proper authorization") + return err + } + } + + gvrs := []schema.GroupVersionResource{} + for _, rl := range resourcelist { + for i := range rl.APIResources { + gv, _ := schema.ParseGroupVersion(rl.GroupVersion) + ResourceName := rl.APIResources[i].Name + g := schema.GroupVersionResource{Group: gv.Group, Version: gv.Version, Resource: ResourceName} + gvrs = append(gvrs, g) + } + } + + var results []map[string]interface{} + for _, g := range gvrs { + ri := cl.ClientSet.Resource(g) + klog.V(2).Infof("Retrieving : %s.%s.%s", g.Resource, g.Version, g.Group) + rs, err := ri.List(context.TODO(), metav1.ListOptions{}) + if err != nil { + klog.Error("Failed to retrieve: ", g, err) + continue + } + + if len(rs.Items) == 0 { + klog.V(2).Infof("No annotations for ResourceVer %s", rs.GetAPIVersion()) + obj := rs.UnstructuredContent() + data, err := json.Marshal(obj) + if err != nil { + klog.Error("Failed to marshal data ", err.Error()) + return err + } + output, err := cl.Instance.IsVersioned(data) + if err != nil { + return err + } + if output == nil { + continue + } + cl.Instance.Outputs = append(cl.Instance.Outputs, output...) + + } else { + for _, r := range rs.Items { + if jsonManifest, ok := r.GetAnnotations()["kubectl.kubernetes.io/last-applied-configuration"]; ok { + var manifest map[string]interface{} + + err := json.Unmarshal([]byte(jsonManifest), &manifest) + if err != nil { + klog.Error("failed to parse 'last-applied-configuration' annotation of resource %s/%s: %s", r.GetNamespace(), r.GetName(), err.Error()) + continue + } + data, err := json.Marshal(manifest) + if err != nil { + klog.Error("Failed to marshal data ", err.Error()) + return err + } + output, err := cl.Instance.IsVersioned(data) + if err != nil { + return err + } + cl.Instance.Outputs = append(cl.Instance.Outputs, output...) + } + } + } + + } + + klog.V(6).Infof("Result from resources: %d", len(results)) + return nil +} diff --git a/pkg/discovery-api/discovery_api_test.go b/pkg/discovery-api/discovery_api_test.go new file mode 100644 index 00000000..60e0112d --- /dev/null +++ b/pkg/discovery-api/discovery_api_test.go @@ -0,0 +1,54 @@ +// Copyright 2022 FairwindsOps Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Copyright 2020 Fairwinds +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +package discoveryapi + +import ( + "testing" + + "k8s.io/apimachinery/pkg/runtime" + discoveryFake "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/dynamic/fake" +) + +func TestNewDiscoveryAPIClientValidEmpty(t *testing.T) { + + scheme := runtime.NewScheme() + clientset := fake.NewSimpleDynamicClient(scheme) + discoveryClient := discoveryFake.FakeDiscovery{} + testOpts := DiscoveryClient{ + ClientSet: clientset, + DiscoveryClient: &discoveryClient, + } + + err := testOpts.GetApiResources() + if err != nil { + t.Errorf("Unable to fetch API Resources") + } + +}