using cli runtime builder (#261)

* issue #263 - resources scan refactoring

Signed-off-by: adisos <[email protected]>

cmd/netpolicy/cmd/command_test.go

@@ -495,17 +495,43 @@ func TestCommands(t *testing.T) {
 		},
 		{
 			name: "test_list_dir_with_severe_error_running_with_fail_stops_and_return_empty_output",
-			//  MalformedYamlDocError is not fatal, but severe, thus stops the run if --fail is on
 			// as we saw in a previous test on same path, when --fail is not used, the test produces connectivity map
 			args: []string{
 				"list",
 				"--dirpath",
 				filepath.Join(getTestsDir(), "bad_yamls", "document_with_syntax_error.yaml"),
 				"--fail",
 			},
-			expectedOutput: "",
-			exact:          true,
-			isErr:          false, // not fatal error but severe
+			expectedOutput: "found character that cannot start any token",
+			exact:          false,
+			isErr:          true, // fatal error because err is issued by the builder and uses stopOnErr
+		},
+		{
+			name: "test_diff_one_dir_with_severe_error_without_fail_produces_output",
+			args: []string{
+				"diff",
+				"--dir1",
+				filepath.Join(getTestsDir(), "onlineboutique"),
+				"--dir2",
+				filepath.Join(getTestsDir(), "onlineboutique_with_pods_severe_error")},
+			expectedOutput: "Connectivity diff:\n" +
+				"diff-type: changed, source: default/frontend-99684f7f8[ReplicaSet], " +
+				"destination: default/adservice-77d5cd745d[ReplicaSet], dir1:  TCP 9555, dir2: TCP 8080",
+			exact: true,
+			isErr: false,
+		},
+		{
+			name: "test_diff_one_dir_with_severe_error_with_fail_returns_empty_output",
+			args: []string{
+				"diff",
+				"--dir1",
+				filepath.Join(getTestsDir(), "onlineboutique"),
+				"--dir2",
+				filepath.Join(getTestsDir(), "onlineboutique_with_pods_severe_error"),
+				"--fail"},
+			expectedOutput: "found character that cannot start any token",
+			exact:          false,
+			isErr:          true,
 		},
 		{
 			name: "test_eval_on_dir_with_severe_error_without_fail_produces_output",
@@ -536,37 +562,10 @@ func TestCommands(t *testing.T) {
 				"-p",
 				"80",
 				"--fail"},
-			expectedOutput: "had processing errors: YAML document is malformed: yaml: line 1828: found character that cannot start any token\n",
+			expectedOutput: "found character that cannot start any token",
 			exact:          false,
 			isErr:          true, // eval command returns err if stopOnFirstError & severe
 		},
-		{
-			name: "test_diff_one_dir_with_severe_error_without_fail_produces_output",
-			args: []string{
-				"diff",
-				"--dir1",
-				filepath.Join(getTestsDir(), "onlineboutique"),
-				"--dir2",
-				filepath.Join(getTestsDir(), "onlineboutique_with_pods_severe_error")},
-			expectedOutput: "Connectivity diff:\n" +
-				"diff-type: changed, source: default/frontend-99684f7f8[ReplicaSet], " +
-				"destination: default/adservice-77d5cd745d[ReplicaSet], dir1:  TCP 9555, dir2: TCP 8080",
-			exact: true,
-			isErr: false,
-		},
-		{
-			name: "test_diff_one_dir_with_severe_error_with_fail_returns_empty_output",
-			args: []string{
-				"diff",
-				"--dir1",
-				filepath.Join(getTestsDir(), "onlineboutique"),
-				"--dir2",
-				filepath.Join(getTestsDir(), "onlineboutique_with_pods_severe_error"),
-				"--fail"},
-			expectedOutput: "",
-			exact:          true,
-			isErr:          false,
-		},
 	}
 
 	for _, test := range tests {

cmd/netpolicy/cmd/diff.go

@@ -56,9 +56,6 @@ func runDiffCommand() error {
 
 func getDiffOptions(l *logger.DefaultLogger) []diff.DiffAnalyzerOption {
 	res := []diff.DiffAnalyzerOption{diff.WithLogger(l), diff.WithOutputFormat(outFormat)}
-	if includeJSONManifests {
-		res = append(res, diff.WithIncludeJSONManifests())
-	}
 	if stopOnFirstError {
 		res = append(res, diff.WithStopOnError())
 	}
@@ -88,6 +85,7 @@ func newCommandDiff() *cobra.Command {
 
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := runDiffCommand(); err != nil {
+				cmd.SilenceUsage = true // don't print usage message when returning an error from running a valid command
 				return err
 			}
 			return nil

cmd/netpolicy/cmd/evaluate.go

@@ -17,16 +17,18 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"path/filepath"
 	"time"
 
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+
 	"github.com/np-guard/netpol-analyzer/pkg/netpol/eval"
 	"github.com/np-guard/netpol-analyzer/pkg/netpol/logger"
-	"github.com/np-guard/netpol-analyzer/pkg/netpol/scan"
+	"github.com/np-guard/netpol-analyzer/pkg/netpol/manifests/fsscanner"
+	"github.com/np-guard/netpol-analyzer/pkg/netpol/manifests/parser"
 )
 
 // TODO: consider using k8s.io/cli-runtime/pkg/genericclioptions to load kube config.
@@ -70,22 +72,36 @@ func validateEvalFlags() error {
 func updatePolicyEngineObjectsFromDirPath(pe *eval.PolicyEngine, podNames []types.NamespacedName) error {
 	// get relevant resources from dir path
 	elogger := logger.NewDefaultLoggerWithVerbosity(detrmineLogVerbosity())
-	scanner := scan.NewResourcesScanner(elogger, stopOnFirstError, filepath.WalkDir, includeJSONManifests)
-	objectsList, processingErrs := scanner.FilesToObjectsListFiltered(dirPath, podNames)
+
+	rList, errs := fsscanner.GetResourceInfosFromDirPath([]string{dirPath}, true, false)
+	if errs != nil {
+		// TODO: consider avoid logging this error because it is already printed to log by the builder
+		if len(rList) == 0 || stopOnFirstError {
+			err := utilerrors.NewAggregate(errs)
+			elogger.Errorf(err, "Error getting resourceInfos from dir path")
+			return err // return as fatal error if rList is empty or if stopOnError is on
+		}
+		// split err if it's an aggregated error to a list of separate errors
+		for _, err := range errs {
+			elogger.Errorf(err, "Error reading file") // print to log the error from builder
+		}
+	}
+	objectsList, processingErrs := parser.ResourceInfoListToK8sObjectsList(rList, elogger, false)
 	for _, err := range processingErrs {
 		if err.IsFatal() || (stopOnFirstError && err.IsSevere()) {
 			return fmt.Errorf("scan dir path %s had processing errors: %v", dirPath, err.Error())
 		}
 	}
+	objectsList = parser.FilterObjectsList(objectsList, podNames)
 
 	var err error
 	for _, obj := range objectsList {
 		switch obj.Kind {
-		case scan.Pod:
+		case parser.Pod:
 			err = pe.UpsertObject(obj.Pod)
-		case scan.Namespace:
+		case parser.Namespace:
 			err = pe.UpsertObject(obj.Namespace)
-		case scan.Networkpolicy:
+		case parser.Networkpolicy:
 			err = pe.UpsertObject(obj.Networkpolicy)
 		default:
 			continue

cmd/netpolicy/cmd/list.go

@@ -83,9 +83,7 @@ func getConnlistOptions(l *logger.DefaultLogger) []connlist.ConnlistAnalyzerOpti
 		connlist.WithFocusWorkload(focusWorkload),
 		connlist.WithOutputFormat(output),
 	}
-	if includeJSONManifests {
-		res = append(res, connlist.WithIncludeJSONManifests())
-	}
+
 	if stopOnFirstError {
 		res = append(res, connlist.WithStopOnError())
 	}
@@ -138,5 +136,6 @@ defined`,
 	c.Flags().StringVarP(&output, "output", "o", common.DefaultFormat, getOutputFormatDescription(supportedFormats))
 	// out file
 	c.Flags().StringVarP(&outFile, "file", "f", "", "Write output to specified file")
+
 	return c
 }

cmd/netpolicy/cmd/root.go

@@ -31,11 +31,10 @@ var (
 	// resources dir information
 	dirPath string
 	// k8s client
-	clientset            *kubernetes.Clientset
-	quiet                bool
-	verbose              bool
-	includeJSONManifests bool
-	stopOnFirstError     bool
+	clientset        *kubernetes.Clientset
+	quiet            bool
+	verbose          bool
+	stopOnFirstError bool
 )
 
 // returns verbosity level based on the -q and -v switches
@@ -97,8 +96,6 @@ func newCommandRoot() *cobra.Command {
 		"Kubernetes context to use when evaluating connections in a live cluster")
 	c.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "runs quietly, reports only severe errors and results")
 	c.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "runs with more informative messages printed to log")
-	c.PersistentFlags().BoolVarP(&includeJSONManifests, "include-json", "", false,
-		"consider JSON manifests (in addition to YAML) when analyzing from dir")
 	c.PersistentFlags().BoolVarP(&stopOnFirstError, "fail", "", false, "fail on the first encountered error")
 
 	// add sub-commands

go.mod

@@ -7,9 +7,9 @@ require (
 	github.com/openshift/api v0.0.0-20230502160752-c71432710382
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
-	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.28.2
 	k8s.io/apimachinery v0.28.2
+	k8s.io/cli-runtime v0.28.2
 	k8s.io/client-go v0.28.2
 	sigs.k8s.io/yaml v1.4.0
 
@@ -18,6 +18,8 @@ require (
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
@@ -27,6 +29,7 @@ require (
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -35,11 +38,16 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
+	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/sync v0.2.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/term v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
@@ -48,9 +56,12 @@ require (
 	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )