[KEP-0009] feat: add expression based assertions

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/docker/docker v27.3.1+incompatible
 	github.com/dustin/go-humanize v1.0.1
 	github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0
+	github.com/google/cel-go v0.20.1
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 	github.com/spf13/cobra v1.8.1
@@ -30,6 +31,7 @@ require (
 	github.com/BurntSushi/toml v1.4.0 // indirect
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
@@ -70,12 +72,14 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
 	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
 	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
@@ -85,6 +89,8 @@ require (
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=
 github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -70,6 +72,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84=
+github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -165,12 +169,15 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
@@ -255,7 +262,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
 google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
 google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA=

pkg/apis/testharness/v1beta1/test_types.go

@@ -152,6 +152,10 @@ type TestAssert struct {
 	Collectors []*TestCollector `json:"collectors,omitempty"`
 	// Commands is a set of commands to be run as assertions for the current step
 	Commands []TestAssertCommand `json:"commands,omitempty"`
+
+	ResourceRefs []TestResourceRef `json:"resourceRefs,omitempty"`
+
+	AssertExpressions AnyAllExpressions `json:"assertExpressions,omitempty"`
 }
 
 // TestAssertCommand an assertion based on the result of the execution of a command
@@ -222,6 +226,19 @@ type TestCollector struct {
 	Cmd string `json:"command,omitempty"`
 }
 
+type TestResourceRef struct {
+	ApiVersion string `json:"apiVersion,omitempty"`
+	Kind       string `json:"kind,omitempty"`
+	Namespace  string `json:"namespace,omitempty"`
+	Name       string `json:"name,omitempty"`
+	Id         string `json:"id,omitempty"`
+}
+
+type AnyAllExpressions struct {
+	Any []string `json:"any,omitempty"`
+	All []string `json:"all,omitempty"`
+}
+
 // DefaultKINDContext defines the default kind context to use.
 const DefaultKINDContext = "kind"
 

pkg/test/case.go

@@ -339,7 +339,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
 			continue
 		}
 
-		cl, err = newClient(testStep.Kubeconfig, testStep.Context)(false)
+		cl, err = testutils.NewClient(testStep.Kubeconfig, testStep.Context)(false)
 		if err != nil {
 			setupReport.Failure = report.NewFailure(err.Error(), nil)
 			ts.AddTestcase(setupReport)
@@ -364,7 +364,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
 		tc := report.NewCase("step " + testStep.String())
 		testStep.Client = t.Client
 		if testStep.Kubeconfig != "" {
-			testStep.Client = newClient(testStep.Kubeconfig, testStep.Context)
+			testStep.Client = testutils.NewClient(testStep.Kubeconfig, testStep.Context)
 		}
 		testStep.DiscoveryClient = t.DiscoveryClient
 		if testStep.Kubeconfig != "" {
@@ -532,19 +532,6 @@ func (t *Case) LoadTestSteps() error {
 	return nil
 }
 
-func newClient(kubeconfig, context string) func(bool) (client.Client, error) {
-	return func(bool) (client.Client, error) {
-		config, err := k8s.BuildConfigWithContext(kubeconfig, context)
-		if err != nil {
-			return nil, err
-		}
-
-		return testutils.NewRetryClient(config, client.Options{
-			Scheme: testutils.Scheme(),
-		})
-	}
-}
-
 func newDiscoveryClient(kubeconfig, context string) func() (discovery.DiscoveryInterface, error) {
 	return func() (discovery.DiscoveryInterface, error) {
 		config, err := k8s.BuildConfigWithContext(kubeconfig, context)

pkg/test/step.go

@@ -412,6 +412,14 @@ func (s *Step) CheckAssertCommands(ctx context.Context, namespace string, comman
 	return testErrors
 }
 
+func (s *Step) CheckAssertExpressions(
+	ctx context.Context,
+	resourceRefs []harness.TestResourceRef,
+	expressions harness.AnyAllExpressions,
+) []error {
+	return testutils.RunAssertExpressions(ctx, s.Logger, resourceRefs, expressions, s.Kubeconfig)
+}
+
 // Check checks if the resources defined in Asserts and Errors are in the correct state.
 func (s *Step) Check(namespace string, timeout int) []error {
 	testErrors := []error{}
@@ -422,6 +430,7 @@ func (s *Step) Check(namespace string, timeout int) []error {
 
 	if s.Assert != nil {
 		testErrors = append(testErrors, s.CheckAssertCommands(context.TODO(), namespace, s.Assert.Commands, timeout)...)
+		testErrors = append(testErrors, s.CheckAssertExpressions(context.TODO(), s.Assert.ResourceRefs, s.Assert.AssertExpressions)...)
 	}
 
 	for _, expected := range s.Errors {

pkg/test/utils/kubernetes.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/cel-go/cel"
 	"github.com/google/shlex"
 	"github.com/pmezard/go-difflib/difflib"
 	"github.com/spf13/pflag"
@@ -57,6 +58,7 @@ import (
 	"github.com/kudobuilder/kuttl/pkg/apis"
 	harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1"
 	"github.com/kudobuilder/kuttl/pkg/env"
+	"github.com/kudobuilder/kuttl/pkg/k8s"
 )
 
 // ensure that we only add to the scheme once.
@@ -1217,6 +1219,81 @@ func RunAssertCommands(ctx context.Context, logger Logger, namespace string, com
 	return RunCommands(ctx, logger, namespace, convertAssertCommand(commands, timeout), workdir, timeout, kubeconfigOverride)
 }
 
+// RunAssertExpressions evaluates a set of CEL expressions specified as AnyAllExpressions
+func RunAssertExpressions(
+	ctx context.Context,
+	logger Logger,
+	resourceRefs []harness.TestResourceRef,
+	expressions harness.AnyAllExpressions,
+	kubeconfigOverride string,
+) []error {
+	errs := []error{}
+
+	actualDir, err := os.Getwd()
+	if err != nil {
+		return []error{fmt.Errorf("failed to get current working director: %w", err)}
+	}
+
+	kubeconfig := kubeconfigPath(actualDir, kubeconfigOverride)
+	cl, err := NewClient(kubeconfig, "")(false)
+	if err != nil {
+		return []error{fmt.Errorf("failed to construct client: %w", err)}
+	}
+
+	variables := make(map[string]interface{})
+	for _, resourceRef := range resourceRefs {
+		gvk := constructGVK(resourceRef.ApiVersion, resourceRef.Kind)
+		referencedResource := &unstructured.Unstructured{}
+		referencedResource.SetGroupVersionKind(gvk)
+
+		if err := cl.Get(
+			ctx,
+			types.NamespacedName{Namespace: resourceRef.Namespace, Name: resourceRef.Name},
+			referencedResource,
+		); err != nil {
+			return []error{fmt.Errorf("failed to get referenced resource '%v': %w", gvk, err)}
+		}
+
+		variables[resourceRef.Id] = referencedResource.Object
+	}
+
+	env, err := cel.NewEnv()
+	if err != nil {
+		return []error{fmt.Errorf("failed to create environment: %w", err)}
+	}
+
+	for k := range variables {
+		env, err = env.Extend(cel.Variable(k, cel.DynType))
+		if err != nil {
+			return []error{fmt.Errorf("failed to add resource parameter '%v' to environment: %w", k, err)}
+		}
+	}
+
+	for _, expr := range expressions.Any {
+		ast, issues := env.Compile(expr)
+		if issues != nil && issues.Err() != nil {
+			return []error{fmt.Errorf("type-check error: %s", issues.Err())}
+		}
+
+		prg, err := env.Program(ast)
+		if err != nil {
+			return []error{fmt.Errorf("program constuction error: %w", err)}
+		}
+
+		out, _, err := prg.Eval(variables)
+		if err != nil {
+			return []error{fmt.Errorf("failed to evaluate program: %w", err)}
+		}
+
+		logger.Logf("expression '%v' evaluated to '%v'", expr, out.Value())
+		if out.Value() != true {
+			errs = append(errs, fmt.Errorf("failed validation, expression '%v' evaluated to '%v'", expr, out.Value()))
+		}
+	}
+
+	return errs
+}
+
 // RunCommands runs a set of commands, returning any errors.
 // If any (non-background) command fails, the following commands are skipped
 // commands running in the background are returned
@@ -1321,3 +1398,29 @@ func Kubeconfig(cfg *rest.Config, w io.Writer) error {
 		},
 	}, w)
 }
+
+func constructGVK(apiVersion, kind string) schema.GroupVersionKind {
+	apiVersionSplit := strings.Split(apiVersion, "/")
+	gvk := schema.GroupVersionKind{
+		Version: apiVersionSplit[len(apiVersionSplit)-1],
+		Kind:    kind,
+	}
+	if len(apiVersion) > 1 {
+		gvk.Group = apiVersionSplit[0]
+	}
+
+	return gvk
+}
+
+func NewClient(kubeconfig, context string) func(bool) (client.Client, error) {
+	return func(bool) (client.Client, error) {
+		config, err := k8s.BuildConfigWithContext(kubeconfig, context)
+		if err != nil {
+			return nil, err
+		}
+
+		return NewRetryClient(config, client.Options{
+			Scheme: Scheme(),
+		})
+	}
+}