feat(assertions): add logical operators (#455)

ovh · Dec 3, 2021 · 9c2e51a · 9c2e51a
1 parent 24cc72f
commit 9c2e51a
Show file tree

Hide file tree

Showing 21 changed files with 243 additions and 19 deletions.
diff --git a/README.md b/README.md
@@ -431,6 +431,30 @@ Available formats: jUnit (xml), json, yaml, tap reports
 * ShouldHappenOnOrAfter - [example](https://github.com/ovh/venom/tree/master/tests/assertions/ShouldHappenOnOrAfter.yml)
 * ShouldHappenBetween - [example](https://github.com/ovh/venom/tree/master/tests/assertions/ShouldHappenBetween.yml)
 
+## Using logical operators
+
+While assertions use `and` operator implicitely, it is possible to use other logical operators to perform complex assertions.
+
+Supported operators are `and`, `or` and `xor`.
+
+```yml
+- name: Assertions operators
+  steps:
+  - script: echo 1
+    assertions:
+      - or:
+        - result.systemoutjson ShouldEqual 1 
+        - result.systemoutjson ShouldEqual 2
+      # Nested operators
+      - or:
+        - result.systemoutjson ShouldBeGreaterThanOrEqualTo 1
+        - result.systemoutjson ShouldBeLessThanOrEqualTo 1
+        - or:
+          - result.systemoutjson ShouldEqual 1
+```
+
+More examples are available in [`tests/assertions_operators.yml`](/tests/assertions_operators.yml).
+
 # Advanced usage
 ## Debug your testsuites
 

diff --git a/assertion.go b/assertion.go
@@ -106,7 +106,99 @@ func parseAssertions(ctx context.Context, s string, input interface{}) (*asserti
 	}, nil
 }
 
-func check(tc TestCase, stepNumber int, assertion string, r interface{}) (*Failure, *Failure) {
+// check selects the correct assertion function to call depending on typing provided by user
+func check(tc TestCase, stepNumber int, assertion Assertion, r interface{}) (*Failure, *Failure) {
+	var errs *Failure
+	var fails *Failure
+	switch t := assertion.(type) {
+	case string:
+		errs, fails = checkString(tc, stepNumber, assertion.(string), r)
+	case map[string]interface{}:
+		errs, fails = checkBranch(tc, stepNumber, assertion.(map[string]interface{}), r)
+	default:
+		errs = newFailure(tc, stepNumber, "", fmt.Errorf("unsupported assertion format: %v", t))
+	}
+	return errs, fails
+}
+
+// checkString evaluate a complex assertion containing logical operators
+// it recursively calls checkAssertion for each operand
+func checkBranch(tc TestCase, stepNumber int, branch map[string]interface{}, r interface{}) (*Failure, *Failure) {
+	// Extract logical operator
+	if len(branch) != 1 {
+		return newFailure(tc, stepNumber, "", fmt.Errorf("expected exactly 1 logical operator but %d were provided", len(branch))), nil
+	}
+	var operator string
+	for k := range branch {
+		operator = k
+	}
+
+	// Extract logical operands
+	var operands []interface{}
+	switch t := branch[operator].(type) {
+	case []interface{}:
+		operands = branch[operator].([]interface{})
+	default:
+		return newFailure(tc, stepNumber, "", fmt.Errorf("expected %s operands to be an []interface{}, got %v", operator, t)), nil
+	}
+	if len(operands) == 0 {
+		return nil, nil
+	}
+
+	// Evaluate assertions (operands)
+	var errsBuf []Failure
+	var failsBuf []Failure
+	var results []string
+	assertionsCount := len(operands)
+	assertionsSuccess := 0
+	for _, assertion := range operands {
+		errs, fails := check(tc, stepNumber, assertion, r)
+		if errs != nil {
+			errsBuf = append(errsBuf, *errs)
+		}
+		if fails != nil {
+			failsBuf = append(failsBuf, *fails)
+			results = append(results, fmt.Sprintf("  - fail: %s", assertion))
+		}
+		if (errs == nil) && (fails == nil) {
+			assertionsSuccess++
+			results = append(results, fmt.Sprintf("  - pass: %s", assertion))
+		}
+	}
+
+	// Evaluate operator behaviour
+	var err error
+	switch operator {
+	case "and":
+		if assertionsSuccess != assertionsCount {
+			err = fmt.Errorf("%d/%d assertions succeeded:\n%s\n", assertionsSuccess, assertionsCount, strings.Join(results, "\n"))
+		}
+	case "or":
+		if assertionsSuccess == 0 {
+			err = fmt.Errorf("no assertions succeeded:\n%s\n", strings.Join(results, "\n"))
+		}
+	case "xor":
+		if assertionsSuccess == 0 {
+			err = fmt.Errorf("no assertions succeeded:\n%s\n", strings.Join(results, "\n"))
+		}
+		if assertionsSuccess > 1 {
+			err = fmt.Errorf("multiple assertions succeeded but expected only one to suceed:\n%s\n", strings.Join(results, "\n"))
+		}
+	case "not":
+		if assertionsSuccess > 0 {
+			err = fmt.Errorf("some assertions succeeded but expected none to suceed:\n%s\n", strings.Join(results, "\n"))
+		}
+	default:
+		return newFailure(tc, stepNumber, "", fmt.Errorf("unsupported assertion operator %s", operator)), nil
+	}
+	if err != nil {
+		return nil, newFailure(tc, stepNumber, "", err)
+	}
+	return nil, nil
+}
+
+// checkString evaluate a single string assertion
+func checkString(tc TestCase, stepNumber int, assertion string, r interface{}) (*Failure, *Failure) {
 	assert, err := parseAssertions(context.Background(), assertion, r)
 	if err != nil {
 		return nil, newFailure(tc, stepNumber, assertion, err)

diff --git a/executors/README.md b/executors/README.md
@@ -42,7 +42,7 @@ type Result struct {
 // GetDefaultAssertions return default assertions for this executor
 // Optional
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.code ShouldEqual 0"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.code ShouldEqual 0"}}
 }
 
 // Run execute TestStep

diff --git a/executors/dbfixtures/dbfixtures.go b/executors/dbfixtures/dbfixtures.go
@@ -115,7 +115,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return the default assertions of the executor.
 func (e Executor) GetDefaultAssertions() venom.StepAssertions {
-	return venom.StepAssertions{Assertions: []string{}}
+	return venom.StepAssertions{Assertions: []venom.Assertion{}}
 }
 
 // loadFixtures loads the fixtures in the database.

diff --git a/executors/exec/exec.go b/executors/exec/exec.go
@@ -49,7 +49,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.code ShouldEqual 0"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.code ShouldEqual 0"}}
 }
 
 // Run execute TestStep of type exec

diff --git a/executors/grpc/grpc.go b/executors/grpc/grpc.go
@@ -94,7 +94,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.code ShouldEqual 0"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.code ShouldEqual 0"}}
 }
 
 // Run execute TestStep of type exec

diff --git a/executors/http/http.go b/executors/http/http.go
@@ -82,7 +82,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for this executor
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.statuscode ShouldEqual 200"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.statuscode ShouldEqual 200"}}
 }
 
 // Run execute TestStep

diff --git a/executors/imap/imap.go b/executors/imap/imap.go
@@ -64,7 +64,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.err ShouldNotExist"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.err ShouldNotExist"}}
 }
 
 // Run execute TestStep of type exec

diff --git a/executors/kafka/kafka.go b/executors/kafka/kafka.go
@@ -111,7 +111,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.err ShouldBeEmpty"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.err ShouldBeEmpty"}}
 }
 
 // Run execute TestStep of type exec

diff --git a/executors/mqtt/mqtt.go b/executors/mqtt/mqtt.go
@@ -69,7 +69,7 @@ type Result struct {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.error ShouldBeEmpty"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.error ShouldBeEmpty"}}
 }
 
 func (Executor) Run(ctx context.Context, step venom.TestStep) (interface{}, error) {

diff --git a/executors/ovhapi/ovhapi.go b/executors/ovhapi/ovhapi.go
@@ -67,7 +67,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for this executor
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.statuscode ShouldEqual 200"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.statuscode ShouldEqual 200"}}
 }
 
 // Run execute TestStep

diff --git a/executors/plugins/README.md b/executors/plugins/README.md
@@ -52,7 +52,7 @@ func (e Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return the default assertions of the executor.
 func (e Executor) GetDefaultAssertions() venom.StepAssertions {
-	return venom.StepAssertions{Assertions: []string{}}
+	return venom.StepAssertions{Assertions: []venom.Assertion{}}
 }
 
 ```

diff --git a/executors/plugins/hello/hello.go b/executors/plugins/hello/hello.go
@@ -45,5 +45,5 @@ func (e Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return the default assertions of the executor.
 func (e Executor) GetDefaultAssertions() venom.StepAssertions {
-	return venom.StepAssertions{Assertions: []string{}}
+	return venom.StepAssertions{Assertions: []venom.Assertion{}}
 }
diff --git a/executors/plugins/odbc/odbc.go b/executors/plugins/odbc/odbc.go
@@ -104,7 +104,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return the default assertions of the executor.
 func (e Executor) GetDefaultAssertions() venom.StepAssertions {
-	return venom.StepAssertions{Assertions: []string{}}
+	return venom.StepAssertions{Assertions: []venom.Assertion{}}
 }
 
 // handleRows iter on each SQL rows result sets and serialize it into a []Row.

diff --git a/executors/rabbitmq/rabbitmq.go b/executors/rabbitmq/rabbitmq.go
@@ -76,7 +76,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.error ShouldBeEmpty"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.error ShouldBeEmpty"}}
 }
 
 // Run execute TestStep of type exec

diff --git a/executors/readfile/readfile.go b/executors/readfile/readfile.go
@@ -56,7 +56,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.err ShouldBeEmpty"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.err ShouldBeEmpty"}}
 }
 
 // Run execute TestStep of type exec

diff --git a/executors/smtp/smtp.go b/executors/smtp/smtp.go
@@ -49,7 +49,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.err ShouldBeEmpty"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.err ShouldBeEmpty"}}
 }
 
 // Run execute TestStep of type exec

diff --git a/executors/sql/sql.go b/executors/sql/sql.go
@@ -112,7 +112,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return the default assertions of the executor.
 func (e Executor) GetDefaultAssertions() venom.StepAssertions {
-	return venom.StepAssertions{Assertions: []string{}}
+	return venom.StepAssertions{Assertions: []venom.Assertion{}}
 }
 
 // handleRows iter on each SQL rows result sets and serialize it into a []Row.

diff --git a/executors/ssh/ssh.go b/executors/ssh/ssh.go
@@ -50,7 +50,7 @@ func (Executor) ZeroValueResult() interface{} {
 
 // GetDefaultAssertions return default assertions for type exec
 func (Executor) GetDefaultAssertions() *venom.StepAssertions {
-	return &venom.StepAssertions{Assertions: []string{"result.code ShouldEqual 0"}}
+	return &venom.StepAssertions{Assertions: []venom.Assertion{"result.code ShouldEqual 0"}}
 }
 
 // Run execute TestStep of type exec

diff --git a/tests/assertions_operators.yml b/tests/assertions_operators.yml
@@ -0,0 +1,105 @@
+name: Assertions testsuite
+testcases:
+
+  - name: AssertionsOperatorAnd
+    steps:
+    - script: echo 1
+      assertions:
+        - and:
+          - result.systemoutjson ShouldEqual 1 
+          - result.systemoutjson ShouldContain 1
+
+  - name: AssertionsOperatorOr
+    steps:
+    - script: echo 1
+      assertions:
+        - or:
+          - result.systemoutjson ShouldEqual 1 
+          - result.systemoutjson ShouldEqual 2
+
+  - name: AssertionsOperatorXor
+    steps:
+    - script: echo 1
+      assertions:
+        - xor:
+          - result.systemoutjson ShouldEqual 1
+          - result.systemoutjson ShouldEqual 2
+
+  - name: AssertionsOperatorNested
+    steps:
+    - script: echo 1
+      assertions:
+        - or:
+          - and:
+            - result.systemoutjson ShouldBeGreaterThanOrEqualTo 1
+            - result.systemoutjson ShouldBeLessThanOrEqualTo 1
+          - or:
+            - result.systemoutjson ShouldEqual 1
+
+  - name: AssertionsOperatorNested2
+    steps:
+    - script: echo 1
+      assertions:
+        - or:
+          - or:
+            - or:
+              - or:
+                - or:
+                  - result.systemoutjson ShouldEqual 1
+
+  - name: AssertionsReadmeExample
+    steps:
+    - script: echo 1
+      assertions:
+        - or:
+          - result.systemoutjson ShouldEqual 1 
+          - result.systemoutjson ShouldEqual 2
+        # Nested operators
+        - or:
+          - result.systemoutjson ShouldBeGreaterThanOrEqualTo 1
+          - result.systemoutjson ShouldBeLessThanOrEqualTo 1
+          - or:
+            - result.systemoutjson ShouldEqual 1
+
+  - name: AssertionsOperatorNot
+    steps:
+    - script: echo 1
+      assertions:
+        - not: 
+          - result.systemoutjson ShouldEqual 0
+
+  - name: AssertionsOperatorOrError
+    steps:
+    - script: echo 1
+      assertions:
+        - not:
+          - or:
+            - result.systemoutjson ShouldEqual 0
+            - result.systemoutjson ShouldEqual 2
+
+  - name: AssertionsOperatorXorErrorMultiple
+    steps:
+    - script: echo 1
+      assertions:
+        - not:
+          - xor:
+            - result.systemoutjson ShouldEqual 1
+            - result.systemoutjson ShouldContain 1 
+
+  - name: AssertionsOperatorXorErrorNone
+    steps:
+    - script: echo 1
+      assertions:
+        - not:
+          - xor:
+            - result.systemoutjson ShouldEqual 0
+            - result.systemoutjson ShouldContain 0
+
+  - name: AssertionsOperatorAndError
+    steps:
+    - script: echo 1
+      assertions:
+        - not:
+          - and:
+            - result.systemoutjson ShouldEqual 1
+            - result.systemoutjson ShouldContain 0