Fearure orm support for subqueries (#731)

ORM support for sub-queries

operations/maintainer.go

@@ -185,8 +185,9 @@ func (om *Maintainer) cleanupExternalOperations() {
 		query.ByField(query.NotEqualsOperator, "platform_id", types.SMPlatform),
 		// check if operation hasn't been updated for the operation's maximum allowed time to live in DB
 		query.ByField(query.LessThanOperator, "updated_at", util.ToRFCNanoFormat(currentTime.Add(-om.settings.Lifespan))),
-		query.ByNotExists(storage.GetSubQuery(storage.QueryForAllLastOperationsPerResource)),
+		query.BySubquery(query.InSubqueryOperator, "id", storage.GetSubQuery(storage.QueryForAllNotLastOperationsPerResource)),
 	}
+
 	if err := om.repository.Delete(om.smCtx, types.OperationType, criteria...); err != nil && err != util.ErrNotFoundInStorage {
 		log.C(om.smCtx).Debugf("Failed to cleanup operations: %s", err)
 		return
@@ -252,7 +253,7 @@ func (om *Maintainer) CleanupFinishedCascadeOperations() {
 		query.ByField(query.InOperator, "state", string(types.SUCCEEDED), string(types.FAILED)),
 		// check if operation hasn't been updated for the operation's maximum allowed time to live in DB//
 		query.ByField(query.LessThanOperator, "updated_at", util.ToRFCNanoFormat(currentTime.Add(-om.settings.Lifespan))),
-		query.ByNotExists(storage.GetSubQuery(storage.QueryForAllLastOperationsPerResource)),
+		query.BySubquery(query.InSubqueryOperator, "id", storage.GetSubQuery(storage.QueryForAllNotLastOperationsPerResource)),
 	}
 
 	roots, err := om.repository.List(om.smCtx, types.OperationType, rootsCriteria...)
@@ -280,7 +281,7 @@ func (om *Maintainer) cleanupInternalSuccessfulOperations() {
 		query.ByField(query.EqualsOrNilOperator, "cascade_root_id", ""),
 		// check if operation hasn't been updated for the operation's maximum allowed time to live in DB
 		query.ByField(query.LessThanOperator, "updated_at", util.ToRFCNanoFormat(currentTime.Add(-om.settings.Lifespan))),
-		query.ByNotExists(storage.GetSubQuery(storage.QueryForAllLastOperationsPerResource)),
+		query.BySubquery(query.InSubqueryOperator, "id", storage.GetSubQuery(storage.QueryForAllNotLastOperationsPerResource)),
 	}
 
 	if err := om.repository.Delete(om.smCtx, types.OperationType, criteria...); err != nil && err != util.ErrNotFoundInStorage {
@@ -302,7 +303,7 @@ func (om *Maintainer) cleanupInternalFailedOperations() {
 		query.ByField(query.EqualsOrNilOperator, "cascade_root_id", ""),
 		// check if operation hasn't been updated for the operation's maximum allowed time to live in DB
 		query.ByField(query.LessThanOperator, "updated_at", util.ToRFCNanoFormat(currentTime.Add(-om.settings.Lifespan))),
-		query.ByNotExists(storage.GetSubQuery(storage.QueryForAllLastOperationsPerResource)),
+		query.BySubquery(query.InSubqueryOperator, "id", storage.GetSubQuery(storage.QueryForAllNotLastOperationsPerResource)),
 	}
 
 	if err := om.repository.Delete(om.smCtx, types.OperationType, criteria...); err != nil && err != util.ErrNotFoundInStorage {

pkg/query/operators.go

@@ -32,7 +32,8 @@ const (
 	// LessThanOrEqualOperator takes two operands and tests if the left is lesser than or equal the right
 	LessThanOrEqualOperator leOperator = "le"
 	// InOperator takes two operands and tests if the left is contained in the right
-	InOperator inOperator = "in"
+	InOperator         inOperator         = "in"
+	InSubqueryOperator inSubqueryOperator = "inSubquery"
 	// NotInOperator takes two operands and tests if the left is not contained in the right
 	NotInOperator notInOperator = "notin"
 	// NotExistsSubquery receives a sub-query as single left-operand and checks the sub-query for rows existence. If there're no rows then it will return TRUE, otherwise FALSE.
@@ -173,6 +174,24 @@ func (containsOperator) IsNumeric() bool {
 	return false
 }
 
+type inSubqueryOperator string
+
+func (o inSubqueryOperator) String() string {
+	return string(o)
+}
+
+func (inSubqueryOperator) Type() OperatorType {
+	return UnivariateOperator
+}
+
+func (inSubqueryOperator) IsNullable() bool {
+	return false
+}
+
+func (inSubqueryOperator) IsNumeric() bool {
+	return false
+}
+
 type existsSubquery string
 
 func (o existsSubquery) String() string {

pkg/query/selection.go

@@ -48,6 +48,8 @@ const (
 	ResultQuery CriterionType = "resultQuery"
 	// ExistQuery denotes that the query should test for the existence of any record in a given sub-query
 	ExistQuery CriterionType = "existQuery"
+	// InQuery denotes that the query
+	Subquery CriterionType = "subQuery"
 )
 
 // OperatorType represents the type of the query operator
@@ -86,7 +88,7 @@ var (
 		InOperator, NotInOperator, EqualsOrNilOperator, ContainsOperator,
 	}
 	// CriteriaTypes returns the supported query criteria types
-	CriteriaTypes = []CriterionType{FieldQuery, LabelQuery, ExistQuery}
+	CriteriaTypes = []CriterionType{FieldQuery, LabelQuery, ExistQuery, Subquery}
 )
 
 // Operator is a query operator
@@ -126,6 +128,10 @@ func ByExists(subQuery string) Criterion {
 	return NewCriterion("", ExistsSubquery, []string{subQuery}, ExistQuery)
 }
 
+func BySubquery(operator Operator, leftOp string, subQuery string) Criterion {
+	return NewCriterion(leftOp, operator, []string{subQuery}, Subquery)
+}
+
 // ByLabel constructs a new criterion for label querying
 func ByLabel(operator Operator, leftOp string, rightOp ...string) Criterion {
 	return NewCriterion(leftOp, operator, rightOp, LabelQuery)
@@ -188,7 +194,7 @@ func (c Criterion) Validate() error {
 		return &util.UnsupportedQueryError{Message: fmt.Sprintf("separator %s is not allowed in %s with left operand \"%s\".", Separator, c.Type, c.LeftOp)}
 	}
 	for _, op := range c.RightOp {
-		if strings.ContainsRune(op, '\n') && c.Type != ExistQuery {
+		if strings.ContainsRune(op, '\n') && c.Type != ExistQuery && c.Type != Subquery {
 			return &util.UnsupportedQueryError{Message: fmt.Sprintf("%s with key \"%s\" has value \"%s\" contaning forbidden new line character", c.Type, c.LeftOp, op)}
 		}
 	}

pkg/query/selection_test.go

@@ -124,106 +124,142 @@ new line`))
 
 	Describe("Parse query", func() {
 		for _, queryType := range CriteriaTypes {
-			Context("With no query", func() {
-				It("Should return empty criteria", func() {
-					criteria, err := Parse(queryType, "")
-					Expect(err).ToNot(HaveOccurred())
-					Expect(len(criteria)).To(Equal(0))
+			Describe(string("queryType - "+queryType), func() {
+				queryType := queryType
+				Context("With no query", func() {
+					It("Should return empty criteria", func() {
+						criteria, err := Parse(queryType, "")
+						Expect(err).ToNot(HaveOccurred())
+						Expect(len(criteria)).To(Equal(0))
+					})
 				})
-			})
 
-			Context("With missing query operator", func() {
-				It("Should return an error", func() {
-					criteria, err := Parse(queryType, "leftop_rightop")
-					Expect(err).To(HaveOccurred())
-					Expect(criteria).To(BeNil())
+				Context("With missing query operator", func() {
+					It("Should return an error", func() {
+						criteria, err := Parse(queryType, "leftop_rightop")
+						Expect(err).To(HaveOccurred())
+						Expect(criteria).To(BeNil())
+					})
 				})
-			})
 
-			Context("When there is an invalid field query", func() {
-				It("Should return an error", func() {
-					criteria, err := Parse(queryType, "leftop lt 'rightop'")
-					Expect(err).To(HaveOccurred())
-					Expect(criteria).To(BeNil())
+				Context("When there is an invalid field query", func() {
+					It("Should return an error", func() {
+						criteria, err := Parse(queryType, "leftop lt 'rightop'")
+						Expect(err).To(HaveOccurred())
+						Expect(criteria).To(BeNil())
+					})
 				})
-			})
 
-			Context("When passing multivariate query", func() {
-				It("Should be ok", func() {
-					criteria, err := Parse(queryType, "leftop in ('rightop', 'rightop2')")
-					Expect(err).ToNot(HaveOccurred())
-					Expect(criteria).To(ConsistOf(NewCriterion("leftop", InOperator, []string{"rightop2", "rightop"}, queryType)))
+				Context("When passing multivariate query", func() {
+					It("Should be ok", func() {
+						criteria, err := Parse(queryType, "leftop in ('rightop', 'rightop2')")
+						Expect(err).ToNot(HaveOccurred())
+						Expect(criteria).To(ConsistOf(NewCriterion("leftop", InOperator, []string{"rightop2", "rightop"}, queryType)))
+					})
 				})
-			})
 
-			Context("When passing multiple queries", func() {
-				It("Should build criteria", func() {
-					criteria, err := Parse(queryType, "leftop1 in ('rightop','rightop2') and leftop2 eq 'rightop3'")
-					Expect(err).ToNot(HaveOccurred())
-					Expect(criteria).To(ConsistOf(NewCriterion("leftop1", InOperator, []string{"rightop2", "rightop"}, queryType), NewCriterion("leftop2", EqualsOperator, []string{"rightop3"}, queryType)))
+				Context("When passing multiple queries", func() {
+					It("Should build criteria", func() {
+						criteria, err := Parse(queryType, "leftop1 in ('rightop','rightop2') and leftop2 eq 'rightop3'")
+						Expect(err).ToNot(HaveOccurred())
+						Expect(criteria).To(ConsistOf(NewCriterion("leftop1", InOperator, []string{"rightop2", "rightop"}, queryType), NewCriterion("leftop2", EqualsOperator, []string{"rightop3"}, queryType)))
+					})
 				})
-			})
 
-			Context("Operator is unsupported", func() {
-				It("Should return error", func() {
-					criteria, err := Parse(queryType, "leftop1 @ ('rightop', 'rightop2')")
-					Expect(err).To(HaveOccurred())
-					Expect(criteria).To(BeNil())
+				Context("Operator is unsupported", func() {
+					It("Should return error", func() {
+						criteria, err := Parse(queryType, "leftop1 @ ('rightop', 'rightop2')")
+						Expect(err).To(HaveOccurred())
+						Expect(criteria).To(BeNil())
+					})
 				})
-			})
 
-			Context("Right operand is empty", func() {
-				It("Should be ok", func() {
-					criteria, err := Parse(queryType, "leftop1 in ()")
-					Expect(err).ToNot(HaveOccurred())
-					Expect(criteria).ToNot(BeNil())
-					Expect(criteria).To(ConsistOf(NewCriterion("leftop1", InOperator, []string{""}, queryType)))
+				Context("Right operand is empty", func() {
+					It("Should be ok", func() {
+						criteria, err := Parse(queryType, "leftop1 in ()")
+						Expect(err).ToNot(HaveOccurred())
+						Expect(criteria).ToNot(BeNil())
+						Expect(criteria).To(ConsistOf(NewCriterion("leftop1", InOperator, []string{""}, queryType)))
+					})
 				})
-			})
-			Context("Multivariate operator with right operand without opening brace", func() {
-				It("Should return error", func() {
-					criteria, err := Parse(queryType, "leftop in 'rightop','rightop2')")
-					Expect(err).To(HaveOccurred())
-					Expect(criteria).To(BeNil())
+				Context("Multivariate operator with right operand without opening brace", func() {
+					It("Should return error", func() {
+						criteria, err := Parse(queryType, "leftop in 'rightop','rightop2')")
+						Expect(err).To(HaveOccurred())
+						Expect(criteria).To(BeNil())
+					})
 				})
-			})
-			Context("Multivariate operator with right operand without closing brace", func() {
-				It("Should return error", func() {
-					criteria, err := Parse(queryType, "leftop in ('rightop','rightop2'")
-					Expect(err).To(HaveOccurred())
-					Expect(criteria).To(BeNil())
+				Context("Multivariate operator with right operand without closing brace", func() {
+					It("Should return error", func() {
+						criteria, err := Parse(queryType, "leftop in ('rightop','rightop2'")
+						Expect(err).To(HaveOccurred())
+						Expect(criteria).To(BeNil())
+					})
 				})
-			})
-			Context("Right operand with escaped quote", func() {
-				It("Should be okay", func() {
-					criteria, err := Parse(queryType, "leftop1 eq 'right''op'")
-					Expect(err).ToNot(HaveOccurred())
-					Expect(criteria).ToNot(BeNil())
-					Expect(criteria).To(ConsistOf(NewCriterion("leftop1", EqualsOperator, []string{"right'op"}, queryType)))
+				Context("Right operand with escaped quote", func() {
+					It("Should be okay", func() {
+						criteria, err := Parse(queryType, "leftop1 eq 'right''op'")
+						Expect(err).ToNot(HaveOccurred())
+						Expect(criteria).ToNot(BeNil())
+						Expect(criteria).To(ConsistOf(NewCriterion("leftop1", EqualsOperator, []string{"right'op"}, queryType)))
+					})
 				})
-			})
-			Context("Complex right operand", func() {
-				It("Should be okay", func() {
-					rightOp := "this is a mixed, input example. It contains symbols   words ! -h@ppy p@rs'ng"
-					escaped := strings.Replace(rightOp, "'", "''", -1)
-					criteria, err := Parse(queryType, fmt.Sprintf("leftop1 eq '%s'", escaped))
-					Expect(err).ToNot(HaveOccurred())
-					Expect(criteria).ToNot(BeNil())
-					Expect(criteria).To(ConsistOf(NewCriterion("leftop1", EqualsOperator, []string{rightOp}, queryType)))
+				Context("Complex right operand", func() {
+					It("Should be okay", func() {
+						rightOp := "this is a mixed, input example. It contains symbols   words ! -h@ppy p@rs'ng"
+						escaped := strings.Replace(rightOp, "'", "''", -1)
+						criteria, err := Parse(queryType, fmt.Sprintf("leftop1 eq '%s'", escaped))
+						Expect(err).ToNot(HaveOccurred())
+						Expect(criteria).ToNot(BeNil())
+						Expect(criteria).To(ConsistOf(NewCriterion("leftop1", EqualsOperator, []string{rightOp}, queryType)))
+					})
 				})
-			})
 
-			Context("Duplicate query key", func() {
-				It("Should return error", func() {
-					//ExistQuery type doesn't support left operands nor it excepts any operators aside from NotExist/Exist operators
-					if queryType != ExistQuery {
-						criteria, err := Parse(queryType, "leftop1 eq 'rightop' and leftop1 eq 'rightop2'")
+				Context("Duplicate query key", func() {
+					It("Should return error", func() {
+						//ExistQuery type doesn't support left operands nor it excepts any operators aside from NotExist/Exist operators
+						if queryType == LabelQuery {
+							criteria, err := Parse(queryType, "leftop1 eq 'rightop' and leftop1 eq 'rightop2'")
+							Expect(err).To(HaveOccurred())
+							Expect(criteria).To(BeNil())
+						}
+					})
+				})
+
+				Context("When separator is not properly escaped in first query value", func() {
+					It("Should return error", func() {
+						criteria, err := Parse(queryType, "leftop1 eq 'not'escaped' and leftOp2 eq 'rightOp'")
 						Expect(err).To(HaveOccurred())
 						Expect(criteria).To(BeNil())
-					}
+					})
 				})
-			})
 
+				Context("When separator is not escaped in value", func() {
+					It("Trims the value to the separator", func() {
+						criteria, err := Parse(queryType, "leftop1 eq 'not'escaped'")
+						Expect(err).To(HaveOccurred())
+						Expect(criteria).To(BeNil())
+					})
+				})
+
+				Context("When using equals or operators", func() {
+					It("should build the right ge query", func() {
+						criteria, err := Parse(FieldQuery, "leftop ge -1.35")
+						Expect(err).ToNot(HaveOccurred())
+						Expect(criteria).ToNot(BeNil())
+						expectedQuery := ByField(GreaterThanOrEqualOperator, "leftop", "-1.35")
+						Expect(criteria).To(ConsistOf(expectedQuery))
+					})
+
+					It("should build the right le query", func() {
+						criteria, err := Parse(FieldQuery, "leftop le 3")
+						Expect(err).ToNot(HaveOccurred())
+						Expect(criteria).ToNot(BeNil())
+						expectedQuery := ByField(LessThanOrEqualOperator, "leftop", "3")
+						Expect(criteria).To(ConsistOf(expectedQuery))
+					})
+				})
+			})
 			for _, op := range Operators {
 				op := op
 				for _, queryType := range []CriterionType{FieldQuery, LabelQuery} {
@@ -260,41 +296,8 @@ new line`))
 					})
 				}
 			}
-
-			Context("When separator is not properly escaped in first query value", func() {
-				It("Should return error", func() {
-					criteria, err := Parse(queryType, "leftop1 eq 'not'escaped' and leftOp2 eq 'rightOp'")
-					Expect(err).To(HaveOccurred())
-					Expect(criteria).To(BeNil())
-				})
-			})
-
-			Context("When separator is not escaped in value", func() {
-				It("Trims the value to the separator", func() {
-					criteria, err := Parse(queryType, "leftop1 eq 'not'escaped'")
-					Expect(err).To(HaveOccurred())
-					Expect(criteria).To(BeNil())
-				})
-			})
-
-			Context("When using equals or operators", func() {
-				It("should build the right ge query", func() {
-					criteria, err := Parse(FieldQuery, "leftop ge -1.35")
-					Expect(err).ToNot(HaveOccurred())
-					Expect(criteria).ToNot(BeNil())
-					expectedQuery := ByField(GreaterThanOrEqualOperator, "leftop", "-1.35")
-					Expect(criteria).To(ConsistOf(expectedQuery))
-				})
-
-				It("should build the right le query", func() {
-					criteria, err := Parse(FieldQuery, "leftop le 3")
-					Expect(err).ToNot(HaveOccurred())
-					Expect(criteria).ToNot(BeNil())
-					expectedQuery := ByField(LessThanOrEqualOperator, "leftop", "3")
-					Expect(criteria).To(ConsistOf(expectedQuery))
-				})
-			})
 		}
+
 	})
 
 	DescribeTable("Validate Criterion",

storage/postgres/query_builder.go

@@ -356,14 +356,20 @@ func (pq *pgQuery) WithCriteria(criteria ...query.Criterion) *pgQuery {
 			})
 		case query.ResultQuery:
 			pq.processResultCriteria(criterion)
-
 		case query.ExistQuery:
 			pq.fieldsWhereClause.children = append(pq.fieldsWhereClause.children, &whereClauseTree{
 				criterion: criterion,
 				dbTags:    pq.entityTags,
 				tableName: pq.entityTableName,
 			})
+		case query.Subquery:
+			pq.fieldsWhereClause.children = append(pq.fieldsWhereClause.children, &whereClauseTree{
+				criterion: criterion,
+				dbTags:    pq.entityTags,
+				tableName: pq.entityTableName,
+			})
 		}
+
 	}
 
 	return pq