Skip to content

Commit

Permalink
feat: resource selector search in config and labels (#1269)
Browse files Browse the repository at this point in the history 
* feat: resource selector search in config and labels

* chore: fix lint
  • Loading branch information
@yashmehrotra
yashmehrotra authored Jan 16, 2025
1 parent 83d9445 commit c6746ec
Show file tree
Hide file tree
Showing 3 changed files with 125 additions and 90 deletions.
150 changes: 62 additions & 88 deletions query/models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package query

import (
"fmt"
"slices"
"strconv"
"strings"

Expand Down Expand Up @@ -35,41 +36,46 @@ var AgentMapper = func(ctx context.Context, id string) (any, error) {
return nil, fmt.Errorf("invalid agent: %s", id)
}

var JSONPathMapper = func(ctx context.Context, tx *gorm.DB, column string, path string, val string) *gorm.DB {
return tx.Where(fmt.Sprintf(`TRIM(BOTH '"' from jsonb_path_query_first(%s, '$.%s')::TEXT) = ?`, column, path), val)
}

var CommonFields = map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error){
"limit": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Limit(i), nil
} else {
return nil, err
}
},
"sort": func(ctx context.Context, tx *gorm.DB, sort string) (*gorm.DB, error) {
return tx.Order(clause.OrderByColumn{Column: clause.Column{Name: sort}}), nil
},
"offset": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Offset(i), nil
} else {
return nil, err
}
},
}

type QueryModel struct {
Table string
LabelsColumn string
DateFields []string
Columns []string
FieldMapper map[string]func(ctx context.Context, id string) (any, error)
Custom map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error)
Aliases map[string]string
Table string
JSONColumns []string
DateFields []string
Columns []string
FieldMapper map[string]func(ctx context.Context, id string) (any, error)
Custom map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error)
Aliases map[string]string
}

var ConfigQueryModel = QueryModel{
Table: "configs",
Custom: map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error){
"limit": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Limit(i), nil
} else {
return nil, err
}
},
"sort": func(ctx context.Context, tx *gorm.DB, sort string) (*gorm.DB, error) {
return tx.Order(clause.OrderByColumn{Column: clause.Column{Name: sort}}), nil
},
"offset": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Offset(i), nil
} else {
return nil, err
}
},
},
Columns: []string{
"name", "source", "type", "status", "health",
},
LabelsColumn: "labels",
JSONColumns: []string{"labels", "tags", "config"},
Aliases: map[string]string{
"created": "created_at",
"updated": "updated_at",
Expand All @@ -92,23 +98,6 @@ var ConfigQueryModel = QueryModel{
var ComponentQueryModel = QueryModel{
Table: "components",
Custom: map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error){
"limit": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Limit(i), nil
} else {
return nil, err
}
},
"sort": func(ctx context.Context, tx *gorm.DB, sort string) (*gorm.DB, error) {
return tx.Order(clause.OrderByColumn{Column: clause.Column{Name: sort}}), nil
},
"offset": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Offset(i), nil
} else {
return nil, err
}
},
"component_config_traverse": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
// search: component_config_traverse=72143d48-da4a-477f-bac1-1e9decf188a6,outgoing
// Args should be componentID, direction and types (compID,direction)
Expand All @@ -127,7 +116,7 @@ var ComponentQueryModel = QueryModel{
Columns: []string{
"name", "topology_id", "type", "status", "health",
},
LabelsColumn: "labels",
JSONColumns: []string{"labels", "properties"},
Aliases: map[string]string{
"created": "created_at",
"updated": "updated_at",
Expand All @@ -149,29 +138,10 @@ var ComponentQueryModel = QueryModel{

var CheckQueryModel = QueryModel{
Table: "checks",
Custom: map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error){
"limit": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Limit(i), nil
} else {
return nil, err
}
},
"sort": func(ctx context.Context, tx *gorm.DB, sort string) (*gorm.DB, error) {
return tx.Order(clause.OrderByColumn{Column: clause.Column{Name: sort}}), nil
},
"offset": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Offset(i), nil
} else {
return nil, err
}
},
},
Columns: []string{
"name", "canary_id", "type", "status",
},
LabelsColumn: "labels",
JSONColumns: []string{"labels"},
Aliases: map[string]string{
"created": "created_at",
"updated": "updated_at",
Expand All @@ -191,25 +161,6 @@ var CheckQueryModel = QueryModel{

var PlaybookQueryModel = QueryModel{
Table: models.Playbook{}.TableName(),
Custom: map[string]func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error){
"limit": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Limit(i), nil
} else {
return nil, err
}
},
"sort": func(ctx context.Context, tx *gorm.DB, sort string) (*gorm.DB, error) {
return tx.Order(clause.OrderByColumn{Column: clause.Column{Name: sort}}), nil
},
"offset": func(ctx context.Context, tx *gorm.DB, val string) (*gorm.DB, error) {
if i, err := strconv.Atoi(val); err == nil {
return tx.Offset(i), nil
} else {
return nil, err
}
},
},
Aliases: map[string]string{
"created": "created_at",
"updated": "updated_at",
Expand Down Expand Up @@ -238,12 +189,19 @@ func GetModelFromTable(table string) (QueryModel, error) {
}
}

var (
// QueryModel.Apply will ignore these fields when converting to clauses
// as we modify the tx directly for them
ignoreFieldsForClauses = []string{"sort", "offset", "limit", "labels", "config", "tags"}
)

func (qm QueryModel) Apply(ctx context.Context, q types.QueryField, tx *gorm.DB) (*gorm.DB, []clause.Expression, error) {
if tx == nil {
tx = ctx.DB().Table(qm.Table)
}
clauses := []clause.Expression{}
var err error

if q.Field != "" {
q.Field = strings.ToLower(q.Field)
if alias, ok := qm.Aliases[q.Field]; ok {
Expand All @@ -257,17 +215,33 @@ func (qm QueryModel) Apply(ctx context.Context, q types.QueryField, tx *gorm.DB)
}
}

if mapper, ok := CommonFields[q.Field]; ok {
tx, err = mapper(ctx, tx, val)
if err != nil {
return nil, nil, errors.Wrapf(err, "Invalid value for %s", q.Field)
}
}

if mapper, ok := qm.Custom[q.Field]; ok {
tx, err = mapper(ctx, tx, val)
if err != nil {
return nil, nil, errors.Wrapf(err, "Invalid value for %s", q.Field)
}
}

if c, err := q.ToClauses(); err != nil {
return nil, nil, err
} else {
clauses = append(clauses, c...)
for _, column := range qm.JSONColumns {
if strings.HasPrefix(q.Field, column) {
tx = JSONPathMapper(ctx, tx, column, strings.TrimPrefix(q.Field, column+"."), val)
q.Field = column
}
}

if !slices.Contains(ignoreFieldsForClauses, q.Field) {
if c, err := q.ToClauses(); err != nil {
return nil, nil, err
} else {
clauses = append(clauses, c...)
}
}
}

Expand Down
40 changes: 39 additions & 1 deletion tests/fixtures/dummy/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,45 @@ var LogisticsAPIDeployment = models.ConfigItem{
ID: uuid.New(),
Name: lo.ToPtr("logistics-api"),
ConfigClass: models.ConfigClassDeployment,
Type: lo.ToPtr("Kubernetes::Deployment"),
Config: lo.ToPtr(`{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": {
"name": "logistics-api",
"labels": {
"app": "logistics-api"
}
},
"spec": {
"replicas": 3,
"selector": {
"matchLabels": {
"app": "logistics-api"
}
},
"template": {
"metadata": {
"labels": {
"app": "logistics-api"
}
},
"spec": {
"containers": [
{
"name": "logistics-api",
"image": "logistics-api:latest",
"ports": [
{
"containerPort": 80
}
]
}
]
}
}
}
}`),
Type: lo.ToPtr("Kubernetes::Deployment"),
Labels: lo.ToPtr(types.JSONStringMap{
"app": "logistics",
"environment": "production",
Expand Down
25 changes: 24 additions & 1 deletion tests/query_resource_selector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
. "github.com/onsi/gomega"
"github.com/samber/lo"

//"github.com/flanksource/commons/logger"
"github.com/flanksource/duty/context"
"github.com/flanksource/duty/models"
"github.com/flanksource/duty/query"
Expand Down Expand Up @@ -402,6 +401,30 @@ var _ = ginkgo.Describe("Resoure Selector with PEG", ginkgo.Ordered, func() {
expectedIDs: []uuid.UUID{},
resource: "component",
},
{
description: "config soft and limit query",
query: `name=node-* type="Kubernetes::Node" limit=1 sort=name`,
expectedIDs: []uuid.UUID{dummy.KubernetesNodeA.ID},
resource: "config",
},
{
description: "config json query",
query: `config.metadata.name=node-a`,
expectedIDs: []uuid.UUID{dummy.KubernetesNodeA.ID},
resource: "config",
},
{
description: "config json integer query",
query: `config.spec.replicas=3`,
expectedIDs: []uuid.UUID{dummy.LogisticsAPIDeployment.ID},
resource: "config",
},
{
description: "config labels query",
query: `labels.account=flanksource labels.environment=production`,
expectedIDs: []uuid.UUID{dummy.EKSCluster.ID, dummy.EC2InstanceB.ID},
resource: "config",
},
}

fmap := map[string]func(context.Context, int, ...types.ResourceSelector) ([]uuid.UUID, error){
Expand Down

0 comments on commit c6746ec

Please sign in to comment.