Skip to content

Commit

Permalink
feat: artifacts (#1453)
Browse files Browse the repository at this point in the history
  • Loading branch information
@adityathebe
adityathebe authored Nov 30, 2023
1 parent 43992a0 commit 4fa6e09
Show file tree
Hide file tree
Showing 41 changed files with 639 additions and 808 deletions.
3 changes: 2 additions & 1 deletion api/context/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/flanksource/canary-checker/pkg"
"github.com/flanksource/commons/logger"
ctemplate "github.com/flanksource/commons/template"
"github.com/flanksource/duty/connection"
dutyCtx "github.com/flanksource/duty/context"
"github.com/flanksource/duty/models"
"github.com/flanksource/duty/types"
Expand Down Expand Up @@ -181,7 +182,7 @@ func (ctx Context) TemplateStruct(o interface{}) error {
return templater.Walk(o)
}

func (ctx Context) GetAuthValues(auth v1.Authentication) (v1.Authentication, error) {
func (ctx Context) GetAuthValues(auth connection.Authentication) (connection.Authentication, error) {
// in case nil we are sending empty string values for username and password
if auth.IsEmpty() {
return auth, nil
Expand Down
282 changes: 51 additions & 231 deletions api/v1/checks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"strings"

"github.com/flanksource/canary-checker/api/external"
"github.com/flanksource/duty/connection"
"github.com/flanksource/duty/models"
"github.com/flanksource/duty/types"
v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -53,6 +53,16 @@ func (c Check) GetLabels() map[string]string {
return c.Labels
}

type Artifact struct {
// Path to the artifact on the check runner.
// Special paths: /dev/stdout & /dev/stdin
Path string `yaml:"path" json:"path"`
}

// type TestResult struct {
// Path string
// }

type Oauth2Config struct {
Scopes []string `json:"scope,omitempty" yaml:"scope,omitempty"`
TokenURL string `json:"tokenURL,omitempty" yaml:"tokenURL,omitempty"`
Expand Down Expand Up @@ -141,9 +151,9 @@ type Bucket struct {
}

type S3Check struct {
Description `yaml:",inline" json:",inline"`
AWSConnection `yaml:",inline" json:",inline"`
BucketName string `yaml:"bucketName" json:"bucketName,omitempty"`
Description `yaml:",inline" json:",inline"`
connection.AWSConnection `yaml:",inline" json:",inline"`
BucketName string `yaml:"bucketName" json:"bucketName,omitempty"`
}

func (c S3Check) GetEndpoint() string {
Expand All @@ -155,10 +165,10 @@ func (c S3Check) GetType() string {
}

type CloudWatchCheck struct {
Description `yaml:",inline" json:",inline"`
AWSConnection `yaml:",inline" json:",inline"`
Templatable `yaml:",inline" json:",inline"`
CloudWatchFilter `yaml:",inline" json:",inline"`
Description `yaml:",inline" json:",inline"`
connection.AWSConnection `yaml:",inline" json:",inline"`
Templatable `yaml:",inline" json:",inline"`
CloudWatchFilter `yaml:",inline" json:",inline"`
}

type CloudWatchFilter struct {
Expand Down Expand Up @@ -619,6 +629,8 @@ type JunitCheck struct {
// +kubebuilder:pruning:PreserveUnknownFields
// +kubebuilder:validation:Type=object
Spec json.RawMessage `yaml:"spec" json:"spec"`
// Artifacts configure the artifacts generated by the check
Artifacts []Artifact `yaml:"artifacts,omitempty" json:"artifacts,omitempty"`
}

func (c JunitCheck) GetEndpoint() string {
Expand Down Expand Up @@ -647,92 +659,6 @@ func (c JunitCheck) GetType() string {
return "junit"
}

type SMBConnection struct {
// ConnectionName of the connection. It'll be used to populate the connection fields.
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
//Port on which smb server is running. Defaults to 445
Port int `yaml:"port,omitempty" json:"port,omitempty"`
Authentication `yaml:",inline" json:",inline"`
//Domain...
Domain string `yaml:"domain,omitempty" json:"domain,omitempty"`
}

func (c SMBConnection) GetPort() int {
if c.Port != 0 {
return c.Port
}
return 445
}

func (c *SMBConnection) HydrateConnection(ctx checkContext) (found bool, err error) {
connection, err := ctx.HydrateConnectionByURL(c.ConnectionName)
if err != nil {
return false, err
}

if connection == nil {
return false, nil
}

c.Authentication = Authentication{
Username: types.EnvVar{ValueStatic: connection.Username},
Password: types.EnvVar{ValueStatic: connection.Password},
}

if domain, ok := connection.Properties["domain"]; ok {
c.Domain = domain
}

if portRaw, ok := connection.Properties["port"]; ok {
if port, err := strconv.Atoi(portRaw); nil == err {
c.Port = port
}
}

return true, nil
}

type SFTPConnection struct {
// ConnectionName of the connection. It'll be used to populate the connection fields.
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
// Port for the SSH server. Defaults to 22
Port int `yaml:"port,omitempty" json:"port,omitempty"`
Host string `yaml:"host" json:"host"`
Authentication `yaml:",inline" json:",inline"`
}

func (c *SFTPConnection) HydrateConnection(ctx checkContext) (found bool, err error) {
connection, err := ctx.HydrateConnectionByURL(c.ConnectionName)
if err != nil {
return false, err
}

if connection == nil {
return false, nil
}

c.Host = connection.URL
c.Authentication = Authentication{
Username: types.EnvVar{ValueStatic: connection.Username},
Password: types.EnvVar{ValueStatic: connection.Password},
}

if portRaw, ok := connection.Properties["port"]; ok {
if port, err := strconv.Atoi(portRaw); nil == err {
c.Port = port
}
}

return true, nil
}

func (c SFTPConnection) GetPort() int {
if c.Port != 0 {
return c.Port
}
return 22
}

/*
[include:datasources/prometheus.yaml]
*/
Expand Down Expand Up @@ -824,114 +750,6 @@ func (c KubernetesCheck) GetEndpoint() string {
return fmt.Sprintf("%v/%v/%v", c.Kind, c.Description.Description, c.Namespace.Name)
}

type AWSConnection struct {
// ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey.
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
AccessKey types.EnvVar `yaml:"accessKey" json:"accessKey,omitempty"`
SecretKey types.EnvVar `yaml:"secretKey" json:"secretKey,omitempty"`
SessionToken types.EnvVar `yaml:"sessionToken,omitempty" json:"sessionToken,omitempty"`
Region string `yaml:"region,omitempty" json:"region,omitempty"`
Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"`
// Skip TLS verify when connecting to aws
SkipTLSVerify bool `yaml:"skipTLSVerify,omitempty" json:"skipTLSVerify,omitempty"`
// glob path to restrict matches to a subset
ObjectPath string `yaml:"objectPath,omitempty" json:"objectPath,omitempty"`
// Use path style path: http://s3.amazonaws.com/BUCKET/KEY instead of http://BUCKET.s3.amazonaws.com/KEY
UsePathStyle bool `yaml:"usePathStyle,omitempty" json:"usePathStyle,omitempty"`
}

func (t *AWSConnection) GetUsername() types.EnvVar {
return t.AccessKey
}

func (t *AWSConnection) GetPassword() types.EnvVar {
return t.SecretKey
}

func (t *AWSConnection) GetProperties() map[string]string {
return map[string]string{
"region": t.Region,
}
}

func (t *AWSConnection) GetURL() types.EnvVar {
return types.EnvVar{ValueStatic: t.Endpoint}
}

// Populate populates an AWSConnection with credentials and other information.
// If a connection name is specified, it'll be used to populate the endpoint, accessKey and secretKey.
func (t *AWSConnection) Populate(ctx checkContext) error {
if t.ConnectionName != "" {
connection, err := ctx.HydrateConnectionByURL(t.ConnectionName)
if err != nil {
return fmt.Errorf("could not parse EC2 access key: %v", err)
}

t.AccessKey.ValueStatic = connection.Username
t.SecretKey.ValueStatic = connection.Password
if t.Endpoint == "" {
t.Endpoint = connection.URL
}

t.SkipTLSVerify = connection.InsecureTLS
if t.Region == "" {
if region, ok := connection.Properties["region"]; ok {
t.Region = region
}
}
}

if accessKey, err := ctx.GetEnvValueFromCache(t.AccessKey); err != nil {
return fmt.Errorf("could not parse AWS access key id: %v", err)
} else {
t.AccessKey.ValueStatic = accessKey
}

if secretKey, err := ctx.GetEnvValueFromCache(t.SecretKey); err != nil {
return fmt.Errorf(fmt.Sprintf("Could not parse AWS secret access key: %v", err))
} else {
t.SecretKey.ValueStatic = secretKey
}

if sessionToken, err := ctx.GetEnvValueFromCache(t.SessionToken); err != nil {
return fmt.Errorf(fmt.Sprintf("Could not parse AWS session token: %v", err))
} else {
t.SessionToken.ValueStatic = sessionToken
}

return nil
}

type GCPConnection struct {
// ConnectionName of the connection. It'll be used to populate the endpoint and credentials.
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint,omitempty"`
Credentials *types.EnvVar `yaml:"credentials" json:"credentials,omitempty"`
}

func (g *GCPConnection) Validate() *GCPConnection {
if g == nil {
return &GCPConnection{}
}
return g
}

// HydrateConnection attempts to find the connection by name
// and populate the endpoint and credentials.
func (g *GCPConnection) HydrateConnection(ctx checkContext) error {
connection, err := ctx.HydrateConnectionByURL(g.ConnectionName)
if err != nil {
return err
}

if connection != nil {
g.Credentials = &types.EnvVar{ValueStatic: connection.Certificate}
g.Endpoint = connection.URL
}

return nil
}

type AzureConnection struct {
ConnectionName string `yaml:"connection,omitempty" json:"connection,omitempty"`
ClientID *types.EnvVar `yaml:"clientID,omitempty" json:"clientID,omitempty"`
Expand Down Expand Up @@ -960,13 +778,13 @@ type FolderCheck struct {
Description `yaml:",inline" json:",inline"`
Templatable `yaml:",inline" json:",inline"`
// Path to folder or object storage, e.g. `s3://<bucket-name>`, `gcs://<bucket-name>`, `/path/tp/folder`
Path string `yaml:"path" json:"path"`
Filter FolderFilter `yaml:"filter,omitempty" json:"filter,omitempty"`
FolderTest `yaml:",inline" json:",inline"`
*AWSConnection `yaml:"awsConnection,omitempty" json:"awsConnection,omitempty"`
*GCPConnection `yaml:"gcpConnection,omitempty" json:"gcpConnection,omitempty"`
*SMBConnection `yaml:"smbConnection,omitempty" json:"smbConnection,omitempty"`
*SFTPConnection `yaml:"sftpConnection,omitempty" json:"sftpConnection,omitempty"`
Path string `yaml:"path" json:"path"`
Filter FolderFilter `yaml:"filter,omitempty" json:"filter,omitempty"`
FolderTest `yaml:",inline" json:",inline"`
*connection.AWSConnection `yaml:"awsConnection,omitempty" json:"awsConnection,omitempty"`
*connection.GCPConnection `yaml:"gcpConnection,omitempty" json:"gcpConnection,omitempty"`
*connection.SMBConnection `yaml:"smbConnection,omitempty" json:"smbConnection,omitempty"`
*connection.SFTPConnection `yaml:"sftpConnection,omitempty" json:"sftpConnection,omitempty"`
}

func (c FolderCheck) GetType() string {
Expand All @@ -978,9 +796,9 @@ func (c FolderCheck) GetEndpoint() string {
}

type ExecConnections struct {
AWS *AWSConnection `yaml:"aws,omitempty" json:"aws,omitempty"`
GCP *GCPConnection `yaml:"gcp,omitempty" json:"gcp,omitempty"`
Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"`
AWS *connection.AWSConnection `yaml:"aws,omitempty" json:"aws,omitempty"`
GCP *connection.GCPConnection `yaml:"gcp,omitempty" json:"gcp,omitempty"`
Azure *AzureConnection `yaml:"azure,omitempty" json:"azure,omitempty"`
}

type GitCheckout struct {
Expand Down Expand Up @@ -1021,6 +839,8 @@ type ExecCheck struct {
EnvVars []types.EnvVar `yaml:"env,omitempty" json:"env,omitempty"`
// Checkout details the git repository that should be mounted to the process
Checkout *GitCheckout `yaml:"checkout,omitempty" json:"checkout,omitempty"`
// Artifacts configure the artifacts generated by the check
Artifacts []Artifact `yaml:"artifacts,omitempty" json:"artifacts,omitempty"`
}

func (c ExecCheck) GetType() string {
Expand All @@ -1039,11 +859,11 @@ func (c ExecCheck) GetTestFunction() Template {
}

type AwsConfigCheck struct {
Description `yaml:",inline" json:",inline"`
Templatable `yaml:",inline" json:",inline"`
Query string `yaml:"query" json:"query"`
*AWSConnection `yaml:",inline" json:",inline"`
AggregatorName *string `yaml:"aggregatorName,omitempty" json:"aggregatorName,omitempty"`
Description `yaml:",inline" json:",inline"`
Templatable `yaml:",inline" json:",inline"`
Query string `yaml:"query" json:"query"`
*connection.AWSConnection `yaml:",inline" json:",inline"`
AggregatorName *string `yaml:"aggregatorName,omitempty" json:"aggregatorName,omitempty"`
}

func (c AwsConfigCheck) GetType() string {
Expand All @@ -1062,8 +882,8 @@ type AwsConfigRuleCheck struct {
// Specify one or more Config rule names to filter the results by rule.
Rules []string `yaml:"rules,omitempty" json:"rules,omitempty"`
// Filters the results by compliance. The allowed values are INSUFFICIENT_DATA, NON_COMPLIANT, NOT_APPLICABLE, COMPLIANT
ComplianceTypes []string `yaml:"complianceTypes,omitempty" json:"complianceTypes,omitempty"`
*AWSConnection `yaml:",inline" json:",inline"`
ComplianceTypes []string `yaml:"complianceTypes,omitempty" json:"complianceTypes,omitempty"`
*connection.AWSConnection `yaml:",inline" json:",inline"`
}

func (c AwsConfigRuleCheck) GetType() string {
Expand All @@ -1082,9 +902,9 @@ type DatabaseBackupCheck struct {
}

type GCPDatabase struct {
Project string `yaml:"project" json:"project"`
Instance string `yaml:"instance" json:"instance"`
*GCPConnection `yaml:"gcpConnection,omitempty" json:"gcpConnection,omitempty"`
Project string `yaml:"project" json:"project"`
Instance string `yaml:"instance" json:"instance"`
*connection.GCPConnection `yaml:"gcpConnection,omitempty" json:"gcpConnection,omitempty"`
}

func (c DatabaseBackupCheck) GetType() string {
Expand Down Expand Up @@ -1346,15 +1166,15 @@ type EC2 struct {
}

type EC2Check struct {
Description `yaml:",inline" json:",inline"`
AWSConnection `yaml:",inline" json:",inline"`
AMI string `yaml:"ami,omitempty" json:"ami,omitempty"`
UserData string `yaml:"userData,omitempty" json:"userData,omitempty"`
SecurityGroup string `yaml:"securityGroup,omitempty" json:"securityGroup,omitempty"`
KeepAlive bool `yaml:"keepAlive,omitempty" json:"keepAlive,omitempty"`
WaitTime int `yaml:"waitTime,omitempty" json:"waitTime,omitempty"`
TimeOut int `yaml:"timeOut,omitempty" json:"timeOut,omitempty"`
CanaryRef []v1.LocalObjectReference `yaml:"canaryRef,omitempty" json:"canaryRef,omitempty"`
Description `yaml:",inline" json:",inline"`
connection.AWSConnection `yaml:",inline" json:",inline"`
AMI string `yaml:"ami,omitempty" json:"ami,omitempty"`
UserData string `yaml:"userData,omitempty" json:"userData,omitempty"`
SecurityGroup string `yaml:"securityGroup,omitempty" json:"securityGroup,omitempty"`
KeepAlive bool `yaml:"keepAlive,omitempty" json:"keepAlive,omitempty"`
WaitTime int `yaml:"waitTime,omitempty" json:"waitTime,omitempty"`
TimeOut int `yaml:"timeOut,omitempty" json:"timeOut,omitempty"`
CanaryRef []v1.LocalObjectReference `yaml:"canaryRef,omitempty" json:"canaryRef,omitempty"`
}

func (c EC2Check) GetEndpoint() string {
Expand Down
Loading

0 comments on commit 4fa6e09

Please sign in to comment.