Fixed integration HCL parsing error handling.

modconfig/flowpipe_integration.go

@@ -173,7 +173,7 @@ func NewIntegration(mod *Mod, block *hcl.Block) IIntegration {
 			HclResourceImpl: HclResourceImpl{
 				// The FullName is the full name of the resource, including the mod name
 				FullName:        integrationName,
-				UnqualifiedName: "trigger." + integrationName,
+				UnqualifiedName: "integration." + block.Labels[0] + "." + block.Labels[1],
 				DeclRange:       block.DefRange,
 				blockType:       block.Type,
 			},
@@ -185,7 +185,7 @@ func NewIntegration(mod *Mod, block *hcl.Block) IIntegration {
 			HclResourceImpl: HclResourceImpl{
 				// The FullName is the full name of the resource, including the mod name
 				FullName:        integrationName,
-				UnqualifiedName: "pipeline." + block.Labels[0],
+				UnqualifiedName: "integration." + block.Labels[0] + "." + block.Labels[1],
 				DeclRange:       block.DefRange,
 				blockType:       block.Type,
 			},

modconfig/flowpipe_pipeline_step.go

@@ -180,48 +180,28 @@ func NewPipelineStep(stepType, stepName string) IPipelineStep {
 	var step IPipelineStep
 	switch stepType {
 	case schema.BlockTypePipelineStepHttp:
-		s := &PipelineStepHttp{}
-		step = s
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
+		step = &PipelineStepHttp{}
 	case schema.BlockTypePipelineStepSleep:
-		s := &PipelineStepSleep{}
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
-		step = s
+		step = &PipelineStepSleep{}
 	case schema.BlockTypePipelineStepEmail:
-		s := &PipelineStepEmail{}
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
-		step = s
+		step = &PipelineStepEmail{}
 	case schema.BlockTypePipelineStepEcho:
-		s := &PipelineStepEcho{}
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
-		step = s
+		step = &PipelineStepEcho{}
 	case schema.BlockTypePipelineStepQuery:
-		s := &PipelineStepQuery{}
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
-		step = s
+		step = &PipelineStepQuery{}
 	case schema.BlockTypePipelineStepPipeline:
-		s := &PipelineStepPipeline{}
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
-		step = s
+		step = &PipelineStepPipeline{}
 	case schema.BlockTypePipelineStepFunction:
-		s := &PipelineStepFunction{}
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
-		step = s
-
+		step = &PipelineStepFunction{}
 	case schema.BlockTypePipelineStepContainer:
-		s := &PipelineStepContainer{}
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
-		step = s
-
+		step = &PipelineStepContainer{}
 	case schema.BlockTypePipelineStepInput:
-		s := &PipelineStepInput{}
-		s.UnresolvedAttributes = make(map[string]hcl.Expression)
-		step = s
-
+		step = &PipelineStepInput{}
 	default:
 		return nil
 	}
 
+	step.Initialize()
 	step.SetName(stepName)
 	step.SetType(stepType)
 
@@ -230,6 +210,7 @@ func NewPipelineStep(stepType, stepName string) IPipelineStep {
 
 // A common interface that all pipeline steps must implement
 type IPipelineStep interface {
+	Initialize()
 	GetFullyQualifiedName() string
 	GetName() string
 	SetName(string)
@@ -240,6 +221,8 @@ type IPipelineStep interface {
 	IsResolved() bool
 	AddUnresolvedAttribute(string, hcl.Expression)
 	GetUnresolvedAttributes() map[string]hcl.Expression
+	AddUnresolvedBody(string, hcl.Body)
+	GetUnresolvedBodies() map[string]hcl.Body
 	GetInputs(*hcl.EvalContext) (map[string]interface{}, error)
 	GetDependsOn() []string
 	AppendDependsOn(...string)
@@ -321,9 +304,23 @@ type PipelineStepBase struct {
 
 	// This cant' be serialised
 	UnresolvedAttributes map[string]hcl.Expression `json:"-"`
+	UnresolvedBodies     map[string]hcl.Body       `json:"-"`
 	ForEach              hcl.Expression            `json:"-"`
 }
 
+func (p *PipelineStepBase) Initialize() {
+	p.UnresolvedAttributes = make(map[string]hcl.Expression)
+	p.UnresolvedBodies = make(map[string]hcl.Body)
+}
+
+func (p *PipelineStepBase) AddUnresolvedBody(name string, body hcl.Body) {
+	p.UnresolvedBodies[name] = body
+}
+
+func (p *PipelineStepBase) GetUnresolvedBodies() map[string]hcl.Body {
+	return p.UnresolvedBodies
+}
+
 func (p *PipelineStepBase) Equals(otherBase *PipelineStepBase) bool {
 	if p == nil || otherBase == nil {
 		return false
@@ -2786,6 +2783,53 @@ func (p *PipelineStepInput) SetAttributes(hclAttributes hcl.Attributes, evalCont
 	return diags
 }
 
+func (p *PipelineStepBase) HandleDecodeBodyDiags(diags hcl.Diagnostics, attributeName string, body hcl.Body) hcl.Diagnostics {
+	resolvedDiags := 0
+
+	for _, e := range diags {
+		if e.Severity == hcl.DiagError {
+			if e.Detail == `There is no variable named "step".` {
+				// traversals := expr.Variables()
+				// dependsOnAdded := false
+				// for _, traversal := range traversals {
+				// 	parts := hclhelpers.TraversalAsStringSlice(traversal)
+				// 	if len(parts) > 0 {
+				// 		// When the expression/traversal is referencing an index, the index is also included in the parts
+				// 		// for example: []string len: 5, cap: 5, ["step","sleep","sleep_1","0","duration"]
+				// 		if parts[0] == schema.BlockTypePipelineStep {
+				// 			dependsOn := parts[1] + "." + parts[2]
+				// 			p.AppendDependsOn(dependsOn)
+				// 			dependsOnAdded = true
+				// 		}
+				// 	}
+				// }
+				// if dependsOnAdded {
+				// 	resolvedDiags++
+				// }
+				resolvedDiags++
+			} else if e.Detail == `There is no variable named "each".` || e.Detail == `There is no variable named "param".` || e.Detail == "Unsuitable value: value must be known" {
+				// hcl.decodeBody returns 2 error messages:
+				// 1. There's no variable named "param", AND
+				// 2. Unsuitable value: value must be known
+				resolvedDiags++
+			} else {
+				return diags
+			}
+		}
+	}
+
+	// check if all diags have been resolved
+	if resolvedDiags == len(diags) {
+
+		// * Don't forget to add this, if you change the logic ensure that the code flow still
+		// * calls AddUnresolvedAttribute
+		p.AddUnresolvedBody(attributeName, body)
+		return hcl.Diagnostics{}
+	}
+	// There's an error here
+	return diags
+
+}
 func (p *PipelineStepInput) SetBlockConfig(blocks hcl.Blocks, evalContext *hcl.EvalContext) hcl.Diagnostics {
 	diags := hcl.Diagnostics{}
 
@@ -2795,8 +2839,11 @@ func (p *PipelineStepInput) SetBlockConfig(blocks hcl.Blocks, evalContext *hcl.E
 			notify := PipelineStepInputNotify{}
 			moreDiags := gohcl.DecodeBody(b.Body, evalContext, &notify)
 			if len(moreDiags) > 0 {
-				diags = append(diags, moreDiags...)
-				continue
+				moreDiags = p.PipelineStepBase.HandleDecodeBodyDiags(moreDiags, schema.BlockTypeNotify, b.Body)
+				if len(moreDiags) > 0 {
+					diags = append(diags, moreDiags...)
+					continue
+				}
 			}
 			p.Notify = &notify
 		default:

modconfig/parsed_property_path.go

@@ -45,6 +45,14 @@ func ParseResourcePropertyPath(propertyPath string) (*ParsedPropertyPath, error)
 	// <resource>.<name>.<property path...>
 	// so either the first or second slice must be a valid resource type
 
+	//
+	// unless they are some flowpipe resources:
+	//
+	// mod.trigger.trigger_type.trigger_name.<property_path>
+	// trigger.trigger_type.trigger_name.<property_path>
+	//
+	// We can have trigger and integration in this current format
+
 	parts := strings.Split(propertyPath, ".")
 	if len(parts) < 2 {
 		return nil, perr.BadRequestWithMessage("invalid property path: " + propertyPath)
@@ -73,10 +81,19 @@ func ParseResourcePropertyPath(propertyPath string) (*ParsedPropertyPath, error)
 		res.ItemType = parts[1]
 		res.Name = parts[2]
 	default:
-		res.Mod = parts[0]
-		res.ItemType = parts[1]
-		res.Name = parts[2]
-		res.PropertyPath = parts[3:]
+		if parts[1] == "integration" || parts[1] == "trigger" {
+			res.Mod = parts[0]
+			res.ItemType = parts[1]
+			res.Name = parts[2] + "." + parts[3]
+			if len(parts) > 4 {
+				res.PropertyPath = parts[3:]
+			}
+		} else {
+			res.Mod = parts[0]
+			res.ItemType = parts[1]
+			res.Name = parts[2]
+			res.PropertyPath = parts[3:]
+		}
 	}
 
 	if !schema.IsValidResourceItemType(res.ItemType) {

parse/mod.go

@@ -191,7 +191,7 @@ func ParseMod(fileData map[string][]byte, pseudoResources []modconfig.MappableRe
 	for attempts := 0; ; attempts++ {
 		diags = decode(parseCtx)
 		if diags.HasErrors() {
-			return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to decode all mod hcl files", diags))
+			return nil, error_helpers.NewErrorsAndWarning(plugin.DiagsToError("Failed to decode mod", diags))
 		}
 		// now retrieve the warning strings
 		res.AddWarning(plugin.DiagsToWarnings(diags)...)

parse/mod_parse_context.go

@@ -775,19 +775,4 @@ func (m *ModParseContext) AddTrigger(trigger *modconfig.Trigger) hcl.Diagnostics
 	return nil
 }
 
-func (m *ModParseContext) AddIntegration(integration modconfig.IIntegration) hcl.Diagnostics {
-
-	// Split and get the last part for pipeline name
-	parts := strings.Split(integration.Name(), ".")
-	integrationNameOnly := parts[len(parts)-1]
-
-	m.IntegrationHcls[integrationNameOnly] = integration
-
-	// remove this resource from unparsed blocks
-	delete(m.UnresolvedBlocks, integration.Name())
-
-	m.buildEvalContext()
-	return nil
-}
-
 // TODO: transition period

parse/pipeline_decode.go

@@ -327,6 +327,7 @@ func decodeIntegration(mod *modconfig.Mod, block *hcl.Block, parseCtx *ModParseC
 
 	diags = decodeHclBody(body, parseCtx.EvalCtx, parseCtx, integration)
 	if len(diags) > 0 {
+		res.handleDecodeDiags(diags)
 		return integration, res
 	}