From 0a4f7ecf6766eaf3731f166e92c8e085641443c0 Mon Sep 17 00:00:00 2001 From: Victor Hadianto Date: Wed, 25 Oct 2023 01:34:46 +0000 Subject: [PATCH] Migrated mod load test from flowpipe repo to pipe-fittings repo. --- modconfig/flowpipe_pipeline_step.go | 4 +- .../.flowpipe/mods/mod_child_a@v1.0.0/mod.sp | 13 + .../.flowpipe/mods/mod_child_b@v2.0.0/mod.hcl | 13 + .../mods/mod_child_b@v2.0.0/pipe_a.hcl | 9 + .../mod_child_b@v2.0.0/pipelines/pipe_a.sp | 9 + .../mod_child_b@v2.0.0/pipelines/pipes.hcl | 9 + .../backward_compatible_mod/.mod.cache.json | 18 + .../backward_compatible_mod/mod.hcl | 32 ++ .../pipelines/pipes.hcl | 9 + .../pipelines/pipes.sp | 9 + .../backward_compatible_mod/pipes.hcl | 20 + .../backward_compatible_mod/pipes.sp | 10 + .../flowpipe_mod_suite_test.go | 466 ++++++++++++++++++ tests/flowpipe_mod_tests/good_mod/mod.hcl | 69 +++ .../.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl | 13 + .../.flowpipe/mods/mod_child_b@v1.0.0/mod.hcl | 35 ++ .../mods/mod_child_b_1@v1.0.0/mod.hcl | 13 + .../mod_dep_one/.mod.cache.json | 18 + tests/flowpipe_mod_tests/mod_dep_one/mod.hcl | 85 ++++ .../.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl | 37 ++ .../mod_dep_simple/.mod.cache.json | 11 + .../mod_dep_simple/flowpipe.pvars | 1 + .../flowpipe_mod_tests/mod_dep_simple/mod.hcl | 60 +++ .../flowpipe_mod_tests/mod_references/mod.hcl | 41 ++ .../mod_variable/flowpipe.pvars | 3 + tests/flowpipe_mod_tests/mod_variable/mod.hcl | 178 +++++++ .../mod_variable/one.auto.pvars | 1 + .../mod_variable/triggers.hcl | 28 ++ .../mod_variable/two.auto.pvars | 2 + .../mod_with_step_output/mod.hcl | 37 ++ 30 files changed, 1252 insertions(+), 1 deletion(-) create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_a@v1.0.0/mod.sp create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/mod.hcl create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipe_a.hcl create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipelines/pipe_a.sp create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipelines/pipes.hcl create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/.mod.cache.json create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/mod.hcl create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/pipelines/pipes.hcl create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/pipelines/pipes.sp create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/pipes.hcl create mode 100644 tests/flowpipe_mod_tests/backward_compatible_mod/pipes.sp create mode 100644 tests/flowpipe_mod_tests/flowpipe_mod_suite_test.go create mode 100644 tests/flowpipe_mod_tests/good_mod/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_b@v1.0.0/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_b_1@v1.0.0/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_dep_one/.mod.cache.json create mode 100644 tests/flowpipe_mod_tests/mod_dep_one/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_dep_simple/.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_dep_simple/.mod.cache.json create mode 100644 tests/flowpipe_mod_tests/mod_dep_simple/flowpipe.pvars create mode 100644 tests/flowpipe_mod_tests/mod_dep_simple/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_references/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_variable/flowpipe.pvars create mode 100644 tests/flowpipe_mod_tests/mod_variable/mod.hcl create mode 100644 tests/flowpipe_mod_tests/mod_variable/one.auto.pvars create mode 100644 tests/flowpipe_mod_tests/mod_variable/triggers.hcl create mode 100644 tests/flowpipe_mod_tests/mod_variable/two.auto.pvars create mode 100644 tests/flowpipe_mod_tests/mod_with_step_output/mod.hcl diff --git a/modconfig/flowpipe_pipeline_step.go b/modconfig/flowpipe_pipeline_step.go index 92932119..7d37dffc 100644 --- a/modconfig/flowpipe_pipeline_step.go +++ b/modconfig/flowpipe_pipeline_step.go @@ -2907,7 +2907,9 @@ func (p *PipelineStepInput) Validate() hcl.Diagnostics { } } - if p.Notify != nil { + // Only validate the notify block if there are no unresolved bodies + // TODO: this is not correct fix this + if p.Notify != nil && p.UnresolvedBodies[schema.BlockTypeNotify] == nil { moreDiags := p.Notify.Validate() if len(moreDiags) > 0 { diags = append(diags, moreDiags...) diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_a@v1.0.0/mod.sp b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_a@v1.0.0/mod.sp new file mode 100644 index 00000000..a998a3bd --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_a@v1.0.0/mod.sp @@ -0,0 +1,13 @@ +mod "mod_child_a" { + title = "Child Mod A" +} + +pipeline "this_pipeline_is_in_the_child" { + step "echo" "foo" { + text = "foo" + } + + output "foo_a" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/mod.hcl b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/mod.hcl new file mode 100644 index 00000000..375477ba --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/mod.hcl @@ -0,0 +1,13 @@ +mod "mod_child_b" { + title = "Child Mod b" +} + +pipeline "this_pipeline_is_in_the_child" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipe_a.hcl b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipe_a.hcl new file mode 100644 index 00000000..0a9df658 --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipe_a.hcl @@ -0,0 +1,9 @@ +pipeline "another_child_pipeline" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipelines/pipe_a.sp b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipelines/pipe_a.sp new file mode 100644 index 00000000..38846c07 --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipelines/pipe_a.sp @@ -0,0 +1,9 @@ +pipeline "second_pipe_in_the_child" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipelines/pipes.hcl b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipelines/pipes.hcl new file mode 100644 index 00000000..c4581d46 --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/.flowpipe/mods/mod_child_b@v2.0.0/pipelines/pipes.hcl @@ -0,0 +1,9 @@ +pipeline "nested_pipe_in_child_hcl" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/.mod.cache.json b/tests/flowpipe_mod_tests/backward_compatible_mod/.mod.cache.json new file mode 100644 index 00000000..9b894bf6 --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/.mod.cache.json @@ -0,0 +1,18 @@ +{ + "mod_parent": { + "mod_child_a": { + "name": "mod_child_a", + "alias": "mod_child_a", + "version": "1.0.0", + "constraint": "*", + "struct_version": 20220411 + }, + "mod_child_b": { + "name": "mod_child_b", + "alias": "mod_child_b", + "version": "2.0.0", + "constraint": "*", + "struct_version": 20220411 + } + } + } \ No newline at end of file diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/mod.hcl b/tests/flowpipe_mod_tests/backward_compatible_mod/mod.hcl new file mode 100644 index 00000000..0f500d52 --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/mod.hcl @@ -0,0 +1,32 @@ +mod "mod_parent" { + title = "Parent Mod" + require { + mod "mod_child_a" { + version = "1.0.0" + } + mod "mod_child_b" { + version = "2.0.0" + } + } +} + +pipeline "json" { + step "echo" "json" { + json = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:Describe*", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) + } + + output "foo" { + value = step.echo.json.json + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/pipelines/pipes.hcl b/tests/flowpipe_mod_tests/backward_compatible_mod/pipelines/pipes.hcl new file mode 100644 index 00000000..fd98ae56 --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/pipelines/pipes.hcl @@ -0,0 +1,9 @@ +pipeline "parent_pipeline_hcl_nested" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/pipelines/pipes.sp b/tests/flowpipe_mod_tests/backward_compatible_mod/pipelines/pipes.sp new file mode 100644 index 00000000..829a5be7 --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/pipelines/pipes.sp @@ -0,0 +1,9 @@ +pipeline "parent_pipeline_sp_nested" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/pipes.hcl b/tests/flowpipe_mod_tests/backward_compatible_mod/pipes.hcl new file mode 100644 index 00000000..e1ebca4f --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/pipes.hcl @@ -0,0 +1,20 @@ +pipeline "parent_pipeline_hcl" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} + + +pipeline "parent_pipeline_hcl_b" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/backward_compatible_mod/pipes.sp b/tests/flowpipe_mod_tests/backward_compatible_mod/pipes.sp new file mode 100644 index 00000000..ba671b1f --- /dev/null +++ b/tests/flowpipe_mod_tests/backward_compatible_mod/pipes.sp @@ -0,0 +1,10 @@ +pipeline "parent_pipeline_sp" { + step "echo" "foo" { + text = "foo" + } + + output "foo_b" { + value = step.echo.foo.text + } +} + diff --git a/tests/flowpipe_mod_tests/flowpipe_mod_suite_test.go b/tests/flowpipe_mod_tests/flowpipe_mod_suite_test.go new file mode 100644 index 00000000..e56edf6a --- /dev/null +++ b/tests/flowpipe_mod_tests/flowpipe_mod_suite_test.go @@ -0,0 +1,466 @@ +package pipeline_test + +import ( + "context" + "os" + "path" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/turbot/pipe-fittings/constants" + "github.com/turbot/pipe-fittings/filepaths" + "github.com/turbot/pipe-fittings/modconfig" + "github.com/turbot/pipe-fittings/workspace" +) + +type FlowpipeModTestSuite struct { + suite.Suite + SetupSuiteRunCount int + TearDownSuiteRunCount int + ctx context.Context +} + +func (suite *FlowpipeModTestSuite) SetupSuite() { + + err := os.Setenv("RUN_MODE", "TEST_ES") + if err != nil { + panic(err) + } + + // Get the current working directory + cwd, err := os.Getwd() + if err != nil { + panic(err) + } + + // clear the output dir before each test + outputPath := path.Join(cwd, "output") + + // Check if the directory exists + _, err = os.Stat(outputPath) + if !os.IsNotExist(err) { + // Remove the directory and its contents + err = os.RemoveAll(outputPath) + if err != nil { + panic(err) + } + + } + + pipelineDirPath := path.Join(cwd, "pipelines") + + viper.GetViper().Set("pipeline.dir", pipelineDirPath) + viper.GetViper().Set("output.dir", outputPath) + viper.GetViper().Set("log.dir", outputPath) + + // Create a single, global context for the application + ctx := context.Background() + + suite.ctx = ctx + + filepaths.PipesComponentWorkspaceDataDir = ".flowpipe" + filepaths.PipesComponentModsFileName = "mod.hcl" + filepaths.PipesComponentDefaultVarsFileName = "flowpipe.pvars" + filepaths.PipesComponentDefaultInstallDir = "~/.flowpipe" + + constants.PipesComponentModDataExtension = ".hcl" + constants.PipesComponentVariablesExtension = ".pvars" + constants.PipesComponentAutoVariablesExtension = ".auto.pvars" + constants.PipesComponentEnvInputVarPrefix = "P_VAR_" + constants.PipesComponentAppName = "flowpipe" + + suite.SetupSuiteRunCount++ +} + +// The TearDownSuite method will be run by testify once, at the very +// end of the testing suite, after all tests have been run. +func (suite *FlowpipeModTestSuite) TearDownSuite() { + suite.TearDownSuiteRunCount++ +} + +func (suite *FlowpipeModTestSuite) TestGoodMod() { + assert := assert.New(suite.T()) + + w, errorAndWarning := workspace.LoadWithParams(suite.ctx, "./good_mod", []string{".hcl"}) + + assert.NotNil(w) + assert.Nil(errorAndWarning.Error) + + mod := w.Mod + if mod == nil { + assert.Fail("mod is nil") + return + } + + // check if all pipelines are there + pipelines := mod.ResourceMaps.Pipelines + assert.Equal(len(pipelines), 3, "wrong number of pipelines") + + jsonForPipeline := pipelines["test_mod.pipeline.json_for"] + if jsonForPipeline == nil { + assert.Fail("json_for pipeline not found") + return + } + + // check if all steps are there + assert.Equal(2, len(jsonForPipeline.Steps), "wrong number of steps") + assert.Equal(jsonForPipeline.Steps[0].GetName(), "json", "wrong step name") + assert.Equal(jsonForPipeline.Steps[0].GetType(), "echo", "wrong step type") + assert.Equal(jsonForPipeline.Steps[1].GetName(), "json_for", "wrong step name") + assert.Equal(jsonForPipeline.Steps[1].GetType(), "echo", "wrong step type") + + // check if all triggers are there + triggers := mod.ResourceMaps.Triggers + assert.Equal(1, len(triggers), "wrong number of triggers") + assert.Equal("test_mod.trigger.schedule.my_hourly_trigger", triggers["test_mod.trigger.schedule.my_hourly_trigger"].FullName, "wrong trigger name") +} + +func (suite *FlowpipeModTestSuite) TestModReferences() { + assert := assert.New(suite.T()) + + w, errorAndWarning := workspace.LoadWithParams(suite.ctx, "./mod_references", []string{".hcl"}) + + assert.NotNil(w) + assert.Nil(errorAndWarning.Error) + + mod := w.Mod + if mod == nil { + assert.Fail("mod is nil") + return + } + + // check if all pipelines are there + pipelines := mod.ResourceMaps.Pipelines + assert.NotNil(pipelines, "pipelines is nil") + assert.Equal(2, len(pipelines), "wrong number of pipelines") + assert.NotNil(pipelines["pipeline_with_references.pipeline.foo"]) + assert.NotNil(pipelines["pipeline_with_references.pipeline.foo_two"]) +} + +func (suite *FlowpipeModTestSuite) TestStepOutputParsing() { + assert := assert.New(suite.T()) + + w, errorAndWarning := workspace.LoadWithParams(suite.ctx, "./mod_with_step_output", []string{".hcl"}) + + assert.NotNil(w) + assert.Nil(errorAndWarning.Error) + + mod := w.Mod + if mod == nil { + assert.Fail("mod is nil") + return + } + + // check if all pipelines are there + pipelines := mod.ResourceMaps.Pipelines + assert.NotNil(pipelines, "pipelines is nil") + assert.Equal(1, len(pipelines), "wrong number of pipelines") + + assert.Equal(2, len(pipelines["test_mod.pipeline.with_step_output"].Steps), "wrong number of steps") + assert.False(pipelines["test_mod.pipeline.with_step_output"].Steps[0].IsResolved()) + assert.False(pipelines["test_mod.pipeline.with_step_output"].Steps[1].IsResolved()) + +} + +func (suite *FlowpipeModTestSuite) TestModDependencies() { + assert := assert.New(suite.T()) + + w, errorAndWarning := workspace.LoadWithParams(suite.ctx, "./mod_dep_one", []string{".hcl"}) + + assert.NotNil(w) + assert.Nil(errorAndWarning.Error) + + mod := w.Mod + if mod == nil { + assert.Fail("mod is nil") + return + } + + pipelines := mod.ResourceMaps.Pipelines + + assert.NotNil(mod, "mod is nil") + jsonForPipeline := pipelines["mod_parent.pipeline.json"] + if jsonForPipeline == nil { + assert.Fail("json pipeline not found") + return + } + + fooPipeline := pipelines["mod_parent.pipeline.foo"] + if fooPipeline == nil { + assert.Fail("foo pipeline not found") + return + } + + fooTwoPipeline := pipelines["mod_parent.pipeline.foo_two"] + if fooTwoPipeline == nil { + assert.Fail("foo_two pipeline not found") + return + } + + referToChildPipeline := pipelines["mod_parent.pipeline.refer_to_child"] + if referToChildPipeline == nil { + assert.Fail("foo pipeline not found") + return + } + + referToChildBPipeline := pipelines["mod_parent.pipeline.refer_to_child_b"] + if referToChildBPipeline == nil { + assert.Fail("refer_to_child_b pipeline not found") + return + } + + childModA := mod.ResourceMaps.Mods["mod_child_a@v1.0.0"] + assert.NotNil(childModA) + + thisPipelineIsInTheChildPipelineModA := childModA.ResourceMaps.Pipelines["mod_child_a.pipeline.this_pipeline_is_in_the_child"] + assert.NotNil(thisPipelineIsInTheChildPipelineModA) + + // check for the triggers + triggers := mod.ResourceMaps.Triggers + myHourlyTrigger := triggers["mod_parent.trigger.schedule.my_hourly_trigger"] + if myHourlyTrigger == nil { + assert.Fail("my_hourly_trigger not found") + return + } + +} + +func (suite *FlowpipeModTestSuite) TestModDependenciesSimple() { + assert := assert.New(suite.T()) + + w, errorAndWarning := workspace.LoadWithParams(suite.ctx, "./mod_dep_simple", []string{".hcl"}) + + assert.NotNil(w) + assert.Nil(errorAndWarning.Error) + + mod := w.Mod + if mod == nil { + assert.Fail("mod is nil") + return + } + + pipelines := mod.ResourceMaps.Pipelines + jsonForPipeline := pipelines["mod_parent.pipeline.json"] + if jsonForPipeline == nil { + assert.Fail("json pipeline not found") + return + } + + fooPipeline := pipelines["mod_parent.pipeline.foo"] + if fooPipeline == nil { + assert.Fail("foo pipeline not found") + return + } + + assert.Equal(2, len(fooPipeline.Steps), "wrong number of steps") + assert.Equal("baz", fooPipeline.Steps[0].GetName()) + assert.Equal("bar", fooPipeline.Steps[1].GetName()) + + referToChildPipeline := pipelines["mod_parent.pipeline.refer_to_child"] + if referToChildPipeline == nil { + assert.Fail("foo pipeline not found") + return + } + + childPipeline := pipelines["mod_child_a.pipeline.this_pipeline_is_in_the_child"] + if childPipeline == nil { + assert.Fail("this_pipeline_is_in_the_child pipeline not found") + return + } + + childPipelineWithVar := pipelines["mod_child_a.pipeline.this_pipeline_is_in_the_child_using_variable"] + if childPipelineWithVar == nil { + assert.Fail("this_pipeline_is_in_the_child pipeline not found") + return + } + + assert.Equal("foo: this is the value of var_one", childPipelineWithVar.Steps[0].(*modconfig.PipelineStepEcho).Text) + + childPipelineWithVarPassedFromParent := pipelines["mod_child_a.pipeline.this_pipeline_is_in_the_child_using_variable_passed_from_parent"] + if childPipelineWithVarPassedFromParent == nil { + assert.Fail("this_pipeline_is_in_the_child pipeline not found") + return + } + + assert.Equal("foo: var_two from parent .pvars file", childPipelineWithVarPassedFromParent.Steps[0].(*modconfig.PipelineStepEcho).Text) +} + +func (suite *FlowpipeModTestSuite) TestModDependenciesBackwardCompatible() { + assert := assert.New(suite.T()) + + w, errorAndWarning := workspace.LoadWithParams(suite.ctx, "./backward_compatible_mod", []string{".hcl", ".sp"}) + + assert.NotNil(w) + assert.Nil(errorAndWarning.Error) + + mod := w.Mod + if mod == nil { + assert.Fail("mod is nil") + return + } + + pipelines := mod.ResourceMaps.Pipelines + + // TODO: need to fix this + assert.Equal(11, len(pipelines), "wrong number of pipelines") + + assert.NotNil(mod, "mod is nil") + jsonForPipeline := pipelines["mod_parent.pipeline.json"] + if jsonForPipeline == nil { + assert.Fail("json pipeline not found") + return + } + + parentPipelineHcl := pipelines["mod_parent.pipeline.parent_pipeline_hcl"] + assert.NotNil(parentPipelineHcl) + + parentPipelineHclB := pipelines["mod_parent.pipeline.parent_pipeline_hcl_b"] + assert.NotNil(parentPipelineHclB) + + parentPipelineHclNested := pipelines["mod_parent.pipeline.parent_pipeline_hcl_nested"] + assert.NotNil(parentPipelineHclNested) + + // This should be nil, there was a bug that was causing the child pipelines to be loaded in the parent mod + thisPipelineIsInTheChildParent := pipelines["mod_parent.pipeline.this_pipeline_is_in_the_child"] + assert.Nil(thisPipelineIsInTheChildParent) + + nestedPipeInChildHclParent := pipelines["mod_parent.pipeline.nested_pipe_in_child_hcl"] + assert.Nil(nestedPipeInChildHclParent) + + // SP file format + parentPipelineSp := pipelines["mod_parent.pipeline.parent_pipeline_sp"] + assert.NotNil(parentPipelineSp) + + parentPipelineSpNested := pipelines["mod_parent.pipeline.parent_pipeline_sp_nested"] + assert.NotNil(parentPipelineSpNested) + + childModA := mod.ResourceMaps.Mods["mod_child_a@v1.0.0"] + assert.NotNil(childModA) + + thisPipelineIsInTheChildPipelineModA := childModA.ResourceMaps.Pipelines["mod_child_a.pipeline.this_pipeline_is_in_the_child"] + assert.NotNil(thisPipelineIsInTheChildPipelineModA) + + childModB := mod.ResourceMaps.Mods["mod_child_b@v2.0.0"] + assert.NotNil(childModB) + + thisPipelineIsInTheChildPipelineModB := childModB.ResourceMaps.Pipelines["mod_child_b.pipeline.this_pipeline_is_in_the_child"] + assert.NotNil(thisPipelineIsInTheChildPipelineModB) + + anotherChildPipelineModB := childModB.ResourceMaps.Pipelines["mod_child_b.pipeline.another_child_pipeline"] + assert.NotNil(anotherChildPipelineModB) + + secondPipeInTheChildModB := childModB.ResourceMaps.Pipelines["mod_child_b.pipeline.second_pipe_in_the_child"] + assert.NotNil(secondPipeInTheChildModB) + + nestedPipeInTheChildModB := childModB.ResourceMaps.Pipelines["mod_child_b.pipeline.nested_pipe_in_child_hcl"] + assert.NotNil(nestedPipeInTheChildModB) +} + +func (suite *FlowpipeModTestSuite) TestModVariable() { + assert := assert.New(suite.T()) + + os.Setenv("P_VAR_var_six", "set from env var") + + w, errorAndWarning := workspace.LoadWithParams(suite.ctx, "./mod_variable", []string{".hcl", ".sp"}) + + assert.NotNil(w) + assert.Nil(errorAndWarning.Error) + + mod := w.Mod + if mod == nil { + assert.Fail("mod is nil") + return + } + + pipelines := mod.ResourceMaps.Pipelines + pipelineOne := pipelines["test_mod.pipeline.one"] + if pipelineOne == nil { + assert.Fail("pipeline one not found") + return + } + + assert.Equal("prefix text here and this is the value of var_one and suffix", pipelineOne.Steps[0].(*modconfig.PipelineStepEcho).Text) + assert.Equal("prefix text here and value from var file and suffix", pipelineOne.Steps[1].(*modconfig.PipelineStepEcho).Text) + assert.Equal("prefix text here and var_three from var file and suffix", pipelineOne.Steps[2].(*modconfig.PipelineStepEcho).Text) + + assert.True(pipelineOne.Steps[0].IsResolved()) + assert.True(pipelineOne.Steps[1].IsResolved()) + assert.True(pipelineOne.Steps[2].IsResolved()) + + // step echo.one_echo should not be resolved, it has reference to echo.one step + assert.False(pipelineOne.Steps[3].IsResolved()) + + assert.Equal("using value from locals: value of locals_one", pipelineOne.Steps[4].(*modconfig.PipelineStepEcho).Text) + assert.Equal("using value from locals: 10", pipelineOne.Steps[5].(*modconfig.PipelineStepEcho).Text) + assert.Equal("using value from locals: value of key_two", pipelineOne.Steps[6].(*modconfig.PipelineStepEcho).Text) + assert.Equal("using value from locals: value of key_two", pipelineOne.Steps[7].(*modconfig.PipelineStepEcho).Text) + assert.Equal("using value from locals: 33", pipelineOne.Steps[8].(*modconfig.PipelineStepEcho).Text) + assert.Equal("var_four value is: value from auto.vars file", pipelineOne.Steps[9].(*modconfig.PipelineStepEcho).Text) + assert.Equal("var_five value is: value from two.auto.vars file", pipelineOne.Steps[10].(*modconfig.PipelineStepEcho).Text) + assert.Equal("var_six value is: set from env var", pipelineOne.Steps[11].(*modconfig.PipelineStepEcho).Text) + + githubIssuePipeline := pipelines["test_mod.pipeline.github_issue"] + if githubIssuePipeline == nil { + assert.Fail("github_issue pipeline not found") + return + } + + assert.Equal(1, len(githubIssuePipeline.Params)) + assert.NotNil(githubIssuePipeline.Params["gh_repo"]) + assert.Equal("hello-world", githubIssuePipeline.Params["gh_repo"].Default.AsString()) + + githubGetIssueWithNumber := pipelines["test_mod.pipeline.github_get_issue_with_number"] + if githubGetIssueWithNumber == nil { + assert.Fail("github_get_issue_with_number pipeline not found") + return + } + + assert.Equal(2, len(githubGetIssueWithNumber.Params)) + assert.Equal("cty.String", githubGetIssueWithNumber.Params["github_token"].Type.GoString()) + assert.Equal("cty.Number", githubGetIssueWithNumber.Params["github_issue_number"].Type.GoString()) + + triggers := mod.ResourceMaps.Triggers + + if len(triggers) == 0 { + assert.Fail("triggers not loaded") + return + } + + reportTrigger := triggers["test_mod.trigger.schedule.report_triggers"] + if reportTrigger == nil { + assert.Fail("report_triggers not found") + return + } + + _, ok := reportTrigger.Config.(*modconfig.TriggerSchedule) + assert.True(ok, "report_triggers is not of type TriggerSchedule") + + // Check the schedule with the default var + reportTriggersWithScheduleVarWithDefaultValue := triggers["test_mod.trigger.schedule.report_triggers_with_schedule_var_with_default_value"] + if reportTriggersWithScheduleVarWithDefaultValue == nil { + assert.Fail("report_triggers_with_schedule_var_with_default_value not found") + return + } + configSchedule := reportTriggersWithScheduleVarWithDefaultValue.Config.(*modconfig.TriggerSchedule) + + // This value is set in the pvar file + assert.Equal("5 * * * *", configSchedule.Schedule) + + reportTriggersWithIntervalVarWithDefaultValue := triggers["test_mod.trigger.interval.report_triggers_with_interval_var_with_default_value"] + if reportTriggersWithIntervalVarWithDefaultValue == nil { + assert.Fail("report_triggers_with_interval_var_with_default_value not found") + return + } + + intervalSchedule := reportTriggersWithIntervalVarWithDefaultValue.Config.(*modconfig.TriggerInterval) + assert.Equal("weekly", intervalSchedule.Schedule) + +} + +// In order for 'go test' to run this suite, we need to create +// a normal test function and pass our suite to suite.Run +func TestFpTestSuite(t *testing.T) { + suite.Run(t, new(FlowpipeModTestSuite)) +} diff --git a/tests/flowpipe_mod_tests/good_mod/mod.hcl b/tests/flowpipe_mod_tests/good_mod/mod.hcl new file mode 100644 index 00000000..754a236f --- /dev/null +++ b/tests/flowpipe_mod_tests/good_mod/mod.hcl @@ -0,0 +1,69 @@ +mod "test_mod" { + title = "my_mod" +} + +trigger "schedule" "my_hourly_trigger" { + schedule = "5 * * * *" + # this is valid, but not necessary + # pipeline = local.pipeline.simple_with_trigger + # this is the recommended way to refer to another resource within the same mod + pipeline = pipeline.simple_with_trigger + + # this is the way to refer to a pipeline in another mod + # pipeline = another_mod.pipeline.another_pipeline + + # you can't refer to nested mods + # pipeline = another_mod.that_other_mod_dependencies.pipeline.that_pipeline + + # http://localhost:7103/api/v0/pipeline/local.pipeline.simple_with_trigger/cmd + + # should this work if simple_with_trigger is the top level mod? + # http://localhost:7103/api/v0/pipeline/simple_with_trigger/cmd + + # if there's no mod.sp + # http://localhost:7103/api/v0/pipeline/foo/cmd + + # .pipeline. +} + + +pipeline "json" { + step "echo" "json" { + json = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:Describe*", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) + } +} + +pipeline "json_for" { + step "echo" "json" { + json = jsonencode({ + Version = "2012-10-17" + Users = ["jeff", "jerry", "jim"] + }) + } + + + step "echo" "json_for" { + for_each = step.echo.json.Users + text = "user: ${each.value}" + } +} + +pipeline "simple_with_trigger" { + description = "simple pipeline that will be referred to by a trigger" + + step "echo" "simple_echo" { + text = "foo bar" + } +} + diff --git a/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl b/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl new file mode 100644 index 00000000..a998a3bd --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl @@ -0,0 +1,13 @@ +mod "mod_child_a" { + title = "Child Mod A" +} + +pipeline "this_pipeline_is_in_the_child" { + step "echo" "foo" { + text = "foo" + } + + output "foo_a" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_b@v1.0.0/mod.hcl b/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_b@v1.0.0/mod.hcl new file mode 100644 index 00000000..412eb94d --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_b@v1.0.0/mod.hcl @@ -0,0 +1,35 @@ +mod "mod_child_b" { + title = "Child Mod B" +} + +pipeline "this_pipeline_is_in_the_child_b" { + step "echo" "foo" { + text = "foo" + } + + step "echo" "baz" { + text = "baz" + } + + output "foo_a" { + value = step.echo.foo.text + } + + # this will test resolving a step in a child pipeline + # when the mod is a dependency of the parent + step "pipeline" "child_pipeline" { + pipeline = pipeline.foo_two + } + +} + + +pipeline "foo_two" { + step "echo" "baz" { + text = "foo" + } + + output "foo" { + value = echo.baz.text + } +} diff --git a/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_b_1@v1.0.0/mod.hcl b/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_b_1@v1.0.0/mod.hcl new file mode 100644 index 00000000..f1dbc005 --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_one/.flowpipe/mods/mod_child_b_1@v1.0.0/mod.hcl @@ -0,0 +1,13 @@ +mod "mod_child_b" { + title = "Child Mod B" +} + +pipeline "this_pipeline_is_in_the_child" { + step "echo" "foo" { + text = "foo" + } + + output "foo_a" { + value = step.echo.foo.text + } +} diff --git a/tests/flowpipe_mod_tests/mod_dep_one/.mod.cache.json b/tests/flowpipe_mod_tests/mod_dep_one/.mod.cache.json new file mode 100644 index 00000000..054e7c1d --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_one/.mod.cache.json @@ -0,0 +1,18 @@ +{ + "mod_parent": { + "mod_child_a": { + "name": "mod_child_a", + "alias": "mod_child_a", + "version": "1.0.0", + "constraint": "*", + "struct_version": 20220411 + }, + "mod_child_b": { + "name": "mod_child_b", + "alias": "mod_child_b", + "version": "1.0.0", + "constraint": "*", + "struct_version": 20220411 + } + } + } \ No newline at end of file diff --git a/tests/flowpipe_mod_tests/mod_dep_one/mod.hcl b/tests/flowpipe_mod_tests/mod_dep_one/mod.hcl new file mode 100644 index 00000000..368789ef --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_one/mod.hcl @@ -0,0 +1,85 @@ +mod "mod_parent" { + title = "Parent Mod" + require { + mod "mod_child_a" { + version = "1.0.0" + } + mod "mod_child_b" { + version = "1.0.0" + } + } +} + +# declare trigger before the pipeline to test forward reference +trigger "schedule" "my_hourly_trigger" { + schedule = "5 * * * *" + pipeline = pipeline.refer_to_child +} + +pipeline "json" { + step "echo" "json" { + json = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:Describe*", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) + } + + output "foo" { + value = step.echo.json.json + } +} + +pipeline "refer_to_child" { + step "pipeline" "child_output" { + pipeline = mod_child_a.pipeline.this_pipeline_is_in_the_child + } +} + +pipeline "refer_to_child_b" { + step "pipeline" "child_output" { + pipeline = mod_child_b.pipeline.foo_two + } +} + + +pipeline "foo" { + + # leave this here to ensure that references that is later than the resource can be resolved + # + # we parse the HCL files from top to bottom, so putting this step `baz` after `bar` is the easier path + # reversing is the a harder parse + step "echo" "baz" { + text = step.echo.bar + } + + step "echo" "bar" { + text = "test" + } + + step "pipeline" "child_pipeline" { + pipeline = pipeline.foo_two + } + + step "echo" "child_pipeline" { + text = step.pipeline.child_pipeline.foo + } +} + + +pipeline "foo_two" { + step "echo" "baz" { + text = "foo" + } + + output "foo" { + value = echo.baz.text + } +} diff --git a/tests/flowpipe_mod_tests/mod_dep_simple/.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl b/tests/flowpipe_mod_tests/mod_dep_simple/.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl new file mode 100644 index 00000000..f789255a --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_simple/.flowpipe/mods/mod_child_a@v1.0.0/mod.hcl @@ -0,0 +1,37 @@ +mod "mod_child_a" { + title = "Child Mod A" +} + +variable "var_one" { + type = string + description = "test variable" + default = "this is the value of var_one" +} + +variable "var_two" { + type = string + description = "test variable" + default = "this is the value of var_two" +} + +pipeline "this_pipeline_is_in_the_child" { + step "echo" "foo" { + text = "foo" + } + + output "foo_a" { + value = step.echo.foo.text + } +} + +pipeline "this_pipeline_is_in_the_child_using_variable" { + step "echo" "foo" { + text = "foo: ${var.var_one}" + } +} + +pipeline "this_pipeline_is_in_the_child_using_variable_passed_from_parent" { + step "echo" "foo" { + text = "foo: ${var.var_two}" + } +} diff --git a/tests/flowpipe_mod_tests/mod_dep_simple/.mod.cache.json b/tests/flowpipe_mod_tests/mod_dep_simple/.mod.cache.json new file mode 100644 index 00000000..2cd81ca8 --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_simple/.mod.cache.json @@ -0,0 +1,11 @@ +{ + "mod_parent": { + "mod_child_a": { + "name": "mod_child_a", + "alias": "mod_child_a", + "version": "1.0.0", + "constraint": "*", + "struct_version": 20220411 + } + } + } \ No newline at end of file diff --git a/tests/flowpipe_mod_tests/mod_dep_simple/flowpipe.pvars b/tests/flowpipe_mod_tests/mod_dep_simple/flowpipe.pvars new file mode 100644 index 00000000..0f8a99fd --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_simple/flowpipe.pvars @@ -0,0 +1 @@ +var_two_parent = "var_two from parent .pvars file" \ No newline at end of file diff --git a/tests/flowpipe_mod_tests/mod_dep_simple/mod.hcl b/tests/flowpipe_mod_tests/mod_dep_simple/mod.hcl new file mode 100644 index 00000000..406cd610 --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_dep_simple/mod.hcl @@ -0,0 +1,60 @@ +mod "mod_parent" { + title = "Parent Mod" + require { + mod "mod_child_a" { + version = "1.0.0" + args = { + var_two = var.var_two_parent, + } + } + } +} + + +variable "var_two_parent" { + type = string + description = "test variable" +} + +pipeline "json" { + step "echo" "json" { + json = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "ec2:Describe*", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) + } + + output "foo" { + value = step.echo.json.json + } +} + +pipeline "foo" { + + # leave this here to ensure that references that is later than the resource can be resolved + # + # we parse the HCL files from top to bottom, so putting this step `baz` after `bar` is the easier path + # reversing is the a harder parse + step "echo" "baz" { + text = step.echo.bar + } + + step "echo" "bar" { + text = "test" + } +} + + +pipeline "refer_to_child" { + step "pipeline" "child_output" { + pipeline = mod_child_a.pipeline.this_pipeline_is_in_the_child + } +} diff --git a/tests/flowpipe_mod_tests/mod_references/mod.hcl b/tests/flowpipe_mod_tests/mod_references/mod.hcl new file mode 100644 index 00000000..1e1108f0 --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_references/mod.hcl @@ -0,0 +1,41 @@ + + +mod "pipeline_with_references" { + title = "Test mod" + description = "Use this mod for testing references within pipeline and from one pipeline to another" +} + + +pipeline "foo" { + + # leave this here to ensure that references that is later than the resource can be resolved + # + # we parse the HCL files from top to bottom, so putting this step `baz` after `bar` is the easier path + # reversing is the a harder parse + step "echo" "baz" { + text = step.echo.bar + } + + step "echo" "bar" { + text = "test" + } + + step "pipeline" "child_pipeline" { + pipeline = pipeline.foo_two + } + + step "echo" "child_pipeline" { + text = step.pipeline.child_pipeline.foo + } +} + + +pipeline "foo_two" { + step "echo" "baz" { + text = "foo" + } + + output "foo" { + value = echo.baz.text + } +} diff --git a/tests/flowpipe_mod_tests/mod_variable/flowpipe.pvars b/tests/flowpipe_mod_tests/mod_variable/flowpipe.pvars new file mode 100644 index 00000000..6bf6c0b2 --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_variable/flowpipe.pvars @@ -0,0 +1,3 @@ + +var_two = "value from var file" +var_three = "var_three from var file" \ No newline at end of file diff --git a/tests/flowpipe_mod_tests/mod_variable/mod.hcl b/tests/flowpipe_mod_tests/mod_variable/mod.hcl new file mode 100644 index 00000000..124a97b0 --- /dev/null +++ b/tests/flowpipe_mod_tests/mod_variable/mod.hcl @@ -0,0 +1,178 @@ +mod "test_mod" { + title = "my_mod" +} + +variable "schedule_default" { + type = string + description = "schedule with default value" + default = "5 * * * *" +} + +variable "interval_default" { + type = string + description = "interval with default value" + default = "weekly" +} + +variable "var_one" { + type = string + description = "test variable" + default = "this is the value of var_one" +} + +# var_two will be overriden in the test +variable "var_two" { + type = string + description = "test variable" + default = "default of var_two" +} + + +# var_three has no default +variable "var_three" { + type = string + description = "test variable" +} + +# var_four has no default +variable "var_four" { + type = string + description = "test variable" +} + +# var_five has no default +variable "var_five" { + type = string + description = "test variable" +} + +# var_six has no default +variable "var_six" { + type = string + description = "test variable" +} + +pipeline "one" { + step "echo" "one" { + text = "prefix text here and ${var.var_one} and suffix" + } + + step "echo" "two" { + text = "prefix text here and ${var.var_two} and suffix" + } + + step "echo" "three" { + text = "prefix text here and ${var.var_three} and suffix" + } + + step "echo" "one_echo" { + text = "got prefix? ${step.echo.one.text} and again ${step.echo.one.text} and var ${var.var_one}" + } + + + step "echo" "four" { + text = "using value from locals: ${local.locals_one}" + } + + step "echo" "five" { + text = "using value from locals: ${local.locals_two}" + } + + step "echo" "six" { + text = "using value from locals: ${local.locals_three.key_two}" + } + + step "echo" "seven" { + text = "using value from locals: ${local.locals_three_merge.key_two}" + } + + step "echo" "eight" { + text = "using value from locals: ${local.locals_three_merge.key_three}" + } + + step "echo" "eight" { + text = "var_four value is: ${var.var_four}" + } + + step "echo" "nine" { + text = "var_five value is: ${var.var_five}" + } + + step "echo" "ten" { + text = "var_six value is: ${var.var_six}" + } +} + + +variable "default_gh_repo" { + type = string + default = "hello-world" +} + +pipeline "github_issue" { + param "gh_repo" { + type = string + default = var.default_gh_repo + } + + step "http" "get_issue" { + url = "https://api.github.com/repos/octocat/${param.gh_repo}/issues/2743" + } +} + + +pipeline "github_get_issue_with_number" { + + param "github_token" { + type = string + } + + param "github_issue_number" { + type = number + } + + step "http" "get_issue" { + title = "Get details about an issue" + method = "post" + url = "https://api.github.com/graphql" + request_headers = { + Content-Type = "application/json" + Authorization = "Bearer ${param.github_token}" + } + + request_body = jsonencode({ + query = <