From 15b43e56709f4ee1d480631cf82d672264208d9f Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:08:29 -0400 Subject: [PATCH] fix(revert): use buildkite as yaml parser (#392) --- go.mod | 2 +- go.sum | 4 +- library/string.go | 2 +- raw/map_test.go | 2 +- raw/slice_test.go | 2 +- yaml/build_test.go | 68 +++++++++++++++++++++++++++++++++- yaml/ruleset_test.go | 2 +- yaml/secret_test.go | 2 +- yaml/service_test.go | 2 +- yaml/stage.go | 64 ++++++++++++++++---------------- yaml/stage_test.go | 2 +- yaml/step_test.go | 2 +- yaml/template_test.go | 2 +- yaml/testdata/merge_anchor.yml | 46 +++++++++++++++++++++++ yaml/ulimit_test.go | 2 +- yaml/volume_test.go | 2 +- 16 files changed, 159 insertions(+), 47 deletions(-) create mode 100644 yaml/testdata/merge_anchor.yml diff --git a/go.mod b/go.mod index 8d545579..cf690832 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,11 @@ go 1.23.1 require ( github.com/adhocore/gronx v1.19.0 + github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 github.com/drone/envsubst v1.0.3 github.com/ghodss/yaml v1.0.0 github.com/lib/pq v1.10.9 github.com/microcosm-cc/bluemonday v1.0.27 - gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/go.sum b/go.sum index 630bcae2..06f57abd 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ github.com/adhocore/gronx v1.19.0 h1:GrEvNMPDwXND+YFadCyFVQPC+/xxoGJaQzu+duNf6aU github.com/adhocore/gronx v1.19.0/go.mod h1:7oUY1WAU8rEJWmAxXR2DN0JaO4gi9khSgKjiRypqteg= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3 h1:q+sMKdA6L8LyGVudTkpGoC73h6ak2iWSPFiFo/pFOU8= +github.com/buildkite/yaml v0.0.0-20181016232759-0caa5f0796e3/go.mod h1:5hCug3EZaHXU3FdCA3gJm0YTNi+V+ooA2qNTiVpky4A= github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -27,5 +29,3 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/library/string.go b/library/string.go index d431f6ac..79dac91d 100644 --- a/library/string.go +++ b/library/string.go @@ -7,8 +7,8 @@ import ( "strconv" "strings" + "github.com/buildkite/yaml" json "github.com/ghodss/yaml" - "gopkg.in/yaml.v3" ) // ToString is a helper function to convert diff --git a/raw/map_test.go b/raw/map_test.go index 918664a6..0494c8ed 100644 --- a/raw/map_test.go +++ b/raw/map_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" ) func TestRaw_StringSliceMap_UnmarshalJSON(t *testing.T) { diff --git a/raw/slice_test.go b/raw/slice_test.go index 0ae76f9d..7a32bd3d 100644 --- a/raw/slice_test.go +++ b/raw/slice_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" ) func TestRaw_StringSlice_UnmarshalJSON(t *testing.T) { diff --git a/yaml/build_test.go b/yaml/build_test.go index 8875d755..5cacc76b 100644 --- a/yaml/build_test.go +++ b/yaml/build_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/library" "github.com/go-vela/types/raw" @@ -596,6 +596,72 @@ func TestYaml_Build_UnmarshalYAML(t *testing.T) { }, }, }, + { + file: "testdata/merge_anchor.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Services: ServiceSlice{ + { + Name: "service-a", + Ports: []string{"5432:5432"}, + Environment: raw.StringSliceMap{ + "REGION": "dev", + }, + Image: "postgres", + Pull: "not_present", + }, + }, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"echo alpha"}, + Name: "alpha", + Image: "alpine:latest", + Pull: "not_present", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Commands: raw.StringSlice{"echo beta"}, + Name: "beta", + Image: "alpine:latest", + Pull: "not_present", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Commands: raw.StringSlice{"echo gamma"}, + Name: "gamma", + Image: "alpine:latest", + Pull: "not_present", + Environment: raw.StringSliceMap{ + "REGION": "dev", + }, + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + }, + }, + }, } // run tests diff --git a/yaml/ruleset_test.go b/yaml/ruleset_test.go index 2728c495..2d674c0f 100644 --- a/yaml/ruleset_test.go +++ b/yaml/ruleset_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/pipeline" ) diff --git a/yaml/secret_test.go b/yaml/secret_test.go index 8825a819..62ad3291 100644 --- a/yaml/secret_test.go +++ b/yaml/secret_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/pipeline" ) diff --git a/yaml/service_test.go b/yaml/service_test.go index a4c8586c..1289095c 100644 --- a/yaml/service_test.go +++ b/yaml/service_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/pipeline" "github.com/go-vela/types/raw" diff --git a/yaml/stage.go b/yaml/stage.go index 5132e399..5e166444 100644 --- a/yaml/stage.go +++ b/yaml/stage.go @@ -5,7 +5,7 @@ package yaml import ( "fmt" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/pipeline" "github.com/go-vela/types/raw" @@ -50,27 +50,33 @@ func (s *StageSlice) ToPipeline() *pipeline.StageSlice { } // UnmarshalYAML implements the Unmarshaler interface for the StageSlice type. -func (s *StageSlice) UnmarshalYAML(v *yaml.Node) error { - if v.Kind != yaml.MappingNode { - return fmt.Errorf("invalid yaml: expected map node for stage") +func (s *StageSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // map slice we try unmarshalling to + mapSlice := new(yaml.MapSlice) + + // attempt to unmarshal as a map slice type + err := unmarshal(mapSlice) + if err != nil { + return err } // iterate through each element in the map slice - for i := 0; i < len(v.Content); i += 2 { - key := v.Content[i] - value := v.Content[i+1] - + for _, v := range *mapSlice { + // stage we try unmarshalling to stage := new(Stage) - // unmarshal value into stage - err := value.Decode(stage) + // marshal interface value from ordered map + out, _ := yaml.Marshal(v.Value) + + // unmarshal interface value as stage + err = yaml.Unmarshal(out, stage) if err != nil { return err } // implicitly set stage `name` if empty if len(stage.Name) == 0 { - stage.Name = fmt.Sprintf("%v", key.Value) + stage.Name = fmt.Sprintf("%v", v.Key) } // implicitly set the stage `needs` @@ -82,42 +88,36 @@ func (s *StageSlice) UnmarshalYAML(v *yaml.Node) error { return needs } } - return append(needs, "clone") }(stage.Needs) } - // append stage to stage slice *s = append(*s, stage) } - return nil } // MarshalYAML implements the marshaler interface for the StageSlice type. func (s StageSlice) MarshalYAML() (interface{}, error) { - output := new(yaml.Node) - output.Kind = yaml.MappingNode + // map slice to return as marshaled output + var output yaml.MapSlice + // loop over the input stages for _, inputStage := range s { - n := new(yaml.Node) - - // create new stage with existing properties - outputStage := &Stage{ - Name: inputStage.Name, - Needs: inputStage.Needs, - Independent: inputStage.Independent, - Steps: inputStage.Steps, - } + // create a new stage + outputStage := new(Stage) - err := n.Encode(outputStage) - if err != nil { - return nil, err - } + // add the existing needs to the new stage + outputStage.Needs = inputStage.Needs + + // add the existing dependent tag to the new stage + outputStage.Independent = inputStage.Independent + + // add the existing steps to the new stage + outputStage.Steps = inputStage.Steps - // append stage to map output - output.Content = append(output.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: inputStage.Name}) - output.Content = append(output.Content, n) + // append stage to MapSlice + output = append(output, yaml.MapItem{Key: inputStage.Name, Value: outputStage}) } return output, nil diff --git a/yaml/stage_test.go b/yaml/stage_test.go index 698bfd3b..a068cd6d 100644 --- a/yaml/stage_test.go +++ b/yaml/stage_test.go @@ -7,8 +7,8 @@ import ( "reflect" "testing" + "github.com/buildkite/yaml" "github.com/google/go-cmp/cmp" - "gopkg.in/yaml.v3" "github.com/go-vela/types/pipeline" ) diff --git a/yaml/step_test.go b/yaml/step_test.go index 3ecc2240..686631a6 100644 --- a/yaml/step_test.go +++ b/yaml/step_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/pipeline" "github.com/go-vela/types/raw" diff --git a/yaml/template_test.go b/yaml/template_test.go index 6599e2db..e7f760cb 100644 --- a/yaml/template_test.go +++ b/yaml/template_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/library" ) diff --git a/yaml/testdata/merge_anchor.yml b/yaml/testdata/merge_anchor.yml new file mode 100644 index 00000000..3de1e64b --- /dev/null +++ b/yaml/testdata/merge_anchor.yml @@ -0,0 +1,46 @@ +# test file that uses the non-standard multiple anchor keys in one step to test custom step unmarshaler + +version: "1" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + postgres: &pg-image + image: postgres + + events: + push: &event-push + ruleset: + event: + - push + env: + dev-env: &dev-environment + environment: + REGION: dev + +services: + - name: service-a + <<: *pg-image + <<: *dev-environment + ports: + - "5432:5432" + +steps: + - name: alpha + <<: *alpine-image + <<: *event-push + commands: + - echo alpha + + - name: beta + <<: [ *alpine-image, *event-push ] + commands: + - echo beta + + - name: gamma + <<: *alpine-image + <<: *event-push + <<: *dev-environment + commands: + - echo gamma \ No newline at end of file diff --git a/yaml/ulimit_test.go b/yaml/ulimit_test.go index 66a96c62..dc6fdac6 100644 --- a/yaml/ulimit_test.go +++ b/yaml/ulimit_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/pipeline" ) diff --git a/yaml/volume_test.go b/yaml/volume_test.go index 3d711088..837703ad 100644 --- a/yaml/volume_test.go +++ b/yaml/volume_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v3" + "github.com/buildkite/yaml" "github.com/go-vela/types/pipeline" )