diff --git a/go.mod b/go.mod index 803e931..ebb6259 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( ) require ( + github.com/dominikbraun/graph v0.23.0 // indirect github.com/kr/pretty v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum index 9f4945c..26454c4 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo= +github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= diff --git a/reference.go b/reference.go index 3f816ee..55f84c2 100644 --- a/reference.go +++ b/reference.go @@ -6,20 +6,24 @@ import ( "regexp" "strings" + "github.com/dominikbraun/graph" + "github.com/lukasjarosch/skipper/data" ) // TODO: handle reference-to-reference // TODO: PathReferences / KeyReferences // TODO: handle cyclic references +// TODO: handle multiple references per path +// TODO: handle reference-to-reference with multiple references on the same path var ( // ReferenceRegex defines the strings which are valid references // See: https://regex101.com/r/lIuuep/1 ReferenceRegex = regexp.MustCompile(`\${(?P[\w-]+(?:\:[\w-]+)*)}`) - ErrUndefinedReference = fmt.Errorf("undefined reference") - ErrReferenceSourceIsNil = fmt.Errorf("reference source is nil") + ErrUndefinedReferenceTarget = fmt.Errorf("undefined reference target path") + ErrReferenceSourceIsNil = fmt.Errorf("reference source is nil") ) // Reference is a reference to a value with a different path. @@ -39,9 +43,9 @@ type ResolvedReference struct { // TargetValue is the value to which the TargetPath points to. // If TargetReference is not nil, this value must be [data.NilValue]. TargetValue data.Value - // TargetReference is non-nil if the Reference points to another [Reference] + // TargetReference is non-nil if the Reference points to another [ResolvedReference] // If the Reference just points to a scalar value, this will be nil. - TargetReference *Reference + TargetReference *ResolvedReference } type ReferenceSourceWalker interface { @@ -72,36 +76,146 @@ func ParseReferences(source ReferenceSourceWalker) ([]Reference, error) { return references, nil } +// ReferencesInValue returns all references within the passed value. +// Note that the returned references do not have a 'Path' set! +func ReferencesInValue(value data.Value) []Reference { + var references []Reference + + referenceMatches := ReferenceRegex.FindAllStringSubmatch(value.String(), -1) + if referenceMatches != nil { + for _, match := range referenceMatches { + references = append(references, Reference{ + TargetPath: ReferencePathToPath(match[1]), + Path: data.Path{}, + }) + } + } + + return references +} + +type referenceVertex struct { + Reference + RawValue data.Value +} + type ReferenceSourceGetter interface { GetPath(path data.Path) (data.Value, error) } -func ResolveReferences(references []Reference, resolveSource ReferenceSourceGetter) ([]ResolvedReference, error) { - if resolveSource == nil { - return nil, ErrReferenceSourceIsNil +// ResolveReferences will resolve dependencies between all given references and return +// a sorted slice of the same references. This represents the order in which references should +// be replaced without causing dependency issues. +// +// Note that the list of references passed must contain all existing references, even duplicates. +func ResolveReferences(references []Reference, resolveSource ReferenceSourceGetter) ([]Reference, error) { + referenceNodeHash := func(ref referenceVertex) string { + return ref.TargetPath.String() } - var errs error - var resolvedReferences []ResolvedReference - for _, reference := range references { - val, err := resolveSource.GetPath(reference.TargetPath) + g := graph.New(referenceNodeHash, graph.Acyclic(), graph.Directed()) + + // Register all references as nodes + var nodes []referenceVertex + for _, ref := range references { + rawValue, err := resolveSource.GetPath(ref.TargetPath) if err != nil { - errs = errors.Join(errs, fmt.Errorf("%w %s at %s: %w", ErrUndefinedReference, reference.Name(), reference.Path, err)) - continue + return nil, fmt.Errorf("%w: %s at path %s", ErrUndefinedReferenceTarget, ref.Name(), ref.Path) + } + node := referenceVertex{Reference: ref, RawValue: rawValue} + err = g.AddVertex(node) + if err != nil { + // References can occur multiple times, but we only need to resolve them once. + // So we can ignore the ErrVertexAlreadyExists error. + if !errors.Is(err, graph.ErrVertexAlreadyExists) { + return nil, err + } + } + nodes = append(nodes, node) + } + + // Create edges between dependent references by looking at the actual value the 'TargetPath' points to. + // If that value contains more references, then these are dependencies of the currently examined node (reference). + for _, refNode := range nodes { + referenceDependencies := ReferencesInValue(refNode.RawValue) + + for _, refDep := range referenceDependencies { + n, err := g.Vertex(refDep.TargetPath.String()) + if err != nil { + return nil, err + } + + if refNode.TargetPath.Equals(n.TargetPath) { + return nil, fmt.Errorf("self-referencing reference %s at path %s", refNode.Name(), refNode.Path) + } + + err = g.AddEdge(refNode.TargetPath.String(), n.TargetPath.String()) + if err != nil { + return nil, err + } } + } - resolvedReferences = append(resolvedReferences, ResolvedReference{ - Reference: reference, - TargetValue: val, - }) + // Perform TopologicalSort which returns a list of strings (TargetPaths) + // starting with the one which has the most dependencies. + order, err := graph.TopologicalSort[string, referenceVertex](g) + if err != nil { + return nil, err } - if errs != nil { - return nil, errs + + // We need to reverse the result from the TopologicalSort. + // This is because the reference without dependencies will be sorted 'at the end'. + // But we want to resolve them first. + var reversedOrder []data.Path + for i := len(order) - 1; i >= 0; i-- { + reversedOrder = append(reversedOrder, data.NewPath(order[i])) + } + + // Now that we have the order in which references must be replaced, + // lets finally re-order the passed (non-deduplicated) input references. + var orderedReferences []Reference + for _, refOrder := range reversedOrder { + for _, ref := range references { + if ref.TargetPath.Equals(refOrder) { + orderedReferences = append(orderedReferences, ref) + } + } + } + + // sanity check + if len(orderedReferences) != len(references) { + return nil, fmt.Errorf("unexpected amount of resolved references, this is a bug") } - return resolvedReferences, nil + return orderedReferences, nil } +// func ResolveReferences(references []Reference, resolveSource ReferenceSourceGetter) ([]ResolvedReference, error) { +// if resolveSource == nil { +// return nil, ErrReferenceSourceIsNil +// } +// +// var errs error +// var resolvedReferences []ResolvedReference +// for _, reference := range references { +// val, err := resolveSource.GetPath(reference.TargetPath) +// if err != nil { +// errs = errors.Join(errs, fmt.Errorf("%w %s at %s: %w", ErrUndefinedReferenceTarget, reference.Name(), reference.Path, err)) +// continue +// } +// +// resolvedReferences = append(resolvedReferences, ResolvedReference{ +// Reference: reference, +// TargetValue: val, +// }) +// } +// if errs != nil { +// return nil, errs +// } +// +// return resolvedReferences, nil +// } + // ReferencePathToPath converts the path used within references (colon-separated) to a proper [data.Path] func ReferencePathToPath(referencePath string) data.Path { referencePath = strings.ReplaceAll(referencePath, ":", ".") diff --git a/reference_test.go b/reference_test.go index 5270252..b24a3e8 100644 --- a/reference_test.go +++ b/reference_test.go @@ -39,60 +39,24 @@ import ( // return inventory // } -var ( - localReferences = []Reference{ - { - Path: data.NewPath("simple.departments.engineering.manager"), - TargetPath: data.NewPath("employees.0.name"), - }, - { - Path: data.NewPath("simple.departments.analytics.manager"), - TargetPath: data.NewPath("simple.employees.1.name"), - }, - { - Path: data.NewPath("simple.departments.marketing.manager"), - TargetPath: data.NewPath("simple.employees.2.name"), - }, - { - Path: data.NewPath("simple.projects.Project_X.department"), - TargetPath: data.NewPath("simple.departments.engineering.name"), - }, - } - localResolvedReferences = []ResolvedReference{ - { - Reference: Reference{ - Path: data.NewPath("simple.departments.engineering.manager"), - TargetPath: data.NewPath("employees.0.name"), - }, - TargetValue: data.NewValue("John Doe"), - TargetReference: nil, - }, - { - Reference: Reference{ - Path: data.NewPath("simple.departments.analytics.manager"), - TargetPath: data.NewPath("simple.employees.1.name"), - }, - TargetValue: data.NewValue("Jane Smith"), - TargetReference: nil, - }, - { - Reference: Reference{ - Path: data.NewPath("simple.departments.marketing.manager"), - TargetPath: data.NewPath("simple.employees.2.name"), - }, - TargetValue: data.NewValue("Michael Johnson"), - TargetReference: nil, - }, - { - Reference: Reference{ - Path: data.NewPath("simple.projects.Project_X.department"), - TargetPath: data.NewPath("simple.departments.engineering.name"), - }, - TargetValue: data.NewValue("Engineering"), - TargetReference: nil, - }, - } -) +var localReferences = []Reference{ + { + Path: data.NewPath("simple.departments.engineering.manager"), + TargetPath: data.NewPath("employees.0.name"), + }, + { + Path: data.NewPath("simple.departments.analytics.manager"), + TargetPath: data.NewPath("simple.employees.1.name"), + }, + { + Path: data.NewPath("simple.departments.marketing.manager"), + TargetPath: data.NewPath("simple.employees.2.name"), + }, + { + Path: data.NewPath("simple.projects.Project_X.department"), + TargetPath: data.NewPath("simple.departments.engineering.name"), + }, +} func TestParseReferences(t *testing.T) { _, err := ParseReferences(nil) @@ -115,10 +79,9 @@ func TestResolveReferencesSimple(t *testing.T) { // Test: resolve all valid references which have a direct TargetValue resolved, err := ResolveReferences(localReferences, class) assert.NoError(t, err) - assert.Len(t, resolved, len(localResolvedReferences), "Every Reference should emit a ResolveReference") + assert.Len(t, resolved, len(localReferences), "Every Reference should emit a ResolveReference") for _, resolved := range resolved { - assert.Contains(t, localResolvedReferences, resolved, "ResolvedReference should be returned") - assert.Nil(t, resolved.TargetReference) + assert.Contains(t, localReferences, resolved, "ResolvedReference should be returned") } // Test: references with invalid TargetPath @@ -133,14 +96,14 @@ func TestResolveReferencesSimple(t *testing.T) { }, } resolved, err = ResolveReferences(invalidReferences, class) - assert.ErrorIs(t, err, ErrUndefinedReference) + assert.ErrorIs(t, err, ErrUndefinedReferenceTarget) assert.Nil(t, resolved) // TODO: reference to reference // TODO: cycle } -func TestResolveReferencesMap(t *testing.T) { +func TestResolveReferencesNested(t *testing.T) { class, err := NewClass("testdata/references/local/nested.yaml", codec.NewYamlCodec(), data.NewPathFromOsPath("nested")) assert.NoError(t, err) @@ -152,44 +115,56 @@ func TestResolveReferencesMap(t *testing.T) { assert.NoError(t, err) assert.Len(t, resolved, len(references)) - expected := []ResolvedReference{ + expected := []Reference{ { - Reference: Reference{ - Path: data.NewPath("nested.target"), - TargetPath: data.NewPath("source"), - }, - TargetReference: nil, - TargetValue: data.NewValue(map[string]interface{}{ - "foo": "bar", - "bar": "baz", - }), + Path: data.NewPath("nested.target"), + TargetPath: data.NewPath("source"), }, { - Reference: Reference{ - Path: data.NewPath("nested.target_array"), - TargetPath: data.NewPath("source_array"), - }, - TargetReference: nil, - TargetValue: data.NewValue([]interface{}{"foo", "bar", "baz"}), + Path: data.NewPath("nested.target_array"), + TargetPath: data.NewPath("source_array"), }, { - Reference: Reference{ - Path: data.NewPath("nested.target_nested_map"), - TargetPath: data.NewPath("nested_map"), - }, - TargetReference: nil, - TargetValue: data.NewValue(map[string]interface{}{ - "foo": map[string]interface{}{ - "bar": map[string]interface{}{ - "baz": "qux", - }, - }, - }), + Path: data.NewPath("nested.target_nested_map"), + TargetPath: data.NewPath("nested_map"), }, } assert.Len(t, resolved, len(expected)) for _, res := range resolved { assert.Contains(t, expected, res) - assert.Nil(t, res.TargetReference) } } + +func TestResolveReferencesChained(t *testing.T) { + class, err := NewClass("testdata/references/local/chained.yaml", codec.NewYamlCodec(), data.NewPathFromOsPath("chained")) + assert.NoError(t, err) + + references, err := ParseReferences(class) + assert.NoError(t, err) + assert.NotNil(t, references) + + resolved, err := ResolveReferences(references, class) + assert.NoError(t, err) + assert.Len(t, resolved, len(references)) + + expected := []Reference{ + { + Path: data.NewPath("chained.gotcha"), + TargetPath: data.NewPath("chained.john.name"), + }, + { + Path: data.NewPath("chained.name_placeholder"), + TargetPath: data.NewPath("gotcha"), + }, + { + Path: data.NewPath("chained.first_name"), + TargetPath: data.NewPath("name_placeholder"), + }, + { + Path: data.NewPath("chained.greeting"), + TargetPath: data.NewPath("first_name"), + }, + } + + assert.Equal(t, resolved, expected) +} diff --git a/testdata/references/local/chained.yaml b/testdata/references/local/chained.yaml new file mode 100644 index 0000000..e867f20 --- /dev/null +++ b/testdata/references/local/chained.yaml @@ -0,0 +1,7 @@ +chained: + greeting: Hello, ${first_name} + gotcha: ${chained:john:name} + john: + name: John + first_name: ${name_placeholder} + name_placeholder: ${gotcha}