Extend the patch package used by registry apply to support lists …

…returned by `registry get` (#939)

cmd/registry/patch/apply.go

@@ -24,6 +24,7 @@ import (
 	"github.com/apigee/registry/cmd/registry/core"
 	"github.com/apigee/registry/log"
 	"github.com/apigee/registry/pkg/connection"
+	"gopkg.in/yaml.v3"
 )
 
 func Apply(ctx context.Context, client connection.RegistryClient, path, parent string, recursive bool, jobs int) error {
@@ -39,11 +40,48 @@ func Apply(ctx context.Context, client connection.RegistryClient, path, parent s
 			} else if !strings.HasSuffix(fileName, ".yaml") {
 				return nil // Skip everything that's not a YAML file.
 			}
-			return patches.add(&applyFileTask{
+			bytes, err := os.ReadFile(fileName)
+			if err != nil {
+				return err
+			}
+			header, items, err := readHeaderWithItems(bytes)
+			if err != nil {
+				return err
+			}
+			if header.ApiVersion != RegistryV1 {
+				return nil
+			}
+			if items.Kind == yaml.SequenceNode {
+				for _, n := range items.Content {
+					itemBytes, err := yaml.Marshal(n)
+					if err != nil {
+						return err
+					}
+					itemHeader, err := readHeader(itemBytes)
+					if err != nil {
+						return err
+					}
+					if itemHeader.ApiVersion != RegistryV1 {
+						continue
+					}
+					patches.add(&applyBytesTask{
+						client: client,
+						path:   fileName,
+						parent: parent,
+						kind:   itemHeader.Kind,
+						bytes:  itemBytes,
+					})
+				}
+				return nil
+			}
+			patches.add(&applyBytesTask{
 				client: client,
 				path:   fileName,
 				parent: parent,
+				kind:   header.Kind,
+				bytes:  bytes,
 			})
+			return nil
 		})
 	if err != nil {
 		return err
@@ -59,17 +97,7 @@ type patchGroup struct {
 	artifactTasks   []core.Task
 }
 
-func (p *patchGroup) add(task *applyFileTask) error {
-	bytes, err := os.ReadFile(task.path)
-	if err != nil {
-		return err
-	}
-	header, err := readHeader(bytes)
-	if err != nil {
-		// Skip YAML files that don't have our expected headers. We assume they aren't ours.
-		return nil
-	}
-	task.kind = header.Kind
+func (p *patchGroup) add(task *applyBytesTask) {
 	switch task.kind {
 	case "API":
 		p.apiTasks = append(p.apiTasks, task)
@@ -82,7 +110,6 @@ func (p *patchGroup) add(task *applyFileTask) error {
 	default: // for everything else, try an artifact type
 		p.artifactTasks = append(p.artifactTasks, task)
 	}
-	return nil
 }
 
 func (p *patchGroup) run(ctx context.Context, jobs int) error {
@@ -103,33 +130,41 @@ func (p *patchGroup) run(ctx context.Context, jobs int) error {
 	return nil
 }
 
-type applyFileTask struct {
+type applyBytesTask struct {
 	client connection.RegistryClient
 	path   string
 	parent string
 	kind   string
+	bytes  []byte
 }
 
-func (task *applyFileTask) String() string {
-	return "apply file " + task.path
+func (task *applyBytesTask) String() string {
+	return "apply " + task.path
 }
 
-func (task *applyFileTask) Run(ctx context.Context) error {
-	log.FromContext(ctx).Infof("Applying %s", task.path)
-	bytes, err := os.ReadFile(task.path)
+func (task *applyBytesTask) Run(ctx context.Context) error {
+	header, err := readHeader(task.bytes)
 	if err != nil {
 		return err
 	}
-	switch task.kind {
+	if header.ApiVersion != RegistryV1 {
+		return nil
+	}
+	name := header.Metadata.Name
+	if header.Metadata.Parent != "" {
+		name = header.Metadata.Parent + "/" + name
+	}
+	log.FromContext(ctx).Infof("Applying %s %s", task.path, name)
+	switch header.Kind {
 	case "API":
-		return applyApiPatchBytes(ctx, task.client, bytes, task.parent)
+		return applyApiPatchBytes(ctx, task.client, task.bytes, task.parent)
 	case "Version":
-		return applyApiVersionPatchBytes(ctx, task.client, bytes, task.parent)
+		return applyApiVersionPatchBytes(ctx, task.client, task.bytes, task.parent)
 	case "Spec":
-		return applyApiSpecPatchBytes(ctx, task.client, bytes, task.parent, task.path)
+		return applyApiSpecPatchBytes(ctx, task.client, task.bytes, task.parent, task.path)
 	case "Deployment":
-		return applyApiDeploymentPatchBytes(ctx, task.client, bytes, task.parent)
+		return applyApiDeploymentPatchBytes(ctx, task.client, task.bytes, task.parent)
 	default: // for everything else, try an artifact type
-		return applyArtifactPatchBytes(ctx, task.client, bytes, task.parent)
+		return applyArtifactPatchBytes(ctx, task.client, task.bytes, task.parent)
 	}
 }

cmd/registry/patch/header.go

@@ -15,8 +15,6 @@
 package patch
 
 import (
-	"fmt"
-
 	"github.com/apigee/registry/pkg/models"
 	"gopkg.in/yaml.v3"
 )
@@ -26,11 +24,15 @@ const RegistryV1 = "apigeeregistry/v1"
 func readHeader(bytes []byte) (models.Header, error) {
 	var header models.Header
 	err := yaml.Unmarshal(bytes, &header)
-	if err != nil {
-		return header, err
-	}
-	if header.ApiVersion != RegistryV1 {
-		return header, fmt.Errorf("unsupported API version: %s", header.ApiVersion)
+	return header, err
+}
+
+func readHeaderWithItems(bytes []byte) (models.Header, yaml.Node, error) {
+	type headerWithItems struct {
+		models.Header `yaml:",inline"`
+		Items         yaml.Node `yaml:"items,omitempty"`
 	}
-	return header, nil
+	var wrapper headerWithItems
+	err := yaml.Unmarshal(bytes, &wrapper)
+	return wrapper.Header, wrapper.Items, err
 }

cmd/registry/patch/patch_lists_test.go

@@ -0,0 +1,106 @@
+// Copyright 2023 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package patch
+
+import (
+	"context"
+	"testing"
+
+	"github.com/apigee/registry/pkg/connection"
+	"github.com/apigee/registry/rpc"
+	"github.com/apigee/registry/server/registry/names"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+func TestArtifactLists(t *testing.T) {
+	tests := []struct {
+		desc string
+		root string
+	}{
+		{
+			desc: "artifacts",
+			root: "testdata/lists",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ctx := context.Background()
+			adminClient, err := connection.NewAdminClient(ctx)
+			if err != nil {
+				t.Fatalf("Setup: failed to create client: %+v", err)
+			}
+			project := names.Project{ProjectID: "patch-lists-test"}
+			if err = adminClient.DeleteProject(ctx, &rpc.DeleteProjectRequest{
+				Name:  project.String(),
+				Force: true,
+			}); err != nil && status.Code(err) != codes.NotFound {
+				t.Errorf("Setup: failed to delete test project: %s", err)
+			}
+			if _, err := adminClient.CreateProject(ctx, &rpc.CreateProjectRequest{
+				ProjectId: project.ProjectID,
+				Project:   &rpc.Project{},
+			}); err != nil {
+				t.Fatalf("Setup: Failed to create test project: %s", err)
+			}
+			t.Cleanup(func() {
+				if err := adminClient.DeleteProject(ctx, &rpc.DeleteProjectRequest{
+					Name:  project.String(),
+					Force: true,
+				}); err != nil {
+					t.Logf("Cleanup: Failed to delete test project: %s", err)
+				}
+				adminClient.Close()
+			})
+
+			// set the configured registry.project to the test project
+			config, err := connection.ActiveConfig()
+			if err != nil {
+				t.Fatalf("Setup: Failed to get registry configuration: %s", err)
+			}
+			config.Project = project.ProjectID
+			connection.SetConfig(config)
+
+			registryClient, err := connection.NewRegistryClient(ctx)
+			if err != nil {
+				t.Fatalf("Setup: Failed to create registry client: %s", err)
+			}
+			defer registryClient.Close()
+
+			if err := Apply(ctx, registryClient, test.root, project.String()+"/locations/global", true, 10); err != nil {
+				t.Fatalf("Apply() returned error: %s", err)
+			}
+
+			artifacts := []string{
+				"apihub-display-settings",
+				"apihub-lifecycle",
+				"apihub-lint-errors",
+				"apihub-lint-summary",
+				"apihub-lint-warnings",
+				"apihub-styleguide",
+				"apihub-taxonomies",
+			}
+			for _, a := range artifacts {
+				_, err := registryClient.GetArtifact(ctx, &rpc.GetArtifactRequest{
+					Name: project.Artifact(a).String(),
+				})
+				if status.Code(err) == codes.NotFound {
+					t.Fatalf("Expected artifact doesn't exist: %s", err)
+				} else if err != nil {
+					t.Fatalf("Failed to verify artifact existence: %s", err)
+				}
+			}
+		})
+	}
+}

cmd/registry/patch/patch_project_test.go

@@ -174,7 +174,6 @@ func TestProjectExport(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Setup: failed to create client: %+v", err)
 			}
-			defer adminClient.Close()
 			project := names.Project{ProjectID: "patch-export-project-test"}
 			if err = adminClient.DeleteProject(ctx, &rpc.DeleteProjectRequest{
 				Name:  project.String(),
@@ -197,6 +196,7 @@ func TestProjectExport(t *testing.T) {
 				}); err != nil {
 					t.Logf("Cleanup: Failed to delete test project: %s", err)
 				}
+				adminClient.Close()
 			})
 
 			// set the configured registry.project to the test project
@@ -276,7 +276,6 @@ func TestApiExport(t *testing.T) {
 			if err != nil {
 				t.Fatalf("Setup: failed to create client: %+v", err)
 			}
-			defer adminClient.Close()
 			project := names.Project{ProjectID: "patch-export-api-test"}
 			if err = adminClient.DeleteProject(ctx, &rpc.DeleteProjectRequest{
 				Name:  project.String(),
@@ -299,6 +298,7 @@ func TestApiExport(t *testing.T) {
 				}); err != nil {
 					t.Logf("Cleanup: Failed to delete test project: %s", err)
 				}
+				adminClient.Close()
 			})
 
 			// set the configured registry.project to the test project