feat: new gcs client using cloud client libraries (GoogleContainerToo…

…ls#9518)

* feat: new gcs client using cloud client libraries

* feat: tests for new native client and changes in other places to make it work with new implementation

integration/remote_config_dependency_test.go

@@ -120,3 +120,105 @@ requires:
 		})
 	}
 }
+
+func TestRenderWithRemoteGCS(t *testing.T) {
+	tests := []struct {
+		description    string
+		configFile     string
+		args           []string
+		shouldErr      bool
+		expectedOutput string
+		expectedErrMsg string
+	}{
+		{
+			description: "download all repo with same folders from subfolder",
+			configFile: `apiVersion: skaffold/v4beta11
+kind: Config
+requires:
+  - googleCloudStorage:
+      source:  gs://skaffold-remote-dependency-e2e-tests/test1/*
+      path: ./skaffold.yaml
+`,
+			args: []string{"--tag", "fixed", "--default-repo=", "--digest-source", "tag"},
+			expectedOutput: `apiVersion: v1
+kind: Pod
+metadata:
+  name: getting-started
+spec:
+  containers:
+    - image: skaffold-example:fixed
+      name: getting-started`,
+		},
+		{
+			description: "download full repo with top sub folder",
+			configFile: `apiVersion: skaffold/v4beta11
+kind: Config
+requires:
+  - googleCloudStorage:
+      source:  gs://skaffold-remote-dependency-e2e-tests/test1
+      path: ./test1/skaffold.yaml
+`,
+			args: []string{"--tag", "fixed", "--default-repo=", "--digest-source", "tag"},
+			expectedOutput: `apiVersion: v1
+kind: Pod
+metadata:
+  name: getting-started
+spec:
+  containers:
+    - image: skaffold-example:fixed
+      name: getting-started`,
+		},
+		{
+			description: "download full repo with bucket name as top folder",
+			configFile: `apiVersion: skaffold/v4beta11
+kind: Config
+requires:
+  - googleCloudStorage:
+      source:  gs://skaffold-remote-dependency-e2e-tests
+      path: ./skaffold-remote-dependency-e2e-tests/test1/skaffold.yaml
+`,
+			args: []string{"--tag", "fixed", "--default-repo=", "--digest-source", "tag"},
+			expectedOutput: `apiVersion: v1
+kind: Pod
+metadata:
+  name: getting-started
+spec:
+  containers:
+    - image: skaffold-example:fixed
+      name: getting-started`,
+		},
+		{
+			description: "download only all yaml files across bucket",
+			configFile: `apiVersion: skaffold/v4beta11
+kind: Config
+requires:
+  - googleCloudStorage:
+      source:  gs://skaffold-remote-dependency-e2e-tests/test1/**.yaml
+      path: ./skaffold.yaml
+`,
+			args: []string{"--tag", "fixed", "--default-repo=", "--digest-source", "tag", "-p", "flat-structure"},
+			expectedOutput: `apiVersion: v1
+kind: Pod
+metadata:
+  name: getting-started
+spec:
+  containers:
+    - image: skaffold-example:fixed
+      name: getting-started`,
+		},
+	}
+
+	for _, test := range tests {
+		testutil.Run(t, test.description, func(t *testutil.T) {
+			MarkIntegrationTest(t.T, NeedsGcp)
+			tmpDirRemoteRepo := t.NewTempDir()
+			tmpDirTest := t.NewTempDir()
+
+			tmpDirTest.Write("skaffold.yaml", test.configFile)
+			args := append(test.args, "--remote-cache-dir", tmpDirRemoteRepo.Root())
+			output, err := skaffold.Render(args...).InDir(tmpDirTest.Root()).RunWithCombinedOutput(t.T)
+			t.CheckNoError(err)
+			t.CheckDeepEqual(test.expectedOutput, string(output), testutil.YamlObj(t.T))
+		})
+	}
+}

pkg/skaffold/deploy/kubectl/kubectl_test.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -40,6 +39,11 @@ import (
 	"github.com/GoogleContainerTools/skaffold/v2/testutil"
 )
 
+type gcsClientMock struct{}
+
+func (g gcsClientMock) DownloadRecursive(ctx context.Context, src, dst string) error {
+	return nil
+}
 func TestKubectlV1RenderDeploy(t *testing.T) {
 	tests := []struct {
 		description                 string
@@ -520,13 +524,15 @@ func TestGCSManifests(t *testing.T) {
 				RawK8s: []string{"gs://dev/deployment.yaml"},
 			},
 			commands: testutil.
-				CmdRunOut(fmt.Sprintf("gsutil cp -r %s %s", "gs://dev/deployment.yaml", filepath.Join(manifest.ManifestTmpDir, manifest.ManifestsFromGCS)), "log").
-				AndRun("kubectl --context kubecontext --namespace testNamespace apply -f -"),
+				CmdRun("kubectl --context kubecontext --namespace testNamespace apply -f -"),
 		}}
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
 			t.Override(&client.Client, deployutil.MockK8sClient)
 			t.Override(&util.DefaultExecCommand, test.commands)
+			t.Override(&manifest.GetGCSClient, func() manifest.GCSClient {
+				return gcsClientMock{}
+			})
 			if err := os.MkdirAll(manifest.ManifestTmpDir, os.ModePerm); err != nil {
 				t.Fatal(err)
 			}