test: ensure prefetched SRPMs are included in source image for multi-arch builds

go.mod

@@ -54,6 +54,7 @@ require (
 	golang.org/x/oauth2 v0.20.0
 	golang.org/x/tools v0.20.0
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.29.4
 	k8s.io/apimachinery v0.29.4
 	k8s.io/cli-runtime v0.29.4
@@ -302,7 +303,6 @@ require (
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.29.3 // indirect
 	k8s.io/apiserver v0.29.4 // indirect
 	k8s.io/component-base v0.29.4 // indirect

magefiles/magefile.go

@@ -573,12 +573,20 @@ func SetupMultiPlatformTests() error {
 			var lastBundle *tektonapi.Param
 			var lastName *tektonapi.Param
 			buildahTask := false
+			prefetchDepsTask := false
+			buildSourceImageTask := false
 			for i, param := range params {
 				if param.Name == "bundle" {
 					lastBundle = &t.TaskRef.Params[i]
 				} else if param.Name == "name" && param.Value.StringVal == "buildah" {
 					lastName = &t.TaskRef.Params[i]
 					buildahTask = true
+				} else if param.Name == "name" && param.Value.StringVal == "prefetch-dependencies" {
+					lastName = &t.TaskRef.Params[i]
+					prefetchDepsTask = true
+				} else if param.Name == "name" && param.Value.StringVal == "build-source-image" {
+					lastName = &t.TaskRef.Params[i]
+					buildSourceImageTask = true
 				}
 			}
 			if buildahTask {
@@ -590,6 +598,24 @@ func SetupMultiPlatformTests() error {
 				t.Params = append(t.Params, tektonapi.Param{Name: "PLATFORM", Value: *tektonapi.NewStructuredValues("$(params.PLATFORM)")})
 				dockerPipelineObject.Spec.Params = append(dockerPipelineObject.PipelineSpec().Params, tektonapi.ParamSpec{Name: "PLATFORM", Default: tektonapi.NewStructuredValues(platformType)})
 				dockerPipelineObject.Name = "buildah-remote-pipeline"
+			} else if prefetchDepsTask {
+				// Always run prefetch-dependencies task for prefetching RPMs
+				t.When = make([]tektonapi.WhenExpression, 0, 5)
+				// There is a test repo is used for testing including prefetched SRPMs
+				t.Params = make([]tektonapi.Param, 0, 5)
+				t.Params = append(t.Params, tektonapi.Param{Name: "input", Value: *tektonapi.NewStructuredValues(`{"type": "rpm"}`)})
+			} else if buildSourceImageTask {
+				// Enable source container build.
+				newWhenExpressions := make([]tektonapi.WhenExpression, 0, 5)
+				for _, expression := range t.When {
+					if expression.Input == "$(params.build-source-image)" {
+						continue
+					}
+					newWhenExpressions = append(newWhenExpressions, expression)
+				}
+				t.When = newWhenExpressions
+			}
+			if buildahTask && prefetchDepsTask && buildSourceImageTask {
 				break
 			}
 		}

tests/build/multi-platform.go

@@ -6,10 +6,12 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/konflux-ci/e2e-tests/pkg/clients/has"
+	"github.com/konflux-ci/e2e-tests/pkg/utils/build"
 	"golang.org/x/crypto/ssh"
 	v1 "k8s.io/api/core/v1"
 
@@ -30,6 +32,8 @@ import (
 
 	"github.com/IBM/go-sdk-core/v5/core"
 	"github.com/IBM/vpc-go-sdk/vpcv1"
+
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -59,6 +63,31 @@ var (
 	interval                     = 10 * time.Second
 )
 
+type RPMLockFile struct {
+	LockFileVersion int    `yaml:"lockfileVersion"`
+	LockFileVendor  string `yaml:"lockfileVendor"`
+	Arches          []struct {
+		Arch     string `yaml:"arch"`
+		Packages []struct {
+			Url       string `yaml:"url"`
+			Repoid    string `yaml:"repoid"`
+			Size      int    `yaml:"size"`
+			Checksum  string `yaml:"checksum"`
+			Name      string `yaml:"name"`
+			EVR       string `yaml:"evr"`
+			SourceRPM string `yaml:"sourcerpm"`
+		}
+		Source []struct {
+			Url      string `yaml:"url"`
+			Repoid   string `yaml:"repoid"`
+			Size     int    `yaml:"size"`
+			Checksum string `yaml:"checksum"`
+			Name     string `yaml:"name"`
+			EVR      string `yaml:"evr"`
+		}
+	}
+}
+
 var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E tests", Pending, Label("multi-platform"), func() {
 	var f *framework.Framework
 	AfterEach(framework.ReportFailure(&f))
@@ -71,6 +100,9 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E
 		var testNamespace, applicationName, componentName, multiPlatformSecretName, host, userDir string
 		var component *appservice.Component
 
+		// TODO: move this repo into redhat-appstudio-qe Github organization
+		const testRepoUrl = "https://github.com/cqi-stonesoup-test/multi-arch-builds"
+
 		AfterAll(func() {
 			// Cleanup aws secet and host-config
 			Expect(f.AsKubeAdmin.CommonController.DeleteSecret(ControllerNamespace, AwsSecretName)).To(Succeed())
@@ -101,7 +133,7 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E
 			err = createSecretForHostPool(f)
 			Expect(err).ShouldNot(HaveOccurred())
 
-			component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "ARM64")
+			component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "ARM64", testRepoUrl, "main")
 		})
 
 		When("the Component with multi-platform-build is created", func() {
@@ -173,8 +205,49 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E
 					return nil
 				}, timeout, interval).Should(Succeed(), "timed out when verifying that the remote host was cleaned up correctly")
 			})
+
+			It("prefetched SRPMs are included", func() {
+				pr, err := f.AsKubeAdmin.HasController.GetComponentPipelineRun(componentName, applicationName, testNamespace, "")
+				Expect(err).Should(Succeed())
+				client := f.AsKubeAdmin.CommonController.KubeRest()
+				sourceImageUrl, err := f.AsKubeAdmin.TektonController.GetTaskRunResult(client, pr, "build-source-image", "IMAGE_URL")
+				Expect(err).Should(Succeed())
+
+				imageDir, err := build.ExtractImage(sourceImageUrl)
+				Expect(err).Should(Succeed())
+				defer os.RemoveAll(imageDir)
+
+				entries, err := os.ReadDir(filepath.Join(imageDir, "rpm_dir"))
+				Expect(err).Should(Succeed())
+
+				srpmSet := make(map[string]int)
+				for _, entry := range entries {
+					srpmSet[entry.Name()] = 1
+				}
+
+				content, err := build.ReadFileFromGitRepo(testRepoUrl, "rpm.lock.file", "main")
+				Expect(err).Should(Succeed())
+
+				rpmLockFile := RPMLockFile{}
+				Expect(yaml.Unmarshal([]byte(content), &rpmLockFile)).Should(Succeed())
+
+				// Ideally, this check should be in the CheckSourceImage function. However, it is
+				// not easy for multi-platform builds tests here to set hermetic parameter for the
+				// triggered PipelineRun, which causes the prefetched sources check for hermetic.
+				// So, before figuring out a good solution, make this check separate here.
+
+				for _, arch := range rpmLockFile.Arches {
+					for _, sourceInfo := range arch.Source {
+						srpmFilename := filepath.Base(sourceInfo.Url)
+						if _, exists := srpmSet[srpmFilename]; !exists {
+							Fail(fmt.Sprintf("SRPM %s is not included for arch %s", srpmFilename, arch.Arch))
+						}
+					}
+				}
+			})
 		})
 	})
+
 	Describe("aws dynamic allocation", Label("aws-dynamic"), func() {
 		var testNamespace, applicationName, componentName, multiPlatformSecretName, multiPlatformTaskName, dynamicInstanceTag, instanceId string
 		var component *appservice.Component
@@ -221,7 +294,7 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E
 			err = createSecretsForDynamicInstance(f)
 			Expect(err).ShouldNot(HaveOccurred())
 
-			component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "ARM64")
+			component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "ARM64", "", "")
 		})
 
 		When("the Component with multi-platform-build is created", func() {
@@ -300,7 +373,7 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E
 			err = createSecretsForIbmDynamicInstance(f)
 			Expect(err).ShouldNot(HaveOccurred())
 
-			component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "S390X")
+			component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "S390X", "", "")
 		})
 
 		When("the Component with multi-platform-build is created", func() {
@@ -380,7 +453,7 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E
 			err = createSecretsForIbmDynamicInstance(f)
 			Expect(err).ShouldNot(HaveOccurred())
 
-			component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "PPC64LE")
+			component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "PPC64LE", "", "")
 		})
 
 		When("the Component with multi-platform-build is created", func() {
@@ -417,12 +490,24 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E
 	})
 })
 
-func createApplicationAndComponent(f *framework.Framework, testNamespace, platform string) (component *appservice.Component, applicationName, componentName string) {
-	applicationName = fmt.Sprintf("multi-platform-suite-application-%s", util.GenerateRandomString(4))
+// createApplicationAndComponent creates Applicaiton and Component CR from given Git URL and revision.
+// createApplicationAndComponent returns a tuple with three values in order, the created Component CR, application name and component name.
+func createApplicationAndComponent(f *framework.Framework, testNamespace, platform, gitUrl, gitRevision string) (*appservice.Component, string, string) {
+    applicationName := fmt.Sprintf("multi-platform-suite-application-%s", util.GenerateRandomString(4))
 	_, err := f.AsKubeAdmin.HasController.CreateApplication(applicationName, testNamespace)
 	Expect(err).NotTo(HaveOccurred())
 
-	componentName = fmt.Sprintf("multi-platform-suite-component-%s", util.GenerateRandomString(4))
+    componentName := fmt.Sprintf("multi-platform-suite-component-%s", util.GenerateRandomString(4))
+
+    gitSourceUrl := gitUrl
+    if gitSourceUrl == "" {
+        gitSourceUrl = multiPlatformProjectGitUrl
+    }
+
+    gitSourceRevision := gitRevision
+    if gitSourceRevision == "" {
+        gitSourceRevision = multiPlatformProjectRevision
+    }
 
 	customBuildahRemotePipeline := os.Getenv(constants.CUSTOM_BUILDAH_REMOTE_PIPELINE_BUILD_BUNDLE_ENV + "_" + platform)
 	Expect(customBuildahRemotePipeline).ShouldNot(BeEmpty())
@@ -436,16 +521,16 @@ func createApplicationAndComponent(f *framework.Framework, testNamespace, platfo
 		Source: appservice.ComponentSource{
 			ComponentSourceUnion: appservice.ComponentSourceUnion{
 				GitSource: &appservice.GitSource{
-					URL:           multiPlatformProjectGitUrl,
-					Revision:      multiPlatformProjectRevision,
+					URL:      gitSourceUrl,
+					Revision: gitSourceRevision,
 					DockerfileURL: constants.DockerFilePath,
 				},
 			},
 		},
 	}
-	component, err = f.AsKubeAdmin.HasController.CreateComponent(componentObj, testNamespace, "", "", applicationName, true, buildPipelineAnnotation)
+    component, err := f.AsKubeAdmin.HasController.CreateComponent(componentObj, testNamespace, "", "", applicationName, true, buildPipelineAnnotation)
 	Expect(err).ShouldNot(HaveOccurred())
-	return
+	return component, applicationName, componentName
 }
 
 func validateMultiPlatformSecretIsPopulated(f *framework.Framework, testNamespace, multiPlatformTaskName, multiPlatformSecretName string) (instanceId string) {