diff --git a/go.mod b/go.mod index abced6437..00834b1f1 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( golang.org/x/oauth2 v0.20.0 golang.org/x/tools v0.20.0 gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.29.4 k8s.io/apimachinery v0.29.4 k8s.io/cli-runtime v0.29.4 @@ -302,7 +303,6 @@ require ( google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.29.3 // indirect k8s.io/apiserver v0.29.4 // indirect k8s.io/component-base v0.29.4 // indirect diff --git a/magefiles/magefile.go b/magefiles/magefile.go index 4b2841de5..059748b1f 100644 --- a/magefiles/magefile.go +++ b/magefiles/magefile.go @@ -573,12 +573,20 @@ func SetupMultiPlatformTests() error { var lastBundle *tektonapi.Param var lastName *tektonapi.Param buildahTask := false + prefetchDepsTask := false + buildSourceImageTask := false for i, param := range params { if param.Name == "bundle" { lastBundle = &t.TaskRef.Params[i] } else if param.Name == "name" && param.Value.StringVal == "buildah" { lastName = &t.TaskRef.Params[i] buildahTask = true + } else if param.Name == "name" && param.Value.StringVal == "prefetch-dependencies" { + lastName = &t.TaskRef.Params[i] + prefetchDepsTask = true + } else if param.Name == "name" && param.Value.StringVal == "build-source-image" { + lastName = &t.TaskRef.Params[i] + buildSourceImageTask = true } } if buildahTask { @@ -590,6 +598,24 @@ func SetupMultiPlatformTests() error { t.Params = append(t.Params, tektonapi.Param{Name: "PLATFORM", Value: *tektonapi.NewStructuredValues("$(params.PLATFORM)")}) dockerPipelineObject.Spec.Params = append(dockerPipelineObject.PipelineSpec().Params, tektonapi.ParamSpec{Name: "PLATFORM", Default: tektonapi.NewStructuredValues(platformType)}) dockerPipelineObject.Name = "buildah-remote-pipeline" + } else if prefetchDepsTask { + // Always run prefetch-dependencies task for prefetching RPMs + t.When = make([]tektonapi.WhenExpression, 0, 5) + // There is a test repo is used for testing including prefetched SRPMs + t.Params = make([]tektonapi.Param, 0, 5) + t.Params = append(t.Params, tektonapi.Param{Name: "input", Value: *tektonapi.NewStructuredValues(`{"type": "rpm"}`)}) + } else if buildSourceImageTask { + // Enable source container build. + newWhenExpressions := make([]tektonapi.WhenExpression, 0, 5) + for _, expression := range t.When { + if expression.Input == "$(params.build-source-image)" { + continue + } + newWhenExpressions = append(newWhenExpressions, expression) + } + t.When = newWhenExpressions + } + if buildahTask && prefetchDepsTask && buildSourceImageTask { break } } diff --git a/tests/build/multi-platform.go b/tests/build/multi-platform.go index 08b3f75fb..7f189ff10 100644 --- a/tests/build/multi-platform.go +++ b/tests/build/multi-platform.go @@ -6,10 +6,12 @@ import ( "fmt" "log" "os" + "path/filepath" "strings" "time" "github.com/konflux-ci/e2e-tests/pkg/clients/has" + "github.com/konflux-ci/e2e-tests/pkg/utils/build" "golang.org/x/crypto/ssh" v1 "k8s.io/api/core/v1" @@ -30,6 +32,8 @@ import ( "github.com/IBM/go-sdk-core/v5/core" "github.com/IBM/vpc-go-sdk/vpcv1" + + "gopkg.in/yaml.v3" ) const ( @@ -59,6 +63,31 @@ var ( interval = 10 * time.Second ) +type RPMLockFile struct { + LockFileVersion int `yaml:"lockfileVersion"` + LockFileVendor string `yaml:"lockfileVendor"` + Arches []struct { + Arch string `yaml:"arch"` + Packages []struct { + Url string `yaml:"url"` + Repoid string `yaml:"repoid"` + Size int `yaml:"size"` + Checksum string `yaml:"checksum"` + Name string `yaml:"name"` + EVR string `yaml:"evr"` + SourceRPM string `yaml:"sourcerpm"` + } + Source []struct { + Url string `yaml:"url"` + Repoid string `yaml:"repoid"` + Size int `yaml:"size"` + Checksum string `yaml:"checksum"` + Name string `yaml:"name"` + EVR string `yaml:"evr"` + } + } +} + var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E tests", Pending, Label("multi-platform"), func() { var f *framework.Framework AfterEach(framework.ReportFailure(&f)) @@ -71,6 +100,9 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E var testNamespace, applicationName, componentName, multiPlatformSecretName, host, userDir string var component *appservice.Component + // TODO: move this repo into redhat-appstudio-qe Github organization + const testRepoUrl = "https://github.com/cqi-stonesoup-test/multi-arch-builds" + AfterAll(func() { // Cleanup aws secet and host-config Expect(f.AsKubeAdmin.CommonController.DeleteSecret(ControllerNamespace, AwsSecretName)).To(Succeed()) @@ -101,7 +133,7 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E err = createSecretForHostPool(f) Expect(err).ShouldNot(HaveOccurred()) - component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "ARM64") + component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "ARM64", testRepoUrl, "main") }) When("the Component with multi-platform-build is created", func() { @@ -173,8 +205,49 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E return nil }, timeout, interval).Should(Succeed(), "timed out when verifying that the remote host was cleaned up correctly") }) + + It("prefetched SRPMs are included", func() { + pr, err := f.AsKubeAdmin.HasController.GetComponentPipelineRun(componentName, applicationName, testNamespace, "") + Expect(err).Should(Succeed()) + client := f.AsKubeAdmin.CommonController.KubeRest() + sourceImageUrl, err := f.AsKubeAdmin.TektonController.GetTaskRunResult(client, pr, "build-source-image", "IMAGE_URL") + Expect(err).Should(Succeed()) + + imageDir, err := build.ExtractImage(sourceImageUrl) + Expect(err).Should(Succeed()) + defer os.RemoveAll(imageDir) + + entries, err := os.ReadDir(filepath.Join(imageDir, "rpm_dir")) + Expect(err).Should(Succeed()) + + srpmSet := make(map[string]int) + for _, entry := range entries { + srpmSet[entry.Name()] = 1 + } + + content, err := build.ReadFileFromGitRepo(testRepoUrl, "rpm.lock.file", "main") + Expect(err).Should(Succeed()) + + rpmLockFile := RPMLockFile{} + Expect(yaml.Unmarshal([]byte(content), &rpmLockFile)).Should(Succeed()) + + // Ideally, this check should be in the CheckSourceImage function. However, it is + // not easy for multi-platform builds tests here to set hermetic parameter for the + // triggered PipelineRun, which causes the prefetched sources check for hermetic. + // So, before figuring out a good solution, make this check separate here. + + for _, arch := range rpmLockFile.Arches { + for _, sourceInfo := range arch.Source { + srpmFilename := filepath.Base(sourceInfo.Url) + if _, exists := srpmSet[srpmFilename]; !exists { + Fail(fmt.Sprintf("SRPM %s is not included for arch %s", srpmFilename, arch.Arch)) + } + } + } + }) }) }) + Describe("aws dynamic allocation", Label("aws-dynamic"), func() { var testNamespace, applicationName, componentName, multiPlatformSecretName, multiPlatformTaskName, dynamicInstanceTag, instanceId string var component *appservice.Component @@ -221,7 +294,7 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E err = createSecretsForDynamicInstance(f) Expect(err).ShouldNot(HaveOccurred()) - component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "ARM64") + component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "ARM64", "", "") }) When("the Component with multi-platform-build is created", func() { @@ -300,7 +373,7 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E err = createSecretsForIbmDynamicInstance(f) Expect(err).ShouldNot(HaveOccurred()) - component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "S390X") + component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "S390X", "", "") }) When("the Component with multi-platform-build is created", func() { @@ -380,7 +453,7 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E err = createSecretsForIbmDynamicInstance(f) Expect(err).ShouldNot(HaveOccurred()) - component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "PPC64LE") + component, applicationName, componentName = createApplicationAndComponent(f, testNamespace, "PPC64LE", "", "") }) When("the Component with multi-platform-build is created", func() { @@ -417,12 +490,24 @@ var _ = framework.MultiPlatformBuildSuiteDescribe("Multi Platform Controller E2E }) }) -func createApplicationAndComponent(f *framework.Framework, testNamespace, platform string) (component *appservice.Component, applicationName, componentName string) { - applicationName = fmt.Sprintf("multi-platform-suite-application-%s", util.GenerateRandomString(4)) +// createApplicationAndComponent creates Applicaiton and Component CR from given Git URL and revision. +// createApplicationAndComponent returns a tuple with three values in order, the created Component CR, application name and component name. +func createApplicationAndComponent(f *framework.Framework, testNamespace, platform, gitUrl, gitRevision string) (*appservice.Component, string, string) { + applicationName := fmt.Sprintf("multi-platform-suite-application-%s", util.GenerateRandomString(4)) _, err := f.AsKubeAdmin.HasController.CreateApplication(applicationName, testNamespace) Expect(err).NotTo(HaveOccurred()) - componentName = fmt.Sprintf("multi-platform-suite-component-%s", util.GenerateRandomString(4)) + componentName := fmt.Sprintf("multi-platform-suite-component-%s", util.GenerateRandomString(4)) + + gitSourceUrl := gitUrl + if gitSourceUrl == "" { + gitSourceUrl = multiPlatformProjectGitUrl + } + + gitSourceRevision := gitRevision + if gitSourceRevision == "" { + gitSourceRevision = multiPlatformProjectRevision + } customBuildahRemotePipeline := os.Getenv(constants.CUSTOM_BUILDAH_REMOTE_PIPELINE_BUILD_BUNDLE_ENV + "_" + platform) Expect(customBuildahRemotePipeline).ShouldNot(BeEmpty()) @@ -436,16 +521,16 @@ func createApplicationAndComponent(f *framework.Framework, testNamespace, platfo Source: appservice.ComponentSource{ ComponentSourceUnion: appservice.ComponentSourceUnion{ GitSource: &appservice.GitSource{ - URL: multiPlatformProjectGitUrl, - Revision: multiPlatformProjectRevision, + URL: gitSourceUrl, + Revision: gitSourceRevision, DockerfileURL: constants.DockerFilePath, }, }, }, } - component, err = f.AsKubeAdmin.HasController.CreateComponent(componentObj, testNamespace, "", "", applicationName, true, buildPipelineAnnotation) + component, err := f.AsKubeAdmin.HasController.CreateComponent(componentObj, testNamespace, "", "", applicationName, true, buildPipelineAnnotation) Expect(err).ShouldNot(HaveOccurred()) - return + return component, applicationName, componentName } func validateMultiPlatformSecretIsPopulated(f *framework.Framework, testNamespace, multiPlatformTaskName, multiPlatformSecretName string) (instanceId string) {