feat: check source images built on top of Konflux images

STONEBLD-2256

This patch onboards a so-called child component which is built on top of
another Konflux image, then check if the built source image includes all the
necessary sources propertly.

By considering the execution time of the whole test suite, the check
runs only for a single component and the child test repository is
associated explicitly by hardcoding the name.

Major changes:

* IsAppSourceFilesExists is updated to check multiple app source
  archives.
* IsPreFetchDependenciesFilesExists checks if prefetched sources are
  included in the source image. For simplicity, it only checks Python
  requirements.
* Relative code are extracted into separate functions OnboardComponent
  and WaitForPipelineRunStarts, so that they can be reused for manual
  onboard to test child component source image build.

Signed-off-by: Chenxiong Qi <[email protected]>

.gitignore

@@ -31,3 +31,4 @@ rp_preproc/
 
 # Files likely to be created by users
 user.env
+*.swp

pkg/clients/github/repositories.go

@@ -87,6 +87,10 @@ func (g *Github) UpdateFile(repository, pathToFile, newContent, branchName, file
 		Content: []byte(newContent),
 		Branch:  github.String(branchName),
 	}
+	fmt.Println("======= debug =======")
+	fmt.Println("--> updating file contents", g.organization, repository, pathToFile)
+	fmt.Println("--> new content:\n", newContent)
+	fmt.Println("======= end of debug =======")
 	updatedFile, _, err := g.client.Repositories.UpdateFile(context.Background(), g.organization, repository, pathToFile, newFileContent)
 	if err != nil {
 		return nil, fmt.Errorf("error when updating a file on github: %v", err)

pkg/utils/build/image.go

@@ -73,27 +73,32 @@ func ImageFromPipelineRun(pipelineRun *pipeline.PipelineRun) (*imageInfo.Image,
 // FetchImageConfig fetches image config from remote registry.
 // It uses the registry authentication credentials stored in default place ~/.docker/config.json
 func FetchImageConfig(imagePullspec string) (*v1.ConfigFile, error) {
+	wrapErr := func(err error) error {
+		return fmt.Errorf("error while fetching image config %s: %w", imagePullspec, err)
+	}
 	ref, err := name.ParseReference(imagePullspec)
 	if err != nil {
-		return nil, err
+		return nil, wrapErr(err)
 	}
 	// Fetch the manifest using default credentials.
 	descriptor, err := remote.Get(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
 	if err != nil {
-		return nil, err
+		return nil, wrapErr(err)
 	}
 
 	image, err := descriptor.Image()
 	if err != nil {
-		return nil, err
+		return nil, wrapErr(err)
 	}
 	configFile, err := image.ConfigFile()
 	if err != nil {
-		return nil, err
+		return nil, wrapErr(err)
 	}
 	return configFile, nil
 }
 
+// FetchImageDigest fetches image manifest digest.
+// Digest is returned as a hex string without sha256: prefix.
 func FetchImageDigest(imagePullspec string) (string, error) {
 	ref, err := name.ParseReference(imagePullspec)
 	if err != nil {

pkg/utils/build/source_image.go

@@ -1,13 +1,17 @@
 package build
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
+	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/go-git/go-git/v5"
@@ -16,7 +20,9 @@ import (
 	"github.com/openshift/library-go/pkg/image/reference"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/redhat-appstudio/e2e-tests/pkg/clients/github"
 	"github.com/redhat-appstudio/e2e-tests/pkg/clients/tekton"
+	"github.com/redhat-appstudio/e2e-tests/pkg/constants"
 	"github.com/redhat-appstudio/e2e-tests/pkg/utils"
 	pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
 )
@@ -104,14 +110,15 @@ func IsSourceFilesExistsInSourceImage(srcImage string, gitUrl string, isHermetic
 			return false, fmt.Errorf("error while untaring %s: %v", tarFile, err)
 		}
 	}
+
 	//Check if application source files exists
 	_, err = IsAppSourceFilesExists(absExtraSourceDirPath, gitUrl)
 	if err != nil {
 		return false, err
 	}
 	// Check the pre-fetch dependency related files
 	if isHermetic {
-		_, err := IsPreFetchDependencysFilesExists(absExtraSourceDirPath, isHermetic, prefetchValue)
+		_, err := IsPreFetchDependenciesFilesExists(gitUrl, absExtraSourceDirPath, isHermetic, prefetchValue)
 		if err != nil {
 			return false, err
 		}
@@ -120,6 +127,8 @@ func IsSourceFilesExistsInSourceImage(srcImage string, gitUrl string, isHermetic
 	return true, nil
 }
 
+// IsAppSourceFilesExists checks if there is app source archive included.
+// For the builds based on Konflux image, multiple app sources could be included.
 func IsAppSourceFilesExists(absExtraSourceDirPath string, gitUrl string) (bool, error) {
 	//Get the file list from extra_src_dir
 	fileNames, err := utils.GetFileNamesFromDir(absExtraSourceDirPath)
@@ -134,28 +143,104 @@ func IsAppSourceFilesExists(absExtraSourceDirPath string, gitUrl string) (bool,
 	if len(resultFiles) == 0 {
 		return false, fmt.Errorf("did not found the component source inside extra_src_dir, files found are: %v", fileNames)
 	}
-	sourceGzTarFileName := resultFiles[0]
 
-	//Untar the <repo-name>-<git-sha>.tar.gz file
-	err = utils.Untar(absExtraSourceDirPath, filepath.Join(absExtraSourceDirPath, sourceGzTarFileName))
+	for _, sourceGzTarFileName := range resultFiles {
+		//Untar the <repo-name>-<git-sha>.tar.gz file
+		err = utils.Untar(absExtraSourceDirPath, filepath.Join(absExtraSourceDirPath, sourceGzTarFileName))
+		if err != nil {
+			return false, fmt.Errorf("error while untaring %s: %v", sourceGzTarFileName, err)
+		}
+
+		//Get the file list from extra_src_dir/<repo-name>-<sha>
+		sourceGzTarDirName := strings.TrimSuffix(sourceGzTarFileName, ".tar.gz")
+		absSourceGzTarPath := filepath.Join(absExtraSourceDirPath, sourceGzTarDirName)
+		fileNames, err = utils.GetFileNamesFromDir(absSourceGzTarPath)
+		if err != nil {
+			return false, fmt.Errorf("error while getting files from %s: %v", sourceGzTarDirName, err)
+		}
+		if len(fileNames) == 0 {
+			return false, fmt.Errorf("no file found under extra_src_dir/<repo-name>-<git-sha>")
+		}
+	}
+
+	return true, nil
+}
+
+// NewGithubClient creates a GitHub client with custom organization.
+// The token is retrieved in the same way as what SuiteController does.
+func NewGithubClient(organization string) (*github.Github, error) {
+	token := utils.GetEnv(constants.GITHUB_TOKEN_ENV, "")
+	if gh, err := github.NewGithubClient(token, organization); err != nil {
+		return nil, err
+	} else {
+		return gh, nil
+	}
+}
+
+// ReadFileFromGitRepo reads a file from a remote Git repository hosted in GitHub.
+// The filePath should be a relative path to the root of the repository.
+// File content is returned. If error occurs, the error will be returned and
+// empty string is returned as nothing is read.
+// If branch is omitted, file is read from the "main" branch.
+func ReadFileFromGitRepo(repoUrl, filePath, branch string) (string, error) {
+	fromBranch := branch
+	if fromBranch == "" {
+		fromBranch = "main"
+	}
+	wrapErr := func(err error) error {
+		return fmt.Errorf("error while reading file %s from repository %s: %v", filePath, repoUrl, err)
+	}
+	parsedUrl, err := url.Parse(repoUrl)
+	if err != nil {
+		return "", wrapErr(err)
+	}
+	org, repo := path.Split(parsedUrl.Path)
+	gh, err := NewGithubClient(strings.Trim(org, "/"))
+	if err != nil {
+		return "", wrapErr(err)
+	}
+	repoContent, err := gh.GetFile(repo, filePath, fromBranch)
 	if err != nil {
-		return false, fmt.Errorf("error while untaring %s: %v", sourceGzTarFileName, err)
+		return "", wrapErr(err)
+	}
+	if content, err := repoContent.GetContent(); err != nil {
+		return "", wrapErr(err)
+	} else {
+		return content, nil
+	}
+}
+
+// ReadRequirements reads dependencies from compiled requirements.txt by pip-compile,
+// and it assumes the requirements.txt is simple in the root of the repository.
+// The requirements are returned a list of strings, each of them is in form name==version.
+func ReadRequirements(repoUrl string) ([]string, error) {
+	const requirementsFile = "requirements.txt"
+
+	wrapErr := func(err error) error {
+		return fmt.Errorf("error while reading requirements.txt from repo %s: %v", repoUrl, err)
 	}
 
-	//Get the file list from extra_src_dir/<repo-name>-<sha>
-	sourceGzTarDirName := strings.TrimSuffix(sourceGzTarFileName, ".tar.gz")
-	absSourceGzTarPath := filepath.Join(absExtraSourceDirPath, sourceGzTarDirName)
-	fileNames, err = utils.GetFileNamesFromDir(absSourceGzTarPath)
+	content, err := ReadFileFromGitRepo(repoUrl, requirementsFile, "")
 	if err != nil {
-		return false, fmt.Errorf("error while getting files from %s: %v", sourceGzTarDirName, err)
+		return nil, wrapErr(err)
 	}
-	if len(fileNames) == 0 {
-		return false, fmt.Errorf("no file found under extra_src_dir/<repo-name>-<git-sha>")
+
+	reqs := make([]string, 0, 5)
+	// Match line: "requests==2.31.0 \"
+	reqRegex := regexp.MustCompile(`^\S.+ \\$`)
+
+	scanner := bufio.NewScanner(strings.NewReader(content))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if reqRegex.MatchString(line) {
+			reqs = append(reqs, strings.TrimSuffix(line, " \\"))
+		}
 	}
-	return true, nil
+
+	return reqs, nil
 }
 
-func IsPreFetchDependencysFilesExists(absExtraSourceDirPath string, isHermetic bool, prefetchValue string) (bool, error) {
+func IsPreFetchDependenciesFilesExists(gitUrl, absExtraSourceDirPath string, isHermetic bool, prefetchValue string) (bool, error) {
 	var absDependencyPath string
 	if prefetchValue == "gomod" {
 		fmt.Println("Checking go dependency files")
@@ -174,6 +259,36 @@ func IsPreFetchDependencysFilesExists(absExtraSourceDirPath string, isHermetic b
 	if len(fileNames) == 0 {
 		return false, fmt.Errorf("no file found under extra_src_dir/deps/")
 	}
+
+	// Easy to check for pip. Check if all requirements are included in the built source image.
+	if prefetchValue == "pip" {
+		fileSet := make(map[string]int)
+		for _, name := range fileNames {
+			fileSet[name] = 1
+		}
+		fmt.Println("file set:", fileSet)
+
+		requirements, err := ReadRequirements(gitUrl)
+		fmt.Println("requirements:", requirements)
+		if err != nil {
+			return false, fmt.Errorf("error while reading requirements.txt from repo %s: %v", gitUrl, err)
+		}
+		var sdistFilename string
+		for _, requirement := range requirements {
+			if strings.Contains(requirement, "==") {
+				sdistFilename = strings.Replace(requirement, "==", "-", 1) + ".tar.gz"
+			} else if strings.Contains(requirement, " @ https://") {
+				sdistFilename = fmt.Sprintf("external-%s", strings.Split(requirement, " ")[0])
+			} else {
+				fmt.Println("unknown requirement form:", requirement)
+				continue
+			}
+			if _, exists := fileSet[sdistFilename]; !exists {
+				return false, fmt.Errorf("requirement '%s' is not included", requirement)
+			}
+		}
+	}
+
 	return true, nil
 }
 
@@ -399,7 +514,7 @@ func SourceBuildTaskRunLogsContain(
 	return false, nil
 }
 
-func ResolveSourceImage(image string) (string, error) {
+func ResolveSourceImageByVersionRelease(image string) (string, error) {
 	config, err := FetchImageConfig(image)
 	if err != nil {
 		return "", err
@@ -425,11 +540,11 @@ func ResolveSourceImage(image string) (string, error) {
 func AllParentSourcesIncluded(parentSourceImage, builtSourceImage string) (bool, error) {
 	parentConfig, err := FetchImageConfig(parentSourceImage)
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("error while getting parent source image manifest %s: %w", parentSourceImage, err)
 	}
 	builtConfig, err := FetchImageConfig(builtSourceImage)
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("error while getting built source image manifest %s: %w", builtSourceImage, err)
 	}
 	srpmSha256Sums := make(map[string]int)
 	var parts []string
@@ -447,3 +562,17 @@ func AllParentSourcesIncluded(parentSourceImage, builtSourceImage string) (bool,
 	}
 	return true, nil
 }
+
+func ResolveKonfluxSourceImage(image string) (string, error) {
+	digest, err := FetchImageDigest(image)
+	if err != nil {
+		return "", fmt.Errorf("error while fetching image digest of %s: %w", image, err)
+	}
+	ref, err := reference.Parse(image)
+	if err != nil {
+		return "", fmt.Errorf("error while parsing image %s: %w", image, err)
+	}
+	ref.ID = ""
+	ref.Tag = fmt.Sprintf("sha256-%s.src", digest)
+	return ref.Exact(), nil
+}