From c8164a0733989ac93ef68bb8cc2431cbdc80e115 Mon Sep 17 00:00:00 2001 From: Akash-Nayak Date: Wed, 20 Mar 2024 19:32:31 +0530 Subject: [PATCH] feat: Allow overriding the default QA Mappings file (#1158) feat: Allow overriding the default QA Mappings file chore: Moved ports questions to a separate category fix: Categories disabled in the QA mappings file were not being respected --------- Signed-off-by: Akash Nayak Co-authored-by: Harikrishnan Balagopal --- assets/built-in/qa/qamappings.yaml | 152 +++++++++++++++-------------- cmd/flags.go | 4 + cmd/transform.go | 57 +++-------- cmd/utils.go | 79 +++++++++++++++ lib/planner.go | 5 - lib/utils.go | 2 +- types/qaengine/qamapping.go | 23 +++-- 7 files changed, 192 insertions(+), 130 deletions(-) diff --git a/assets/built-in/qa/qamappings.yaml b/assets/built-in/qa/qamappings.yaml index 381fbbcd4..ebdef6109 100644 --- a/assets/built-in/qa/qamappings.yaml +++ b/assets/built-in/qa/qamappings.yaml @@ -1,72 +1,80 @@ -categories: - - name: imageregistry - # or use the flags --enable imageregistry/--disable imageregistry - enabled: true - questions: - - move2kube.target.imageregistry.url - - move2kube.target.imageregistry.namespace - - move2kube.target.imageregistry.*.logintype - - move2kube.target.imageregistry.*.password - - name: sshkeys - enabled: true - questions: - - move2kube.repo.keys.*.key - - move2kube.repo.keys.pub.domain.*.pubkey - - move2kube.repo.keys.pub.load - - move2kube.repo.keys.load - - move2kube.repo.keys.paths - - move2kube.repo.keys.priv.*.password - - name: storage - enabled: true - questions: - - move2kube.storage.type.*.options - - name: sourceanalyzer - enabled: true - questions: - - move2kube.services.*.enable - - move2kube.services.*.statefulset - - move2kube.services.*.containerizationoption - - move2kube.services.*.childProjects.*.publishprofile - - move2kube.services.*.apacheconfig - - move2kube.services.*.pythonmainfile - - move2kube.services.*.pythonstartingfile - - move2kube.services.*.dockerfileType - - move2kube.services.*.childModules.*.enable - - move2kube.services.*.childProjects.*.enable - - move2kube.services.*.childModules.*.springBootProfiles - - move2kube.services.*.mavenProfiles - - name: cluster - enabled: true - questions: - - move2kube.target.*.clustertype - - move2kube.minreplicas - - name: network - enabled: true - questions: - - move2kube.services.*.*.servicetype - - move2kube.services.*.*.urlpath - - move2kube.services.*.ports - - move2kube.services.*.port - - move2kube.target.*.ingress.ingressclassname - - move2kube.target.*.ingress.host - - move2kube.target.*.ingress.tls - - name: git - enabled: true - questions: - - move2kube.vcs.git.name - - move2kube.vcs.git.username - - move2kube.vcs.git.email - - move2kube.vcs.git.pass - - name: cicd - enabled: true - questions: - - move2kube.target.cicd.tekton.gitreposshsecret - - move2kube.target.cicd.tekton.gitrepobasicauthsecret - - move2kube.target.cicd.tekton.registrypushsecret - - move2kube.transformers.kubernetes.argocd.namespace - - name: transformers - enabled: true - questions: - - move2kube.transformerselector - - move2kube.spawncontainers - - move2kube.transformers.types +apiVersion: move2kube.konveyor.io/v1alpha1 +kind: QAMappings +metadata: + name: DefaultQA-Mappings +spec: + categories: + - name: imageregistry + # or use the flags --enable imageregistry/--disable imageregistry + enabled: true + questions: + - move2kube.target.imageregistry.url + - move2kube.target.imageregistry.namespace + - move2kube.target.imageregistry.*.logintype + - move2kube.target.imageregistry.*.password + - name: sshkeys + enabled: true + questions: + - move2kube.repo.keys.*.key + - move2kube.repo.keys.pub.domain.*.pubkey + - move2kube.repo.keys.pub.load + - move2kube.repo.keys.load + - move2kube.repo.keys.paths + - move2kube.repo.keys.priv.*.password + - name: storage + enabled: true + questions: + - move2kube.storage.type.*.options + - name: sourceanalyzer + enabled: true + questions: + - move2kube.services.*.enable + - move2kube.services.*.statefulset + - move2kube.services.*.containerizationoption + - move2kube.services.*.childProjects.*.publishprofile + - move2kube.services.*.apacheconfig + - move2kube.services.*.pythonmainfile + - move2kube.services.*.pythonstartingfile + - move2kube.services.*.dockerfileType + - move2kube.services.*.childModules.*.enable + - move2kube.services.*.childProjects.*.enable + - move2kube.services.*.childModules.*.springBootProfiles + - move2kube.services.*.mavenProfiles + - name: cluster + enabled: true + questions: + - move2kube.target.*.clustertype + - move2kube.minreplicas + - name: network + enabled: true + questions: + - move2kube.services.*.*.servicetype + - move2kube.services.*.*.urlpath + - move2kube.target.*.ingress.ingressclassname + - move2kube.target.*.ingress.host + - move2kube.target.*.ingress.tls + - name: ports + enabled: true + questions: + - move2kube.services.*.ports + - move2kube.services.*.port + - name: git + enabled: true + questions: + - move2kube.vcs.git.name + - move2kube.vcs.git.username + - move2kube.vcs.git.email + - move2kube.vcs.git.pass + - name: cicd + enabled: true + questions: + - move2kube.target.cicd.tekton.gitreposshsecret + - move2kube.target.cicd.tekton.gitrepobasicauthsecret + - move2kube.target.cicd.tekton.registrypushsecret + - move2kube.transformers.kubernetes.argocd.namespace + - name: transformers + enabled: true + questions: + - move2kube.transformerselector + - move2kube.spawncontainers + - move2kube.transformers.types diff --git a/cmd/flags.go b/cmd/flags.go index a30c728a7..e05579b07 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -77,4 +77,8 @@ type qaflags struct { preSets []string // persistPasswords sets whether to persist the password or not persistPasswords bool + // qaEnabledCategories contains list of categories to be enabled + qaEnabledCategories []string + // qaDisabledCategories contains list of categories to be disabled + qaDisabledCategories []string } diff --git a/cmd/transform.go b/cmd/transform.go index 152f266d2..683c247dd 100644 --- a/cmd/transform.go +++ b/cmd/transform.go @@ -23,17 +23,14 @@ import ( "path/filepath" "runtime/pprof" - "github.com/konveyor/move2kube/assets" "github.com/konveyor/move2kube/common" "github.com/konveyor/move2kube/common/download" "github.com/konveyor/move2kube/common/vcs" "github.com/konveyor/move2kube/lib" "github.com/konveyor/move2kube/types/plan" - "github.com/konveyor/move2kube/types/qaengine" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" - "gopkg.in/yaml.v3" ) type transformFlags struct { @@ -59,10 +56,8 @@ type transformFlags struct { // maxIterations is the maximum number of iterations to allow before aborting with an error maxIterations int // CustomizationsPaths contains the path to the customizations directory - customizationsPath string - transformerSelector string - qaEnabledCategories []string - qaDisabledCategories []string + customizationsPath string + transformerSelector string } func transformHandler(cmd *cobra.Command, flags transformFlags) { @@ -133,46 +128,9 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) { } } - // qa-disable and qa=enable are mutually exclusive - if len(flags.qaEnabledCategories) > 0 && len(flags.qaDisabledCategories) > 0 { - logrus.Fatalf("--qa-enable and --qa-disable cannot be used together.\n") - } - - // Read the QA categories from the QA mapping file - var qaMapping qaengine.QAMapping - qaMappingFilepath := filepath.Join("built-in/qa", "qamappings.yaml") - file, err := assets.AssetsDir.ReadFile(qaMappingFilepath) - if err != nil { - logrus.Fatalf("failed to read qa-mapping file at %s. Error: %q\n", qaMappingFilepath, err) - } - - if err := yaml.Unmarshal(file, &qaMapping); err != nil { - logrus.Fatalf("failed to decode qa-mapping file. Error: %q\n", err) - } - - for _, mapping := range qaMapping.Categories { - common.QACategoryMap[mapping.Name] = mapping.Questions - } - common.QACategoryMap["default"] = []string{} - common.QACategoryMap["external"] = []string{} - // Global settings common.IgnoreEnvironment = flags.ignoreEnv common.DisableLocalExecution = flags.disableLocalExecution - // if --qa-enable is passed, all categories are disabled by default. Otherwise, only categories passed to --qa-disable - // are disabled - if len(flags.qaEnabledCategories) > 0 { - for k := range common.QACategoryMap { - if !common.IsStringPresent(flags.qaEnabledCategories, k) { - common.DisabledCategories = append(common.DisabledCategories, k) - } - } - } else { - for _, cat := range flags.qaDisabledCategories { - common.DisabledCategories = append(common.DisabledCategories, cat) - } - } - // Parameter cleaning and curate plan transformationPlan := plan.Plan{} preExistingPlan := false @@ -182,7 +140,7 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) { _, err = os.Stat(flags.planfile) } if err != nil { - logrus.Debugf("No plan file found.") + logrus.Infof("No plan file found.") if cmd.Flags().Changed(planFlag) { logrus.Fatalf("Error while accessing plan file at path %s Error: %q", flags.planfile, err) } @@ -201,6 +159,11 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) { logrus.Fatalf("Failed to create the output directory at path %s Error: %q", flags.outpath, err) } } + if flags.customizationsPath != "" { + if err := lib.CheckAndCopyCustomizations(flags.customizationsPath); err != nil { + logrus.Fatalf("Failed to check and copy the customizations. Error: %q", err) + } + } startQA(flags.qaflags) logrus.Debugf("Creating a new plan.") transformationPlan, err = lib.CreatePlan(ctx, flags.srcpath, flags.outpath, flags.customizationsPath, flags.transformerSelector, flags.name) @@ -240,7 +203,9 @@ func transformHandler(cmd *cobra.Command, flags transformFlags) { if transformationPlan.Spec.SourceDir != "" { checkSourcePath(transformationPlan.Spec.SourceDir) } - lib.CheckAndCopyCustomizations(transformationPlan.Spec.CustomizationsDir) + if err := lib.CheckAndCopyCustomizations(transformationPlan.Spec.CustomizationsDir); err != nil { + logrus.Fatalf("Failed to check and copy the customizations. Error: %q", err) + } if !isRemoteOutPath { flags.outpath = filepath.Join(flags.outpath, transformationPlan.Name) checkOutputPath(flags.outpath, flags.overwrite) diff --git a/cmd/utils.go b/cmd/utils.go index b2b72e1b8..1a23451b7 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -18,16 +18,20 @@ package cmd import ( "encoding/json" + "fmt" "net/http" "os" "path/filepath" "strings" "github.com/gorilla/mux" + "github.com/konveyor/move2kube/assets" "github.com/konveyor/move2kube/common" "github.com/konveyor/move2kube/qaengine" + qaenginetypes "github.com/konveyor/move2kube/types/qaengine" "github.com/sirupsen/logrus" "github.com/spf13/cast" + "gopkg.in/yaml.v3" ) // checkSourcePath checks if the source path is an existing directory. @@ -82,7 +86,82 @@ Exiting.`, outpath, overwriteFlag, outputFlag) logrus.Infof("Output directory '%s' exists. The contents might get overwritten.", outpath) } +func getCustomMappingFilePath() (qaenginetypes.QAMappings, error) { + // Read the QA categories from the QA mapping file + var qaMapping qaenginetypes.QAMappings + customizationsDir := filepath.Join(common.AssetsPath, "custom") + _, err := os.Stat(customizationsDir) + if err == nil { + yamlPaths, err := common.GetFilesByExt(customizationsDir, []string{".yml", ".yaml"}) + if err != nil { + return qaMapping, fmt.Errorf("failed to look for yaml files in the directory '%s' . Error: %w", customizationsDir, err) + } + for _, yamlPath := range yamlPaths { + if err := common.ReadMove2KubeYaml(yamlPath, &qaMapping); err != nil { + logrus.Debugf("failed to read the mappings file metadata from the yaml file at path '%s' . Error: %q", yamlPath, err) + continue + } + if qaMapping.Kind != qaenginetypes.QAMappingsKind { + logrus.Debugf( + "the file at path '%s' is not a valid cluster metadata. Expected kind: '%s' Actual kind: '%s'", + yamlPath, qaenginetypes.QAMappingsKind, qaMapping.Kind, + ) + continue + } + logrus.Infof("Found custom QA mappings file '%s' at path '%s'", qaMapping.ObjectMeta.Name, yamlPath) + return qaMapping, nil + } + } + logrus.Infof("Using the default QA mappings file") + qaMappingFilepath := filepath.Join("built-in/qa", "qamappings.yaml") + file, err := assets.AssetsDir.ReadFile(qaMappingFilepath) + if err != nil { + return qaMapping, fmt.Errorf("failed to read the mappings file metadata from the yaml file at path '%s' . Error: %w", qaMappingFilepath, err) + } + if err := yaml.Unmarshal(file, &qaMapping); err != nil { + return qaMapping, fmt.Errorf("failed to decode qa-mapping file. Error: %w", err) + } + return qaMapping, nil +} + +func initDisabledCategories(flags qaflags) { + // qa-disable and qa=enable are mutually exclusive + if len(flags.qaEnabledCategories) > 0 && len(flags.qaDisabledCategories) > 0 { + logrus.Fatalf("--qa-enable and --qa-disable cannot be used together.\n") + } + // Read the QA categories from the QA mapping file + qaMapping, err := getCustomMappingFilePath() + if err != nil { + logrus.Fatalf("failed to read the QAMappings file. Error: %q", err) + } + for _, category := range qaMapping.Spec.Categories { + common.QACategoryMap[category.Name] = category.Questions + } + common.QACategoryMap["default"] = []string{} + common.QACategoryMap["external"] = []string{} + // if --qa-enable is passed, all categories are disabled by default. Otherwise, only categories passed to --qa-disable + // are disabled + for _, category := range qaMapping.Spec.Categories { + if !category.Enabled { + common.DisabledCategories = append(common.DisabledCategories, category.Name) + } + } + if len(flags.qaEnabledCategories) > 0 { + for k := range common.QACategoryMap { + if !common.IsStringPresent(flags.qaEnabledCategories, k) { + common.DisabledCategories = append(common.DisabledCategories, k) + } + } + } else { + common.DisabledCategories = append(common.DisabledCategories, flags.qaDisabledCategories...) + } + if len(common.DisabledCategories) > 0 { + logrus.Infof("Disabling the questions in the following categories: %v", common.DisabledCategories) + } +} + func startQA(flags qaflags) { + initDisabledCategories(flags) qaengine.StartEngine(flags.qaskip, flags.qaport, flags.qadisablecli) if flags.configOut == "" { qaengine.SetupConfigFile("", flags.setconfigs, flags.configs, flags.preSets, flags.persistPasswords) diff --git a/lib/planner.go b/lib/planner.go index ad2f03a41..de55c6564 100644 --- a/lib/planner.go +++ b/lib/planner.go @@ -54,11 +54,6 @@ func CreatePlan(ctx context.Context, inputPath, outputPath string, customization if remoteOutputFSPath != "" { outputFSPath = remoteOutputFSPath } - if customizationsPath != "" { - if err := CheckAndCopyCustomizations(customizationsPath); err != nil { - return plan, fmt.Errorf("failed to check and copy the customizations. Error: %w", err) - } - } transformerSelectorObj, err := metav1.ParseToLabelSelector(transformerSelector) if err != nil { return plan, fmt.Errorf("failed to parse the string '%s' as a transformer selector. Error: %w", transformerSelector, err) diff --git a/lib/utils.go b/lib/utils.go index b0461ca5e..48bcb44f1 100644 --- a/lib/utils.go +++ b/lib/utils.go @@ -67,7 +67,7 @@ func CheckAndCopyCustomizations(customizationsPath string) error { // check if the customization path has files other than YAMLs yamls, err := common.GetYamlsWithTypeMeta(customizationsFSPath, TransformerTypeMeta) if err == nil && len(yamls) == 0 { - logrus.Warnf("no manifests for external transformers found in %s, the transformers won't be loaded.", customizationsFSPath) + logrus.Infof("No manifests for external transformers found in the customizations directory '%s'.", customizationsFSPath) } if err = CopyCustomizationsAssetsData(customizationsFSPath); err != nil { return fmt.Errorf("failed to copy the customizations data from the directory '%s' . Error: %w", customizationsFSPath, err) diff --git a/types/qaengine/qamapping.go b/types/qaengine/qamapping.go index 33e0a0101..e4ad9a3ea 100644 --- a/types/qaengine/qamapping.go +++ b/types/qaengine/qamapping.go @@ -21,19 +21,30 @@ import ( "github.com/gobwas/glob" "github.com/konveyor/move2kube/common" + "github.com/konveyor/move2kube/types" "github.com/sirupsen/logrus" ) -// QAMapping defines the structure of the QA Category mapping file -type QAMapping struct { - Categories []QACategory `yaml:"categories"` +// QAMappingsKind represents the QAMappings kind +const QAMappingsKind = "QAMappings" + +// QAMappings defines the structure of the QA Category mapping file +type QAMappings struct { + types.TypeMeta `yaml:",inline" json:",inline"` + types.ObjectMeta `yaml:"metadata,omitempty" json:"metadata,omitempty"` + Spec QAMappingsSpec `yaml:"spec" json:"spec"` +} + +// QAMappingsSpec defines the structure of the QA mapping spec file +type QAMappingsSpec struct { + Categories []QACategory `yaml:"categories" json:"categories"` } // QACategory represents a category of QA engine questions type QACategory struct { - Name string `yaml:"name"` - Enabled bool `yaml:"enabled"` - Questions []string `yaml:"questions"` + Name string `yaml:"name" json:"name"` + Enabled bool `yaml:"enabled" json:"enabled"` + Questions []string `yaml:"questions" json:"questions"` } // GetProblemCategories returns the list of categories a particular question belongs to