Merge branch 'master' into sync-subpipeline-sas

.github/workflows/radix-operator-pr.yml

@@ -4,6 +4,45 @@ on:
     branches:
     - master
 jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: Install dependencies
+        run: go mod download
+      - name: Run Tests
+        run: CGO_ENABLED=0 GOOS=linux go test `go list ./... | grep -v "pkg/client"` -timeout 2m
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.21'
+      - name: Install dependencies
+        run: go mod download
+      - name: Install GolangCI Lint
+        run: go install github.com/golangci/golangci-lint/cmd/[email protected]
+      - name: Install StaticCheck
+        run: go install honnef.co/go/tools/cmd/[email protected]
+
+      - name: golangci-lint
+        run: golangci-lint run --timeout=30m --max-same-issues=0 --out-format=github-actions --new
+
+      # Add legacy checks since golangci-lint only tests changed files for now
+      - name: StaticCheck
+        run: staticcheck -f text `go list ./... | grep -v "pkg/client"`
+      - name: Go Vet
+        run: go vet `go list ./... | grep -v "pkg/client"`
+
   build-operator:
     name: Build-operator
     runs-on: ubuntu-latest
@@ -14,6 +53,7 @@ jobs:
         REF: ${{ github. sha }}
         DOCKER_BUILDKIT: 1
       run: docker build -t radix-operator:${REF##*/} -f operator.Dockerfile .
+
   build-pipeline:
     name: Build-pipeline
     runs-on: ubuntu-latest
@@ -23,4 +63,4 @@ jobs:
       env:
         REF: ${{ github. sha }}
         DOCKER_BUILDKIT: 1
-      run: docker build -t radix-operator:${REF##*/} -f pipeline.Dockerfile .
+      run: docker build -t radix-operator:${REF##*/} -f pipeline.Dockerfile .

.vscode/launch.json

@@ -40,14 +40,15 @@
       "program": "${workspaceFolder}/pipeline-runner/main.go",
       "env": {},
       "args": [
-        "--RADIX_APP=radix-job-demo",
+        "--RADIX_APP=oauth-demo",
         "--IMAGE_TAG=abcdef",
-        "--JOB_NAME=radix-pipeline-promotion-1",
+        "--JOB_NAME=radix-pipeline-20231121120818-sb2xq",
         "--PIPELINE_TYPE=promote",
         "--RADIX_TEKTON_IMAGE=radix-tekton:main-latest",
-        "--FROM_ENVIRONMENT=qa",
+        "--FROM_ENVIRONMENT=dev",
         "--TO_ENVIRONMENT=prod",
-        "--DEPLOYMENT_NAME=qa-etxkt-ac6rxchq",
+        "--DEPLOYMENT_NAME=dev-hyxzv-j9pg34k2",
+        "--RADIX_FILE_NAME=/workspace/radixconfig.yaml",
         "--DEBUG=true",
         "--RADIX_CONTAINER_REGISTRY=radixdev.azurecr.io"
       ]
@@ -97,7 +98,7 @@
         "RADIXOPERATOR_APP_ROLLING_UPDATE_MAX_SURGE": "25%",
         "RADIXOPERATOR_APP_READINESS_PROBE_INITIAL_DELAY_SECONDS": "5",
         "RADIXOPERATOR_APP_READINESS_PROBE_PERIOD_SECONDS": "10",
-        "RADIX_ACTIVE_CLUSTERNAME": "weekly-45",
+        "RADIX_ACTIVE_CLUSTERNAME": "weekly-47",
         "RADIX_IMAGE_BUILDER": "radix-image-builder:master-latest",
         "RADIX_TEKTON_IMAGE": "radix-tekton:main-latest",
         "RADIXOPERATOR_JOB_SCHEDULER": "radix-job-scheduler:main-latest",

charts/radix-operator/templates/radix-operator-rbac.yaml

@@ -44,7 +44,7 @@ rules:
   verbs:
   - get
   - list
-  - watch 
+  - watch
   - create
   - delete
   - update # required since radix operator grants this to others

charts/radix-operator/templates/seccomp-profile-cm.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ .Values.seccompProfile.configMapName }}
-  namespace: {{ .Release.Namespace | quote }}
+  namespace: kube-system
   annotations:
     description: "configmap with seccomp profile suitable for running buildah. It's a copy of the moby container runtime's default seccomp profile, but with the addition of two syscalls: clone3 and unshare. https://github.com/moby/moby/blob/b335e3d305be86bd28089a057d8be6a346445549/profiles/seccomp/default.json"
 data:
@@ -837,4 +837,4 @@ data:
           }
         }
       ]
-    }
+    }

charts/radix-operator/templates/seccomp-profile-daemonset.yaml

@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: {{ .Values.seccompProfile.daemonSetName }}
-  namespace: {{ .Release.Namespace | quote }}
+  namespace: kube-system
 spec:
   selector:
     matchLabels:

operator.Dockerfile

@@ -13,16 +13,6 @@ RUN go mod download
 COPY ./radix-operator ./radix-operator
 COPY ./pkg ./pkg
 
-FROM base as run-staticcheck
-RUN go install honnef.co/go/tools/cmd/[email protected]
-RUN staticcheck `go list ./... | grep -v "pkg/client"` && touch /staticcheck.done
-
-FROM base as tester
-# Run tests
-RUN go vet `go list ./... | grep -v "pkg/client"` && \
-    CGO_ENABLED=0 GOOS=linux go test `go list ./... | grep -v "pkg/client"` && \
-    touch /tests.done
-
 FROM base as builder
 # Build
 WORKDIR /go/src/github.com/equinor/radix-operator/radix-operator/
@@ -35,8 +25,6 @@ FROM scratch
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /etc/passwd /etc/passwd
 COPY --from=builder /go/src/github.com/equinor/radix-operator/radix-operator/rootfs/radix-operator /usr/local/bin/radix-operator
-# This will make sure staticcheck and tests are run before the final stage is built
-COPY --from=run-staticcheck /staticcheck.done /staticcheck.done
-COPY --from=tester /tests.done /tests.done
+
 USER radix-operator
 ENTRYPOINT ["/usr/local/bin/radix-operator"]

pipeline-runner/internal/hash/encoding.go

@@ -6,7 +6,7 @@ import (
 	"math"
 	"reflect"
 
-	yamlk8s "sigs.k8s.io/yaml"
+	"sigs.k8s.io/yaml"
 )
 
 type encoder func(v any) ([]byte, error)
@@ -39,7 +39,7 @@ func stringEncoder(v any) ([]byte, error) {
 }
 
 func structEncoder(v any) ([]byte, error) {
-	b, err := yamlk8s.Marshal(v)
+	b, err := yaml.Marshal(v)
 	return b, err
 }
 

pipeline-runner/internal/test/utils.go

@@ -0,0 +1,81 @@
+package test
+
+import (
+	"context"
+
+	"github.com/equinor/radix-operator/pipeline-runner/internal/hash"
+	"github.com/equinor/radix-operator/pipeline-runner/model"
+	pipelineDefaults "github.com/equinor/radix-operator/pipeline-runner/model/defaults"
+	"github.com/equinor/radix-operator/pkg/apis/defaults"
+	radixv1 "github.com/equinor/radix-operator/pkg/apis/radix/v1"
+	"github.com/equinor/radix-operator/pkg/apis/utils"
+	"gopkg.in/yaml.v3"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	yamlk8s "sigs.k8s.io/yaml"
+)
+
+func CreatePreparePipelineConfigMapResponse(kubeClient kubernetes.Interface, configMapName, appName string, ra *radixv1.RadixApplication, buildCtx *model.PrepareBuildContext) error {
+	raBytes, err := yamlk8s.Marshal(ra)
+	if err != nil {
+		return err
+	}
+	data := map[string]string{
+		pipelineDefaults.PipelineConfigMapContent: string(raBytes),
+	}
+
+	if buildCtx != nil {
+		buildCtxBytes, err := yaml.Marshal(buildCtx)
+		if err != nil {
+			return err
+		}
+		data[pipelineDefaults.PipelineConfigMapBuildContext] = string(buildCtxBytes)
+	}
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{Name: configMapName},
+		Data:       data,
+	}
+	_, err = kubeClient.CoreV1().ConfigMaps(utils.GetAppNamespace(appName)).Create(context.Background(), cm, metav1.CreateOptions{})
+	return err
+}
+
+func CreateGitInfoConfigMapResponse(kubeClient kubernetes.Interface, configMapName, appName, gitHash, gitTags string) error {
+	cm := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{Name: configMapName},
+		Data: map[string]string{
+			defaults.RadixGitCommitHashKey: gitHash,
+			defaults.RadixGitTagsKey:       gitTags,
+		},
+	}
+	_, err := kubeClient.CoreV1().ConfigMaps(utils.GetAppNamespace(appName)).Create(context.Background(), cm, metav1.CreateOptions{})
+	return err
+}
+
+func CreateBuildSecret(kubeClient kubernetes.Interface, appName string, data map[string][]byte) error {
+	secret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{Name: defaults.BuildSecretsName},
+		Data:       data,
+	}
+
+	_, err := kubeClient.CoreV1().Secrets(utils.GetAppNamespace(appName)).Create(context.Background(), secret, metav1.CreateOptions{})
+	return err
+}
+
+func GetRadixApplicationHash(ra *radixv1.RadixApplication) string {
+	if ra == nil {
+		hash, _ := hash.ToHashString(hash.SHA256, "0nXSg9l6EUepshGFmolpgV3elB0m8Mv7")
+		return hash
+	}
+	hash, _ := hash.ToHashString(hash.SHA256, ra.Spec)
+	return hash
+}
+
+func GetBuildSecretHash(secret *corev1.Secret) string {
+	if secret == nil {
+		hash, _ := hash.ToHashString(hash.SHA256, "34Wd68DsJRUzrHp2f63o3U5hUD6zl8Tj")
+		return hash
+	}
+	hash, _ := hash.ToHashString(hash.SHA256, secret.Data)
+	return hash
+}

pipeline-runner/main.go

@@ -103,9 +103,9 @@ func setPipelineArgsFromArguments(cmd *cobra.Command, pipelineArgs *model.Pipeli
 	cmd.Flags().StringVar(&pipelineArgs.RadixConfigFile, defaults.RadixConfigFileEnvironmentVariable, "", "Radix config file name. Example: /workspace/radixconfig.yaml")
 	cmd.Flags().StringVar(&pipelineArgs.ImageTag, defaults.RadixImageTagEnvironmentVariable, "latest", "Docker image tag")
 	cmd.Flags().StringVar(&pipelineArgs.LogLevel, defaults.LogLevel, "INFO", "Log level: ERROR, INFO (default), DEBUG")
-	cmd.Flags().StringVar(&pipelineArgs.Builder.ResourcesLimitsMemory, defaults.OperatorAppBuilderResourcesLimitsMemoryEnvironmentVariable, "500M", "Image builder resource limit memory")
+	cmd.Flags().StringVar(&pipelineArgs.Builder.ResourcesLimitsMemory, defaults.OperatorAppBuilderResourcesLimitsMemoryEnvironmentVariable, "2000M", "Image builder resource limit memory")
 	cmd.Flags().StringVar(&pipelineArgs.Builder.ResourcesRequestsCPU, defaults.OperatorAppBuilderResourcesRequestsCPUEnvironmentVariable, "200m", "Image builder resource requests CPU")
-	cmd.Flags().StringVar(&pipelineArgs.Builder.ResourcesRequestsMemory, defaults.OperatorAppBuilderResourcesRequestsMemoryEnvironmentVariable, "2000M", "Image builder resource requests memory")
+	cmd.Flags().StringVar(&pipelineArgs.Builder.ResourcesRequestsMemory, defaults.OperatorAppBuilderResourcesRequestsMemoryEnvironmentVariable, "500M", "Image builder resource requests memory")
 	var useCache string
 	cmd.Flags().StringVar(&useCache, defaults.RadixUseCacheEnvironmentVariable, "0", "Use cache")
 	var pushImage string

pipeline-runner/pipelines/app.go

@@ -13,12 +13,12 @@ import (
 	radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned"
 	monitoring "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	secretsstorevclient "sigs.k8s.io/secrets-store-csi-driver/pkg/client/clientset/versioned"
+	"sigs.k8s.io/yaml"
 )
 
 // PipelineRunner Instance variables

pipeline-runner/steps/apply_radixconfig.go

@@ -2,7 +2,7 @@ package steps
 
 import (
 	"context"
-	"errors"
+	stderrors "errors"
 	"fmt"
 	"path/filepath"
 	"strings"
@@ -21,12 +21,12 @@ import (
 	operatorutils "github.com/equinor/radix-operator/pkg/apis/utils"
 	"github.com/equinor/radix-operator/pkg/apis/utils/git"
 	radixclient "github.com/equinor/radix-operator/pkg/client/clientset/versioned"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
 	kubeerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	yamlk8s "sigs.k8s.io/yaml"
+	"sigs.k8s.io/yaml"
 )
 
 const (
@@ -90,11 +90,6 @@ func (cli *ApplyConfigStepImplementation) Run(pipelineInfo *model.PipelineInfo)
 	applicationConfig := application.NewApplicationConfig(cli.GetKubeclient(), cli.GetKubeutil(),
 		cli.GetRadixclient(), cli.GetRegistration(), ra)
 
-	err = applicationConfig.ApplyConfigToApplicationNamespace()
-	if err != nil {
-		return err
-	}
-
 	pipelineInfo.SetApplicationConfig(applicationConfig)
 
 	if err := cli.setBuildSecret(pipelineInfo); err != nil {
@@ -105,10 +100,6 @@ func (cli *ApplyConfigStepImplementation) Run(pipelineInfo *model.PipelineInfo)
 		return err
 	}
 
-	if pipelineInfo.IsPipelineType(radixv1.Deploy) && len(pipelineInfo.BuildComponentImages) > 0 {
-		return errors.New("deploy pipeline does not support building components and jobs")
-	}
-
 	if pipelineInfo.IsPipelineType(radixv1.BuildDeploy) {
 		gitCommitHash, gitTags := cli.getHashAndTags(namespace, pipelineInfo)
 		err = validate.GitTagsContainIllegalChars(gitTags)
@@ -119,7 +110,11 @@ func (cli *ApplyConfigStepImplementation) Run(pipelineInfo *model.PipelineInfo)
 		pipelineInfo.StopPipeline, pipelineInfo.StopPipelineMessage = getPipelineShouldBeStopped(pipelineInfo.PrepareBuildContext)
 	}
 
-	return nil
+	if err := cli.validatePipelineInfo(pipelineInfo); err != nil {
+		return err
+	}
+
+	return applicationConfig.ApplyConfigToApplicationNamespace()
 }
 
 func (cli *ApplyConfigStepImplementation) setBuildSecret(pipelineInfo *model.PipelineInfo) error {
@@ -157,6 +152,14 @@ func (cli *ApplyConfigStepImplementation) setBuildAndDeployImages(pipelineInfo *
 	return nil
 }
 
+func (cli ApplyConfigStepImplementation) validatePipelineInfo(pipelineInfo *model.PipelineInfo) error {
+	if pipelineInfo.IsPipelineType(radixv1.Deploy) && len(pipelineInfo.BuildComponentImages) > 0 {
+		return ErrDeployOnlyPipelineDoesNotSupportBuild
+	}
+
+	return validateDeployComponentImages(pipelineInfo.DeployEnvironmentComponentImages, pipelineInfo.RadixApplication)
+}
+
 func printEnvironmentComponentImageSources(imageSources environmentComponentSourceMap) {
 	log.Info("Component image source in environments:")
 	for envName, envInfo := range imageSources {
@@ -521,7 +524,7 @@ func CreateRadixApplication(radixClient radixclient.Interface,
 	// Important: Must use sigs.k8s.io/yaml decoder to correctly unmarshal Kubernetes objects.
 	// This package supports encoding and decoding of yaml for CRD struct types using the json tag.
 	// The gopkg.in/yaml.v3 package requires the yaml tag.
-	if err := yamlk8s.Unmarshal([]byte(configFileContent), ra); err != nil {
+	if err := yaml.Unmarshal([]byte(configFileContent), ra); err != nil {
 		return nil, err
 	}
 
@@ -552,3 +555,26 @@ func getValueFromConfigMap(key string, configMap *corev1.ConfigMap) (string, err
 	}
 	return value, nil
 }
+
+func validateDeployComponentImages(deployComponentImages pipeline.DeployEnvironmentComponentImages, ra *radixv1.RadixApplication) error {
+	var errs []error
+
+	for envName, components := range deployComponentImages {
+		for componentName, imageInfo := range components {
+			if strings.HasSuffix(imageInfo.ImagePath, radixv1.DynamicTagNameInEnvironmentConfig) {
+				if len(imageInfo.ImageTagName) > 0 {
+					continue
+				}
+
+				env := ra.GetCommonComponentByName(componentName).GetEnvironmentConfigByName(envName)
+				if !commonutils.IsNil(env) && len(env.GetImageTagName()) > 0 {
+					continue
+				}
+
+				errs = append(errs, errors.WithMessagef(ErrMissingRequiredImageTagName, "component %s in environment %s", componentName, envName))
+			}
+		}
+	}
+
+	return stderrors.Join(errs...)
+}