From 50e24024ca374802737d9e465fcadf852b62b8cb Mon Sep 17 00:00:00 2001 From: Eric Hernandez Date: Wed, 5 Jun 2024 15:18:13 -0500 Subject: [PATCH] allows for image repos to be set via Environment Variables - useful when using a mirrored repo or no access to upstream repos --- clusterloader2/testing/load/config.yaml | 14 ++++++++++++-- clusterloader2/testing/load/daemonset.yaml | 3 +-- clusterloader2/testing/load/deployment.yaml | 8 ++++---- clusterloader2/testing/load/job.yaml | 4 ++-- .../testing/load/modules/reconcile-objects.yaml | 15 ++++++++++++++- .../load/modules/scheduler-throughput.yaml | 5 +++-- .../testing/load/simple-deployment.yaml | 3 +-- clusterloader2/testing/load/statefulset.yaml | 4 ++-- 8 files changed, 39 insertions(+), 17 deletions(-) diff --git a/clusterloader2/testing/load/config.yaml b/clusterloader2/testing/load/config.yaml index 12fd0c105a..c05b2fe76a 100644 --- a/clusterloader2/testing/load/config.yaml +++ b/clusterloader2/testing/load/config.yaml @@ -74,8 +74,11 @@ {{$EXEC_TIMEOUT := DefaultParam .CL2_EXEC_TIMEOUT "3600s"}} {{$SLEEP_AFTER_EXEC_DURATION := DefaultParam .CL2_SLEEP_AFTER_EXEC_DURATION "0s"}} -{{$registry := DefaultParam .CL2_LATENCY_POD_REGISTRY "registry.k8s.io"}} -{{$latencyPodImage := DefaultParam .CL2_LATENCY_POD_IMAGE (Concat $registry "/pause:3.9")}} +## Image Repositories +{{$registryK8sRepo := DefaultParam .CL2_REGISTRY_K8S_REPO "registry.k8s.io"}} +{{$gcrRepo := DefaultParam .CL2_GCR_REPO "gcr.io"}} + +{{$latencyPodImage := DefaultParam .CL2_LATENCY_POD_IMAGE "pause:3.9"}} {{$defaultQps := DefaultParam .CL2_DEFAULT_QPS (IfThenElse (le .Nodes 500) 10 100)}} {{$ADDITIONAL_MEASUREMENT_MODULES := DefaultParam .CL2_ADDITIONAL_MEASUREMENT_MODULES nil}} @@ -211,6 +214,8 @@ steps: mediumJobsPerNamespace: 1 smallJobSize: {{$SMALL_GROUP_SIZE}} smallJobsPerNamespace: 1 + registryK8sRepo: {{$registryK8sRepo}} + gcrRepo: {{$gcrRepo}} {{if $ADDITIONAL_PHASES_MODULES}} {{range $ADDITIONAL_PHASES_MODULES}} @@ -232,6 +237,7 @@ steps: schedulerThroughputNamespaces: {{$schedulerThroughputNamespaces}} schedulerThroughputPodsPerDeployment: {{$schedulerThroughputPodsPerDeployment}} deploymentImage: {{$latencyPodImage}} + registryK8sRepo: {{$registryK8sRepo}} {{end}} - module: @@ -356,6 +362,8 @@ steps: mediumJobsPerNamespace: 1 smallJobSize: {{$SMALL_GROUP_SIZE}} smallJobsPerNamespace: 1 + registryK8sRepo: {{$registryK8sRepo}} + gcrRepo: {{$gcrRepo}} {{if $ADDITIONAL_PHASES_MODULES}} {{range $ADDITIONAL_PHASES_MODULES}} @@ -402,6 +410,8 @@ steps: smallJobsPerNamespace: 0 pvSmallStatefulSetSize: {{$SMALL_STATEFUL_SETS_PER_NAMESPACE}} pvMediumStatefulSetSize: {{$MEDIUM_STATEFUL_SETS_PER_NAMESPACE}} + registryK8sRepo: {{$registryK8sRepo}} + gcrRepo: {{$gcrRepo}} {{if $ADDITIONAL_PHASES_MODULES}} {{range $ADDITIONAL_PHASES_MODULES}} diff --git a/clusterloader2/testing/load/daemonset.yaml b/clusterloader2/testing/load/daemonset.yaml index 5c69397693..d8bfff5898 100644 --- a/clusterloader2/testing/load/daemonset.yaml +++ b/clusterloader2/testing/load/daemonset.yaml @@ -1,5 +1,4 @@ {{$HostNetworkMode := DefaultParam .CL2_USE_HOST_NETWORK_PODS false}} -{{$Image := DefaultParam .Image "registry.k8s.io/pause:3.9"}} {{$Env := DefaultParam .Env ""}} {{$DaemonSetSurge := DefaultParam .CL2_DS_SURGE (MaxInt 10 (DivideInt .Nodes 20))}} # 5% of nodes, but not less than 10 {{$RUN_ON_ARM_NODES := DefaultParam .CL2_RUN_ON_ARM_NODES false}} @@ -29,7 +28,7 @@ spec: hostNetwork: {{$HostNetworkMode}} containers: - name: {{.Name}} - image: {{$Image}} + image: {{.RegistryK8sRepo}}/pause:3.9 env: - name: TEST_ENV value: {{$Env}} diff --git a/clusterloader2/testing/load/deployment.yaml b/clusterloader2/testing/load/deployment.yaml index f665c09b7c..635c363d5e 100644 --- a/clusterloader2/testing/load/deployment.yaml +++ b/clusterloader2/testing/load/deployment.yaml @@ -15,7 +15,7 @@ {{$NetPolServerOnEveryNthPod := DefaultParam .NetPolServerOnEveryNthPod 3}} {{$RunNetPolicyTest := and $EnableNetworkPolicyEnforcementLatencyTest (eq (Mod .Index $NetPolServerOnEveryNthPod) 0)}} -{{$Image := DefaultParam .Image "registry.k8s.io/pause:3.9"}} +{{$Image := DefaultParam .Image "pause:3.9"}} apiVersion: apps/v1 kind: Deployment @@ -47,13 +47,13 @@ spec: containers: {{if .EnableDNSTests}} {{if $USE_ADVANCED_DNSTEST}} - - image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.4.0 + - image: {{.GcrRepo}}/k8s-staging-perf-tests/dnsperfgo:v1.4.0 ports: - containerPort: 9153 name: dnsperfmetrics protocol: TCP {{else}} - - image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.2.0 + - image: {{.GcrRepo}}/k8s-staging-perf-tests/dnsperfgo:v1.2.0 {{end}} # Fetches the dns server from /etc/resolv.conf and # sends 1 query per second. @@ -74,7 +74,7 @@ spec: ports: - containerPort: 80 {{else}} - - image: {{$Image}} + - image: {{.RegistryK8sRepo}}/{{$Image}} name: {{.Name}} {{end}} {{end}} diff --git a/clusterloader2/testing/load/job.yaml b/clusterloader2/testing/load/job.yaml index 338dc6f483..4710a140c2 100644 --- a/clusterloader2/testing/load/job.yaml +++ b/clusterloader2/testing/load/job.yaml @@ -1,6 +1,6 @@ {{$HostNetworkMode := DefaultParam .CL2_USE_HOST_NETWORK_PODS false}} {{$RUN_ON_ARM_NODES := DefaultParam .CL2_RUN_ON_ARM_NODES false}} -{{$Image := DefaultParam .Image "registry.k8s.io/pause:3.9"}} +{{$Image := DefaultParam .Image "pause:3.9"}} apiVersion: batch/v1 kind: Job @@ -26,7 +26,7 @@ spec: containers: - name: {{.Name}} # TODO(#799): We should test the "run-to-completion" workflow and hence don't use pause pods. - image: {{$Image}} + image: {{.RegistryK8sRepo}}/{{$Image}} resources: # Keep the CpuRequest/MemoryRequest request equal percentage of 1-core, 4GB node. # For now we're setting it to 0.5%. diff --git a/clusterloader2/testing/load/modules/reconcile-objects.yaml b/clusterloader2/testing/load/modules/reconcile-objects.yaml index e96ef37657..9e795f37c1 100644 --- a/clusterloader2/testing/load/modules/reconcile-objects.yaml +++ b/clusterloader2/testing/load/modules/reconcile-objects.yaml @@ -13,7 +13,7 @@ {{$operationTimeout := .operationTimeout}} # DaemonSets -{{$daemonSetImage := DefaultParam .daemonSetImage "registry.k8s.io/pause:3.9"}} +{{$daemonSetImage := DefaultParam .daemonSetImage "$registryK8sRepo/pause:3.9"}} {{$daemonSetReplicas := .daemonSetReplicas}} {{$daemonSetEnv := .daemonSetEnv}} @@ -58,6 +58,9 @@ {{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE := DefaultParam .CL2_NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE "enforcement-latency"}} {{$NET_POLICY_SERVER_EVERY_NTH_POD := DefaultParam .CL2_NET_POLICY_SERVER_EVERY_NTH_POD 3}} +{{$registryK8sRepo := .registryK8sRepo}} +{{$gcrRepo := .gcrRepo}} + steps: - name: Starting measurement for '{{$actionName}}' measurements: @@ -99,6 +102,7 @@ steps: templateFillMap: Image: {{$daemonSetImage}} Env: {{$daemonSetEnv}} + RegistryK8sRepo: {{$registryK8sRepo}} {{end}} - namespaceRange: min: 1 @@ -119,6 +123,7 @@ steps: ReplicasMax: {{MultiplyInt $bigDeploymentSize $maxReplicaFactor}} SvcName: big-service Image: {{$deploymentImage}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -137,6 +142,7 @@ steps: ReplicasMax: {{MultiplyInt $mediumDeploymentSize $maxReplicaFactor}} SvcName: medium-service Image: {{$deploymentImage}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -160,6 +166,8 @@ steps: TargetLabelValue: {{$NET_POLICY_ENFORCEMENT_LATENCY_TARGET_LABEL_VALUE}} NetPolServerOnEveryNthPod: {{$NET_POLICY_SERVER_EVERY_NTH_POD}} Image: {{$deploymentImage}} + RegistryK8sRepo: {{$registryK8sRepo}} + GcrRepo: {{$gcrRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -174,6 +182,7 @@ steps: ReplicasMin: {{MultiplyInt $smallStatefulSetSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $smallStatefulSetSize $maxReplicaFactor}} Image: {{$statefulSetImage}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -188,6 +197,7 @@ steps: ReplicasMin: {{MultiplyInt $mediumStatefulSetSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $mediumStatefulSetSize $maxReplicaFactor}} Image: {{$statefulSetImage}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -201,6 +211,7 @@ steps: ReplicasMin: {{MultiplyInt $smallJobSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $smallJobSize $maxReplicaFactor}} Image: {{$jobImage}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -214,6 +225,7 @@ steps: ReplicasMin: {{MultiplyInt $mediumJobSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $mediumJobSize $maxReplicaFactor}} Image: {{$jobImage}} + RegistryK8sRepo: {{$registryK8sRepo}} - namespaceRange: min: 1 max: {{$namespaces}} @@ -227,6 +239,7 @@ steps: ReplicasMin: {{MultiplyInt $bigJobSize $minReplicaFactor}} ReplicasMax: {{MultiplyInt $bigJobSize $maxReplicaFactor}} Image: {{$jobImage}} + RegistryK8sRepo: {{$registryK8sRepo}} {{if and $is_deleting $ENABLE_PVS}} - namespaceRange: min: 1 diff --git a/clusterloader2/testing/load/modules/scheduler-throughput.yaml b/clusterloader2/testing/load/modules/scheduler-throughput.yaml index 41e778cc9f..28b402cabd 100644 --- a/clusterloader2/testing/load/modules/scheduler-throughput.yaml +++ b/clusterloader2/testing/load/modules/scheduler-throughput.yaml @@ -14,7 +14,8 @@ {{$CHECK_IF_PODS_ARE_UPDATED := DefaultParam .CL2_CHECK_IF_PODS_ARE_UPDATED true}} -{{$deploymentImage := DefaultParam .deploymentImage "registry.k8s.io/pause:3.9"}} +{{$deploymentImage := DefaultParam .deploymentImage "pause:3.9"}} +{{$registryK8sRepo := .registryK8sRepo}} steps: {{if $is_creating}} @@ -57,7 +58,7 @@ steps: templateFillMap: Replicas: {{$schedulerThroughputPodsPerDeployment}} Group: scheduler-throughput - Image: {{$deploymentImage}} + Image: {{$registryK8sRepo}}/{{$deploymentImage}} - name: Waiting for scheduler throughput pods to be {{$action}}d measurements: - Identifier: WaitForSchedulerThroughputDeployments diff --git a/clusterloader2/testing/load/simple-deployment.yaml b/clusterloader2/testing/load/simple-deployment.yaml index 92685b93e7..c19bac2124 100644 --- a/clusterloader2/testing/load/simple-deployment.yaml +++ b/clusterloader2/testing/load/simple-deployment.yaml @@ -4,7 +4,6 @@ {{$CpuRequest := DefaultParam .CpuRequest "5m"}} {{$EnvVar := DefaultParam .EnvVar "a"}} {{$MemoryRequest := DefaultParam .MemoryRequest "20M"}} -{{$Image := DefaultParam .Image "registry.k8s.io/pause:3.9"}} {{$RUN_ON_ARM_NODES := DefaultParam .CL2_RUN_ON_ARM_NODES false}} apiVersion: apps/v1 @@ -32,7 +31,7 @@ spec: - env: - name: ENV_VAR value: {{$EnvVar}} - image: {{$Image}} + image: {{.Image}} imagePullPolicy: IfNotPresent name: {{.Name}} ports: diff --git a/clusterloader2/testing/load/statefulset.yaml b/clusterloader2/testing/load/statefulset.yaml index 174d3c66cb..54ecba41c7 100644 --- a/clusterloader2/testing/load/statefulset.yaml +++ b/clusterloader2/testing/load/statefulset.yaml @@ -1,7 +1,7 @@ {{$HostNetworkMode := DefaultParam .CL2_USE_HOST_NETWORK_PODS false}} {{$EnablePVs := DefaultParam .CL2_ENABLE_PVS true}} {{$RUN_ON_ARM_NODES := DefaultParam .CL2_RUN_ON_ARM_NODES false}} -{{$Image := DefaultParam .Image "registry.k8s.io/pause:3.9"}} +{{$Image := DefaultParam .Image "pause:3.9"}} apiVersion: apps/v1 kind: StatefulSet @@ -26,7 +26,7 @@ spec: hostNetwork: {{$HostNetworkMode}} containers: - name: {{.Name}} - image: {{$Image}} + image: {{.RegistryK8sRepo}}/{{$Image}} ports: - containerPort: 80 name: web