feat(load-test): Use AWS s3 for tekton results in dev mode

Signed-off-by: Pavel Macík <[email protected]>

cmd/loadTests.go

@@ -219,7 +219,7 @@ func (u *UserAppsCompsMap) GetIntegrationTestScenarios(userName, appName string)
 
 var (
 	componentRepoUrl              string = "https://github.com/devfile-samples/devfile-sample-code-with-quarkus"
-	componentDockerFilePath       string = "src/main/docker/Dockerfile.jvm.staged"
+	componentDockerfilePath       string = "src/main/docker/Dockerfile.jvm.staged"
 	componentsCount               int    = 1
 	usernamePrefix                string = "testuser"
 	numberOfUsers                 int
@@ -248,7 +248,6 @@ var (
 	UserCreationTimeMaxPerThread         []time.Duration
 	ApplicationCreationTimeMaxPerThread  []time.Duration
 	ItsCreationTimeMaxPerThread          []time.Duration
-	CDQCreationTimeMaxPerThread          []time.Duration
 	ComponentCreationTimeMaxPerThread    []time.Duration
 	PipelineRunSucceededTimeMaxPerThread []time.Duration
 
@@ -258,7 +257,6 @@ var (
 	UserCreationTimeSumPerThread          []time.Duration
 	ApplicationCreationTimeSumPerThread   []time.Duration
 	ItsCreationTimeSumPerThread           []time.Duration
-	CDQCreationTimeSumPerThread           []time.Duration
 	ComponentCreationTimeSumPerThread     []time.Duration
 	PipelineRunSucceededTimeSumPerThread  []time.Duration
 	PipelineRunFailedTimeSumPerThread     []time.Duration
@@ -272,7 +270,6 @@ var (
 	SuccessfulUserCreationsPerThread                []int64
 	SuccessfulApplicationCreationsPerThread         []int64
 	SuccessfulItsCreationsPerThread                 []int64
-	SuccessfulCDQCreationsPerThread                 []int64
 	SuccessfulComponentCreationsPerThread           []int64
 	SuccessfulPipelineRunsPerThread                 []int64
 	SuccessfulDeploymentsPerThread                  []int64
@@ -282,7 +279,6 @@ var (
 	FailedUserCreationsPerThread                []int64
 	FailedApplicationCreationsPerThread         []int64
 	FailedItsCreationsPerThread                 []int64
-	FailedCDQCreationsPerThread                 []int64
 	FailedComponentCreationsPerThread           []int64
 	FailedPipelineRunsPerThread                 []int64
 	FailedDeploymentsPerThread                  []int64
@@ -322,6 +318,7 @@ type LogData struct {
 	MachineName                       string  `json:"machineName"`
 	BinaryDetails                     string  `json:"binaryDetails"`
 	ComponentRepoUrl                  string  `json:"componentRepoUrl"`
+	ComponentDockerfilePath           string  `json:"componentDockerfilePath"`
 	ComponentsCount                   int     `json:"componentsCount"`
 	NumberOfThreads                   int     `json:"threads"`
 	NumberOfUsersPerThread            int     `json:"usersPerThread"`
@@ -335,8 +332,6 @@ type LogData struct {
 	MaxTimeToCreateApplications       float64 `json:"createApplicationsTimeMax"`
 	AverageTimeToCreateIts            float64 `json:"createItsTimeAvg"`
 	MaxTimeToCreateIts                float64 `json:"createItsTimeMax"`
-	AverageTimeToCreateCDQs           float64 `json:"createCDQsTimeAvg"`
-	MaxTimeToCreateCDQs               float64 `json:"createCDQsTimeMax"`
 	AverageTimeToCreateComponents     float64 `json:"createComponentsTimeAvg"`
 	MaxTimeToCreateComponents         float64 `json:"createComponentsTimeMax"`
 	AverageTimeToRunPipelineSucceeded float64 `json:"runPipelineSucceededTimeAvg"`
@@ -361,9 +356,6 @@ type LogData struct {
 	ItsCreationSuccessCount         int64   `json:"createItsSuccesses"`
 	ItsCreationFailureCount         int64   `json:"createItsFailures"`
 	ItsCreationFailureRate          float64 `json:"createItsFailureRate"`
-	CDQCreationSuccessCount         int64   `json:"createCDQsSuccesses"`
-	CDQCreationFailureCount         int64   `json:"createCDQsFailures"`
-	CDQCreationFailureRate          float64 `json:"createCDQsFailureRate"`
 	ComponentCreationSuccessCount   int64   `json:"createComponentsSuccesses"`
 	ComponentCreationFailureCount   int64   `json:"createComponentsFailures"`
 	ComponentCreationFailureRate    float64 `json:"createComponentsFailureRate"`
@@ -395,7 +387,6 @@ type JourneyContext struct {
 	AppStudioUsersBar            *uiprogress.Bar
 	ApplicationsBar              *uiprogress.Bar
 	ItsBar                       *uiprogress.Bar
-	CDQsBar                      *uiprogress.Bar
 	ComponentsBar                *uiprogress.Bar
 	PipelinesBar                 *uiprogress.Bar
 	IntegrationTestsPipelinesBar *uiprogress.Bar
@@ -436,7 +427,6 @@ var rootCmd = &cobra.Command{
 var AppStudioUsersBar *uiprogress.Bar
 var ApplicationsBar *uiprogress.Bar
 var itsBar *uiprogress.Bar
-var CDQsBar *uiprogress.Bar
 var ComponentsBar *uiprogress.Bar
 var PipelinesBar *uiprogress.Bar
 var IntegrationTestsPipelinesBar *uiprogress.Bar
@@ -451,6 +441,7 @@ func ExecuteLoadTest() {
 
 func init() {
 	rootCmd.Flags().StringVar(&componentRepoUrl, "component-repo", componentRepoUrl, "the component repo URL to be used")
+	rootCmd.Flags().StringVar(&componentDockerfilePath, "component-dockerfile-path", componentDockerfilePath, "the path to Dockerfile within component repo")
 	rootCmd.Flags().IntVar(&componentsCount, "components-count", componentsCount, "number of components to create per application")
 	rootCmd.Flags().StringVar(&usernamePrefix, "username", usernamePrefix, "the prefix used for usersignup names")
 	rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "if 'debug' traces should be displayed in the console")
@@ -590,6 +581,7 @@ func setup(cmd *cobra.Command, args []string) {
 		MachineName:                 machineName,
 		BinaryDetails:               binaryDetails,
 		ComponentRepoUrl:            componentRepoUrl,
+		ComponentDockerfilePath:     componentDockerfilePath,
 		ComponentsCount:             componentsCount,
 		NumberOfThreads:             threadCount,
 		NumberOfUsersPerThread:      numberOfUsers,
@@ -623,11 +615,6 @@ func setup(cmd *cobra.Command, args []string) {
 		})
 		itsBar = itsProgress
 
-		cdqProgress := uip.AddBar(overallCount).AppendCompleted().PrependFunc(func(b *uiprogress.Bar) string {
-			return strutil.PadLeft(fmt.Sprintf("Creating AppStudio CDQs (%d/%d) [%d failed]", b.Current(), overallCount, sumFromArray(FailedCDQCreationsPerThread)), barLength, ' ')
-		})
-		CDQsBar = cdqProgress
-
 		componentProgress := uip.AddBar(overallCount).AppendCompleted().PrependFunc(func(b *uiprogress.Bar) string {
 			return strutil.PadLeft(fmt.Sprintf("Creating AppStudio Components (%d/%d) [%d failed]", b.Current(), overallCount, sumFromArray(FailedComponentCreationsPerThread)), barLength, ' ')
 		})
@@ -660,7 +647,6 @@ func setup(cmd *cobra.Command, args []string) {
 	UserCreationTimeMaxPerThread = make([]time.Duration, threadCount)
 	ApplicationCreationTimeMaxPerThread = make([]time.Duration, threadCount)
 	ItsCreationTimeMaxPerThread = make([]time.Duration, threadCount)
-	CDQCreationTimeMaxPerThread = make([]time.Duration, threadCount)
 	ComponentCreationTimeMaxPerThread = make([]time.Duration, threadCount)
 	PipelineRunSucceededTimeMaxPerThread = make([]time.Duration, threadCount)
 
@@ -670,7 +656,6 @@ func setup(cmd *cobra.Command, args []string) {
 	UserCreationTimeSumPerThread = make([]time.Duration, threadCount)
 	ApplicationCreationTimeSumPerThread = make([]time.Duration, threadCount)
 	ItsCreationTimeSumPerThread = make([]time.Duration, threadCount)
-	CDQCreationTimeSumPerThread = make([]time.Duration, threadCount)
 	ComponentCreationTimeSumPerThread = make([]time.Duration, threadCount)
 	PipelineRunSucceededTimeSumPerThread = make([]time.Duration, threadCount)
 	PipelineRunFailedTimeSumPerThread = make([]time.Duration, threadCount)
@@ -685,7 +670,6 @@ func setup(cmd *cobra.Command, args []string) {
 	SuccessfulUserCreationsPerThread = make([]int64, threadCount)
 	SuccessfulApplicationCreationsPerThread = make([]int64, threadCount)
 	SuccessfulItsCreationsPerThread = make([]int64, threadCount)
-	SuccessfulCDQCreationsPerThread = make([]int64, threadCount)
 	SuccessfulComponentCreationsPerThread = make([]int64, threadCount)
 	SuccessfulPipelineRunsPerThread = make([]int64, threadCount)
 	SuccessfulPVCCreationsPerThread = make([]int64, threadCount)
@@ -696,7 +680,6 @@ func setup(cmd *cobra.Command, args []string) {
 	FailedUserCreationsPerThread = make([]int64, threadCount)
 	FailedApplicationCreationsPerThread = make([]int64, threadCount)
 	FailedItsCreationsPerThread = make([]int64, threadCount)
-	FailedCDQCreationsPerThread = make([]int64, threadCount)
 	FailedComponentCreationsPerThread = make([]int64, threadCount)
 	FailedPipelineRunsPerThread = make([]int64, threadCount)
 
@@ -718,7 +701,6 @@ func setup(cmd *cobra.Command, args []string) {
 			AppStudioUsersBar:            AppStudioUsersBar,
 			ApplicationsBar:              ApplicationsBar,
 			ItsBar:                       itsBar,
-			CDQsBar:                      CDQsBar,
 			ComponentsBar:                ComponentsBar,
 			PipelinesBar:                 PipelinesBar,
 			IntegrationTestsPipelinesBar: IntegrationTestsPipelinesBar,
@@ -804,24 +786,6 @@ func setup(cmd *cobra.Command, args []string) {
 	itsCreationFailureRate := float64(itsCreationFailureCount) / float64(overallCount)
 	logData.ItsCreationFailureRate = itsCreationFailureRate
 
-	// Compiling data about CDQs
-	cdqCreationSuccessCount := sumFromArray(SuccessfulCDQCreationsPerThread)
-	logData.CDQCreationSuccessCount = cdqCreationSuccessCount
-
-	cdqCreationFailureCount := sumFromArray(FailedCDQCreationsPerThread)
-	logData.CDQCreationFailureCount = cdqCreationFailureCount
-
-	averageTimeToCreateCDQs := float64(0)
-	if cdqCreationSuccessCount > 0 {
-		averageTimeToCreateCDQs = sumDurationFromArray(CDQCreationTimeSumPerThread).Seconds() / float64(cdqCreationSuccessCount)
-	}
-	logData.AverageTimeToCreateCDQs = averageTimeToCreateCDQs
-
-	logData.MaxTimeToCreateCDQs = maxDurationFromArray(CDQCreationTimeMaxPerThread).Seconds()
-
-	cdqCreationFailureRate := float64(cdqCreationFailureCount) / float64(overallCount)
-	logData.CDQCreationFailureRate = cdqCreationFailureRate
-
 	// Compiling data about Components
 	componentCreationSuccessCount := sumFromArray(SuccessfulComponentCreationsPerThread)
 	logData.ComponentCreationSuccessCount = componentCreationSuccessCount
@@ -831,7 +795,7 @@ func setup(cmd *cobra.Command, args []string) {
 
 	averageTimeToCreateComponents := float64(0)
 	if componentCreationSuccessCount > 0 {
-		averageTimeToCreateComponents = sumDurationFromArray(ComponentCreationTimeSumPerThread).Seconds() / float64(cdqCreationSuccessCount)
+		averageTimeToCreateComponents = sumDurationFromArray(ComponentCreationTimeSumPerThread).Seconds() / float64(componentCreationSuccessCount)
 	}
 	logData.AverageTimeToCreateComponents = averageTimeToCreateComponents
 
@@ -912,7 +876,7 @@ func setup(cmd *cobra.Command, args []string) {
 	deploymentFailureRate := float64(deploymentFailureCount) / float64(overallCount*componentsCount)
 	logData.DeploymentFailureRate = deploymentFailureRate
 
-	workloadKPI := logData.AverageTimeToCreateApplications + logData.AverageTimeToCreateCDQs + logData.AverageTimeToCreateComponents + logData.AverageTimeToRunPipelineSucceeded + logData.AverageTimeToDeploymentSucceeded
+	workloadKPI := logData.AverageTimeToCreateApplications + logData.AverageTimeToCreateComponents + logData.AverageTimeToRunPipelineSucceeded + logData.AverageTimeToDeploymentSucceeded
 	logData.WorkloadKPI = workloadKPI
 	if stage {
 		StageCleanup(journeyContexts)
@@ -926,7 +890,6 @@ func setup(cmd *cobra.Command, args []string) {
 	klog.Infof("Avg/max time to spin up users: %.2f s/%.2f s", averageTimeToSpinUpUsers, logData.MaxTimeToSpinUpUsers)
 	klog.Infof("Avg/max time to create application: %.2f s/%.2f s", averageTimeToCreateApplications, logData.MaxTimeToCreateApplications)
 	klog.Infof("Avg/max time to create integration test: %.2f s/%.2f s", averageTimeToCreateIts, logData.MaxTimeToCreateIts)
-	klog.Infof("Avg/max time to create cdq: %.2f s/%.2f s", averageTimeToCreateCDQs, logData.MaxTimeToCreateCDQs)
 	klog.Infof("Avg/max time to create component: %.2f s/%.2f s", averageTimeToCreateComponents, logData.MaxTimeToCreateComponents)
 	klog.Infof("Avg/max time to complete pipelinesrun: %.2f s/%.2f s", averageTimeToRunPipelineSucceeded, logData.MaxTimeToRunPipelineSucceeded)
 	klog.Infof("Avg/max time to complete integration test: %.2f s/%.2f s", IntegrationTestsAverageTimeToRunPipelineSucceeded, logData.IntegrationTestsMaxTimeToRunPipelineSucceeded)
@@ -943,7 +906,6 @@ func setup(cmd *cobra.Command, args []string) {
 	klog.Infof("Number of times application creation worked/failed: %d/%d (%.2f %%)", applicationCreationSuccessCount, applicationCreationFailureCount, applicationCreationFailureRate*100)
 	klog.Infof("Number of times integration tests creation worked/failed: %d/%d (%.2f %%)", itsCreationSuccessCount, itsCreationFailureCount, itsCreationFailureRate*100)
 
-	klog.Infof("Number of times cdq creation worked/failed: %d/%d (%.2f %%)", cdqCreationSuccessCount, cdqCreationFailureCount, cdqCreationFailureRate*100)
 	klog.Infof("Number of times component creation worked/failed: %d/%d (%.2f %%)", componentCreationSuccessCount, componentCreationFailureCount, componentCreationFailureRate*100)
 	klog.Infof("Number of times pipeline run worked/failed: %d/%d (%.2f %%)", pipelineRunSuccessCount, pipelineRunFailureCount, pipelineRunFailureRate*100)
 	klog.Infof("Number of times integration tests' pipeline run worked/failed: %d/%d (%.2f %%)", integrationTestsPipelineRunSuccessCount, integrationTestsPipelineRunFailureCount, IntegrationTestsPipelineRunFailureRate*100)
@@ -1478,7 +1440,7 @@ func (h *ConcreteHandlerResources) handleComponentCreation(ctx *JourneyContext,
 					GitSource: &appstudioApi.GitSource{
 						URL:           componentRepoUrl,
 						Revision:      "",
-						DockerfileURL: componentDockerFilePath,
+						DockerfileURL: componentDockerfilePath,
 					},
 				},
 			},

tests/load-tests/ci-scripts/collect-results.sh

@@ -15,7 +15,7 @@ source "./tests/load-tests/ci-scripts/user-prefix.sh"
 
 echo "Collecting load test results"
 load_test_log=$ARTIFACT_DIR/load-tests.log
-find "$output_dir" -type f -name '*.log' -exec cp -vf {} "${ARTIFACT_DIR}" \;
+find "$output_dir" -type f -name '*.logs?' -exec cp -vf {} "${ARTIFACT_DIR}" \;
 find "$output_dir" -type f -name 'load-tests.json' -exec cp -vf {} "${ARTIFACT_DIR}" \;
 find "$output_dir" -type f -name 'gh-rate-limits-remaining.csv' -exec cp -vf {} "${ARTIFACT_DIR}" \;
 find "$output_dir" -type f -name '*.pprof' -exec cp -vf {} "${ARTIFACT_DIR}" \;

tests/load-tests/ci-scripts/load-test.sh

@@ -11,7 +11,7 @@ pushd "${2:-./tests/load-tests}"
 
 source "./ci-scripts/user-prefix.sh"
 
-export QUAY_E2E_ORGANIZATION MY_GITHUB_ORG GITHUB_TOKEN TEKTON_PERF_ENABLE_PROFILING TEKTON_PERF_ENABLE_CPU_PROFILING TEKTON_PERF_ENABLE_MEMORY_PROFILING TEKTON_PERF_PROFILE_CPU_PERIOD KUBE_SCHEDULER_LOG_LEVEL
+export QUAY_E2E_ORGANIZATION MY_GITHUB_ORG GITHUB_USER GITHUB_TOKEN TEKTON_PERF_ENABLE_PROFILING TEKTON_PERF_ENABLE_CPU_PROFILING TEKTON_PERF_ENABLE_MEMORY_PROFILING TEKTON_PERF_PROFILE_CPU_PERIOD KUBE_SCHEDULER_LOG_LEVEL
 QUAY_E2E_ORGANIZATION=$(cat /usr/local/ci-secrets/redhat-appstudio-load-test/quay-org)
 MY_GITHUB_ORG=$(cat /usr/local/ci-secrets/redhat-appstudio-load-test/github-org)
 
@@ -26,6 +26,8 @@ for kv in "${kvs[@]}"; do
 done
 echo >>"$rate_limits_csv"
 
+echo -e "[INFO] Start tests with user: ${GITHUB_USER}"
+
 while true; do
     timestamp=$(printf "%s" "$(date -u +'%FT%T')")
     echo -n "$timestamp" >>"$rate_limits_csv"

tests/load-tests/ci-scripts/max-concurrency/cluster_read_config.yaml

@@ -45,4 +45,26 @@
 
 {{ monitor_pod('tekton-results', 'tekton-results-watcher', 20, '-.*') }}
 {{ monitor_pod_container('tekton-results', 'tekton-results-watcher', 'watcher', 20, '-.*') }}
+{{ monitor_pod('tekton-results', 'tekton-results-api', 20, '-.*') }}
+{{ monitor_pod_container('tekton-results', 'tekton-results-api', 'api', 20, '-.*') }}
 {{ pv_stats('tekton-results', 'data-postgres-postgresql-0', 20) }}
+
+- name: measurements.tekton-results-watcher.watcher_workqueue_depth
+  monitoring_query: sum(watcher_workqueue_depth{job="tekton-results-watcher"})
+  monitoring_step: 20
+
+- name: measurements.tekton-results-watcher.watcher_reconcile_latency_bucket
+  monitoring_query: histogram_quantile(0.99, sum(rate(watcher_reconcile_latency_bucket{job="tekton-results-watcher"}[30m])) by (le) ) / 1000
+  monitoring_step: 20
+
+- name: measurements.cluster_cpu_usage_seconds_total_rate
+  monitoring_query: sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=""})
+  monitoring_step: 20
+
+- name: measurements.cluster_memory_usage_rss_total
+  monitoring_query: sum(container_memory_rss{job="kubelet", metrics_path="/metrics/cadvisor", cluster="", container!=""})
+  monitoring_step: 20
+
+- name: measurements.cluster_disk_throughput_total
+  monitoring_query: sum (rate(container_fs_reads_bytes_total{id!="", device=~"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+", cluster=""}[5m]) + rate(container_fs_writes_bytes_total{id!="", device=~"(/dev.+)|mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+", cluster=""}[5m]))
+  monitoring_step: 20