[MM-24005] Increase limits for ltagent (#220)

* Increase limits for ltagent instances

* Remove unused settings from config

* Decrease idle timeout for user http connections

* Implement uploadBatch utility function

* Upload sysctl config to agent instances

config/config.default.json

@@ -3,10 +3,7 @@
     "ServerURL": "http://localhost:8065",
     "WebSocketURL": "ws://localhost:8065",
     "AdminEmail": "[email protected]",
-    "AdminPassword": "Sys@dmin-sample1",
-    "MaxIdleConns": 100,
-    "MaxIdleConnsPerHost": 128,
-    "IdleConnTimeoutMilliseconds": 90000
+    "AdminPassword": "Sys@dmin-sample1"
   },
   "UserControllerConfiguration": {
     "Type": "simple",

deployment/terraform/agent.go

@@ -66,21 +66,23 @@ func (t *Terraform) configureAndRunAgents(extAgent *ssh.ExtAgent, output *terraf
 			}
 		}
 
-		mlog.Info("Uploading agent service file")
-		rdr := strings.NewReader(strings.TrimSpace(agentServiceFile))
-		if out, err := sshc.Upload(rdr, "/lib/systemd/system/ltagent.service", true); err != nil {
-			return fmt.Errorf("error uploading file, output: %q: %w", out, err)
+		batch := []uploadInfo{
+			{srcData: strings.TrimPrefix(agentServiceFile, "\n"), dstPath: "/lib/systemd/system/ltagent.service", msg: "Uploading agent service file"},
+			{srcData: strings.TrimPrefix(sysctlConfig, "\n"), dstPath: "/etc/sysctl.conf"},
+			{srcData: strings.TrimPrefix(limitsConfig, "\n"), dstPath: "/etc/security/limits.conf"},
+			{srcData: strings.TrimPrefix(coordinatorServiceFile, "\n"), dstPath: "/lib/systemd/system/ltcoordinator.service", msg: "Uploading coordinator service file"},
 		}
 
-		mlog.Info("Uploading coordinator service file")
-		rdr = strings.NewReader(strings.TrimSpace(coordinatorServiceFile))
-		if out, err := sshc.Upload(rdr, "/lib/systemd/system/ltcoordinator.service", true); err != nil {
-			return fmt.Errorf("error uploading file, output: %q: %w", out, err)
+		if err := uploadBatch(sshc, batch); err != nil {
+			return fmt.Errorf("batch upload failed: %w", err)
+		}
+
+		if out, err := sshc.RunCommand("sudo sysctl -p"); err != nil {
+			return fmt.Errorf("error running command, got output: %q: %w", out, err)
 		}
 
 		mlog.Info("Starting agent")
-		cmd := "sudo service ltagent start"
-		if out, err := sshc.RunCommand(cmd); err != nil {
+		if out, err := sshc.RunCommand("sudo service ltagent start"); err != nil {
 			return fmt.Errorf("error running command, got output: %q: %w", out, err)
 		}
 	}

deployment/terraform/create.go

@@ -267,21 +267,15 @@ func (t *Terraform) setupProxyServer(output *terraformOutput, extAgent *ssh.ExtA
 			backends += "server " + addr.PrivateIP + ":8065;\n"
 		}
 
-		files := []struct {
-			content string
-			path    string
-		}{
-			{content: strings.TrimSpace(fmt.Sprintf(nginxSiteConfig, backends)), path: "/etc/nginx/sites-available/mattermost"},
-			{content: strings.TrimSpace(sysctlConfig), path: "/etc/sysctl.conf"},
-			{content: strings.TrimSpace(nginxConfig), path: "/etc/nginx/nginx.conf"},
-			{content: strings.TrimSpace(limitsConfig), path: "/etc/security/limits.conf"},
+		batch := []uploadInfo{
+			{srcData: strings.TrimSpace(fmt.Sprintf(nginxSiteConfig, backends)), dstPath: "/etc/nginx/sites-available/mattermost"},
+			{srcData: strings.TrimSpace(sysctlConfig), dstPath: "/etc/sysctl.conf"},
+			{srcData: strings.TrimSpace(nginxConfig), dstPath: "/etc/nginx/nginx.conf"},
+			{srcData: strings.TrimSpace(limitsConfig), dstPath: "/etc/security/limits.conf"},
 		}
-		for _, fileInfo := range files {
-			rdr := strings.NewReader(fileInfo.content)
-			if out, err := sshc.Upload(rdr, fileInfo.path, true); err != nil {
-				mlog.Error("error uploading file", mlog.String("output", string(out)), mlog.Err(err), mlog.String("file", fileInfo.path))
-				return
-			}
+		if err := uploadBatch(sshc, batch); err != nil {
+			mlog.Error("batch upload failed", mlog.Err(err))
+			return
 		}
 
 		cmd := "sudo sysctl -p && sudo service nginx reload"

deployment/terraform/strings.go

@@ -149,6 +149,7 @@ RestartSec=1
 WorkingDirectory=/home/ubuntu/mattermost-load-test-ng
 User=ubuntu
 Group=ubuntu
+LimitNOFILE=65536
 
 [Install]
 WantedBy=multi-user.target

deployment/terraform/utils.go

@@ -0,0 +1,41 @@
+// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package terraform
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/mattermost/mattermost-load-test-ng/deployment/terraform/ssh"
+
+	"github.com/mattermost/mattermost-server/v5/mlog"
+)
+
+type uploadInfo struct {
+	msg     string
+	srcData string
+	dstPath string
+}
+
+func uploadBatch(sshc *ssh.Client, batch []uploadInfo) error {
+	if sshc == nil {
+		return errors.New("sshc should not be nil")
+	}
+	if len(batch) == 0 {
+		return errors.New("batch should not be empty")
+	}
+
+	for _, info := range batch {
+		if info.msg != "" {
+			mlog.Info(info.msg)
+		}
+		rdr := strings.NewReader(info.srcData)
+		if out, err := sshc.Upload(rdr, info.dstPath, true); err != nil {
+			return fmt.Errorf("error uploading file, dstPath: %s, output: %q: %w", info.dstPath, out, err)
+		}
+	}
+
+	return nil
+}

docs/loadtest_config.md

@@ -27,24 +27,6 @@ The e-mail for the system admin of the target Mattermost instance.
 
 The password for the system admin of the target Mattermost instance.
 
-### MaxIdleConns
-
-*int*
-
-The maximum number of idle connections held open from the load-test agent to all servers.
-
-### MaxIdleConnsPerHost
-
-*int*
-
-The maximum number of idle connections held open from the load-test agent to any given server.
-
-### IdleConnTimeoutMilliseconds
-
-*int*
-
-The number of milliseconds to leave an idle connection open between the load-test agent an another server.
-
 ## UserControllerConfiguration
 
 ### Type

loadtest/config.go

@@ -14,13 +14,10 @@ import (
 )
 
 type ConnectionConfiguration struct {
-	ServerURL                   string
-	WebSocketURL                string
-	AdminEmail                  string
-	AdminPassword               string
-	MaxIdleConns                int
-	MaxIdleConnsPerHost         int
-	IdleConnTimeoutMilliseconds int
+	ServerURL     string
+	WebSocketURL  string
+	AdminEmail    string
+	AdminPassword string
 }
 
 // IsValid reports whether a given ConnectionConfiguration is valid or not.
@@ -181,9 +178,6 @@ func ReadConfig(configFilePath string) (*Config, error) {
 	v.SetDefault("LogSettings.FileLevel", "INFO")
 	v.SetDefault("LogSettings.FileJson", true)
 	v.SetDefault("LogSettings.FileLocation", "loadtest.log")
-	v.SetDefault("ConnectionConfiguration.MaxIdleConns", 100)
-	v.SetDefault("ConnectionConfiguration.MaxIdleConnsPerHost", 128)
-	v.SetDefault("ConnectionConfiguration.IdleConnTimeoutMilliseconds", 90000)
 
 	if configFilePath != "" {
 		v.SetConfigFile(configFilePath)

loadtest/user/userentity/user.go

@@ -66,7 +66,7 @@ func New(store store.MutableUserStore, config Config) *UserEntity {
 		}).DialContext,
 		MaxIdleConns:          1000,
 		MaxIdleConnsPerHost:   1000,
-		IdleConnTimeout:       30 * time.Second,
+		IdleConnTimeout:       15 * time.Second,
 		TLSHandshakeTimeout:   10 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
 	}