[MM-24469] deployment/terraform: add an option for saving agent output (

#249)

* deployment/terraform: add util command for saving agent output

* deployment/terraform: revert log command

* deployment/terraform: add logging option for service cmd

* config/deployer.default: make EnableAgentLogs true by default

* Update config/deployer.default.json

* deployment/terraform: apply review comments

config/deployer.default.json

@@ -4,6 +4,7 @@
   "AppInstanceType": "c5.xlarge",
   "AgentInstanceCount": 2,
   "AgentInstanceType": "t3.xlarge",
+  "EnableAgentFullLogs": true,
   "ProxyInstanceType": "m4.xlarge",
   "SSHPublicKey": "~/.ssh/id_rsa.pub",
   "DBInstanceCount": 1,

deployment/config.go

@@ -18,18 +18,19 @@ import (
 // Config contains the necessary data
 // to deploy and provision a load test environment.
 type Config struct {
-	ClusterName        string // Name of the cluster.
-	AppInstanceCount   int    // Number of application instances.
-	AppInstanceType    string // Type of the EC2 instance for app.
-	AgentInstanceCount int    // Number of agents, first agent and coordinator will share the same instance.
-	AgentInstanceType  string // Type of the EC2 instance for agent.
-	ProxyInstanceType  string // Type of the EC2 instance for proxy.
-	SSHPublicKey       string // Path to the SSH public key.
-	DBInstanceCount    int    // Number of DB instances.
-	DBInstanceType     string // Type of the DB instance.
-	DBInstanceEngine   string // Type of the DB instance - postgres or mysql.
-	DBUserName         string // Username to connect to the DB.
-	DBPassword         string // Password to connect to the DB.
+	ClusterName         string // Name of the cluster.
+	AppInstanceCount    int    // Number of application instances.
+	AppInstanceType     string // Type of the EC2 instance for app.
+	AgentInstanceCount  int    // Number of agents, first agent and coordinator will share the same instance.
+	AgentInstanceType   string // Type of the EC2 instance for agent.
+	EnableAgentFullLogs bool   // Logs the command output (stdout & stderr) to home directory.
+	ProxyInstanceType   string // Type of the EC2 instance for proxy.
+	SSHPublicKey        string // Path to the SSH public key.
+	DBInstanceCount     int    // Number of DB instances.
+	DBInstanceType      string // Type of the DB instance.
+	DBInstanceEngine    string // Type of the DB instance - postgres or mysql.
+	DBUserName          string // Username to connect to the DB.
+	DBPassword          string // Password to connect to the DB.
 	// URL from where to download Mattermost release.
 	// This can also point to a local binary path if the user wants to run loadtest
 	// on a custom build. The path should be prefixed with "file://". In that case,

deployment/terraform/agent.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"text/template"
 
 	"github.com/mattermost/mattermost-load-test-ng/deployment/terraform/ssh"
 	"github.com/mattermost/mattermost-load-test-ng/loadtest"
@@ -70,8 +71,21 @@ func (t *Terraform) configureAndRunAgents(extAgent *ssh.ExtAgent, output *Output
 			}
 		}
 
+		tpl, err := template.New("").Parse(agentServiceFile)
+		if err != nil {
+			return fmt.Errorf("could not parse agent service template: %w", err)
+		}
+
+		agentCmd := baseAgentCmd
+		if t.config.EnableAgentFullLogs {
+			agentCmd = fmt.Sprintf("/bin/bash -c '%s &>> /home/ubuntu/agent.log'", baseAgentCmd)
+		}
+
+		buf := bytes.NewBufferString("")
+		tpl.Execute(buf, agentCmd)
+
 		batch := []uploadInfo{
-			{srcData: strings.TrimPrefix(agentServiceFile, "\n"), dstPath: "/lib/systemd/system/ltagent.service", msg: "Uploading agent service file"},
+			{srcData: strings.TrimPrefix(buf.String(), "\n"), dstPath: "/lib/systemd/system/ltagent.service", msg: "Uploading agent service file"},
 			{srcData: strings.TrimPrefix(sysctlConfig, "\n"), dstPath: "/etc/sysctl.conf"},
 			{srcData: strings.TrimPrefix(limitsConfig, "\n"), dstPath: "/etc/security/limits.conf"},
 			{srcData: strings.TrimPrefix(coordinatorServiceFile, "\n"), dstPath: "/lib/systemd/system/ltcoordinator.service", msg: "Uploading coordinator service file"},

deployment/terraform/strings.go

@@ -136,6 +136,8 @@ net.ipv4.ip_local_port_range = 1024 65000
 net.ipv4.tcp_fin_timeout = 30
 `
 
+const baseAgentCmd = `/home/ubuntu/mattermost-load-test-ng/bin/ltagent server`
+
 const agentServiceFile = `
 [Unit]
 Description=Mattermost Load Test Agent
@@ -144,7 +146,7 @@ After=network.target
 [Service]
 Type=simple
 Environment="GOGC=50"
-ExecStart=/home/ubuntu/mattermost-load-test-ng/bin/ltagent server
+ExecStart={{ printf "%s" .}}
 Restart=always
 RestartSec=1
 WorkingDirectory=/home/ubuntu/mattermost-load-test-ng

docs/deployer_config.md

@@ -30,6 +30,12 @@ The number of load-test agent instances. The first instance will also host the [
 
 The type of the EC2 instance of the loadtest agent. See type [here](https://aws.amazon.com/ec2/instance-types/).
 
+## EnableAgentFullLogs
+
+*bool*
+
+Allows to log the agent service command output (`stdout` & `stderr`) to home directory.
+
 ## ProxyInstanceType
 
 *string*