Feature/multi loader

cmd/loader.go

@@ -51,6 +51,7 @@ var (
 	verbosity     = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]")
 	iatGeneration = flag.Bool("iatGeneration", false, "Generate iats only or run invocations as well")
 	iatFromFile   = flag.Bool("generated", false, "True if iats were already generated")
+	dryRun		  = flag.Bool("dryRun", false, "Dry run mode - do not deploy functions or generate invocations")
 )
 
 func init() {
@@ -104,6 +105,10 @@ func main() {
 		log.Fatal("Unsupported platform!")
 	}
 
+	if cfg.Platform == "Knative" || cfg.Platform == "Knative-RPS" {
+		common.CheckCPULimit(cfg.CPULimit)
+	}
+
 	if !strings.HasSuffix(cfg.Platform, "-RPS") {
 		runTraceMode(&cfg, *iatFromFile, *iatGeneration)
 	} else {
@@ -200,6 +205,10 @@ func runTraceMode(cfg *config.LoaderConfiguration, readIATFromFile bool, justGen
 		Functions: functions,
 	})
 
+	if *dryRun {
+		return
+	}
+
 	log.Infof("Using %s as a service YAML specification file.\n", experimentDriver.Configuration.YAMLPath)
 
 	experimentDriver.RunExperiment(justGenerateIAT, readIATFromFile)

docs/loader.md

@@ -119,6 +119,8 @@ $ go run cmd/loader.go --config cmd/config_knative_trace.json
 
 Additionally, one can specify log verbosity argument as `--verbosity [info, debug, trace]`. The default value is `info`.
 
+To execute in a dry run mode without generating any load, set the `--dry-run` flag to `true`. This is useful for testing and validating configurations without executing actual requests.
+
 For to configure the workload for load generator, please refer to `docs/configuration.md`.
 
 There are a couple of constants that should not be exposed to the users. They can be examined and changed

docs/multi_loader.md

@@ -0,0 +1,107 @@
+# Multi-Loader
+
+A wrapper around loader to run multiple experiments at once with additional features like validation, dry-run, log collection
+
+## Prerequisites
+As a wrapper around loader, multi-loader requires the initial cluster setup to be completed. See [vHive Loader to create a cluster](https://github.com/vhive-serverless/invitro/blob/main/docs/loader.md#create-a-cluster)
+
+## Configuration
+### Multi-Loader Configuration
+| Parameter name      | Data type          | Possible values | Default value | Description                                                |
+|---------------------|--------------------|-----------------|---------------|------------------------------------------------------------|
+| Studies         | []LoaderStudy | N/A             | N/A           | A list of loader studies with their respective configurations. See [LoaderStudy](#loaderstudy) |
+| BaseConfigPath      | string             | "tools/multi_loader/base_loader_config.json" | N/A           | Path to the base configuration file                         |
+| PreScript           | string             | any bash command | ""           | (Optional) A global script that runs once before all experiments |
+| PostScript          | string             | any bash command | ""           | (Optional) A global script that runs once after all experiments  |
+
+### LoaderStudy
+| Parameter name        | Data type              | Possible values               | Default value | Description                                                        |
+|-----------------------|------------------------|-------------------------------|---------------|--------------------------------------------------------------------|
+| Config                | map[string]interface{} | Any field in [LoaderConfiguration](https://github.com/vhive-serverless/invitro/blob/main/docs/configuration.md#loader-configuration-file-format) | N/A           | The configuration for each loader experiment which overrides configurations in baseLoaderConfig                      |
+| Name                  | string                 | N/A                           | N/A           | The name of the loader experiment                                  |
+| TracesDir             | string                 | N/A                           | N/A           | Directory containing the traces for the experiment                 |
+| TracesFormat          | string                 | "data/traces/example_{}"      | N/A           | Format of the trace files **The format string "{}" is required** |
+| TraceValues           | []interface{}          | ["any", 0, 1.1]               | N/A           | Values of the trace files Replaces the "{}" in TraceFormat             |
+| OutputDir             | string                 | any                           | data/out/{Name} | (Optional) Output directory for experiment results                 |
+| Verbosity             | string                 | "info", "debug", "trace"      | "info"        | (Optional) Verbosity level for logging the experiment             |
+| IatGeneration         | bool                   | true, false                   | false         | (Optional) Whether to Generate iats only and skip invocations |
+| Generated             | bool                   | true, false                   | false         | (Optional) if iats were already generated         |
+| PreScript             | string                 | any bash Command              | ""           | (Optional) Local script that runs this specific experiment |
+| PostScript            | string                 | any bash Command              | ""           | (Optional) Local script that runs this specific experiment |
+
+> **_Important_**: Only one of the following is required:
+> 1. `TracesDir`, or
+> 2. `TracesFormat` and `TraceValues`, or
+> 3. `TracePath` within the `LoaderExperiment`'s `Config` field
+>
+> If more than one is defined, the order of precedence is as follows:  
+> 1. `TracesDir`,  
+> 2. `TracesFormat` and `TraceValues`,  
+> 3. `TracePath`
+
+> **_Note_**: 
+> The `Config` field follows the same structure as the [LoaderConfiguration](https://github.com/vhive-serverless/invitro/blob/main/docs/configuration.md#loader-configuration-file-format). 
+> Any field defined in `Config` will override the corresponding value from the configuration in `BaseConfigPath`, but only for that specific experiment. 
+> For example, if `BaseConfigPath` has `ExperimentDuration` set to 5 minutes, and you define `ExperimentDuration` as 10 minutes in `Config`, that particular experiment will run for 10 minutes instead.
+
+## Command Flags
+
+The multi-loader accepts the almost the same command-line flags as loader. 
+
+> **_Note_**: These flags will subsequently be used during the execution of loader.go for **<u>every experiment</u>**. If you would like to define these flag for specific experiments only, define it in [LoaderStudy](#loaderstudy)
+
+Available flags:
+
+- **`--multiLoaderConfig`** *(default: `tools/multi_loader/multi_loader_config.json`)*:  
+  Specifies the path to the multi-loader configuration file. This file contains settings and parameters that define how the multi-loader operates [see above](#multi-loader-configuration)
+
+- **`--verbosity`** *(default: `info`)*:  
+    Sets the logging verbosity level. You can choose from the following options:
+    - `info`: Standard information messages.
+    - `debug`: Detailed debugging messages.
+    - `trace`: Extremely verbose logging, including detailed execution traces.
+
+- **`--iatGeneration`** *(default: `false`)*:  
+  If set to `true`, the multi-loader will generate inter-arrival times (IATs) only and skip the invocation of actual workloads. This is useful for scenarios where you want to analyze or generate IATs without executing the associated load.
+
+- **`--generated`** *(default: `false`)*:  
+  Indicates whether IATs have already been generated. If set to `true`, the multi-loader will use the existing IATs instead of generating new ones.
+
+
+## Multi-loader Overall Flow
+
+1. **Initialization**
+    - Flags for configuration file path, verbosity, IAT generation, and execution mode are parsed
+    - Logger is initialized based on verbosity level
+
+3. **Experiment Execution Flow**
+    - The multi-loader runner is instantiated with the provided configuration path.
+    - A dry run is executed to validate the setup for all studies:
+        - If any dry run fails, the execution terminates.
+    - If all dry runs succeed, proceed to actual runs:
+        - Global pre-scripts are executed.
+        - Each experiment undergoes the following steps:
+            1. **Pre-Execution Setup**
+                - Experiment-specific pre-scripts are executed.
+                - Necessary directories and folders are created.
+                - Each sub-experiment is unpacked and prepared
+            2. **Experiment Invocation**
+                - The loader is executed with generated configurations and related flags
+            3. **Post-Execution Steps**
+                - Experiment-specific post-scripts are executed
+                - Cleanup tasks are performed
+
+4. **Completion**
+    - Global post-scripts are executed.
+    - Run Make Clean
+
+### How the Dry Run Works
+
+The dry run mode executes the loader with the `--dryRun` flag set to true after the unpacking of experiments defined in the multi-loader configurations.
+
+In this mode, the loader performs the following actions:
+
+- **Configuration Validation**: It verifies the experiment configurations without deploying any functions or generating invocations.
+- **Error Handling**: If a fatal error occurs at any point, the experiment will halt immediately.
+
+The purpose is to ensure that your configurations are correct and to identify any potential issues before actual execution.

pkg/common/constants.go

@@ -97,3 +97,11 @@ const (
 	AwsRegion                  = "us-east-1"
 	AwsTraceFuncRepositoryName = "invitro_trace_function_aws"
 )
+
+// CPULimits
+const (
+	CPULimit1vCPU string = "1vCPU"
+	CPULimitGCP   string = "GCP"
+)
+
+var ValidCPULimits = []string{CPULimit1vCPU, CPULimitGCP}

pkg/common/utilities.go

@@ -25,11 +25,15 @@
 package common
 
 import (
+	"encoding/json"
 	"hash/fnv"
 	"log"
 	"math/rand"
+	"os/exec"
 	"strconv"
 	"strings"
+
+	logger "github.com/sirupsen/logrus"
 )
 
 type Pair struct {
@@ -135,3 +139,41 @@ func SumNumberOfInvocations(withWarmup bool, totalDuration int, functions []*Fun
 
 	return result
 }
+
+func DeepCopy[T any](a T) (T, error) {
+	var b T
+	byt, err := json.Marshal(a)
+	if err != nil {
+		return b, err
+	}
+	err = json.Unmarshal(byt, &b)
+	return b, err
+}
+
+func RunScript(command string) {
+	if command == "" {
+		return
+	}
+	logger.Info("Running command ", command)
+	cmd, err := exec.Command("/bin/sh", command).Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	logger.Info(string(cmd))
+}
+
+func ParseLogType(logString string) string {
+	logTypeArr := strings.Split(logString, "level=")
+	if len(logTypeArr) > 1 {
+		return strings.Split(logTypeArr[1], " ")[0]
+	}
+	return "info"
+}
+
+func ParseLogMessage(logString string) string {
+	message := strings.Split(logString, "msg=")
+	if len(message) > 1 {
+		return message[1][1 : len(message[1])-1]
+	}
+	return logString
+}

pkg/common/validators.go

@@ -0,0 +1,45 @@
+package common
+
+import (
+	"bytes"
+	"net"
+	"os"
+	"os/exec"
+	"slices"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func CheckNode(node string) {
+	if !IsValidIP(node) {
+		log.Fatal("Invalid IP address for node ", node)
+	}
+	cmd := exec.Command("ssh", "-oStrictHostKeyChecking=no", "-p", "22", node, "exit")
+	// -oStrictHostKeyChecking=no -p 22
+	out, err := cmd.CombinedOutput()
+	if bytes.Contains(out, []byte("Permission denied")) || err != nil {
+		log.Error(string(out))
+		log.Fatal("Failed to connect to node ", node)
+	}
+}
+
+func CheckPath(path string) {
+	if (path) == "" {
+		return
+	}
+	_, err := os.Stat(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func IsValidIP(ip string) bool {
+	parsedIP := net.ParseIP(ip)
+	return parsedIP != nil
+}
+
+func CheckCPULimit(cpuLimit string) {
+	if !slices.Contains(ValidCPULimits, cpuLimit) {
+		log.Fatal("Invalid CPU Limit ", cpuLimit)
+	}
+}

tools/multi_loader/base_loader_config.json

@@ -0,0 +1,23 @@
+{
+  "Seed": 42,
+  "Platform": "Knative",
+  "InvokeProtocol": "grpc",
+  "YAMLSelector": "container",
+  "EndpointPort": 80,
+  "BusyLoopOnSandboxStartup": false,
+  "TracePath": "data/traces/example",
+  "Granularity": "minute",
+  "OutputPathPrefix": "data/out/experiment",
+  "IATDistribution": "exponential",
+  "CPULimit": "1vCPU",
+  "ExperimentDuration": 5,
+  "WarmupDuration": 0,
+  "IsPartiallyPanic": false,
+  "EnableZipkinTracing": false,
+  "EnableMetricsScrapping": false,
+  "MetricScrapingPeriodSeconds": 15,
+  "AutoscalingMetric": "concurrency",
+  "GRPCConnectionTimeoutSeconds": 15,
+  "GRPCFunctionTimeoutSeconds": 900,
+  "DAGMode": false
+}

tools/multi_loader/common/constants.go

@@ -0,0 +1,15 @@
+package common
+
+const (
+	TraceFormatString = "{}"
+)
+
+// Multi-loader possible collectable metrics
+const (
+	Activator  string = "activator"
+	AutoScaler string = "autoscaler"
+	TOP        string = "top"
+	Prometheus string = "prometheus"
+)
+
+var ValidCollectableMetrics = []string{Activator, AutoScaler, TOP, Prometheus}

tools/multi_loader/common/utils.go

@@ -0,0 +1,40 @@
+package common
+
+import (
+	"encoding/json"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vhive-serverless/loader/pkg/config"
+	"github.com/vhive-serverless/loader/tools/multi_loader/types"
+)
+
+func ReadMultiLoaderConfigurationFile(path string) types.MultiLoaderConfiguration {
+	byteValue, err := os.ReadFile(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var config types.MultiLoaderConfiguration
+	err = json.Unmarshal(byteValue, &config)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return config
+}
+
+func DeterminePlatformFromConfig(multiLoaderConfig types.MultiLoaderConfiguration) string {
+	// Determine platform
+	baseConfigByteValue, err := os.ReadFile(multiLoaderConfig.BaseConfigPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var loaderConfig config.LoaderConfiguration
+	// Unmarshal base configuration
+	if err = json.Unmarshal(baseConfigByteValue, &loaderConfig); err != nil {
+		log.Fatal(err)
+	}
+	return loaderConfig.Platform
+}

tools/multi_loader/common/validators.go

@@ -0,0 +1,52 @@
+package common
+
+import (
+	"path"
+	"slices"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/vhive-serverless/loader/pkg/common"
+	"github.com/vhive-serverless/loader/tools/multi_loader/types"
+)
+
+// Check general multi-loader configuration that applies to all platforms
+func CheckMultiLoaderConfig(multiLoaderConfig types.MultiLoaderConfiguration) {
+	log.Info("Checking multi-loader configuration")
+	// Check if all paths are valid
+	common.CheckPath(multiLoaderConfig.BaseConfigPath)
+	// Check each study
+	if len(multiLoaderConfig.Studies) == 0 {
+		log.Fatal("No study found in configuration file")
+	}
+	for _, study := range multiLoaderConfig.Studies {
+		// Check trace directory
+		// if configs does not have TracePath or OutputPathPreix, either TracesDir or (TracesFormat and TraceValues) should be defined along with OutputDir
+		if study.TracesDir == "" && (study.TracesFormat == "" || len(study.TraceValues) == 0) {
+			if _, ok := study.Config["TracePath"]; !ok {
+				log.Fatal("Missing one of TracesDir, TracesFormat & TraceValues, Config.TracePath in multi_loader_config ", study.Name)
+			}
+		}
+		if study.TracesFormat != "" {
+			// check if trace format contains TRACE_FORMAT_STRING
+			if !strings.Contains(study.TracesFormat, TraceFormatString) {
+				log.Fatal("Invalid TracesFormat in multi_loader_config ", study.Name, ". Missing ", TraceFormatString, " in format")
+			}
+		}
+		if study.OutputDir == "" {
+			if _, ok := study.Config["OutputPathPrefix"]; !ok {
+				log.Warn("Missing one of OutputDir or Config.OutputPathPrefix in multi_loader_config ", study.Name)
+				// set default output directory
+				study.OutputDir = path.Join("data", "out", study.Name)
+				log.Warn("Setting default output directory to ", study.OutputDir)
+			}
+		}
+	}
+	log.Info("All experiments configs are valid")
+}
+
+func CheckCollectableMetrics(metrics string) {
+	if !slices.Contains(ValidCollectableMetrics, metrics) {
+		log.Fatal("Invalid metrics ", metrics)
+	}
+}